You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by st...@apache.org on 2025/10/17 04:42:06 UTC
(doris) branch dev_rec updated (cc1f62548e1 -> aa680e5c127)
This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a change to branch dev_rec
in repository https://gitbox.apache.org/repos/asf/doris.git
discard cc1f62548e1 fix fe plan
discard 6dd5e0c531f fix some bug in fe
discard e6f133b2aec recursive cte fe part
discard 6e02ee0c2c5 add TRecCTEScanNode
discard c1e13dbc902 add result_expr_lists to TRecCTENode
discard e26120d2325 update thrift
discard 9040bad7b91 update thrift
discard 6f5d84efaf6 add rec cte thrift struct
add e3181d5cc88 [fix](nereids)ReorderJoin rule should not absorb mark join to multi join (#56837)
add 3e91ad98535 [chore](jsonb) remove unused code from jsonb_document (#56496)
add 063f3534c2e [fix](docker) modify readiness probe for ipv6 on k8s (#56917)
add 7e886ee237a [opt](fe) Add alias element_at as struct_element (#56898)
add c36027bec4e [test](sqlcache)Add sql cache with mtmv test case (#56753)
add ed674c7fe8f [test](nereids)fix prune partitions test case (#56892)
add 74109427066 [improvement](cloud) only add filecache profile for olap table in cloud deploy mode (#52118)
add 2594f8c7927 [Improve](fix) add streamjob case with create alter select delete and resume (#56849)
add fc38d1f0c65 [feature](regression-plugin) support version compare plugin with cloud (#56850)
add 16fc34e0cb2 [fix](mysql buffer) fix buffer overflow of output complex type columns (#56922)
add 7f41367e2f0 [opt](function) Make coalesce support more types (#56430)
add 807a87fe970 [fix](search) fix search result for non-tokenizer match (#56914)
add a91c067fabb [Enhancement](nereids)remove the class TableName (#56862)
add dca62cd5862 [fix](field) implement move constructor (#56803)
add ed2e880b44c [chore](code)Remove unnecessary accurate code and reduce template parameters (#56909)
add 27687930852 [feat](nereids) add table alias for catalog relation (#56802)
add d26e4953148 [Feature](func) Support function EXPORT_SET (#56798)
add 032ebaed6bd [fix](hdfs)Fix be coredump in HDFS reader during profile collection. (#56806)
add 588c37bc678 [fix](nereids)fix some show command's redirect policy (#55185)
add 483cd1c6448 [fix](nereids)forbid create sync mv based on annother sync mv (#56912)
add a5cf96230f4 [refactor](query) Clear query context once it is finished (#56911)
add fbb64e1f5ca [test](cache) add regression test cases for filecache statistics and cache eviction patterns (#54062)
add 8c1e26037fe [fix](insert) do not generate next id when fe is not master (#56959)
add 4971c1dc0b6 [feature] (nereids) generate sql digest for nereids prased stmt (#56256)
add ed6fb629e4c [fix](cloud) Fix unhandled error when get_partition_boundaries (#56935)
add 071a7e3fd30 [chore](build) Add feature list to version info (#56604)
add ac66ed0d232 [chore](checker) Add `get_all_tablet_ids` for MetaReader skeleton (#56888)
add 6ca89e31b17 [fix](schema-change) Fix distribution columns with varchar type could not increase length (#56743)
add 1d91724d5a6 [enhance](cloud)enable tablets meta url show rs_meta for cloud (#55167)
add b8688964673 [fix](iceberg) support reading iceberg partition table without partition value stored in file (#56918)
add fffbeabab16 [fix](nereids) partition topn optimization requires all window expressions are in the same order (#56622)
add 886fef76e73 [Enhancement](Snapshot) Add some snapshot interface (#56953)
add 31079ef49ca [Enhancement](nereids) remove class PartitionNames (#56947)
add c4ff082d9c3 [fix](search) fix mow support for search function (#56927)
add d1b0a19c083 [opt](inverted index) add custom analyzer support with char_filter, basic and icu tokenizer (#56243)
add 23ada5ec5c9 [fix](editlog) do not write editlog if not master (#56896)
add 996a54482d1 [opt](if) vec exec for more type in function if (#56905)
add 106e5b3f294 [refactor](insert) unified INSERT_JOB to INSERT for EtlJobType (#56780)
add 9bdc9695a3c [feature](agg) Add a knob to shuffled streaming agg (#56956)
add 7b9c3fea6cf [feat](snapshot) fix Unknown snapshot status when select snapshots (#56726)
add db0593bc591 [fix](function) Crash caused by explode function (#56991)
add e138c3e25b0 [Fix](Repository)Handle non-StorageException errors in Storage parameter validation (#56929)
add 753f3467f6e [fix](string) fix string len check of nested types (#56963)
add 3038a44b8fc [fix](inverted index) fix is null predicate for inverted index evaluate (#56964)
add f0190dbb459 [fix](ubsan) fix ub error of uninitialized variable (#56854)
add ce665f71aa3 [fix](iceberg) fix logic of getting namespaces (#56874)
add 65b2a5f0aa0 [Opt](sys table) Add a system table column_data_sizes (#56815)
add 8d7da73b0e4 [Improvement](execute) optimize for ColumnNullable's serialize_vec/deserialize_vec (#56885)
add 0a2b8d8a9af [enhance](memtable) support adaptive memtable write buffer size (#56948)
add 9a16dcd3fec [opt](case) move some time consuming case to p1 (#56986)
add 3199414db51 [chore](cloud) Add decode_*_key (#57008)
add fd47c36ceb8 [test](cloud) Add more multi cluster case for cloud (#56840)
add d57b02adb42 [chore](tde) Pick rotate open source code (#56838)
add 58203c8e278 [feat](case when) rewrite case when to compound predicate (#56756)
add c436e9224ad [chore](cloud) expose split_tablet_stats_into_load_and_compact_parts (#57009)
add 286b47f04a2 [feat](cloud) Add snapshot data migrator & chain compactor skeleton (#56939)
add ce7b17aea79 [opt](memory) remove useless replicaToTabletMap in TabletInvertedIndex (#57011)
add aad5e485bdc [refine](float) Add NormalizeFloat to handle special values of floating-point numbers. (#56937)
add 01f83399600 [test](auth)Fix backup backup of auth test case (#56988)
add 46d27a21f43 [fix](pipeline) Crashing caused by repeated spill operations (#56755)
add 456af82122a [opt](nereids)move runtime filter info from instance level to BE level (#56978)
add 9451276b524 [feature](search) add variant subcolumn suppport for search function (#56718)
add 7f908857eaa [chore](UT) Make cloud mgr test stable (#57033)
add 02cd275d7f3 [Improve](variant) optimize cast performance when variant only contains null values (#57003)
add 16c62be4598 [fix](nereids) fix rule AvgDistinctToSumDivCount need add cast (#56887)
add 302aec1316a [Enhancement](auto partition) Auto partition could use auto bucket (#56921)
add ea1b7fabe66 [refine](DECIMALV2)Only DECIMALV2 has the member variables original_precision and original_scale. (#56913)
add bdca890342a [Improve](job) add priv check for streaming job (#56965)
add 0578af22b20 [improvement](cache) (log) add logs for mode switching and optimize log output frequency (#54085)
add 1f4cd9a22ab [chore](MS)Rename HMSProperties to HiveProperties to avoid confusion with Hive/HMS configs (#57064)
add 04e5604246c [improve](case) Fix spark flink case in jdk17 (#56992)
add b31c45444ae [fix](schema_change) fix incorrect unique id in rollup after schema change (#57073)
add 7165d4ee6f6 [Fix](metrics) Fix FE's incorrect prometheus metrics format (#57059)
add 490ade94225 [Fix](startup) Skip wrong file when loading tzdata (#57061)
add 78d8cb8ff98 [fix](exception) avoid not set required field exception (#57069)
add b6d920081ca [fix](fe)show config command use readline instead of ready (#54791)
add 508b5712db8 [Feature](func) Support GET_FORAMT function (#56691)
new 5d0a8bffda5 add rec cte thrift struct
new 9b4c8904629 update thrift
new ddb9501dabc update thrift
new 061083ced87 add result_expr_lists to TRecCTENode
new 24a7585fde6 add TRecCTEScanNode
new 379f7c7a6be recursive cte fe part
new c94e85b20c9 fix some bug in fe
new c94b717b92e fix fe plan
new aa680e5c127 update fe code
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (cc1f62548e1)
\
N -- N -- N refs/heads/dev_rec (aa680e5c127)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/cloud/cloud_snapshot_mgr.cpp | 2 +-
be/src/common/compare.h | 12 +
be/src/common/config.cpp | 4 +
be/src/common/config.h | 4 +
be/src/common/version_internal.cpp | 3 +
be/src/common/version_internal.h | 2 +
be/src/exec/schema_scanner.cpp | 3 +
.../schema_cluster_snapshots_scanner.cpp | 19 +-
.../schema_column_data_sizes_scanner.cpp | 363 +++++++
.../schema_column_data_sizes_scanner.h | 71 ++
be/src/http/action/check_encryption_action.cpp | 80 +-
be/src/http/action/meta_action.cpp | 2 +-
be/src/io/cache/block_file_cache.cpp | 79 +-
be/src/io/fs/hdfs_file_reader.cpp | 10 +
be/src/olap/base_tablet.cpp | 16 +-
be/src/olap/base_tablet.h | 5 +-
be/src/olap/inverted_index_parser.cpp | 1 -
be/src/olap/inverted_index_parser.h | 1 +
be/src/olap/memtable.cpp | 21 +-
be/src/olap/memtable.h | 6 +
be/src/olap/memtable_writer.cpp | 23 +-
be/src/olap/memtable_writer.h | 2 +
be/src/olap/rowset/segment_v2/binary_dict_page.cpp | 15 +-
be/src/olap/rowset/segment_v2/binary_dict_page.h | 3 +
be/src/olap/rowset/segment_v2/binary_plain_page.h | 5 +
.../olap/rowset/segment_v2/binary_prefix_page.cpp | 1 +
be/src/olap/rowset/segment_v2/binary_prefix_page.h | 4 +
be/src/olap/rowset/segment_v2/bitshuffle_page.h | 5 +
be/src/olap/rowset/segment_v2/column_writer.cpp | 24 +-
be/src/olap/rowset/segment_v2/column_writer.h | 119 ++-
.../rowset/segment_v2/frame_of_reference_page.h | 5 +
.../inverted_index/analysis_factory_mgr.cpp | 12 +
.../inverted_index/analyzer/analyzer.cpp | 23 +-
.../segment_v2/inverted_index/analyzer/analyzer.h | 5 +-
.../inverted_index/analyzer/basic/basic_analyzer.h | 47 +-
.../inverted_index/analyzer/custom_analyzer.cpp | 51 +-
.../inverted_index/analyzer/custom_analyzer.h | 28 +-
.../analyzer/custom_analyzer_config.cpp | 10 +
.../analyzer/custom_analyzer_config.h | 4 +
.../inverted_index/analyzer/icu/icu_analyzer.h | 49 +-
.../tokenizer.h => char_filter/char_filter.h} | 41 +-
.../char_filter/char_filter_factory.h | 24 +-
.../char_filter/char_replace_char_filter.cpp | 26 +-
.../char_filter/char_replace_char_filter.h | 16 +-
.../char_filter/char_replace_char_filter_factory.h | 75 ++
.../rowset/segment_v2/inverted_index/setting.h | 61 +-
.../inverted_index/token_filter/token_filter.h | 2 -
.../token_filter/word_delimiter_filter_factory.h | 2 +
.../segment_v2/inverted_index/token_stream.h | 37 +-
.../basic/basic_tokenizer.cpp | 39 +-
.../basic/basic_tokenizer.h | 30 +-
.../tokenizer/basic/basic_tokenizer_factory.h | 50 +
.../icu/break_iterator_wrapper.cpp | 4 +-
.../icu/break_iterator_wrapper.h | 4 +-
.../icu/composite_break_iterator.cpp | 4 +-
.../icu/composite_break_iterator.h | 4 +-
.../icu/default_icu_tokenizer_config.cpp | 4 +-
.../icu/default_icu_tokenizer_config.h | 4 +-
.../{analyzer => tokenizer}/icu/icu_common.h | 4 +-
.../{analyzer => tokenizer}/icu/icu_tokenizer.cpp | 18 +-
.../{analyzer => tokenizer}/icu/icu_tokenizer.h | 13 +-
.../icu/icu_tokenizer_config.h | 4 +-
.../icu/icu_tokenizer_factory.h} | 21 +-
.../icu/script_iterator.cpp | 4 +-
.../{analyzer => tokenizer}/icu/script_iterator.h | 4 +-
.../tokenizer/standard/standard_tokenizer_impl.h | 7 +-
.../inverted_index/tokenizer/tokenizer.h | 9 +-
.../rowset/segment_v2/inverted_index/util/reader.h | 25 +-
.../rowset/segment_v2/inverted_index_writer.cpp | 17 +-
.../olap/rowset/segment_v2/inverted_index_writer.h | 8 +-
be/src/olap/rowset/segment_v2/page_builder.h | 4 +
be/src/olap/rowset/segment_v2/plain_page.h | 5 +
be/src/olap/rowset/segment_v2/rle_page.h | 5 +
be/src/olap/rowset/segment_v2/segment.cpp | 9 +
be/src/olap/rowset/segment_v2/segment.h | 2 +
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 10 +-
be/src/olap/rowset/segment_v2/segment_writer.cpp | 14 +
.../variant/variant_column_writer_impl.cpp | 2 +-
.../rowset/segment_v2/vertical_segment_writer.cpp | 14 +
be/src/olap/snapshot_manager.cpp | 2 +-
be/src/olap/tablet_meta.cpp | 10 +-
be/src/olap/tablet_meta.h | 2 +-
be/src/olap/task/engine_storage_migration_task.cpp | 2 +-
be/src/pipeline/exec/aggregation_sink_operator.h | 5 +-
be/src/pipeline/exec/analytic_sink_operator.h | 2 +-
be/src/pipeline/exec/assert_num_rows_operator.h | 2 +-
.../exec/distinct_streaming_aggregation_operator.h | 4 +-
be/src/pipeline/exec/exchange_source_operator.h | 2 +-
be/src/pipeline/exec/hashjoin_build_sink.h | 2 +-
be/src/pipeline/exec/hashjoin_probe_operator.h | 2 +-
.../exec/nested_loop_join_build_operator.h | 2 +-
.../exec/nested_loop_join_probe_operator.h | 2 +-
be/src/pipeline/exec/operator.cpp | 6 +-
be/src/pipeline/exec/operator.h | 5 +-
.../pipeline/exec/partition_sort_sink_operator.h | 2 +-
.../exec/partitioned_aggregation_sink_operator.h | 4 +-
.../exec/partitioned_hash_join_probe_operator.h | 2 +-
.../exec/partitioned_hash_join_sink_operator.h | 2 +-
be/src/pipeline/exec/result_sink_operator.h | 4 +-
be/src/pipeline/exec/scan_operator.h | 2 +-
be/src/pipeline/exec/set_probe_sink_operator.h | 2 +-
be/src/pipeline/exec/set_sink_operator.h | 2 +-
be/src/pipeline/exec/sort_sink_operator.h | 2 +-
be/src/pipeline/exec/spill_sort_sink_operator.h | 4 +-
be/src/pipeline/exec/spill_utils.h | 4 +-
.../exec/streaming_aggregation_operator.cpp | 20 +-
.../pipeline/exec/streaming_aggregation_operator.h | 16 +-
be/src/pipeline/exec/table_function_operator.h | 2 +-
be/src/pipeline/exec/union_sink_operator.h | 4 +-
be/src/pipeline/exec/union_source_operator.h | 2 +-
be/src/pipeline/pipeline.h | 4 +-
be/src/pipeline/pipeline_fragment_context.cpp | 28 +-
be/src/pipeline/pipeline_task.cpp | 73 +-
be/src/pipeline/pipeline_task.h | 40 +-
be/src/pipeline/revokable_task.h | 76 ++
be/src/pipeline/task_scheduler.cpp | 11 +-
be/src/runtime/fragment_mgr.cpp | 7 +
be/src/runtime/index_policy/index_policy_mgr.cpp | 84 +-
be/src/runtime/index_policy/index_policy_mgr.h | 7 +
be/src/runtime/query_context.cpp | 3 +-
be/src/runtime/query_context.h | 6 +
be/src/tools/meta_tool.cpp | 2 +-
be/src/util/debug_util.cpp | 4 +
be/src/util/jsonb_document.h | 20 -
be/src/util/mysql_row_buffer.cpp | 11 +-
be/src/util/mysql_row_buffer.h | 7 +-
be/src/util/timezone_utils.cpp | 21 +-
be/src/vec/columns/column.cpp | 41 +
be/src/vec/columns/column.h | 15 +-
be/src/vec/columns/column_array.cpp | 4 +-
be/src/vec/columns/column_array.h | 4 +-
be/src/vec/columns/column_const.h | 2 +-
be/src/vec/columns/column_decimal.cpp | 48 +-
be/src/vec/columns/column_decimal.h | 8 +-
be/src/vec/columns/column_map.cpp | 6 +-
be/src/vec/columns/column_map.h | 4 +-
be/src/vec/columns/column_nullable.cpp | 36 +-
be/src/vec/columns/column_nullable.h | 4 +-
be/src/vec/columns/column_string.cpp | 47 +-
be/src/vec/columns/column_string.h | 9 +-
be/src/vec/columns/column_struct.cpp | 4 +-
be/src/vec/columns/column_struct.h | 4 +-
be/src/vec/columns/column_variant.cpp | 9 +-
be/src/vec/columns/column_variant.h | 2 +-
be/src/vec/columns/column_vector.cpp | 61 +-
be/src/vec/columns/column_vector.h | 8 +-
be/src/vec/common/hash_table/hash_map_context.h | 4 +-
be/src/vec/core/accurate_comparison.h | 303 +-----
be/src/vec/core/decimal_comparison.h | 7 +-
be/src/vec/core/field.cpp | 261 ++++-
be/src/vec/core/field.h | 7 +-
be/src/vec/data_types/data_type_decimal.h | 195 ++--
be/src/vec/exec/scan/file_scanner.cpp | 11 +
be/src/vec/exec/scan/olap_scanner.cpp | 4 +-
be/src/vec/exprs/table_function/vexplode_v2.cpp | 4 +-
be/src/vec/exprs/vsearch.cpp | 87 +-
be/src/vec/exprs/vsearch.h | 2 +
be/src/vec/functions/cast/cast_to_variant.h | 3 +-
be/src/vec/functions/function_coalesce.cpp | 17 +-
.../function_date_or_datetime_computation.cpp | 1 +
.../function_date_or_datetime_computation.h | 111 +++
be/src/vec/functions/function_jsonb_transform.cpp | 8 +-
be/src/vec/functions/function_search.cpp | 68 +-
be/src/vec/functions/function_search.h | 20 +-
be/src/vec/functions/function_string.cpp | 1 +
be/src/vec/functions/function_string.h | 128 +++
be/src/vec/functions/function_tokenize.cpp | 4 +-
be/src/vec/functions/functions_comparison.h | 82 +-
be/src/vec/functions/if.cpp | 75 +-
be/src/vec/functions/if.h | 39 +-
be/src/vec/functions/is_null.h | 23 +-
be/src/vec/functions/least_greast.cpp | 37 +-
be/src/vec/functions/match.cpp | 6 +-
be/src/vec/sink/vtablet_block_convertor.cpp | 156 ++-
be/test/cloud/cloud_meta_mgr_test.cpp | 135 ++-
.../common/compare_test.cpp} | 43 +-
be/test/olap/index_builder_test.cpp | 6 +-
.../inverted_index/analyzer/icu_analyzer_test.cpp | 59 +-
.../analyzer/simple_analyzer_test.cpp | 8 +-
.../ananlyzer/custom_analyzer_test.cpp | 328 ++++---
.../char_filter/char_filter_test.cpp | 76 ++
.../char_replace_char_filter_factory_test.cpp | 193 ++++
.../inverted_index/query_v2/boolean_query_test.cpp | 3 +-
.../segment_v2/inverted_index/setting_test.cpp | 95 +-
.../ascii_folding_filter_factory_test.cpp | 6 +-
.../lower_case_filter_factory_test.cpp | 6 +-
.../token_filter/word_delimiter_filter_test.cpp | 6 +-
.../tokenizer/basic_tokenizer_factory_test.cpp | 189 ++++
.../char_group_tokenizer_factory_test.cpp | 6 +-
.../tokenizer/edge_ngram_tokenizer_test.cpp | 6 +-
.../tokenizer/icu_tokenizer_factory_test.cpp | 212 ++++
.../tokenizer/keyword_analyzer_test.cpp | 6 +-
.../tokenizer/ngram_tokenizer_test.cpp | 6 +-
.../tokenizer/standard_tokenizer_factory_test.cpp | 6 +-
.../segment_v2/inverted_index/util/reader_test.cpp | 92 ++
be/test/olap/tablet_meta_test.cpp | 10 +-
.../operator/hashjoin_probe_operator_test.cpp | 6 +-
.../operator/spill_sort_sink_operator_test.cpp | 5 +-
be/test/pipeline/pipeline_test.cpp | 46 +-
be/test/util/mysql_row_buffer_test.cpp | 29 +
.../column_nullable_seriazlization_test.cpp | 72 +-
be/test/vec/columns/column_variant_test.cpp | 11 +-
be/test/vec/columns/common_column_test.h | 24 +-
be/test/vec/core/accurate_comparison_test.cpp | 63 +-
be/test/vec/core/field_test.cpp | 88 ++
be/test/vec/function/function_is_null_test.cpp | 147 +++
be/test/vec/function/function_num_if_test.cpp | 6 +-
build.sh | 11 +
cloud/src/common/config.h | 5 +
cloud/src/common/stopwatch.h | 4 +
cloud/src/main.cpp | 6 +-
cloud/src/meta-service/meta_service_job.cpp | 23 -
.../src/meta-service/meta_service_tablet_stats.cpp | 23 +
cloud/src/meta-service/meta_service_tablet_stats.h | 3 +
cloud/src/meta-store/keys.cpp | 281 +++++-
cloud/src/meta-store/keys.h | 52 +-
cloud/src/meta-store/meta_reader.cpp | 160 ++-
cloud/src/meta-store/meta_reader.h | 22 +-
cloud/src/meta-store/txn_kv.cpp | 19 +-
cloud/src/recycler/azure_obj_client.cpp | 7 +-
cloud/src/recycler/checker.cpp | 2 +-
cloud/src/recycler/checker.h | 4 +-
cloud/src/recycler/checker_snapshot.cpp | 2 +-
cloud/src/recycler/recycler.cpp | 31 +-
cloud/src/recycler/recycler.h | 4 +
cloud/src/recycler/snapshot_chain_compactor.cpp | 323 +++++++
cloud/src/recycler/snapshot_chain_compactor.h | 111 +++
cloud/src/recycler/snapshot_data_migrator.cpp | 362 +++++++
cloud/src/recycler/snapshot_data_migrator.h | 132 +++
cloud/src/snapshot/snapshot_manager.cpp | 8 +
cloud/src/snapshot/snapshot_manager.h | 10 +
cloud/test/keys_test.cpp | 258 ++++-
cloud/test/meta_reader_test.cpp | 74 ++
.../runtime/be/resource/be_disaggregated_probe.sh | 4 +-
.../runtime/fe/resource/fe_disaggregated_probe.sh | 4 +-
.../main/java/org/apache/doris/common/Config.java | 2 +-
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 12 +-
.../org/apache/doris/nereids/search/SearchLexer.g4 | 1 +
.../apache/doris/nereids/search/SearchParser.g4 | 7 +-
.../src/main/java/org/apache/doris/DorisFE.java | 5 +
.../apache/doris/alter/SchemaChangeHandler.java | 1 +
.../org/apache/doris/analysis/AddColumnClause.java | 8 +-
.../apache/doris/analysis/AlterTableClause.java | 7 +-
.../apache/doris/analysis/BuildIndexClause.java | 10 +-
.../apache/doris/analysis/ChannelDescription.java | 4 +-
.../org/apache/doris/analysis/CopyFromParam.java | 15 +-
.../apache/doris/analysis/CreateIndexClause.java | 17 +-
.../org/apache/doris/analysis/DataDescription.java | 44 +-
.../org/apache/doris/analysis/DropIndexClause.java | 13 +-
.../org/apache/doris/analysis/InsertTarget.java | 17 +-
.../java/org/apache/doris/analysis/LockTable.java | 10 +-
.../apache/doris/analysis/ModifyColumnClause.java | 8 +-
.../analysis/ModifyTablePropertiesClause.java | 14 +-
.../apache/doris/analysis/PartitionExprUtil.java | 28 +-
.../org/apache/doris/analysis/PartitionNames.java | 142 ---
.../org/apache/doris/analysis/SchemaTableType.java | 4 +-
.../org/apache/doris/analysis/SearchPredicate.java | 24 +-
.../java/org/apache/doris/analysis/SlotRef.java | 53 +-
.../java/org/apache/doris/analysis/TableName.java | 195 ----
.../org/apache/doris/analysis/TableSnapshot.java | 8 +
.../org/apache/doris/analysis/TupleDescriptor.java | 5 +-
.../java/org/apache/doris/backup/Repository.java | 17 +-
.../doris/catalog/BuiltinScalarFunctions.java | 44 +-
.../main/java/org/apache/doris/catalog/Env.java | 4 +-
.../main/java/org/apache/doris/catalog/MTMV.java | 8 +-
.../org/apache/doris/catalog/MetadataViewer.java | 179 ----
.../java/org/apache/doris/catalog/SchemaTable.java | 16 +
.../apache/doris/catalog/TabletInvertedIndex.java | 34 -
.../doris/clone/DynamicPartitionScheduler.java | 42 +-
.../apache/doris/cloud/storage/AzureRemote.java | 50 +-
.../apache/doris/cloud/storage/DefaultRemote.java | 116 +--
.../doris/common/util/AutoBucketCalculator.java | 205 ++++
.../apache/doris/common/util/AutoBucketUtils.java | 10 +-
.../org/apache/doris/datasource/CatalogIf.java | 4 +-
.../apache/doris/datasource/ExternalCatalog.java | 4 +-
.../apache/doris/datasource/InternalCatalog.java | 6 +-
.../doris/datasource/hive/HMSCachedClient.java | 4 +-
.../doris/datasource/hive/HMSExternalCatalog.java | 10 +-
.../doris/datasource/hive/HiveTransaction.java | 4 +-
.../datasource/hive/ThriftHMSCachedClient.java | 8 +-
.../datasource/iceberg/IcebergMetadataOps.java | 28 +-
.../doris/datasource/jdbc/source/JdbcScanNode.java | 2 +-
.../doris/datasource/odbc/source/OdbcScanNode.java | 2 +-
...Properties.java => AbstractHiveProperties.java} | 4 +-
.../property/metastore/HMSBaseProperties.java | 3 +
....java => HiveAliyunDLFMetaStoreProperties.java} | 4 +-
...rties.java => HiveGlueMetaStoreProperties.java} | 6 +-
.../{HMSProperties.java => HiveHMSProperties.java} | 4 +-
...tiesFactory.java => HivePropertiesFactory.java} | 18 +-
.../property/metastore/IcebergRestProperties.java | 10 +-
.../property/metastore/MetastoreProperties.java | 2 +-
.../org/apache/doris/encryption/EncryptionKey.java | 17 +
.../apache/doris/encryption/KeyManagerStore.java | 11 +-
.../org/apache/doris/encryption/RootKeyInfo.java | 2 +-
.../httpv2/controller/HardwareInfoController.java | 1 +
.../doris/indexpolicy/BasicTokenizerValidator.java | 54 ++
.../CharReplaceCharFilterValidator.java | 63 ++
...icyTypeEnum.java => ICUTokenizerValidator.java} | 38 +-
.../org/apache/doris/indexpolicy/IndexPolicy.java | 6 +-
.../apache/doris/indexpolicy/IndexPolicyMgr.java | 56 +-
.../doris/indexpolicy/IndexPolicyTypeEnum.java | 3 +-
.../org/apache/doris/info/PartitionNamesInfo.java | 3 +
.../java/org/apache/doris/info/TableNameInfo.java | 22 +-
.../org/apache/doris/job/common/FailureReason.java | 6 +
.../insert/streaming/StreamingInsertJob.java | 59 +-
.../insert/streaming/StreamingInsertTask.java | 2 +-
.../insert/streaming/StreamingJobProperties.java | 69 +-
.../org/apache/doris/job/manager/JobManager.java | 10 +-
.../org/apache/doris/load/BrokerFileGroup.java | 10 +-
.../main/java/org/apache/doris/load/DeleteJob.java | 5 -
.../java/org/apache/doris/load/EtlJobType.java | 1 +
.../main/java/org/apache/doris/load/ExportJob.java | 4 +-
.../main/java/org/apache/doris/load/ExportMgr.java | 6 +-
.../org/apache/doris/load/RoutineLoadDesc.java | 12 +-
.../apache/doris/load/loadv2/InsertLoadJob.java | 6 +-
.../org/apache/doris/load/loadv2/LoadManager.java | 5 -
.../apache/doris/load/loadv2/MysqlLoadManager.java | 6 +-
.../load/routineload/KafkaRoutineLoadJob.java | 8 +-
.../doris/load/routineload/RoutineLoadJob.java | 30 +-
.../doris/metric/PrometheusMetricVisitor.java | 4 +-
.../org/apache/doris/mtmv/MTMVPartitionUtil.java | 18 +-
.../java/org/apache/doris/mtmv/MTMVService.java | 8 +-
.../mysql/privilege/AccessControllerManager.java | 6 -
.../doris/nereids/analyzer/UnboundAlias.java | 10 +
.../doris/nereids/analyzer/UnboundFunction.java | 14 +
.../doris/nereids/analyzer/UnboundInlineTable.java | 6 +
.../nereids/analyzer/UnboundOneRowRelation.java | 12 +
.../doris/nereids/analyzer/UnboundRelation.java | 23 +
.../doris/nereids/analyzer/UnboundResultSink.java | 5 +
.../apache/doris/nereids/analyzer/UnboundSlot.java | 5 +
.../apache/doris/nereids/analyzer/UnboundStar.java | 15 +
.../doris/nereids/analyzer/UnboundTVFRelation.java | 7 +
.../doris/nereids/analyzer/UnboundTableSink.java | 11 +
.../doris/nereids/analyzer/UnboundVariable.java | 5 +
.../glue/translator/PhysicalPlanTranslator.java | 8 +-
.../doris/nereids/jobs/executor/Rewriter.java | 7 +-
.../doris/nereids/load/NereidsBrokerFileGroup.java | 10 +-
.../doris/nereids/load/NereidsBrokerLoadTask.java | 12 +-
.../doris/nereids/load/NereidsDataDescription.java | 68 +-
.../nereids/load/NereidsLoadPlanInfoCollector.java | 12 +-
.../doris/nereids/load/NereidsLoadTaskInfo.java | 4 +-
.../doris/nereids/load/NereidsLoadUtils.java | 10 +-
.../nereids/load/NereidsLoadingTaskPlanner.java | 14 +-
.../nereids/load/NereidsRoutineLoadTaskInfo.java | 12 +-
.../nereids/load/NereidsStreamLoadPlanner.java | 6 +-
.../doris/nereids/load/NereidsStreamLoadTask.java | 14 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 57 +-
.../processor/pre/PullUpSubqueryAliasToCTE.java | 8 +-
.../apache/doris/nereids/properties/OrderKey.java | 18 +
.../doris/nereids/rules/analysis/AnalyzeCTE.java | 50 +-
.../rules/analysis/AvgDistinctToSumDivCount.java | 3 +-
.../nereids/rules/analysis/CheckSearchUsage.java | 29 +-
.../nereids/rules/analysis/ExpressionAnalyzer.java | 26 +-
.../rules/analysis/GetFormatFunctionBinder.java | 78 ++
.../LogicalSubQueryAliasToLogicalProject.java | 20 +-
.../rules/expression/ExpressionOptimization.java | 2 +
.../rules/expression/ExpressionRuleType.java | 2 +
.../rules/CaseWhenToCompoundPredicate.java | 110 +++
.../expression/rules/FoldConstantRuleOnFE.java | 9 +-
.../rules/OneListPartitionEvaluator.java | 5 +-
.../LogicalEsScanToPhysicalEsScan.java | 9 +-
.../LogicalJdbcScanToPhysicalJdbcScan.java | 10 +-
.../LogicalOdbcScanToPhysicalOdbcScan.java | 10 +-
.../LogicalOlapScanToPhysicalOlapScan.java | 5 +-
.../doris/nereids/rules/rewrite/ColumnPruning.java | 39 +-
.../doris/nereids/rules/rewrite/ReorderJoin.java | 4 +
.../rules/rewrite/RewriteSearchToSlots.java | 61 +-
.../org/apache/doris/nereids/trees/TreeNode.java | 4 +
.../doris/nereids/trees/expressions/Alias.java | 12 +
.../nereids/trees/expressions/BinaryOperator.java | 9 +
.../doris/nereids/trees/expressions/CaseWhen.java | 14 +
.../doris/nereids/trees/expressions/Cast.java | 11 +
.../trees/expressions/CompoundPredicate.java | 10 +
.../trees/expressions/DefaultValueSlot.java | 5 +
.../doris/nereids/trees/expressions/Exists.java | 10 +
.../nereids/trees/expressions/InPredicate.java | 25 +
.../nereids/trees/expressions/InSubquery.java | 10 +
.../doris/nereids/trees/expressions/IsNull.java | 8 +
.../doris/nereids/trees/expressions/Not.java | 7 +
.../nereids/trees/expressions/OrderExpression.java | 5 +
.../nereids/trees/expressions/Placeholder.java | 4 +
.../nereids/trees/expressions/Properties.java | 5 +
.../nereids/trees/expressions/ScalarSubquery.java | 9 +
.../trees/expressions/SearchExpression.java | 8 +-
.../trees/expressions/StringRegexPredicate.java | 11 +
.../doris/nereids/trees/expressions/Subtract.java | 13 +
.../trees/expressions/TimestampArithmetic.java | 15 +
.../nereids/trees/expressions/UnaryOperator.java | 8 +
.../nereids/trees/expressions/WhenClause.java | 7 +
.../trees/expressions/WindowExpression.java | 18 +
.../nereids/trees/expressions/WindowFrame.java | 38 +
.../trees/expressions/functions/BoundFunction.java | 11 +
.../functions/agg/AggregateFunction.java | 14 +
.../trees/expressions/functions/agg/Count.java | 8 +
.../executable/DateTimeExtractAndTransform.java | 80 ++
.../functions/executable/StringArithmetic.java | 101 ++
.../expressions/functions/scalar/ElementAt.java | 15 +-
.../expressions/functions/scalar/ExportSet.java | 93 ++
.../scalar/{ElementAt.java => GetFormat.java} | 43 +-
.../trees/expressions/functions/scalar/Lambda.java | 12 +
.../functions/scalar/SearchDslParser.java | 93 +-
.../trees/expressions/literal/ArrayLiteral.java | 7 +
.../trees/expressions/literal/Interval.java | 9 +
.../nereids/trees/expressions/literal/Literal.java | 5 +
.../trees/expressions/literal/NullLiteral.java | 4 +-
.../expressions/visitor/ScalarFunctionVisitor.java | 10 +
.../apache/doris/nereids/trees/plans/PlanType.java | 3 +
.../trees/plans/commands/AlterJobCommand.java | 57 +-
.../plans/commands/AlterJobStatusCommand.java | 23 +-
.../trees/plans/commands/AlterTableCommand.java | 2 +-
...mand.java => CreateIndexCharFilterCommand.java} | 64 +-
.../commands/CreateMaterializedViewCommand.java | 3 +
...ommand.java => DropIndexCharFilterCommand.java} | 37 +-
.../trees/plans/commands/DropJobCommand.java | 24 +-
.../trees/plans/commands/ExplainCommand.java | 8 +
.../trees/plans/commands/ExportCommand.java | 70 +-
.../trees/plans/commands/PauseJobCommand.java | 7 -
.../trees/plans/commands/ResumeJobCommand.java | 7 -
.../trees/plans/commands/ShowBackupCommand.java | 6 +
.../trees/plans/commands/ShowConfigCommand.java | 20 +-
...ommand.java => ShowIndexCharFilterCommand.java} | 36 +-
.../commands/ShowQueuedAnalyzeJobsCommand.java | 9 +-
.../trees/plans/commands/ShowRestoreCommand.java | 6 +
.../plans/commands/ShowWorkloadGroupsCommand.java | 10 +
.../plans/commands/UpdateMvByPartitionCommand.java | 3 +-
.../trees/plans/commands/info/AlterViewInfo.java | 5 +-
.../trees/plans/commands/info/BuildIndexOp.java | 3 +-
.../trees/plans/commands/info/CopyFromDesc.java | 6 +-
.../trees/plans/commands/info/CopyIntoInfo.java | 17 +-
.../trees/plans/commands/info/CreateIndexOp.java | 3 +-
.../trees/plans/commands/info/CreateJobInfo.java | 7 +-
.../plans/commands/info/CreateRoutineLoadInfo.java | 10 +-
.../trees/plans/commands/info/CreateTableInfo.java | 8 +
.../trees/plans/commands/info/CreateViewInfo.java | 5 +-
.../trees/plans/commands/info/DropIndexOp.java | 3 +-
.../trees/plans/commands/info/ModifyColumnOp.java | 4 -
.../commands/insert/AbstractInsertExecutor.java | 26 +-
.../insert/BaseExternalTableInsertExecutor.java | 4 +-
.../insert/BatchInsertIntoTableCommand.java | 7 +
.../commands/insert/BlackholeInsertExecutor.java | 4 +-
.../commands/insert/DictionaryInsertExecutor.java | 4 +-
.../plans/commands/insert/HiveInsertExecutor.java | 4 +-
.../commands/insert/IcebergInsertExecutor.java | 4 +-
.../commands/insert/InsertIntoTableCommand.java | 31 +-
.../insert/InsertOverwriteTableCommand.java | 12 +
.../plans/commands/insert/JdbcInsertExecutor.java | 4 +-
.../plans/commands/insert/OlapInsertExecutor.java | 3 -
.../trees/plans/logical/LogicalAggregate.java | 68 +-
.../nereids/trees/plans/logical/LogicalCTE.java | 41 +-
.../plans/logical/LogicalCatalogRelation.java | 49 +-
.../trees/plans/logical/LogicalCheckPolicy.java | 5 +
.../logical/LogicalDeferMaterializeOlapScan.java | 14 +-
.../nereids/trees/plans/logical/LogicalEsScan.java | 39 +-
.../nereids/trees/plans/logical/LogicalExcept.java | 9 +
.../trees/plans/logical/LogicalFileScan.java | 38 +-
.../trees/plans/logical/LogicalFileSink.java | 10 +
.../nereids/trees/plans/logical/LogicalFilter.java | 12 +
.../trees/plans/logical/LogicalGenerate.java | 22 +
.../nereids/trees/plans/logical/LogicalHaving.java | 13 +
.../trees/plans/logical/LogicalIntersect.java | 9 +
.../trees/plans/logical/LogicalJdbcScan.java | 29 +-
.../nereids/trees/plans/logical/LogicalJoin.java | 18 +
.../nereids/trees/plans/logical/LogicalLimit.java | 11 +
.../trees/plans/logical/LogicalOdbcScan.java | 25 +-
.../trees/plans/logical/LogicalOlapScan.java | 63 +-
.../trees/plans/logical/LogicalProject.java | 19 +
.../trees/plans/logical/LogicalQualify.java | 10 +
.../trees/plans/logical/LogicalRecursiveCte.java | 183 ----
.../logical/LogicalRecursiveCteRecursiveChild.java | 5 +
.../plans/logical/LogicalRecursiveCteScan.java | 27 +-
.../nereids/trees/plans/logical/LogicalRepeat.java | 45 +-
.../trees/plans/logical/LogicalSchemaScan.java | 53 +-
.../trees/plans/logical/LogicalSelectHint.java | 5 +
.../nereids/trees/plans/logical/LogicalSort.java | 12 +
.../trees/plans/logical/LogicalSubQueryAlias.java | 66 +-
.../trees/plans/logical/LogicalTestScan.java | 25 +-
.../nereids/trees/plans/logical/LogicalUnion.java | 9 +
.../trees/plans/logical/LogicalUsingJoin.java | 16 +
.../nereids/trees/plans/logical/LogicalWindow.java | 16 +
.../plans/physical/PhysicalCatalogRelation.java | 33 +-
.../trees/plans/physical/PhysicalEsScan.java | 29 +-
.../trees/plans/physical/PhysicalJdbcScan.java | 33 +-
.../trees/plans/physical/PhysicalOdbcScan.java | 31 +-
.../trees/plans/physical/PhysicalOlapScan.java | 43 +-
.../PhysicalRecursiveCteRecursiveChild.java | 5 +
.../trees/plans/visitor/CommandVisitor.java | 18 +
.../apache/doris/nereids/util/ExpressionUtils.java | 11 +-
.../java/org/apache/doris/nereids/util/Utils.java | 5 +-
.../org/apache/doris/planner/RecursiveCteNode.java | 27 +-
.../apache/doris/planner/RecursiveCteScanNode.java | 9 +
.../java/org/apache/doris/qe/SessionVariable.java | 94 +-
.../doris/qe/runtime/ThriftPlansBuilder.java | 53 +-
.../apache/doris/service/FrontendServiceImpl.java | 203 ++--
.../apache/doris/statistics/AnalysisManager.java | 32 +-
.../doris/statistics/StatisticsAutoCollector.java | 20 +-
.../doris/statistics/StatisticsJobAppender.java | 17 +-
.../doris/statistics/util/StatisticsUtil.java | 24 +-
.../tablefunction/HudiTableValuedFunction.java | 6 +-
.../tablefunction/IcebergTableValuedFunction.java | 6 +-
.../tablefunction/JobsTableValuedFunction.java | 7 -
.../doris/tablefunction/MetadataGenerator.java | 23 +
.../tablefunction/PaimonTableValuedFunction.java | 6 +-
.../tablefunction/TasksTableValuedFunction.java | 7 -
.../java/org/apache/doris/task/LoadTaskInfo.java | 4 +-
.../doris/transaction/PublishVersionDaemon.java | 3 +
.../org/apache/doris/alter/CloudIndexTest.java | 12 +-
.../org/apache/doris/alter/IndexChangeJobTest.java | 25 +-
.../doris/analysis/PartitionExprUtilTest.java | 223 +++++
.../doris/catalog/ListPartitionInfoTest.java | 6 +-
.../apache/doris/catalog/MetadataViewerTest.java | 4 +-
.../doris/catalog/RangePartitionInfoTest.java | 4 +-
.../doris/common/proc/IndexSchemaProcNodeTest.java | 6 +-
.../common/util/AutoBucketCalculatorTest.java | 106 ++
.../doris/datasource/TestHMSCachedClient.java | 4 +-
.../datasource/iceberg/IcebergMetadataOpTest.java | 48 +
.../datasource/iceberg/IcebergPredicateTest.java | 20 +-
.../datasource/lakesoul/LakeSoulPredicateTest.java | 20 +-
.../HMSAliyunDLFMetaStorePropertiesTest.java | 4 +-
.../datasource/property/metastore/HMSGlueIT.java | 2 +-
.../metastore/HMSGlueMetaStorePropertiesTest.java | 16 +-
.../property/metastore/HMSPropertiesTest.java | 14 +-
.../metastore/IcebergRestPropertiesTest.java | 4 +-
.../TrinoConnectorPredicateTest.java | 36 +-
.../apache/doris/load/loadv2/ExportMgrTest.java | 4 +-
.../load/routineload/KafkaRoutineLoadJobTest.java | 6 +-
.../apache/doris/mtmv/MTMVPartitionUtilTest.java | 52 +-
.../org/apache/doris/mtmv/MTMVRewriteUtilTest.java | 12 +-
.../java/org/apache/doris/mtmv/MTMVTaskTest.java | 8 +-
.../test/java/org/apache/doris/mtmv/MTMVTest.java | 16 +-
.../doris/nereids/jobs/RewriteTopDownJobTest.java | 5 +
.../nereids/parser/NereidsParserDigestTest.java | 300 ++++++
.../rules/CaseWhenToCompoundPredicateTest.java | 59 ++
.../GeneratePartitionTopnFromWindowTest.java | 40 +
.../nereids/rules/rewrite/ReorderJoinTest.java | 19 +
.../functions/scalar/SearchDslParserTest.java | 22 +
.../org/apache/doris/planner/OlapScanNodeTest.java | 8 +-
.../statistics/StatisticsAutoCollectorTest.java | 14 +-
.../statistics/StatisticsJobAppenderTest.java | 58 +-
gensrc/proto/cloud.proto | 30 +
gensrc/proto/segment_v2.proto | 4 +
gensrc/script/gen_build_version.sh | 5 +
gensrc/thrift/AgentService.thrift | 5 +-
gensrc/thrift/Descriptors.thrift | 1 +
gensrc/thrift/Exprs.thrift | 5 +-
gensrc/thrift/PaloInternalService.thrift | 6 +-
.../routine_load/test_routine_load.out | 33 +
.../test_schema_change_with_readd_cluster.out} | 7 +-
.../multi_cluster/stream_load}/all_types.csv | 0
.../multi_cluster/stream_load/stream_load.out} | 14 +-
.../multi_cluster/stream_load/stream_load_2pc.out | 25 +
.../multi_cluster/stream_load/stream_load_lb.out | 25 +
.../test_apsaradb_internal_stage.out} | 7 +-
.../data/correctness/test_compare_float.out | 8 -
.../one_level_nestedtypes_with_s3data.out | 1020 ++++++++++----------
.../string_len/test_string_len_complex.out | 43 +
.../cache/test_file_cache_statistics.out | 5 +-
.../analyzer/test_custom_analyzer.out | 6 +
.../analyzer/test_custom_analyzer1.out | 6 +
.../test_streaming_insert_job_alter.out | 23 +
.../test_streaming_insert_job_crud.out | 23 +
.../partition_topn/check_orderkey.out} | 4 +-
.../datetime_functions/test_date_function_v2.out | 81 ++
.../agg_skew_rewrite/agg_skew_rewrite.out | 10 +-
.../avg_distinct_to_sum_div_count.out} | 4 +-
.../nereids_rules_p0/eager_aggregate/basic.out | 32 +-
.../eager_aggregate/basic_one_side.out | 32 +-
...h_down_count_distinct_through_join_one_side.out | 20 +-
.../push_down_count_through_join.out | 352 +++----
.../push_down_count_through_join_one_side.out | 352 +++----
.../eager_aggregate/push_down_max_through_join.out | 196 ++--
...ush_down_min_distinct_through_join_one_side.out | 20 +-
.../eager_aggregate/push_down_min_through_join.out | 196 ++--
...ush_down_sum_distinct_through_join_one_side.out | 16 +-
.../eager_aggregate/push_down_sum_through_join.out | 196 ++--
.../push_down_sum_through_join_one_side.out | 196 ++--
.../eliminate_gby_key/eliminate_group_by.out | 84 +-
.../eliminate_group_by_key_by_uniform.out | 2 +-
.../eliminate_join_condition.out | 24 +-
.../eliminate_outer_join/eliminate_outer_join.out | 130 +--
...simplify_comparison_predicate_int_vs_double.out | 4 +-
.../extend_infer_equal_predicate.out | 302 +++---
.../infer_predicate/infer_intersect_except.out | 8 +-
.../infer_predicate/infer_unequal_predicates.out | 68 +-
.../infer_predicate/pull_up_predicate_agg.out | 4 +-
.../infer_predicate/pull_up_predicate_literal.out | 258 ++---
.../infer_predicate/pull_up_predicate_set_op.out | 84 +-
.../data/nereids_rules_p0/pkfk/eliminate_inner.out | 24 +-
.../predicate_infer/infer_predicate.out | 152 +--
.../pull_up_join_from_union.out | 82 +-
.../push_down_distinct_through_join.out | 10 +-
.../extract_from_disjunction_in_join.out | 32 +-
.../push_down_limit_distinct_through_join.out | 4 +-
.../push_down_top_n_distinct_through_join.out | 6 +-
.../push_down_top_n_distinct_through_union.out | 44 +-
.../push_down_top_n_through_union.out | 44 +-
.../data/nereids_rules_p0/salt_join/salt_join.out | 72 +-
.../join/eliminate_const_join_condition.out | 24 +-
.../conditional_functions/test_coalesce.out | 28 +
.../string_functions/test_string_function.out | 217 +++++
.../sql_functions/table_function/explode.out | 72 ++
.../system/test_query_sys_column_data_sizes.out | 15 +
.../adjust_virtual_slot_nullable.out | 4 +-
.../search/test_search_vs_match_consistency.out | 6 +
.../data/shape_check/ssb_sf100/shape/flat.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query10.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query14.out | 12 +-
.../tpcds_sf100/noStatsRfPrune/query16.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query17.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query18.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query25.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query29.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query3.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query35.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query36.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query41.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query42.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query44.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query46.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query49.out | 12 +-
.../tpcds_sf100/noStatsRfPrune/query50.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query52.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query59.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query6.out | 12 +-
.../tpcds_sf100/noStatsRfPrune/query64.out | 22 +-
.../tpcds_sf100/noStatsRfPrune/query68.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query69.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query70.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query72.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query85.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query86.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query94.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query95.out | 6 +-
.../tpcds_sf100/no_stats_shape/query10.out | 4 +-
.../tpcds_sf100/no_stats_shape/query14.out | 12 +-
.../tpcds_sf100/no_stats_shape/query16.out | 6 +-
.../tpcds_sf100/no_stats_shape/query17.out | 6 +-
.../tpcds_sf100/no_stats_shape/query18.out | 4 +-
.../tpcds_sf100/no_stats_shape/query25.out | 6 +-
.../tpcds_sf100/no_stats_shape/query29.out | 6 +-
.../tpcds_sf100/no_stats_shape/query3.out | 2 +-
.../tpcds_sf100/no_stats_shape/query35.out | 4 +-
.../tpcds_sf100/no_stats_shape/query36.out | 2 +-
.../tpcds_sf100/no_stats_shape/query41.out | 2 +-
.../tpcds_sf100/no_stats_shape/query42.out | 2 +-
.../tpcds_sf100/no_stats_shape/query44.out | 4 +-
.../tpcds_sf100/no_stats_shape/query46.out | 2 +-
.../tpcds_sf100/no_stats_shape/query49.out | 12 +-
.../tpcds_sf100/no_stats_shape/query50.out | 4 +-
.../tpcds_sf100/no_stats_shape/query52.out | 2 +-
.../tpcds_sf100/no_stats_shape/query59.out | 4 +-
.../tpcds_sf100/no_stats_shape/query6.out | 12 +-
.../tpcds_sf100/no_stats_shape/query64.out | 22 +-
.../tpcds_sf100/no_stats_shape/query68.out | 2 +-
.../tpcds_sf100/no_stats_shape/query69.out | 4 +-
.../tpcds_sf100/no_stats_shape/query70.out | 2 +-
.../tpcds_sf100/no_stats_shape/query72.out | 6 +-
.../tpcds_sf100/no_stats_shape/query85.out | 4 +-
.../tpcds_sf100/no_stats_shape/query86.out | 2 +-
.../tpcds_sf100/no_stats_shape/query94.out | 6 +-
.../tpcds_sf100/no_stats_shape/query95.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query10.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query14.out | 12 +-
.../shape_check/tpcds_sf100/rf_prune/query16.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query17.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query18.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query25.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query29.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query3.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query35.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query36.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query41.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query42.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query44.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query46.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query49.out | 12 +-
.../shape_check/tpcds_sf100/rf_prune/query50.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query52.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query59.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query6.out | 12 +-
.../shape_check/tpcds_sf100/rf_prune/query64.out | 22 +-
.../shape_check/tpcds_sf100/rf_prune/query68.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query69.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query70.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query72.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query85.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query86.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query94.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query95.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query10.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query14.out | 12 +-
.../data/shape_check/tpcds_sf100/shape/query16.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query17.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query18.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query25.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query29.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query3.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query35.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query36.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query41.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query42.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query44.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query46.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query49.out | 12 +-
.../data/shape_check/tpcds_sf100/shape/query50.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query52.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query59.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query6.out | 12 +-
.../data/shape_check/tpcds_sf100/shape/query64.out | 22 +-
.../data/shape_check/tpcds_sf100/shape/query68.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query69.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query70.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query72.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query85.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query86.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query94.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query95.out | 6 +-
.../tpcds_sf1000/bs_downgrade_shape/query44.out | 4 +-
.../tpcds_sf1000/bs_downgrade_shape/query6.out | 12 +-
.../tpcds_sf1000/bs_downgrade_shape/query68.out | 2 +-
.../tpcds_sf1000/bs_downgrade_shape/query95.out | 6 +-
.../tpcds_sf1000/eliminate_empty/query10_empty.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query10.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query14.out | 12 +-
.../data/shape_check/tpcds_sf1000/hint/query16.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query17.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query18.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query25.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query29.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query3.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query36.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query41.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query42.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query44.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query46.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query49.out | 12 +-
.../data/shape_check/tpcds_sf1000/hint/query50.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query52.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query59.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query6.out | 12 +-
.../data/shape_check/tpcds_sf1000/hint/query64.out | 22 +-
.../data/shape_check/tpcds_sf1000/hint/query68.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query69.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query70.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query72.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query85.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query86.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query94.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query95.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query10.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query14.out | 12 +-
.../shape_check/tpcds_sf1000/shape/query16.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query17.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query18.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query25.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query29.out | 6 +-
.../data/shape_check/tpcds_sf1000/shape/query3.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query35.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query36.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query41.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query42.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query44.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query46.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query49.out | 12 +-
.../shape_check/tpcds_sf1000/shape/query50.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query52.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query59.out | 4 +-
.../data/shape_check/tpcds_sf1000/shape/query6.out | 12 +-
.../shape_check/tpcds_sf1000/shape/query64.out | 22 +-
.../shape_check/tpcds_sf1000/shape/query68.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query69.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query70.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query72.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query85.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query86.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query94.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query95.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query10.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query14.out | 12 +-
.../shape_check/tpcds_sf10t_orc/shape/query16.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query17.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query18.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query25.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query29.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query3.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query35.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query36.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query41.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query42.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query44.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query46.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query49.out | 12 +-
.../shape_check/tpcds_sf10t_orc/shape/query50.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query52.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query59.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query6.out | 12 +-
.../shape_check/tpcds_sf10t_orc/shape/query64.out | 22 +-
.../shape_check/tpcds_sf10t_orc/shape/query68.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query69.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query70.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query72.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query85.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query86.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query94.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query95.out | 6 +-
.../data/shape_check/tpch_sf1000/hint/q7.out | 4 +-
.../data/shape_check/tpch_sf1000/hint/q8.out | 4 +-
.../tpch_sf1000/nostats_rf_prune/q21.out | 6 +-
.../tpch_sf1000/nostats_rf_prune/q7.out | 4 +-
.../tpch_sf1000/nostats_rf_prune/q8.out | 4 +-
.../data/shape_check/tpch_sf1000/rf_prune/q21.out | 6 +-
.../data/shape_check/tpch_sf1000/rf_prune/q7.out | 4 +-
.../data/shape_check/tpch_sf1000/rf_prune/q8.out | 4 +-
.../data/shape_check/tpch_sf1000/shape/q21.out | 6 +-
.../data/shape_check/tpch_sf1000/shape/q7.out | 4 +-
.../data/shape_check/tpch_sf1000/shape/q8.out | 4 +-
.../shape_check/tpch_sf1000/shape_no_stats/q21.out | 6 +-
.../shape_check/tpch_sf1000/shape_no_stats/q7.out | 4 +-
.../shape_check/tpch_sf1000/shape_no_stats/q8.out | 4 +-
.../sql/q01.out | 0
.../sql/q02.out | 0
.../sql/q03.out | 0
.../sql/q04.out | 0
.../sql/q05.out | 0
.../sql/q06.out | 0
.../sql/q07.out | 0
.../sql/q08.out | 0
.../sql/q09.out | 0
.../sql/q10.out | 0
.../sql/q11.out | 0
.../sql/q12.out | 0
.../sql/q13.out | 0
.../sql/q14.out | 0
.../sql/q15.out | 0
.../sql/q16.out | 0
.../sql/q17.out | 0
.../sql/q18.out | 0
.../sql/q19.out | 0
.../sql/q20.out | 0
.../sql/q21.out | 0
.../sql/q22.out | 0
.../sql/test_left_anti_join_batch_size.out | 0
.../variant_p0/test_variant_search_subcolumn.out | 32 +
.../cloud_p0/conf/regression-conf-custom.groovy | 1 +
.../nonConcurrent/conf/regression-conf.groovy | 1 +
.../plugins/plugin_cluster_version.groovy | 46 +-
.../suites/auth_call/test_ddl_backup_auth.groovy | 12 +-
.../suites/auth_call/test_ddl_job_auth.groovy | 22 +-
.../async_copy_into/async_load.groovy | 275 ++++++
.../async_copy_into}/ddl/customer.sql | 6 +-
.../async_copy_into}/ddl/customer_delete.sql | 0
.../async_copy_into}/ddl/lineitem.sql | 4 -
.../async_copy_into}/ddl/lineitem_delete.sql | 0
.../multi_cluster/async_copy_into}/ddl/nation.sql | 6 +-
.../async_copy_into}/ddl/nation_delete.sql | 0
.../multi_cluster/async_copy_into}/ddl/orders.sql | 6 +-
.../async_copy_into}/ddl/orders_delete.sql | 0
.../multi_cluster/async_copy_into}/ddl/part.sql | 6 +-
.../async_copy_into}/ddl/part_delete.sql | 0
.../async_copy_into}/ddl/partsupp.sql | 4 -
.../async_copy_into}/ddl/partsupp_delete.sql | 0
.../multi_cluster/async_copy_into}/ddl/region.sql | 6 +-
.../async_copy_into}/ddl/region_delete.sql | 0
.../async_copy_into}/ddl/revenue1.sql | 0
.../async_copy_into}/ddl/revenue1_delete.sql | 0
.../async_copy_into}/ddl/supplier.sql | 6 +-
.../async_copy_into}/ddl/supplier_delete.sql | 0
.../multi_cluster/copy_into}/ddl/customer.sql | 6 +-
.../copy_into}/ddl/customer_delete.sql | 0
.../multi_cluster/copy_into}/ddl/lineitem.sql | 4 -
.../copy_into}/ddl/lineitem_delete.sql | 0
.../multi_cluster/copy_into}/ddl/nation.sql | 6 +-
.../ddl/nation_delete.sql | 0
.../multi_cluster/copy_into}/ddl/orders.sql | 6 +-
.../multi_cluster/copy_into}/ddl/orders_delete.sql | 0
.../multi_cluster/copy_into}/ddl/part.sql | 6 +-
.../multi_cluster/copy_into}/ddl/part_delete.sql | 0
.../multi_cluster/copy_into}/ddl/partsupp.sql | 4 -
.../copy_into}/ddl/partsupp_delete.sql | 0
.../multi_cluster/copy_into}/ddl/region.sql | 6 +-
.../multi_cluster/copy_into}/ddl/region_delete.sql | 0
.../multi_cluster/copy_into}/ddl/revenue1.sql | 0
.../copy_into}/ddl/revenue1_delete.sql | 0
.../multi_cluster/copy_into}/ddl/supplier.sql | 6 +-
.../copy_into}/ddl/supplier_delete.sql | 0
.../multi_cluster/copy_into/sync_laod.groovy | 846 ++++++++++++++++
.../cloud_p0/multi_cluster/default_cluster.groovy | 172 ++++
.../multi_cluster_s3_load}/ddl/customer.sql | 8 +-
.../multi_cluster_s3_load}/ddl/customer_delete.sql | 0
.../multi_cluster_s3_load}/ddl/customer_load.sql | 0
.../multi_cluster_s3_load}/ddl/lineitem.sql | 8 +-
.../multi_cluster_s3_load}/ddl/lineitem_delete.sql | 0
.../multi_cluster_s3_load/ddl/lineitem_load.sql | 6 +
.../multi_cluster_s3_load}/ddl/nation.sql | 7 +-
.../ddl/nation_delete.sql | 0
.../ddl/nation_load.sql | 0
.../multi_cluster_s3_load}/ddl/orders.sql | 9 +-
.../multi_cluster_s3_load}/ddl/orders_delete.sql | 0
.../multi_cluster_s3_load}/ddl/orders_load.sql | 0
.../multi_cluster_s3_load}/ddl/part.sql | 9 +-
.../multi_cluster_s3_load}/ddl/part_delete.sql | 0
.../multi_cluster_s3_load}/ddl/part_load.sql | 0
.../multi_cluster_s3_load}/ddl/partsupp.sql | 9 +-
.../multi_cluster_s3_load}/ddl/partsupp_delete.sql | 0
.../multi_cluster_s3_load}/ddl/partsupp_load.sql | 0
.../multi_cluster_s3_load}/ddl/region.sql | 7 +-
.../multi_cluster_s3_load}/ddl/region_delete.sql | 0
.../multi_cluster_s3_load}/ddl/region_load.sql | 0
.../multi_cluster_s3_load}/ddl/supplier.sql | 9 +-
.../multi_cluster_s3_load}/ddl/supplier_delete.sql | 0
.../multi_cluster_s3_load}/ddl/supplier_load.sql | 0
.../multi_cluster_s3_load/load.groovy | 166 ++++
.../op/test_drop_and_add_cluster.groovy | 221 +++++
.../cloud_p0/multi_cluster/rename_cluster.groovy | 115 +++
.../routine_load/test_routine_load.groovy | 189 ++++
.../test_bitmap_index_with_drop_cluster.groovy | 143 +++
.../test_bitmap_index_with_readd_cluster.groovy | 154 +++
...test_materialized_view_with_drop_cluster.groovy | 123 +++
...est_materialized_view_with_readd_cluster.groovy | 155 +++
.../test_rollup_with_drop_cluster.groovy | 122 +++
.../test_rollup_with_readd_cluster.groovy | 156 +++
.../test_schema_change_with_drop_cluster.groovy | 120 +++
.../test_schema_change_with_readd_cluster.groovy | 169 ++++
.../multi_cluster/smooth_upgrade/ddl/customer.sql | 15 +
.../smooth_upgrade}/ddl/customer_delete.sql | 0
.../smooth_upgrade}/ddl/customer_load.sql | 0
.../smooth_upgrade}/ddl/customer_load_sequence.sql | 0
.../smooth_upgrade}/ddl/customer_part_delete.sql | 0
.../smooth_upgrade/ddl/customer_sequence.sql | 17 +
.../multi_cluster/smooth_upgrade/ddl/lineitem.sql | 23 +
.../smooth_upgrade}/ddl/lineitem_delete.sql | 0
.../smooth_upgrade/ddl/lineitem_load.sql | 6 +
.../smooth_upgrade/ddl/lineitem_load_sequence.sql | 7 +
.../smooth_upgrade}/ddl/lineitem_part_delete.sql | 0
.../smooth_upgrade/ddl/lineitem_sequence.sql | 25 +
.../multi_cluster/smooth_upgrade/ddl/nation.sql | 11 +
.../smooth_upgrade}/ddl/nation_delete.sql | 0
.../smooth_upgrade}/ddl/nation_load.sql | 0
.../smooth_upgrade}/ddl/nation_load_sequence.sql | 0
.../smooth_upgrade}/ddl/nation_part_delete.sql | 0
.../smooth_upgrade/ddl/nation_sequence.sql | 13 +
.../multi_cluster/smooth_upgrade/ddl/orders.sql | 15 +
.../smooth_upgrade}/ddl/orders_delete.sql | 0
.../smooth_upgrade}/ddl/orders_load.sql | 0
.../smooth_upgrade}/ddl/orders_load_sequence.sql | 0
.../smooth_upgrade}/ddl/orders_part_delete.sql | 0
.../smooth_upgrade/ddl/orders_sequence.sql | 17 +
.../multi_cluster/smooth_upgrade/ddl/part.sql | 16 +
.../smooth_upgrade}/ddl/part_delete.sql | 0
.../smooth_upgrade}/ddl/part_load.sql | 0
.../smooth_upgrade}/ddl/part_load_sequence.sql | 0
.../smooth_upgrade}/ddl/part_part_delete.sql | 0
.../smooth_upgrade/ddl/part_sequence.sql | 18 +
.../multi_cluster/smooth_upgrade/ddl/partsupp.sql | 12 +
.../smooth_upgrade}/ddl/partsupp_delete.sql | 0
.../smooth_upgrade}/ddl/partsupp_load.sql | 0
.../smooth_upgrade}/ddl/partsupp_load_sequence.sql | 0
.../smooth_upgrade}/ddl/partsupp_part_delete.sql | 0
.../smooth_upgrade/ddl/partsupp_sequence.sql | 14 +
.../multi_cluster/smooth_upgrade/ddl/region.sql | 10 +
.../smooth_upgrade}/ddl/region_delete.sql | 0
.../smooth_upgrade}/ddl/region_load.sql | 0
.../smooth_upgrade}/ddl/region_load_sequence.sql | 0
.../smooth_upgrade}/ddl/region_part_delete.sql | 0
.../smooth_upgrade/ddl/region_sequence.sql | 12 +
.../multi_cluster/smooth_upgrade/ddl/supplier.sql | 13 +
.../smooth_upgrade}/ddl/supplier_delete.sql | 0
.../smooth_upgrade}/ddl/supplier_load.sql | 0
.../smooth_upgrade}/ddl/supplier_load_sequence.sql | 0
.../smooth_upgrade}/ddl/supplier_part_delete.sql | 0
.../smooth_upgrade/ddl/supplier_sequence.sql | 15 +
.../smooth_upgrade/smooth_upgrade.groovy | 165 ++++
.../multi_cluster/stream_load/stream_load.groovy | 322 ++++++
.../stream_load/stream_load_2pc.groovy | 396 ++++++++
.../stream_load/stream_load_lb.groovy | 227 +++++
.../cloud_p0/multi_cluster/table_rebalance.groovy | 175 ++++
.../test_apsaradb_internal_stage.groovy | 199 ++++
.../multi_cluster/test_cloud_cluster.groovy | 202 ++++
.../multi_cluster/test_drop_cluster.groovy | 96 ++
.../cloud_p0/multi_cluster/test_drop_userid.groovy | 121 +++
.../test_group_commit_multi_cluster.groovy | 129 +++
.../multi_cluster/test_overdue_instance.groovy | 243 +++++
.../spark_connector/spark_connector.groovy | 41 -
.../spark_connector/spark_connector_arrow.groovy | 15 +-
.../spark_connector_read_type.groovy | 16 +-
.../suites/correctness/test_compare_float.groovy | 2 +-
.../one_level_nestedtypes_with_s3data.groovy | 6 +-
.../string_len/test_string_len_complex.groovy | 229 +++++
.../cache/test_file_cache_features.groovy | 222 +++++
.../cache/test_file_cache_statistics.groovy | 199 +++-
.../memtable/test_memtable_too_many_rows.groovy | 49 +
.../flink_connector_p0/flink_connector.groovy | 8 +-
.../flink_connector_syncdb.groovy | 14 +-
.../flink_connector_p0/flink_connector_type.groovy | 108 +--
.../test_iot_auto_detect_fail.groovy | 4 +-
.../analyzer/test_custom_analyzer.groovy | 26 +-
.../analyzer/test_custom_analyzer1.groovy | 69 +-
.../inverted_index_p0/test_inverted_is_null.groovy | 72 ++
.../test_streaming_insert_job_alter.groovy | 155 +++
.../test_streaming_insert_job_crud.groovy | 471 +++++++++
.../test_streaming_insert_job_priv.groovy | 204 ++++
.../mv_p0/test_create_mv/test_create_mv.groovy | 12 +
.../nereids_p0/cache/mtmv_with_sql_cache.groovy | 457 +++++++++
.../nereids_p0/cache/mv_with_sql_cache.groovy | 129 +++
.../cache/parse_sql_from_sql_cache.groovy | 31 +
.../partition_topn/check_orderkey.groovy | 61 ++
.../test_date_function_v2.groovy | 68 ++
.../nereids_p0/stats/partitionRowCount.groovy | 2 +-
.../avg_distinct_to_sum_div_count.groovy | 23 +-
.../expression/test_simplify_range.groovy | 2 +-
.../partition_prune/one_key_list_part_test.groovy | 1 +
.../one_key_list_part_update_test.groovy | 1 +
.../partition_prune/one_key_range_part_test.groovy | 1 +
.../one_key_range_part_update_test.groovy | 3 +-
.../auto_partition/test_auto_list_partition.groovy | 2 +-
.../test_auto_partition_behavior.groovy | 14 +
...uto_partition_with_single_replica_insert.groovy | 4 +-
.../test_col_data_type_boundary.groovy | 20 +-
...t_dynamic_partition_mod_distribution_key.groovy | 1 +
.../query_p0/show/test_show_partitions.groovy | 2 +-
.../conditional_functions/test_coalesce.groovy | 88 ++
.../string_functions/test_string_function.groovy | 99 ++
.../struct_functions/test_struct_functions.groovy | 8 +-
.../sql_functions/table_function/explode.groovy | 25 +
.../system/test_query_sys_column_data_sizes.groovy | 130 +++
.../test_double_write_when_schema_change.groovy | 2 +-
.../test_alter_rollup_table.groovy | 9 +-
.../test_alter_table_modify_column.groovy | 2 +-
.../schema_change_p0/test_schema_change_agg.groovy | 4 +-
.../suites/search/test_search_mow_support.groovy | 78 ++
.../search/test_search_vs_match_consistency.groovy | 40 +
.../suites/show_p0/test_show_data_warehouse.groovy | 10 +-
.../ddl/customer.sql | 0
.../ddl/customer_delete.sql | 0
.../ddl/lineitem.sql | 0
.../ddl/lineitem_delete.sql | 0
.../ddl/nation.sql | 0
.../ddl/nation_delete.sql | 0
.../ddl/orders.sql | 0
.../ddl/orders_delete.sql | 0
.../ddl/part.sql | 0
.../ddl/part_delete.sql | 0
.../ddl/partsupp.sql | 0
.../ddl/partsupp_delete.sql | 0
.../ddl/region.sql | 0
.../ddl/region_delete.sql | 0
.../ddl/revenue1.sql | 0
.../ddl/revenue1_delete.sql | 0
.../ddl/supplier.sql | 0
.../ddl/supplier_delete.sql | 0
.../load.groovy | 0
.../sql/q01.sql | 0
.../sql/q02.sql | 0
.../sql/q03.sql | 0
.../sql/q04.sql | 0
.../sql/q05.sql | 0
.../sql/q06.sql | 0
.../sql/q07.sql | 0
.../sql/q08.sql | 0
.../sql/q09.sql | 0
.../sql/q10.sql | 0
.../sql/q11.sql | 0
.../sql/q12.sql | 0
.../sql/q13.sql | 0
.../sql/q14.sql | 0
.../sql/q15.sql | 0
.../sql/q16.sql | 0
.../sql/q17.sql | 0
.../sql/q18.sql | 0
.../sql/q19.sql | 0
.../sql/q20.sql | 0
.../sql/q21.sql | 0
.../sql/q22.sql | 0
.../sql/test_left_anti_join_batch_size.sql | 0
.../test_key_bounds_truncation_basic.groovy | 12 +
...est_key_bounds_truncation_read_scenarios.groovy | 12 +
.../test_variant_search_subcolumn.groovy | 152 +++
run-regression-test.sh | 2 +-
1078 files changed, 23724 insertions(+), 6926 deletions(-)
create mode 100644 be/src/exec/schema_scanner/schema_column_data_sizes_scanner.cpp
create mode 100644 be/src/exec/schema_scanner/schema_column_data_sizes_scanner.h
copy be/src/olap/rowset/segment_v2/inverted_index/{tokenizer/tokenizer.h => char_filter/char_filter.h} (52%)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory.h
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/basic/basic_tokenizer.cpp (73%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/basic/basic_tokenizer.h (65%)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer_factory.h
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/break_iterator_wrapper.cpp (97%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/break_iterator_wrapper.h (94%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/composite_break_iterator.cpp (97%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/composite_break_iterator.h (94%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/default_icu_tokenizer_config.cpp (98%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/default_icu_tokenizer_config.h (94%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/icu_common.h (93%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/icu_tokenizer.cpp (84%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/icu_tokenizer.h (82%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/icu_tokenizer_config.h (93%)
copy be/src/olap/rowset/segment_v2/inverted_index/{token_filter/token_filter.h => tokenizer/icu/icu_tokenizer_factory.h} (61%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/script_iterator.cpp (97%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/script_iterator.h (95%)
copy fe/fe-core/src/main/java/org/apache/doris/load/EtlJobType.java => be/src/olap/rowset/segment_v2/inverted_index/util/reader.h (76%)
create mode 100644 be/src/pipeline/revokable_task.h
copy be/{src/common/version_internal.h => test/common/compare_test.cpp} (61%)
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/char_filter/char_filter_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/tokenizer/basic_tokenizer_factory_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/tokenizer/icu_tokenizer_factory_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/util/reader_test.cpp
create mode 100644 cloud/src/recycler/snapshot_chain_compactor.cpp
create mode 100644 cloud/src/recycler/snapshot_chain_compactor.h
create mode 100644 cloud/src/recycler/snapshot_data_migrator.cpp
create mode 100644 cloud/src/recycler/snapshot_data_migrator.h
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/TableName.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketCalculator.java
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{AbstractHMSProperties.java => AbstractHiveProperties.java} (93%)
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{HMSAliyunDLFMetaStoreProperties.java => HiveAliyunDLFMetaStoreProperties.java} (94%)
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{HMSGlueMetaStoreProperties.java => HiveGlueMetaStoreProperties.java} (96%)
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{HMSProperties.java => HiveHMSProperties.java} (95%)
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{HMSPropertiesFactory.java => HivePropertiesFactory.java} (70%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/indexpolicy/BasicTokenizerValidator.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/indexpolicy/CharReplaceCharFilterValidator.java
copy fe/fe-core/src/main/java/org/apache/doris/indexpolicy/{IndexPolicyTypeEnum.java => ICUTokenizerValidator.java} (56%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/GetFormatFunctionBinder.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/CaseWhenToCompoundPredicate.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ExportSet.java
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{ElementAt.java => GetFormat.java} (58%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/{AlterJobStatusCommand.java => CreateIndexCharFilterCommand.java} (57%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/{DropJobCommand.java => DropIndexCharFilterCommand.java} (71%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/{PauseJobCommand.java => ShowIndexCharFilterCommand.java} (67%)
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionExprUtilTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketCalculatorTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/CaseWhenToCompoundPredicateTest.java
create mode 100644 regression-test/data/cloud_p0/multi_cluster/routine_load/test_routine_load.out
copy regression-test/data/{tpch_unique_sql_zstd_bucket1_p0/sql/q08.out => cloud_p0/multi_cluster/schema_change/test_schema_change_with_readd_cluster.out} (66%)
copy regression-test/data/{cloud/multi_cluster/vcluster => cloud_p0/multi_cluster/stream_load}/all_types.csv (100%)
copy regression-test/data/{tpch_unique_sql_zstd_bucket1_p0/sql/q08.out => cloud_p0/multi_cluster/stream_load/stream_load.out} (58%)
create mode 100644 regression-test/data/cloud_p0/multi_cluster/stream_load/stream_load_2pc.out
create mode 100644 regression-test/data/cloud_p0/multi_cluster/stream_load/stream_load_lb.out
copy regression-test/data/{tpch_unique_sql_zstd_bucket1_p0/sql/q17.out => cloud_p0/multi_cluster/test_apsaradb_internal_stage.out} (76%)
create mode 100644 regression-test/data/datatype_p0/nested_types/string_len/test_string_len_complex.out
create mode 100644 regression-test/data/job_p0/streaming_job/test_streaming_insert_job_alter.out
create mode 100644 regression-test/data/job_p0/streaming_job/test_streaming_insert_job_crud.out
copy regression-test/data/{tpch_unique_sql_zstd_bucket1_p0/sql/q19.out => nereids_p0/partition_topn/check_orderkey.out} (80%)
copy regression-test/data/{tpch_unique_sql_zstd_bucket1_p0/sql/q17.out => nereids_rules_p0/avg_distinct_to_sum_div_count/avg_distinct_to_sum_div_count.out} (78%)
create mode 100644 regression-test/data/query_p0/system/test_query_sys_column_data_sizes.out
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q01.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q02.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q03.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q04.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q05.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q06.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q07.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q08.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q09.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q10.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q11.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q12.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q13.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q14.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q15.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q16.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q17.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q18.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q19.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q20.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q21.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q22.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/test_left_anti_join_batch_size.out (100%)
create mode 100644 regression-test/data/variant_p0/test_variant_search_subcolumn.out
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/async_copy_into/async_load.groovy
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/customer.sql (79%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/customer_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/lineitem.sql (91%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/lineitem_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/nation.sql (68%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/nation_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/orders.sql (81%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/orders_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/part.sql (81%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/part_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/partsupp.sql (82%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/partsupp_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/region.sql (64%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/region_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/revenue1.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/async_copy_into}/ddl/revenue1_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/supplier.sql (77%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/supplier_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/customer.sql (79%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/customer_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/lineitem.sql (91%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/lineitem_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/nation.sql (68%)
copy regression-test/suites/cloud_p0/multi_cluster/{virtual_compute_group => copy_into}/ddl/nation_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/orders.sql (81%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/orders_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/part.sql (81%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/part_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/partsupp.sql (82%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/partsupp_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/region.sql (64%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/region_delete.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/revenue1.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/revenue1_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/supplier.sql (77%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/supplier_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/copy_into/sync_laod.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/default_cluster.groovy
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/customer.sql (71%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/customer_delete.sql (100%)
copy regression-test/suites/cloud_p0/{cache/multi_cluster/read_write => multi_cluster/multi_cluster_s3_load}/ddl/customer_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/lineitem.sql (78%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/lineitem_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/multi_cluster_s3_load/ddl/lineitem_load.sql
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/nation.sql (68%)
copy regression-test/suites/cloud_p0/multi_cluster/{virtual_compute_group => multi_cluster_s3_load}/ddl/nation_delete.sql (100%)
copy regression-test/suites/cloud_p0/multi_cluster/{virtual_compute_group => multi_cluster_s3_load}/ddl/nation_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/orders.sql (73%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/orders_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/orders_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/part.sql (72%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/part_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/part_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/partsupp.sql (61%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/partsupp_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/partsupp_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/region.sql (64%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/region_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/region_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/supplier.sql (68%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/supplier_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/supplier_load.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/multi_cluster_s3_load/load.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/op/test_drop_and_add_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/rename_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/routine_load/test_routine_load.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_bitmap_index_with_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_bitmap_index_with_readd_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_materialized_view_with_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_materialized_view_with_readd_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_rollup_with_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_rollup_with_readd_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_schema_change_with_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_schema_change_with_readd_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/customer.sql
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/smooth_upgrade}/ddl/customer_delete.sql (100%)
copy regression-test/suites/cloud_p0/{cache/multi_cluster/read_write => multi_cluster/smooth_upgrade}/ddl/customer_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/customer_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/customer_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/customer_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/lineitem.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/lineitem_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/lineitem_load.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/lineitem_load_sequence.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/lineitem_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/lineitem_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/nation.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/nation_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/nation_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/nation_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/nation_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/nation_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/orders.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/orders_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/orders_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/orders_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/orders_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/orders_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/part.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/part_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/part_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/part_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/part_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/part_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/partsupp.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/partsupp_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/partsupp_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/partsupp_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/partsupp_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/partsupp_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/region.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/region_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/region_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/region_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/region_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/region_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/supplier.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/supplier_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/supplier_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/supplier_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/supplier_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/supplier_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/smooth_upgrade.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/stream_load/stream_load.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/stream_load/stream_load_2pc.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/stream_load/stream_load_lb.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/table_rebalance.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_apsaradb_internal_stage.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_cloud_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_drop_userid.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_group_commit_multi_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_overdue_instance.groovy
delete mode 100644 regression-test/suites/connector_p0/spark_connector/spark_connector.groovy
create mode 100644 regression-test/suites/datatype_p0/nested_types/string_len/test_string_len_complex.groovy
create mode 100644 regression-test/suites/external_table_p0/cache/test_file_cache_features.groovy
create mode 100644 regression-test/suites/fault_injection_p0/memtable/test_memtable_too_many_rows.groovy
create mode 100644 regression-test/suites/inverted_index_p0/test_inverted_is_null.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_alter.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_crud.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_priv.groovy
create mode 100644 regression-test/suites/nereids_p0/cache/mtmv_with_sql_cache.groovy
create mode 100644 regression-test/suites/nereids_p0/cache/mv_with_sql_cache.groovy
create mode 100644 regression-test/suites/nereids_p0/partition_topn/check_orderkey.groovy
copy fe/fe-core/src/main/java/org/apache/doris/load/EtlJobType.java => regression-test/suites/nereids_rules_p0/avg_distinct_to_sum_div_count/avg_distinct_to_sum_div_count.groovy (62%)
create mode 100644 regression-test/suites/query_p0/system/test_query_sys_column_data_sizes.groovy
create mode 100644 regression-test/suites/search/test_search_mow_support.groovy
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/customer.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/customer_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/lineitem.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/lineitem_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/nation.sql (100%)
copy regression-test/suites/{cloud_p0/multi_cluster/virtual_compute_group => tpch_unique_sql_zstd_bucket1_p1}/ddl/nation_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/orders.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/orders_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/part.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/part_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/partsupp.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/partsupp_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/region.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/region_delete.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/revenue1.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/revenue1_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/supplier.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/supplier_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/load.groovy (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q01.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q02.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q03.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q04.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q05.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q06.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q07.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q08.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q09.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q10.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q11.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q12.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q13.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q14.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q15.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q16.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q17.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q18.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q19.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q20.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q21.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q22.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/test_left_anti_join_batch_size.sql (100%)
create mode 100644 regression-test/suites/variant_p0/test_variant_search_subcolumn.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 09/09: update fe code
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch dev_rec
in repository https://gitbox.apache.org/repos/asf/doris.git
commit aa680e5c1271c64f7e7815315067562f12cfc298
Author: lichi <li...@selectdb.com>
AuthorDate: Fri Oct 17 12:41:40 2025 +0800
update fe code
---
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 5 +-
.../glue/translator/PhysicalPlanTranslator.java | 8 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 6 +-
.../processor/pre/PullUpSubqueryAliasToCTE.java | 8 +-
.../doris/nereids/rules/analysis/AnalyzeCTE.java | 50 ++++--
.../doris/nereids/rules/rewrite/ColumnPruning.java | 39 +----
.../plans/commands/UpdateMvByPartitionCommand.java | 3 +-
.../nereids/trees/plans/logical/LogicalCTE.java | 27 ++-
.../trees/plans/logical/LogicalRecursiveCte.java | 183 ---------------------
.../logical/LogicalRecursiveCteRecursiveChild.java | 5 +
.../plans/logical/LogicalRecursiveCteScan.java | 27 ++-
.../trees/plans/logical/LogicalSubQueryAlias.java | 53 +++---
.../PhysicalRecursiveCteRecursiveChild.java | 5 +
.../org/apache/doris/planner/RecursiveCteNode.java | 27 ++-
.../apache/doris/planner/RecursiveCteScanNode.java | 9 +
.../doris/qe/runtime/ThriftPlansBuilder.java | 14 +-
gensrc/thrift/PaloInternalService.thrift | 1 +
17 files changed, 182 insertions(+), 288 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 162c6f21688..4add5f55556 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -1200,11 +1200,11 @@ querySpecification
;
cte
- : WITH aliasQuery (COMMA aliasQuery)*
+ : WITH RECURSIVE? aliasQuery (COMMA aliasQuery)*
;
aliasQuery
- : RECURSIVE? identifier columnAliases? AS LEFT_PAREN query RIGHT_PAREN
+ : identifier columnAliases? AS LEFT_PAREN query RIGHT_PAREN
;
columnAliases
@@ -2128,6 +2128,7 @@ nonReserved
| RANDOM
| RECENT
| RECOVER
+ | RECURSIVE
| RECYCLE
| REFRESH
| REPEATABLE
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 3c0007a12c0..8c35e65d1f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -2256,8 +2256,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
if (inputPlanNode instanceof OlapScanNode) {
((OlapScanNode) inputPlanNode).updateRequiredSlots(context, requiredByProjectSlotIdSet);
}
- updateScanSlotsMaterialization((ScanNode) inputPlanNode, requiredSlotIdSet,
- requiredByProjectSlotIdSet, context);
+ if (!(inputPlanNode instanceof RecursiveCteScanNode)) {
+ updateScanSlotsMaterialization((ScanNode) inputPlanNode, requiredSlotIdSet,
+ requiredByProjectSlotIdSet, context);
+ }
} else {
if (project.child() instanceof PhysicalDeferMaterializeTopN) {
inputFragment.setOutputExprs(allProjectionExprs);
@@ -2339,6 +2341,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
recursiveCteFragment.setHasColocatePlanNode(true);
recursiveCteNode.setColocate(true);
}
+ recursiveCteFragment.updateDataPartition(DataPartition.UNPARTITIONED);
+ recursiveCteFragment.setOutputPartition(DataPartition.UNPARTITIONED);
return recursiveCteFragment;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 3a3481132b3..ebcbe07c7bb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -2134,7 +2134,8 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
if (ctx == null) {
return plan;
}
- return new LogicalCTE<>((List) visit(ctx.aliasQuery(), LogicalSubQueryAlias.class), plan);
+ return new LogicalCTE<>(ctx.RECURSIVE() != null,
+ (List) visit(ctx.aliasQuery(), LogicalSubQueryAlias.class), plan);
}
/**
@@ -2149,8 +2150,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
.map(RuleContext::getText)
.collect(ImmutableList.toImmutableList())
);
- return new LogicalSubQueryAlias<>(ctx.identifier().getText(), columnNames, ctx.RECURSIVE() != null,
- queryPlan);
+ return new LogicalSubQueryAlias<>(ctx.identifier().getText(), columnNames, queryPlan);
});
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java
index 31a205d5ed5..1dfe63c9e67 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java
@@ -47,10 +47,11 @@ public class PullUpSubqueryAliasToCTE extends PlanPreprocessor {
subQueryAliases.addAll(logicalCTE.getAliasQueries());
subQueryAliases.addAll(aliasQueries);
return topPlan.withChildren(
- new LogicalCTE<>(subQueryAliases, (LogicalPlan) ((UnboundResultSink) topPlan).child()));
+ new LogicalCTE<>(logicalCTE.isRecursiveCte(), subQueryAliases,
+ (LogicalPlan) ((UnboundResultSink) topPlan).child()));
}
return topPlan.withChildren(
- new LogicalCTE<>(aliasQueries, (LogicalPlan) ((UnboundResultSink) topPlan).child()));
+ new LogicalCTE<>(false, aliasQueries, (LogicalPlan) ((UnboundResultSink) topPlan).child()));
}
return topPlan;
}
@@ -86,7 +87,8 @@ public class PullUpSubqueryAliasToCTE extends PlanPreprocessor {
subQueryAliasesOfCte.addAll(logicalCTE.getAliasQueries());
subQueryAliasesOfCte.addAll(aliasQueries);
aliasQueries = new ArrayList<>();
- return new LogicalCTE<>(subQueryAliasesOfCte, (LogicalPlan) newLogicalCTE.child());
+ return new LogicalCTE<>(newLogicalCTE.isRecursiveCte(), subQueryAliasesOfCte,
+ (LogicalPlan) newLogicalCTE.child());
}
return cte;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
index 906d572c6c1..b7ef605c395 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
@@ -39,6 +39,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteRecursiveChild;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.trees.plans.logical.ProjectProcessor;
@@ -108,7 +109,7 @@ public class AnalyzeCTE extends OneAnalysisRuleFactory {
List<LogicalCTEProducer<Plan>> cteProducerPlans = new ArrayList<>();
for (LogicalSubQueryAlias<Plan> aliasQuery : aliasQueries) {
// we should use a chain to ensure visible of cte
- if (aliasQuery.isRecursiveCte()) {
+ if (aliasQuery.isRecursiveCte() && logicalCTE.isRecursiveCte()) {
Pair<CTEContext, LogicalCTEProducer<Plan>> result = analyzeRecursiveCte(aliasQuery, outerCteCtx,
cascadesContext);
outerCteCtx = result.first;
@@ -150,13 +151,10 @@ public class AnalyzeCTE extends OneAnalysisRuleFactory {
});
cascadesContext.addPlanProcesses(innerAnchorCascadesCtx.getPlanProcesses());
LogicalPlan analyzedAnchorChild = (LogicalPlan) innerAnchorCascadesCtx.getRewritePlan();
- List<NamedExpression> anchorNullableOutputs = new ArrayList<>(analyzedAnchorChild.getOutput().size());
- for (Slot slot : analyzedAnchorChild.getOutput()) {
- anchorNullableOutputs.add(new Alias(new Nullable(slot), slot.getName()));
- }
- analyzedAnchorChild = new LogicalProject<>(anchorNullableOutputs, analyzedAnchorChild);
checkColumnAlias(aliasQuery, analyzedAnchorChild.getOutput());
-
+ // make all output nullable
+ analyzedAnchorChild = forceOutputNullable(analyzedAnchorChild,
+ aliasQuery.getColumnAliases().orElse(ImmutableList.of()));
// analyze recursive child
LogicalPlan recursiveChild = (LogicalPlan) parsedCtePlan.child(1);
CascadesContext innerRecursiveCascadesCtx = CascadesContext.newContextWithCteContext(
@@ -167,26 +165,28 @@ public class AnalyzeCTE extends OneAnalysisRuleFactory {
});
cascadesContext.addPlanProcesses(innerRecursiveCascadesCtx.getPlanProcesses());
LogicalPlan analyzedRecursiveChild = (LogicalPlan) innerRecursiveCascadesCtx.getRewritePlan();
- List<DataType> anchorChildOutputTypes = new ArrayList<>(analyzedAnchorChild.getOutput().size());
- for (Slot slot : analyzedAnchorChild.getOutput()) {
+ List<LogicalRecursiveCteScan> recursiveCteScanList = analyzedRecursiveChild
+ .collectToList(LogicalRecursiveCteScan.class::isInstance);
+ if (recursiveCteScanList.size() > 1) {
+ throw new AnalysisException(String.format("can have only 1 recursive cte instance, but there is %d",
+ recursiveCteScanList.size()));
+ }
+ List<Slot> anchorChildOutputs = analyzedAnchorChild.getOutput();
+ List<DataType> anchorChildOutputTypes = new ArrayList<>(anchorChildOutputs.size());
+ for (Slot slot : anchorChildOutputs) {
anchorChildOutputTypes.add(slot.getDataType());
}
List<Slot> recursiveChildOutputs = analyzedRecursiveChild.getOutput();
for (int i = 0; i < recursiveChildOutputs.size(); ++i) {
- if (recursiveChildOutputs.get(i).getDataType() != anchorChildOutputTypes.get(i)) {
+ if (!recursiveChildOutputs.get(i).getDataType().equals(anchorChildOutputTypes.get(i))) {
throw new AnalysisException(String.format("recursive child's %d column's datatype in select list %s "
+ "is different from anchor child's output datatype %s, please add cast manually "
+ "to get expect datatype",
i + 1, recursiveChildOutputs.get(i).getDataType(), anchorChildOutputTypes.get(i)));
}
}
-
- List<NamedExpression> recursiveNullableOutputs = new ArrayList<>(analyzedRecursiveChild.getOutput().size());
- for (Slot slot : analyzedRecursiveChild.getOutput()) {
- recursiveNullableOutputs.add(new Alias(new Nullable(slot), slot.getName()));
- }
- analyzedRecursiveChild = new LogicalProject<>(recursiveNullableOutputs, analyzedRecursiveChild);
- analyzedRecursiveChild = new LogicalRecursiveCteRecursiveChild<>(analyzedRecursiveChild);
+ analyzedRecursiveChild = new LogicalRecursiveCteRecursiveChild<>(forceOutputNullable(analyzedRecursiveChild,
+ ImmutableList.of()));
// create LogicalRecursiveCte
LogicalUnion logicalUnion = (LogicalUnion) parsedCtePlan;
@@ -223,6 +223,22 @@ public class AnalyzeCTE extends OneAnalysisRuleFactory {
return Pair.of(outerCteCtx, cteProducer);
}
+ private LogicalPlan forceOutputNullable(LogicalPlan logicalPlan, List<String> aliasNames) {
+ List<Slot> oldOutputs = logicalPlan.getOutput();
+ int size = oldOutputs.size();
+ List<NamedExpression> newOutputs = new ArrayList<>(oldOutputs.size());
+ if (!aliasNames.isEmpty()) {
+ for (int i = 0; i < size; ++i) {
+ newOutputs.add(new Alias(new Nullable(oldOutputs.get(i)), aliasNames.get(i)));
+ }
+ } else {
+ for (Slot slot : oldOutputs) {
+ newOutputs.add(new Alias(new Nullable(slot), slot.getName()));
+ }
+ }
+ return new LogicalProject<>(newOutputs, logicalPlan);
+ }
+
/**
* check columnAliases' size and name
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
index 30a00314fbf..dfbf13cedb7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
@@ -216,43 +216,8 @@ public class ColumnPruning extends DefaultPlanRewriter<PruneContext> implements
@Override
public Plan visitLogicalRecursiveCte(LogicalRecursiveCte recursiveCte, PruneContext context) {
- // LogicalRecursiveCte is basically like LogicalUnion, so just do same as LogicalUnion
- if (!recursiveCte.isUnionAll()) {
- return skipPruneThisAndFirstLevelChildren(recursiveCte);
- }
- LogicalRecursiveCte prunedOutputRecursiveCte = pruneRecursiveCteOutput(recursiveCte, context);
- // start prune children of recursiveCte
- List<Slot> originOutput = recursiveCte.getOutput();
- Set<Slot> prunedOutput = prunedOutputRecursiveCte.getOutputSet();
- List<Integer> prunedOutputIndexes = IntStream.range(0, originOutput.size())
- .filter(index -> prunedOutput.contains(originOutput.get(index)))
- .boxed()
- .collect(ImmutableList.toImmutableList());
-
- ImmutableList.Builder<Plan> prunedChildren = ImmutableList.builder();
- ImmutableList.Builder<List<SlotReference>> prunedChildrenOutputs = ImmutableList.builder();
- for (int i = 0; i < prunedOutputRecursiveCte.arity(); i++) {
- List<SlotReference> regularChildOutputs = prunedOutputRecursiveCte.getRegularChildOutput(i);
-
- RoaringBitmap prunedChildOutputExprIds = new RoaringBitmap();
- Builder<SlotReference> prunedChildOutputBuilder
- = ImmutableList.builderWithExpectedSize(regularChildOutputs.size());
- for (Integer index : prunedOutputIndexes) {
- SlotReference slot = regularChildOutputs.get(index);
- prunedChildOutputBuilder.add(slot);
- prunedChildOutputExprIds.add(slot.getExprId().asInt());
- }
-
- List<SlotReference> prunedChildOutput = prunedChildOutputBuilder.build();
- Plan prunedChild = doPruneChild(
- prunedOutputRecursiveCte, prunedOutputRecursiveCte.child(i), prunedChildOutputExprIds,
- prunedChildOutput, true
- );
- prunedChildrenOutputs.add(prunedChildOutput);
- prunedChildren.add(prunedChild);
- }
- return prunedOutputRecursiveCte.withChildrenAndTheirOutputs(prunedChildren.build(),
- prunedChildrenOutputs.build());
+ // keep LogicalRecursiveCte's output unchanged
+ return skipPruneThisAndFirstLevelChildren(recursiveCte);
}
// union can not prune children by the common logic, we must override visit method to write special code.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
index 27758175527..c33db474d06 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
@@ -267,7 +267,8 @@ public class UpdateMvByPartitionCommand extends InsertOverwriteTableCommand {
);
rewrittenSubQueryAlias.add(subQueryAlias.withChildren(subQueryAliasChildren));
}
- return super.visitLogicalCTE(new LogicalCTE<>(rewrittenSubQueryAlias, cte.child()), predicates);
+ return super.visitLogicalCTE(new LogicalCTE<>(cte.isRecursiveCte(),
+ rewrittenSubQueryAlias, cte.child()), predicates);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java
index 4f810c3b6de..5ceb0e55bca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java
@@ -41,21 +41,28 @@ import java.util.stream.Collectors;
public class LogicalCTE<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE> implements PropagateFuncDeps {
private final List<LogicalSubQueryAlias<Plan>> aliasQueries;
+ private final boolean isRecursiveCte;
- public LogicalCTE(List<LogicalSubQueryAlias<Plan>> aliasQueries, CHILD_TYPE child) {
- this(aliasQueries, Optional.empty(), Optional.empty(), child);
+ public LogicalCTE(boolean isRecursiveCte, List<LogicalSubQueryAlias<Plan>> aliasQueries, CHILD_TYPE child) {
+ this(isRecursiveCte, aliasQueries, Optional.empty(), Optional.empty(), child);
}
- public LogicalCTE(List<LogicalSubQueryAlias<Plan>> aliasQueries, Optional<GroupExpression> groupExpression,
- Optional<LogicalProperties> logicalProperties, CHILD_TYPE child) {
+ public LogicalCTE(boolean isRecursiveCte, List<LogicalSubQueryAlias<Plan>> aliasQueries,
+ Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
+ CHILD_TYPE child) {
super(PlanType.LOGICAL_CTE, groupExpression, logicalProperties, child);
this.aliasQueries = ImmutableList.copyOf(Objects.requireNonNull(aliasQueries, "aliasQueries can not be null"));
+ this.isRecursiveCte = isRecursiveCte;
}
public List<LogicalSubQueryAlias<Plan>> getAliasQueries() {
return aliasQueries;
}
+ public boolean isRecursiveCte() {
+ return isRecursiveCte;
+ }
+
@Override
public List<? extends Plan> extraPlans() {
return aliasQueries;
@@ -74,6 +81,7 @@ public class LogicalCTE<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE
@Override
public String toString() {
return Utils.toSqlString("LogicalCTE",
+ "isRecursiveCte", isRecursiveCte,
"aliasQueries", aliasQueries
);
}
@@ -105,18 +113,18 @@ public class LogicalCTE<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE
return false;
}
LogicalCTE that = (LogicalCTE) o;
- return aliasQueries.equals(that.aliasQueries);
+ return aliasQueries.equals(that.aliasQueries) && isRecursiveCte == that.isRecursiveCte;
}
@Override
public int hashCode() {
- return Objects.hash(aliasQueries);
+ return Objects.hash(aliasQueries, isRecursiveCte);
}
@Override
public Plan withChildren(List<Plan> children) {
Preconditions.checkArgument(aliasQueries.size() > 0);
- return new LogicalCTE<>(aliasQueries, children.get(0));
+ return new LogicalCTE<>(isRecursiveCte, aliasQueries, children.get(0));
}
@Override
@@ -131,13 +139,14 @@ public class LogicalCTE<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE
@Override
public LogicalCTE<CHILD_TYPE> withGroupExpression(Optional<GroupExpression> groupExpression) {
- return new LogicalCTE<>(aliasQueries, groupExpression, Optional.of(getLogicalProperties()), child());
+ return new LogicalCTE<>(isRecursiveCte, aliasQueries, groupExpression,
+ Optional.of(getLogicalProperties()), child());
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
Preconditions.checkArgument(aliasQueries.size() > 0);
- return new LogicalCTE<>(aliasQueries, groupExpression, logicalProperties, children.get(0));
+ return new LogicalCTE<>(isRecursiveCte, aliasQueries, groupExpression, logicalProperties, children.get(0));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java
index 3958d5b4df1..4a727476d35 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java
@@ -17,14 +17,10 @@
package org.apache.doris.nereids.trees.plans.logical;
-import org.apache.doris.common.Pair;
-import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.DataTrait;
import org.apache.doris.nereids.properties.LogicalProperties;
-import org.apache.doris.nereids.properties.PhysicalProperties;
-import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ExprId;
@@ -33,30 +29,20 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
-import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.algebra.RecursiveCte;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.types.DataType;
-import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.TypeCoercionUtils;
import org.apache.doris.nereids.util.Utils;
-import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Lists;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Objects;
import java.util.Optional;
-import java.util.Set;
/**
* LogicalRecursiveCte is basically like LogicalUnion
@@ -269,44 +255,10 @@ public class LogicalRecursiveCte extends AbstractLogicalPlan implements Recursiv
@Override
public void computeUnique(DataTrait.Builder builder) {
- if (!isUnionAll) {
- builder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
- }
}
@Override
public void computeUniform(DataTrait.Builder builder) {
- final Optional<ExpressionRewriteContext> context = ConnectContext.get() == null ? Optional.empty()
- : Optional.of(new ExpressionRewriteContext(CascadesContext.initContext(
- ConnectContext.get().getStatementContext(), this, PhysicalProperties.ANY)));
- for (int i = 0; i < getOutputs().size(); i++) {
- Optional<Literal> value = Optional.empty();
- for (int childIdx = 0; childIdx < children.size(); childIdx++) {
- // TODO: use originOutputs = child(childIdx).getOutput() ?
- List<? extends Slot> originOutputs = regularChildrenOutputs.get(childIdx);
- Slot slot = originOutputs.get(i);
- Optional<Expression> childValue = child(childIdx).getLogicalProperties()
- .getTrait().getUniformValue(slot);
- if (childValue == null || !childValue.isPresent() || !childValue.get().isConstant()) {
- value = Optional.empty();
- break;
- }
- Optional<Literal> constExprOpt = ExpressionUtils.checkConstantExpr(childValue.get(), context);
- if (!constExprOpt.isPresent()) {
- value = Optional.empty();
- break;
- }
- if (!value.isPresent()) {
- value = constExprOpt;
- } else if (!value.equals(constExprOpt)) {
- value = Optional.empty();
- break;
- }
- }
- if (value.isPresent()) {
- builder.addUniformSlotAndLiteral(getOutputs().get(i).toSlot(), value.get());
- }
- }
}
@Override
@@ -314,147 +266,12 @@ public class LogicalRecursiveCte extends AbstractLogicalPlan implements Recursiv
return outputs.isEmpty();
}
- private List<BitSet> mapSlotToIndex(Plan plan, List<Set<Slot>> equalSlotsList) {
- Map<Slot, Integer> slotToIndex = new HashMap<>();
- for (int i = 0; i < plan.getOutput().size(); i++) {
- slotToIndex.put(plan.getOutput().get(i), i);
- }
- List<BitSet> equalSlotIndicesList = new ArrayList<>();
- for (Set<Slot> equalSlots : equalSlotsList) {
- BitSet equalSlotIndices = new BitSet();
- for (Slot slot : equalSlots) {
- if (slotToIndex.containsKey(slot)) {
- equalSlotIndices.set(slotToIndex.get(slot));
- }
- }
- if (equalSlotIndices.cardinality() > 1) {
- equalSlotIndicesList.add(equalSlotIndices);
- }
- }
- return equalSlotIndicesList;
- }
-
@Override
public void computeEqualSet(DataTrait.Builder builder) {
- if (children.isEmpty()) {
- return;
- }
-
- // Get the list of equal slot sets and their corresponding index mappings for the first child
- List<Set<Slot>> childEqualSlotsList = child(0).getLogicalProperties()
- .getTrait().calAllEqualSet();
- List<BitSet> childEqualSlotsIndicesList = mapSlotToIndex(child(0), childEqualSlotsList);
- List<BitSet> unionEqualSlotIndicesList = new ArrayList<>(childEqualSlotsIndicesList);
-
- // Traverse all children and find the equal sets that exist in all children
- for (int i = 1; i < children.size(); i++) {
- Plan child = children.get(i);
-
- // Get the equal slot sets for the current child
- childEqualSlotsList = child.getLogicalProperties().getTrait().calAllEqualSet();
-
- // Map slots to indices for the current child
- childEqualSlotsIndicesList = mapSlotToIndex(child, childEqualSlotsList);
-
- // Only keep the equal pairs that exist in all children of the union
- // This is done by calculating the intersection of all children's equal slot indices
- for (BitSet unionEqualSlotIndices : unionEqualSlotIndicesList) {
- BitSet intersect = new BitSet();
- for (BitSet childEqualSlotIndices : childEqualSlotsIndicesList) {
- if (unionEqualSlotIndices.intersects(childEqualSlotIndices)) {
- intersect = childEqualSlotIndices;
- break;
- }
- }
- unionEqualSlotIndices.and(intersect);
- }
- }
-
- // Build the functional dependencies for the output slots
- List<Slot> outputList = getOutput();
- for (BitSet equalSlotIndices : unionEqualSlotIndicesList) {
- if (equalSlotIndices.cardinality() <= 1) {
- continue;
- }
- int first = equalSlotIndices.nextSetBit(0);
- int next = equalSlotIndices.nextSetBit(first + 1);
- while (next > 0) {
- builder.addEqualPair(outputList.get(first), outputList.get(next));
- next = equalSlotIndices.nextSetBit(next + 1);
- }
- }
}
@Override
public void computeFd(DataTrait.Builder builder) {
// don't generate
}
-
- /** castCommonDataTypeAndNullableByConstants */
- public static Pair<List<List<NamedExpression>>, List<Boolean>> castCommonDataTypeAndNullableByConstants(
- List<List<NamedExpression>> constantExprsList) {
- int columnCount = constantExprsList.isEmpty() ? 0 : constantExprsList.get(0).size();
- Pair<List<DataType>, List<Boolean>> commonInfo = computeCommonDataTypeAndNullable(constantExprsList,
- columnCount);
- List<List<NamedExpression>> castedRows = castToCommonType(constantExprsList, commonInfo.key(), columnCount);
- List<Boolean> nullables = commonInfo.second;
- return Pair.of(castedRows, nullables);
- }
-
- private static Pair<List<DataType>, List<Boolean>> computeCommonDataTypeAndNullable(
- List<List<NamedExpression>> constantExprsList, int columnCount) {
- List<Boolean> nullables = Lists.newArrayListWithCapacity(columnCount);
- List<DataType> commonDataTypes = Lists.newArrayListWithCapacity(columnCount);
- List<NamedExpression> firstRow = constantExprsList.get(0);
- for (int columnId = 0; columnId < columnCount; columnId++) {
- Expression constant = firstRow.get(columnId).child(0);
- Pair<DataType, Boolean> commonDataTypeAndNullable = computeCommonDataTypeAndNullable(constant, columnId,
- constantExprsList);
- commonDataTypes.add(commonDataTypeAndNullable.first);
- nullables.add(commonDataTypeAndNullable.second);
- }
- return Pair.of(commonDataTypes, nullables);
- }
-
- private static Pair<DataType, Boolean> computeCommonDataTypeAndNullable(
- Expression firstRowExpr, int columnId, List<List<NamedExpression>> constantExprsList) {
- DataType commonDataType = firstRowExpr.getDataType();
- boolean nullable = firstRowExpr.nullable();
- for (int rowId = 1; rowId < constantExprsList.size(); rowId++) {
- NamedExpression namedExpression = constantExprsList.get(rowId).get(columnId);
- Expression otherConstant = namedExpression.child(0);
- nullable |= otherConstant.nullable();
- DataType otherDataType = otherConstant.getDataType();
- commonDataType = LogicalSetOperation.getAssignmentCompatibleType(commonDataType, otherDataType);
- }
- return Pair.of(commonDataType, nullable);
- }
-
- private static List<List<NamedExpression>> castToCommonType(
- List<List<NamedExpression>> rows, List<DataType> commonDataTypes, int columnCount) {
- ImmutableList.Builder<List<NamedExpression>> castedConstants = ImmutableList
- .builderWithExpectedSize(rows.size());
- for (List<NamedExpression> row : rows) {
- castedConstants.add(castToCommonType(row, commonDataTypes));
- }
- return castedConstants.build();
- }
-
- private static List<NamedExpression> castToCommonType(List<NamedExpression> row, List<DataType> commonTypes) {
- ImmutableList.Builder<NamedExpression> castedRow = ImmutableList.builderWithExpectedSize(row.size());
- boolean changed = false;
- for (int columnId = 0; columnId < row.size(); columnId++) {
- NamedExpression constantAlias = row.get(columnId);
- Expression constant = constantAlias.child(0);
- DataType commonType = commonTypes.get(columnId);
- if (commonType.equals(constant.getDataType())) {
- castedRow.add(constantAlias);
- } else {
- changed = true;
- Expression expression = TypeCoercionUtils.castIfNotSameTypeStrict(constant, commonType);
- castedRow.add((NamedExpression) constantAlias.withChildren(expression));
- }
- }
- return changed ? castedRow.build() : row;
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteRecursiveChild.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteRecursiveChild.java
index f73323b50bb..55747f5ba12 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteRecursiveChild.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteRecursiveChild.java
@@ -76,6 +76,11 @@ public class LogicalRecursiveCteRecursiveChild<CHILD_TYPE extends Plan> extends
return new LogicalRecursiveCteRecursiveChild<>(groupExpression, logicalProperties, children);
}
+ @Override
+ public String toString() {
+ return "LogicalRecursiveCteRecursiveChild(MUST_SHUFFLE)";
+ }
+
@Override
public void computeUnique(DataTrait.Builder builder) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteScan.java
index 7b1fc4862ee..110a6297858 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteScan.java
@@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
import java.util.List;
import java.util.Optional;
@@ -41,22 +42,42 @@ public class LogicalRecursiveCteScan extends LogicalCatalogRelation {
super(relationId, PlanType.LOGICAL_RECURSIVE_CTE_SCAN, table, qualifier, groupExpression, logicalProperties);
}
+ private LogicalRecursiveCteScan(RelationId relationId, TableIf table, List<String> qualifier,
+ Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
+ String tableAlias) {
+ super(relationId, PlanType.LOGICAL_RECURSIVE_CTE_SCAN, table, qualifier, groupExpression,
+ logicalProperties, tableAlias);
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlString("LogicalRecursiveCteScan",
+ "recursive cte name", table.getName());
+ }
+
@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalRecursiveCteScan(relationId, table, qualifier,
- groupExpression, Optional.ofNullable(getLogicalProperties()));
+ groupExpression, Optional.ofNullable(getLogicalProperties()), tableAlias);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
- return new LogicalRecursiveCteScan(relationId, table, qualifier, groupExpression, logicalProperties);
+ return new LogicalRecursiveCteScan(relationId, table, qualifier, groupExpression, logicalProperties,
+ tableAlias);
}
@Override
public LogicalCatalogRelation withRelationId(RelationId relationId) {
return new LogicalRecursiveCteScan(relationId, table, qualifier,
- groupExpression, Optional.ofNullable(getLogicalProperties()));
+ groupExpression, Optional.ofNullable(getLogicalProperties()), tableAlias);
+ }
+
+ @Override
+ public LogicalCatalogRelation withTableAlias(String tableAlias) {
+ return new LogicalRecursiveCteScan(relationId, table, qualifier,
+ groupExpression, Optional.ofNullable(getLogicalProperties()), tableAlias);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
index f7f8d718e07..8b18d3ce49d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.logical;
+import org.apache.doris.nereids.analyzer.UnboundRelation;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.DataTrait;
import org.apache.doris.nereids.properties.LogicalProperties;
@@ -27,6 +28,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.LazyCompute;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
@@ -42,6 +44,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
/**
@@ -55,37 +58,31 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
protected RelationId relationId;
private final List<String> qualifier;
private final Optional<List<String>> columnAliases;
-
- private final boolean isRecursiveCte;
+ private final Supplier<Boolean> isRecursiveCte;
public LogicalSubQueryAlias(String tableAlias, CHILD_TYPE child) {
- this(ImmutableList.of(tableAlias), Optional.empty(), false, Optional.empty(), Optional.empty(), child);
+ this(ImmutableList.of(tableAlias), Optional.empty(), Optional.empty(), Optional.empty(), child);
}
public LogicalSubQueryAlias(List<String> qualifier, CHILD_TYPE child) {
- this(qualifier, Optional.empty(), false, Optional.empty(), Optional.empty(), child);
+ this(qualifier, Optional.empty(), Optional.empty(), Optional.empty(), child);
}
public LogicalSubQueryAlias(String tableAlias, Optional<List<String>> columnAliases, CHILD_TYPE child) {
- this(ImmutableList.of(tableAlias), columnAliases, false, Optional.empty(), Optional.empty(), child);
- }
-
- public LogicalSubQueryAlias(String tableAlias, Optional<List<String>> columnAliases, boolean isRecursiveCte,
- CHILD_TYPE child) {
- this(ImmutableList.of(tableAlias), columnAliases, isRecursiveCte, Optional.empty(), Optional.empty(), child);
+ this(ImmutableList.of(tableAlias), columnAliases, Optional.empty(), Optional.empty(), child);
}
public LogicalSubQueryAlias(List<String> qualifier, Optional<List<String>> columnAliases, CHILD_TYPE child) {
- this(qualifier, columnAliases, false, Optional.empty(), Optional.empty(), child);
+ this(qualifier, columnAliases, Optional.empty(), Optional.empty(), child);
}
- public LogicalSubQueryAlias(List<String> qualifier, Optional<List<String>> columnAliases, boolean isRecursiveCte,
+ public LogicalSubQueryAlias(List<String> qualifier, Optional<List<String>> columnAliases,
Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, CHILD_TYPE child) {
super(PlanType.LOGICAL_SUBQUERY_ALIAS, groupExpression, logicalProperties, child);
this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier is null"));
this.columnAliases = columnAliases;
- this.isRecursiveCte = isRecursiveCte;
+ this.isRecursiveCte = computeIsRecursiveCte();
}
@Override
@@ -129,6 +126,23 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
return currentOutput.build();
}
+ private Supplier<Boolean> computeIsRecursiveCte() {
+ return LazyCompute.of(() -> {
+ List<UnboundRelation> relationList = collectToList(UnboundRelation.class::isInstance);
+ for (UnboundRelation relation : relationList) {
+ List<String> nameParts = relation.getNameParts();
+ if (nameParts.size() == 1 && nameParts.get(0).equalsIgnoreCase(getAlias())) {
+ return true;
+ }
+ }
+ return false;
+ });
+ }
+
+ public boolean isRecursiveCte() {
+ return isRecursiveCte.get();
+ }
+
public String getAlias() {
return qualifier.get(qualifier.size() - 1);
}
@@ -137,14 +151,10 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
return columnAliases;
}
- public boolean isRecursiveCte() {
- return isRecursiveCte;
- }
-
@Override
public String toString() {
return columnAliases.map(strings -> Utils.toSqlString("LogicalSubQueryAlias",
- "qualifier", qualifier, "isRecursiveCte", isRecursiveCte,
+ "qualifier", qualifier,
"columnAliases", StringUtils.join(strings, ",")
)).orElseGet(() -> Utils.toSqlString("LogicalSubQueryAlias",
"qualifier", qualifier
@@ -183,8 +193,7 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
@Override
public LogicalSubQueryAlias<Plan> withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
- return new LogicalSubQueryAlias<>(qualifier, columnAliases, isRecursiveCte, Optional.empty(), Optional.empty(),
- children.get(0));
+ return new LogicalSubQueryAlias<>(qualifier, columnAliases, children.get(0));
}
@Override
@@ -199,7 +208,7 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
@Override
public LogicalSubQueryAlias<CHILD_TYPE> withGroupExpression(Optional<GroupExpression> groupExpression) {
- return new LogicalSubQueryAlias<>(qualifier, columnAliases, isRecursiveCte, groupExpression,
+ return new LogicalSubQueryAlias<>(qualifier, columnAliases, groupExpression,
Optional.of(getLogicalProperties()), child());
}
@@ -207,7 +216,7 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
- return new LogicalSubQueryAlias<>(qualifier, columnAliases, isRecursiveCte, groupExpression, logicalProperties,
+ return new LogicalSubQueryAlias<>(qualifier, columnAliases, groupExpression, logicalProperties,
children.get(0));
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteRecursiveChild.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteRecursiveChild.java
index 5f598d8feac..25903905b7a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteRecursiveChild.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteRecursiveChild.java
@@ -54,6 +54,11 @@ public class PhysicalRecursiveCteRecursiveChild<CHILD_TYPE extends Plan> extends
statistics, child);
}
+ @Override
+ public String toString() {
+ return "PhysicalRecursiveCteRecursiveChild(MUST_SHUFFLE)";
+ }
+
@Override
public Plan withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java
index b4075d66663..7d9eecd35be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java
@@ -14,18 +14,19 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-// This file is copied from
-// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/UnionNode.java
-// and modified by Doris
package org.apache.doris.planner;
+import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.statistics.StatisticalType;
+import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPlanNodeType;
import org.apache.doris.thrift.TRecCTENode;
+import com.google.common.base.MoreObjects;
+
public class RecursiveCteNode extends SetOperationNode {
private boolean isUnionAll;
@@ -49,4 +50,24 @@ public class RecursiveCteNode extends SetOperationNode {
msg.node_type = TPlanNodeType.REC_CTE_NODE;
msg.rec_cte_node = tRecCTENode;
}
+
+ @Override
+ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
+ StringBuilder output = new StringBuilder();
+ output.append(prefix).append("Recursive Cte: ").append("\n");
+ output.append(prefix).append("isUnionAll: ").append(isUnionAll).append("\n");
+ if (!conjuncts.isEmpty()) {
+ Expr expr = convertConjunctsToAndCompoundPredicate(conjuncts);
+ output.append(prefix).append("PREDICATES: ").append(expr.toSql()).append("\n");
+ }
+ return output.toString();
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("id", getId().asInt())
+ .add("tid", tupleId.asInt())
+ .add("isUnionAll", isUnionAll).toString();
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
index eb63ba6fcb7..9603e3cfa63 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
@@ -31,6 +31,7 @@ import org.apache.doris.thrift.TScanRange;
import org.apache.doris.thrift.TScanRangeLocation;
import org.apache.doris.thrift.TScanRangeLocations;
+import com.google.common.base.MoreObjects;
import com.google.common.collect.Lists;
import java.util.Collections;
@@ -101,6 +102,14 @@ public class RecursiveCteScanNode extends ScanNode {
return output.toString();
}
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("recursiveCteName", recursiveCteName)
+ .add("id", getId().asInt())
+ .add("tid", desc.getId().asInt()).toString();
+ }
+
@Override
protected void toThrift(TPlanNode msg) {
msg.node_type = TPlanNodeType.REC_CTE_SCAN_NODE;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
index df71172baad..e556c3555ee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
@@ -83,6 +83,7 @@ import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -99,7 +100,7 @@ public class ThriftPlansBuilder {
CoordinatorContext coordinatorContext) {
List<PipelineDistributedPlan> distributedPlans = coordinatorContext.distributedPlans;
- setParamsForRecursiveCteNode(distributedPlans);
+ Set<Integer> fragmentToNotifyClose = setParamsForRecursiveCteNode(distributedPlans);
// we should set runtime predicate first, then we can use heap sort and to thrift
setRuntimePredicateIfNeed(coordinatorContext.scanNodes);
@@ -125,7 +126,7 @@ public class ThriftPlansBuilder {
TPipelineFragmentParams currentFragmentParam = fragmentToThriftIfAbsent(
currentFragmentPlan, instanceJob, workerToCurrentFragment,
instancesPerWorker, exchangeSenderNum, sharedFileScanRangeParams,
- workerProcessInstanceNum, coordinatorContext);
+ workerProcessInstanceNum, fragmentToNotifyClose, coordinatorContext);
TPipelineInstanceParams instanceParam = instanceToThrift(
currentFragmentParam, instanceJob, currentInstanceIndex++);
@@ -331,6 +332,7 @@ public class ThriftPlansBuilder {
Map<Integer, Integer> exchangeSenderNum,
Map<Integer, TFileScanRangeParams> fileScanRangeParamsMap,
Multiset<DistributedPlanWorker> workerProcessInstanceNum,
+ Set<Integer> fragmentToNotifyClose,
CoordinatorContext coordinatorContext) {
DistributedPlanWorker worker = assignedJob.getAssignedWorker();
return workerToFragmentParams.computeIfAbsent(worker, w -> {
@@ -344,6 +346,9 @@ public class ThriftPlansBuilder {
params.setDescTbl(coordinatorContext.descriptorTable);
params.setQueryId(coordinatorContext.queryId);
params.setFragmentId(fragment.getFragmentId().asInt());
+ if (fragmentToNotifyClose.contains(params.getFragmentId())) {
+ params.setNeedNotifyClose(true);
+ }
// Each tParam will set the total number of Fragments that need to be executed on the same BE,
// and the BE will determine whether all Fragments have been executed based on this information.
@@ -582,7 +587,8 @@ public class ThriftPlansBuilder {
}
}
- private static void setParamsForRecursiveCteNode(List<PipelineDistributedPlan> distributedPlans) {
+ private static Set<Integer> setParamsForRecursiveCteNode(List<PipelineDistributedPlan> distributedPlans) {
+ Set<Integer> fragmentToNotifyClose = new HashSet<>();
Map<PlanFragmentId, TRecCTETarget> fragmentIdToRecCteTargetMap = new TreeMap<>();
Map<PlanFragmentId, Set<TNetworkAddress>> fragmentIdToNetworkAddressMap = new TreeMap<>();
for (PipelineDistributedPlan plan : distributedPlans) {
@@ -634,6 +640,7 @@ public class ThriftPlansBuilder {
planFragment.getChild(0).collectAll(PlanFragment.class::isInstance, childFragments);
for (PlanFragment child : childFragments) {
PlanFragmentId childFragmentId = child.getFragmentId();
+ fragmentToNotifyClose.add(childFragmentId.asInt());
TRecCTETarget tRecCTETarget = fragmentIdToRecCteTargetMap.getOrDefault(childFragmentId, null);
if (tRecCTETarget != null) {
targets.add(tRecCTETarget);
@@ -665,6 +672,7 @@ public class ThriftPlansBuilder {
recursiveCteNode.settRecCTENode(tRecCTENode);
}
}
+ return fragmentToNotifyClose;
}
private static class PerNodeScanParams {
diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
index c5982ed4b0f..37bbc177b27 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -653,6 +653,7 @@ struct TPipelineFragmentParams {
// Used by 2.1
44: optional list<i32> topn_filter_source_node_ids
45: optional map<string, TAIResource> ai_resources
+ 46: optional bool need_notify_close
// For cloud
1000: optional bool is_mow_table;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 05/09: add TRecCTEScanNode
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch dev_rec
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 24a7585fde68e3ef7505f3aec2c4c519127d5732
Author: BiteTheDDDDt <xl...@selectdb.com>
AuthorDate: Sun Sep 28 18:24:43 2025 +0800
add TRecCTEScanNode
---
gensrc/thrift/PlanNodes.thrift | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index c41bc50add3..4036ad7ea49 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -712,6 +712,9 @@ struct TRecCTENode {
4: optional list<list<Exprs.TExpr>> result_expr_lists
}
+struct TRecCTEScanNode {
+}
+
struct TEsScanNode {
1: required Types.TTupleId tuple_id
2: optional map<string,string> properties
@@ -1470,6 +1473,8 @@ struct TPlanNode {
50: optional list<list<Exprs.TExpr>> distribute_expr_lists
51: optional bool is_serial_operator
+ 52: optional TRecCTEScanNode rec_cte_scan_node
+
// projections is final projections, which means projecting into results and materializing them into the output block.
101: optional list<Exprs.TExpr> projections
102: optional Types.TTupleId output_tuple_id
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 07/09: fix some bug in fe
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch dev_rec
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c94e85b20c91f38f5063d1384429937fa85cbfbe
Author: lichi <li...@selectdb.com>
AuthorDate: Sat Oct 11 16:55:51 2025 +0800
fix some bug in fe
---
.../org/apache/doris/analysis/DescriptorTable.java | 5 +
.../doris/catalog/RecursiveCteTempTable.java | 4 +-
.../glue/translator/PhysicalPlanTranslator.java | 77 ++++++-
.../processor/post/RuntimeFilterPruner.java | 9 +
.../properties/ChildOutputPropertyDeriver.java | 2 +-
.../doris/nereids/rules/analysis/AnalyzeCTE.java | 9 +-
.../LogicalRecursiveCteToPhysicalRecursiveCte.java | 3 +-
.../nereids/rules/rewrite/AdjustNullable.java | 63 ++++--
.../doris/nereids/rules/rewrite/ColumnPruning.java | 22 +-
.../doris/nereids/stats/StatsCalculator.java | 70 ++++++-
.../trees/copier/LogicalPlanDeepCopier.java | 8 +-
.../trees/plans/algebra/RecursiveCte.java} | 20 +-
.../commands/CreateMaterializedViewCommand.java | 2 +-
.../distribute/worker/job/AssignedJobBuilder.java | 93 ---------
.../worker/job/UnassignedRecursiveCteScanJob.java | 5 +-
.../trees/plans/logical/LogicalRecursiveCte.java | 223 ++++++++++++---------
.../trees/plans/physical/PhysicalRecursiveCte.java | 113 +++++++----
.../nereids/trees/plans/visitor/PlanVisitor.java | 2 +-
.../apache/doris/planner/RecursiveCteScanNode.java | 18 +-
.../doris/qe/runtime/ThriftPlansBuilder.java | 96 +++++++++
20 files changed, 547 insertions(+), 297 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java
index fb6cc7df0a8..197293ac69a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java
@@ -20,6 +20,7 @@
package org.apache.doris.analysis;
+import org.apache.doris.catalog.RecursiveCteTempTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.IdGenerator;
@@ -212,6 +213,10 @@ public class DescriptorTable {
}
for (TableIf tbl : referencedTbls.values()) {
+ if (tbl instanceof RecursiveCteTempTable) {
+ // skip recursive cte temp table
+ continue;
+ }
result.addToTableDescriptors(tbl.toThrift());
}
thriftDescTable = result;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RecursiveCteTempTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RecursiveCteTempTable.java
index 9f36b04dfc4..ee924466fbb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RecursiveCteTempTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RecursiveCteTempTable.java
@@ -17,10 +17,12 @@
package org.apache.doris.catalog;
+import org.apache.doris.common.SystemIdGenerator;
+
import java.util.List;
public class RecursiveCteTempTable extends Table {
public RecursiveCteTempTable(String tableName, List<Column> fullSchema) {
- super(-1, tableName, TableType.RECURSIVE_CTE_TEMP_TABLE, fullSchema);
+ super(SystemIdGenerator.getNextId(), tableName, TableType.RECURSIVE_CTE_TEMP_TABLE, fullSchema);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 21f9bbf2126..a27268a5af7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -116,7 +116,6 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PreAggStatus;
import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
import org.apache.doris.nereids.trees.plans.algebra.Relation;
-import org.apache.doris.nereids.trees.plans.algebra.SetOperation;
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin;
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort;
import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows;
@@ -2269,6 +2268,79 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return inputFragment;
}
+ @Override
+ public PlanFragment visitPhysicalRecursiveCte(PhysicalRecursiveCte recursiveCte, PlanTranslatorContext context) {
+ List<PlanFragment> childrenFragments = new ArrayList<>();
+ for (Plan plan : recursiveCte.children()) {
+ childrenFragments.add(plan.accept(this, context));
+ }
+
+ TupleDescriptor setTuple = generateTupleDesc(recursiveCte.getOutput(), null, context);
+ List<SlotDescriptor> outputSlotDescs = new ArrayList<>(setTuple.getSlots());
+
+ RecursiveCteNode recursiveCteNode = new RecursiveCteNode(context.nextPlanNodeId(), setTuple.getId(),
+ recursiveCte.isUnionAll());
+ List<List<Expr>> distributeExprLists = getDistributeExprs(recursiveCte.children().toArray(new Plan[0]));
+ recursiveCteNode.setChildrenDistributeExprLists(distributeExprLists);
+ recursiveCteNode.setNereidsId(recursiveCte.getId());
+ List<List<Expression>> resultExpressionLists = Lists.newArrayList();
+ context.getNereidsIdToPlanNodeIdMap().put(recursiveCte.getId(), recursiveCteNode.getId());
+ for (List<SlotReference> regularChildrenOutput : recursiveCte.getRegularChildrenOutputs()) {
+ resultExpressionLists.add(new ArrayList<>(regularChildrenOutput));
+ }
+
+ for (PlanFragment childFragment : childrenFragments) {
+ recursiveCteNode.addChild(childFragment.getPlanRoot());
+ }
+
+ List<List<Expr>> materializedResultExprLists = Lists.newArrayList();
+ for (int i = 0; i < resultExpressionLists.size(); ++i) {
+ List<Expression> resultExpressionList = resultExpressionLists.get(i);
+ List<Expr> exprList = Lists.newArrayList();
+ Preconditions.checkState(resultExpressionList.size() == outputSlotDescs.size());
+ for (int j = 0; j < resultExpressionList.size(); ++j) {
+ if (outputSlotDescs.get(j).isMaterialized()) {
+ exprList.add(ExpressionTranslator.translate(resultExpressionList.get(j), context));
+ // TODO: reconsider this, we may change nullable info in previous nereids rules not here.
+ outputSlotDescs.get(j)
+ .setIsNullable(outputSlotDescs.get(j).getIsNullable() || exprList.get(j).isNullable());
+ }
+ }
+ materializedResultExprLists.add(exprList);
+ }
+ recursiveCteNode.setMaterializedResultExprLists(materializedResultExprLists);
+ Preconditions.checkState(recursiveCteNode.getMaterializedResultExprLists().size()
+ == recursiveCteNode.getChildren().size());
+
+ PlanFragment recursiveCteFragment;
+ if (childrenFragments.isEmpty()) {
+ recursiveCteFragment = createPlanFragment(recursiveCteNode,
+ DataPartition.UNPARTITIONED, recursiveCte);
+ context.addPlanFragment(recursiveCteFragment);
+ } else {
+ int childrenSize = childrenFragments.size();
+ recursiveCteFragment = childrenFragments.get(childrenSize - 1);
+ for (int i = childrenSize - 2; i >= 0; i--) {
+ context.mergePlanFragment(childrenFragments.get(i), recursiveCteFragment);
+ for (PlanFragment child : childrenFragments.get(i).getChildren()) {
+ recursiveCteFragment.addChild(child);
+ }
+ }
+ setPlanRoot(recursiveCteFragment, recursiveCteNode, recursiveCte);
+ }
+
+ // in pipeline engine, we use parallel scan by default, but it broke the rule of data distribution
+ // we need turn of parallel scan to ensure to get correct result.
+ // TODO: nereids forbid all parallel scan under PhysicalSetOperation temporary
+ if (!recursiveCte.getPhysicalProperties().equals(PhysicalProperties.ANY)
+ && findOlapScanNodesByPassExchangeAndJoinNode(recursiveCteFragment.getPlanRoot())) {
+ recursiveCteFragment.setHasColocatePlanNode(true);
+ recursiveCteNode.setColocate(true);
+ }
+
+ return recursiveCteFragment;
+ }
+
/**
* Returns a new fragment with a UnionNode as its root. The data partition of the
* returned fragment and how the data of the child fragments is consumed depends on the
@@ -2300,9 +2372,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
setOperationNode = new ExceptNode(context.nextPlanNodeId(), setTuple.getId());
} else if (setOperation instanceof PhysicalIntersect) {
setOperationNode = new IntersectNode(context.nextPlanNodeId(), setTuple.getId());
- } else if (setOperation instanceof PhysicalRecursiveCte) {
- setOperationNode = new RecursiveCteNode(context.nextPlanNodeId(), setTuple.getId(),
- setOperation.getQualifier().equals(SetOperation.Qualifier.ALL));
} else {
throw new RuntimeException("not support set operation type " + setOperation);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
index 51a9003b78f..d7e4e5b67cb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
@@ -33,6 +33,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN;
@@ -76,6 +77,14 @@ public class RuntimeFilterPruner extends PlanPostProcessor {
return plan;
}
+ @Override
+ public PhysicalRecursiveCte visitPhysicalRecursiveCte(PhysicalRecursiveCte recursiveCte, CascadesContext context) {
+ for (Plan child : recursiveCte.children()) {
+ child.accept(this, context);
+ }
+ return recursiveCte;
+ }
+
@Override
public PhysicalSetOperation visitPhysicalSetOperation(PhysicalSetOperation setOperation, CascadesContext context) {
for (Plan child : setOperation.children()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
index d3592388dae..0751f14b715 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
@@ -151,7 +151,7 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
@Override
public PhysicalProperties visitPhysicalRecursiveCteScan(PhysicalRecursiveCteScan cteScan, PlanContext context) {
- return PhysicalProperties.GATHER;
+ return PhysicalProperties.MUST_SHUFFLE;
}
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
index 8ac4234e846..a6259f0b24c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
@@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.SetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
@@ -159,9 +160,10 @@ public class AnalyzeCTE extends OneAnalysisRuleFactory {
LogicalPlan analyzedRecursiveChild = (LogicalPlan) innerRecursiveCascadesCtx.getRewritePlan();
LogicalUnion logicalUnion = (LogicalUnion) parsedCtePlan;
- // manually bind LogicalRecursiveCte, see bindSetOperation in BindExpression.java
+ // create LogicalRecursiveCte
LogicalRecursiveCte analyzedCtePlan = new LogicalRecursiveCte(
- logicalUnion.getQualifier(), ImmutableList.of(analyzedAnchorChild, analyzedRecursiveChild));
+ logicalUnion.getQualifier() == SetOperation.Qualifier.ALL,
+ ImmutableList.of(analyzedAnchorChild, analyzedRecursiveChild));
List<List<NamedExpression>> childrenProjections = analyzedCtePlan.collectChildrenProjections();
int childrenProjectionSize = childrenProjections.size();
ImmutableList.Builder<List<SlotReference>> childrenOutputs = ImmutableList
@@ -180,8 +182,7 @@ public class AnalyzeCTE extends OneAnalysisRuleFactory {
newChildren.add(newChild);
childrenOutputs.add((List<SlotReference>) (List) newChild.getOutput());
}
- analyzedCtePlan = (LogicalRecursiveCte) analyzedCtePlan.withChildrenAndTheirOutputs(newChildren.build(),
- childrenOutputs.build());
+ analyzedCtePlan = analyzedCtePlan.withChildrenAndTheirOutputs(newChildren.build(), childrenOutputs.build());
List<NamedExpression> newOutputs = analyzedCtePlan.buildNewOutputs();
analyzedCtePlan = analyzedCtePlan.withNewOutputs(newOutputs);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteToPhysicalRecursiveCte.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteToPhysicalRecursiveCte.java
index c8960aa9e51..467092895a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteToPhysicalRecursiveCte.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteToPhysicalRecursiveCte.java
@@ -28,10 +28,9 @@ public class LogicalRecursiveCteToPhysicalRecursiveCte extends OneImplementation
@Override
public Rule build() {
return logicalRecursiveCte().then(recursiveCte ->
- new PhysicalRecursiveCte(recursiveCte.getQualifier(),
+ new PhysicalRecursiveCte(recursiveCte.isUnionAll(),
recursiveCte.getOutputs(),
recursiveCte.getRegularChildrenOutputs(),
- recursiveCte.getConstantExprsList(),
recursiveCte.getLogicalProperties(),
recursiveCte.children())
).toRule(RuleType.LOGICAL_RECURSIVE_CTE_TO_PHYSICAL_RECURSIVE_CTE);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
index 63d63f32b6f..204a018fbc7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
@@ -290,6 +290,53 @@ public class AdjustNullable extends DefaultPlanRewriter<Map<ExprId, Slot>> imple
return repeat.withGroupSetsAndOutput(repeat.getGroupingSets(), newOutputs).recomputeLogicalProperties();
}
+ @Override
+ public Plan visitLogicalRecursiveCte(LogicalRecursiveCte recursiveCte, Map<ExprId, Slot> replaceMap) {
+ recursiveCte = (LogicalRecursiveCte) super.visit(recursiveCte, replaceMap);
+ ImmutableList.Builder<List<SlotReference>> newChildrenOutputs = ImmutableList.builder();
+ List<Boolean> inputNullable = null;
+ if (!recursiveCte.children().isEmpty()) {
+ inputNullable = Lists.newArrayListWithCapacity(recursiveCte.getOutputs().size());
+ for (int i = 0; i < recursiveCte.getOutputs().size(); i++) {
+ inputNullable.add(false);
+ }
+ for (int i = 0; i < recursiveCte.arity(); i++) {
+ List<Slot> childOutput = recursiveCte.child(i).getOutput();
+ List<SlotReference> setChildOutput = recursiveCte.getRegularChildOutput(i);
+ ImmutableList.Builder<SlotReference> newChildOutputs = ImmutableList.builder();
+ for (int j = 0; j < setChildOutput.size(); j++) {
+ for (Slot slot : childOutput) {
+ if (slot.getExprId().equals(setChildOutput.get(j).getExprId())) {
+ inputNullable.set(j, slot.nullable() || inputNullable.get(j));
+ newChildOutputs.add((SlotReference) slot);
+ break;
+ }
+ }
+ }
+ newChildrenOutputs.add(newChildOutputs.build());
+ }
+ }
+ if (inputNullable == null) {
+ // this is a fail-safe
+ // means there is no children and having no getConstantExprsList
+ // no way to update the nullable flag, so just do nothing
+ return recursiveCte;
+ }
+ List<NamedExpression> outputs = recursiveCte.getOutputs();
+ List<NamedExpression> newOutputs = Lists.newArrayListWithCapacity(outputs.size());
+ for (int i = 0; i < inputNullable.size(); i++) {
+ NamedExpression ne = outputs.get(i);
+ Slot slot = ne instanceof Alias ? (Slot) ((Alias) ne).child() : (Slot) ne;
+ slot = slot.withNullable(inputNullable.get(i));
+ NamedExpression newOutput = ne instanceof Alias ? (NamedExpression) ne.withChildren(slot) : slot;
+ newOutputs.add(newOutput);
+ replaceMap.put(newOutput.getExprId(), newOutput.toSlot());
+ }
+ return recursiveCte.withNewOutputs(newOutputs)
+ .withChildrenAndTheirOutputs(recursiveCte.children(), newChildrenOutputs.build())
+ .recomputeLogicalProperties();
+ }
+
@Override
public Plan visitLogicalSetOperation(LogicalSetOperation setOperation, Map<ExprId, Slot> replaceMap) {
setOperation = (LogicalSetOperation) super.visit(setOperation, replaceMap);
@@ -331,22 +378,6 @@ public class AdjustNullable extends DefaultPlanRewriter<Map<ExprId, Slot>> imple
inputNullable.set(j, inputNullable.get(j) || constantExprs.get(j).nullable());
}
}
- } else if (setOperation instanceof LogicalRecursiveCte) {
- // LogicalRecursiveCte is basically like LogicalUnion, so just do same as LogicalUnion
- LogicalRecursiveCte logicalRecursiveCte = (LogicalRecursiveCte) setOperation;
- if (!logicalRecursiveCte.getConstantExprsList().isEmpty() && setOperation.children().isEmpty()) {
- int outputSize = logicalRecursiveCte.getConstantExprsList().get(0).size();
- // create the inputNullable list and fill it with all FALSE values
- inputNullable = Lists.newArrayListWithCapacity(outputSize);
- for (int i = 0; i < outputSize; i++) {
- inputNullable.add(false);
- }
- }
- for (List<NamedExpression> constantExprs : logicalRecursiveCte.getConstantExprsList()) {
- for (int j = 0; j < constantExprs.size(); j++) {
- inputNullable.set(j, inputNullable.get(j) || constantExprs.get(j).nullable());
- }
- }
}
if (inputNullable == null) {
// this is a fail-safe
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
index 96daceedfe4..30a00314fbf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
@@ -217,7 +217,7 @@ public class ColumnPruning extends DefaultPlanRewriter<PruneContext> implements
@Override
public Plan visitLogicalRecursiveCte(LogicalRecursiveCte recursiveCte, PruneContext context) {
// LogicalRecursiveCte is basically like LogicalUnion, so just do same as LogicalUnion
- if (recursiveCte.getQualifier() == Qualifier.DISTINCT) {
+ if (!recursiveCte.isUnionAll()) {
return skipPruneThisAndFirstLevelChildren(recursiveCte);
}
LogicalRecursiveCte prunedOutputRecursiveCte = pruneRecursiveCteOutput(recursiveCte, context);
@@ -451,7 +451,6 @@ public class ColumnPruning extends DefaultPlanRewriter<PruneContext> implements
return recursiveCte;
}
List<NamedExpression> prunedOutputs = Lists.newArrayList();
- List<List<NamedExpression>> constantExprsList = recursiveCte.getConstantExprsList();
List<List<SlotReference>> regularChildrenOutputs = recursiveCte.getRegularChildrenOutputs();
List<Plan> children = recursiveCte.children();
List<Integer> extractColumnIndex = Lists.newArrayList();
@@ -463,8 +462,6 @@ public class ColumnPruning extends DefaultPlanRewriter<PruneContext> implements
}
}
- ImmutableList.Builder<List<NamedExpression>> prunedConstantExprsList
- = ImmutableList.builderWithExpectedSize(constantExprsList.size());
if (prunedOutputs.isEmpty()) {
// process prune all columns
NamedExpression originSlot = originOutput.get(0);
@@ -472,7 +469,7 @@ public class ColumnPruning extends DefaultPlanRewriter<PruneContext> implements
TinyIntType.INSTANCE, false, originSlot.getQualifier()));
regularChildrenOutputs = Lists.newArrayListWithCapacity(regularChildrenOutputs.size());
children = Lists.newArrayListWithCapacity(children.size());
- for (int i = 0; i < recursiveCte.getArity(); i++) {
+ for (int i = 0; i < recursiveCte.children().size(); i++) {
Plan child = recursiveCte.child(i);
List<NamedExpression> newProjectOutput = ImmutableList.of(new Alias(new TinyIntLiteral((byte) 1)));
LogicalProject<?> project;
@@ -487,25 +484,12 @@ public class ColumnPruning extends DefaultPlanRewriter<PruneContext> implements
regularChildrenOutputs.add((List) project.getOutput());
children.add(project);
}
- for (int i = 0; i < constantExprsList.size(); i++) {
- prunedConstantExprsList.add(ImmutableList.of(new Alias(new TinyIntLiteral((byte) 1))));
- }
- } else {
- int len = extractColumnIndex.size();
- for (List<NamedExpression> row : constantExprsList) {
- ImmutableList.Builder<NamedExpression> newRow = ImmutableList.builderWithExpectedSize(len);
- for (int idx : extractColumnIndex) {
- newRow.add(row.get(idx));
- }
- prunedConstantExprsList.add(newRow.build());
- }
}
if (prunedOutputs.equals(originOutput) && !context.requiredSlotsIds.isEmpty()) {
return recursiveCte;
} else {
- return recursiveCte.withNewOutputsChildrenAndConstExprsList(prunedOutputs, children,
- regularChildrenOutputs, prunedConstantExprsList.build());
+ return recursiveCte.withNewOutputsAndChildren(prunedOutputs, children, regularChildrenOutputs);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index a14c2dfc9cb..9ddcc7e2e40 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -60,6 +60,7 @@ import org.apache.doris.nereids.trees.plans.algebra.Limit;
import org.apache.doris.nereids.trees.plans.algebra.OlapScan;
import org.apache.doris.nereids.trees.plans.algebra.PartitionTopN;
import org.apache.doris.nereids.trees.plans.algebra.Project;
+import org.apache.doris.nereids.trees.plans.algebra.RecursiveCte;
import org.apache.doris.nereids.trees.plans.algebra.Relation;
import org.apache.doris.nereids.trees.plans.algebra.Repeat;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation;
@@ -925,7 +926,7 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
@Override
public Statistics visitLogicalRecursiveCte(
LogicalRecursiveCte recursiveCte, Void context) {
- return computeUnion(recursiveCte,
+ return computeRecursiveCte(recursiveCte,
groupExpression.children()
.stream().map(Group::getStatistics).collect(Collectors.toList()));
}
@@ -1108,7 +1109,7 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
@Override
public Statistics visitPhysicalRecursiveCte(PhysicalRecursiveCte recursiveCte, Void context) {
- return computeUnion(recursiveCte, groupExpression.children()
+ return computeRecursiveCte(recursiveCte, groupExpression.children()
.stream().map(Group::getStatistics).collect(Collectors.toList()));
}
@@ -1493,6 +1494,71 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
return new Statistics(rowCount, 1, columnStatsMap);
}
+ /**
+ * computeRecursiveCte
+ */
+ public Statistics computeRecursiveCte(RecursiveCte recursiveCte, List<Statistics> childStats) {
+ // TODO: refactor this for one row relation
+ List<SlotReference> head;
+ Statistics headStats;
+ List<List<SlotReference>> childOutputs = Lists.newArrayList(recursiveCte.getRegularChildrenOutputs());
+
+ head = childOutputs.get(0);
+ headStats = new StatisticsBuilder(childStats.get(0)).build();
+
+ StatisticsBuilder statisticsBuilder = new StatisticsBuilder();
+ List<NamedExpression> unionOutput = recursiveCte.getOutputs();
+ double unionRowCount = childStats.stream().mapToDouble(Statistics::getRowCount).sum();
+ statisticsBuilder.setRowCount(unionRowCount);
+
+ for (int i = 0; i < head.size(); i++) {
+ Slot headSlot = head.get(i);
+ ColumnStatisticBuilder colStatsBuilder = new ColumnStatisticBuilder(
+ headStats.findColumnStatistics(headSlot));
+ for (int j = 1; j < childOutputs.size(); j++) {
+ Slot slot = childOutputs.get(j).get(i);
+ ColumnStatistic rightStatistic = childStats.get(j).findColumnStatistics(slot);
+ double rightRowCount = childStats.get(j).getRowCount();
+ colStatsBuilder = unionColumn(colStatsBuilder,
+ headStats.getRowCount(), rightStatistic, rightRowCount, headSlot.getDataType());
+ }
+
+ //update hot values
+ Map<Literal, Float> unionHotValues = new HashMap<>();
+ for (int j = 0; j < childOutputs.size(); j++) {
+ Slot slot = childOutputs.get(j).get(i);
+ ColumnStatistic slotStats = childStats.get(j).findColumnStatistics(slot);
+ if (slotStats.getHotValues() != null) {
+ for (Map.Entry<Literal, Float> entry : slotStats.getHotValues().entrySet()) {
+ Float value = unionHotValues.get(entry.getKey());
+ if (value == null) {
+ unionHotValues.put(entry.getKey(),
+ (float) (entry.getValue() * childStats.get(j).getRowCount()));
+ } else {
+ unionHotValues.put(entry.getKey(),
+ (float) (value + entry.getValue() * childStats.get(j).getRowCount()));
+ }
+ }
+ }
+ }
+
+ Map<Literal, Float> resultHotValues = new LinkedHashMap<>();
+ for (Literal hot : unionHotValues.keySet()) {
+ float ratio = (float) (unionHotValues.get(hot) / unionRowCount);
+ if (ratio * colStatsBuilder.getNdv() >= SessionVariable.getSkewValueThreshold()
+ || ratio >= SessionVariable.getHotValueThreshold()) {
+ resultHotValues.put(hot, ratio);
+ }
+ }
+ if (!resultHotValues.isEmpty()) {
+ colStatsBuilder.setHotValues(resultHotValues);
+ }
+ statisticsBuilder.putColumnStatistics(unionOutput.get(i), colStatsBuilder.build());
+ }
+
+ return statisticsBuilder.setWidthInJoinCluster(1).build();
+ }
+
/**
* computeUnion
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
index c9cf877f709..739047f4a77 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
@@ -360,11 +360,6 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
List<Plan> children = recursiveCte.children().stream()
.map(c -> c.accept(this, context))
.collect(ImmutableList.toImmutableList());
- List<List<NamedExpression>> constantExprsList = recursiveCte.getConstantExprsList().stream()
- .map(l -> l.stream()
- .map(e -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(e, context))
- .collect(ImmutableList.toImmutableList()))
- .collect(ImmutableList.toImmutableList());
List<NamedExpression> outputs = recursiveCte.getOutputs().stream()
.map(o -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(o, context))
.collect(ImmutableList.toImmutableList());
@@ -373,8 +368,7 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
.map(o -> (SlotReference) ExpressionDeepCopier.INSTANCE.deepCopy(o, context))
.collect(ImmutableList.toImmutableList()))
.collect(ImmutableList.toImmutableList());
- return new LogicalRecursiveCte(recursiveCte.getQualifier(), outputs, childrenOutputs,
- constantExprsList, recursiveCte.hasPushedFilter(), children);
+ return new LogicalRecursiveCte(recursiveCte.isUnionAll(), outputs, childrenOutputs, children);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RecursiveCteTempTable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/RecursiveCte.java
similarity index 64%
copy from fe/fe-core/src/main/java/org/apache/doris/catalog/RecursiveCteTempTable.java
copy to fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/RecursiveCte.java
index 9f36b04dfc4..d7fd8f985a1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RecursiveCteTempTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/RecursiveCte.java
@@ -15,12 +15,22 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.catalog;
+package org.apache.doris.nereids.trees.plans.algebra;
+
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import java.util.List;
-public class RecursiveCteTempTable extends Table {
- public RecursiveCteTempTable(String tableName, List<Column> fullSchema) {
- super(-1, tableName, TableType.RECURSIVE_CTE_TEMP_TABLE, fullSchema);
- }
+/**
+ * Common interface for logical/physical recursive cte.
+ */
+public interface RecursiveCte {
+ boolean isUnionAll();
+
+ List<SlotReference> getRegularChildOutput(int i);
+
+ List<NamedExpression> getOutputs();
+
+ List<List<SlotReference>> getRegularChildrenOutputs();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java
index e8387d675dc..a7f99e101ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java
@@ -326,7 +326,7 @@ public class CreateMaterializedViewCommand extends Command implements ForwardWit
}
try {
Expr defineExpr = translateToLegacyExpr(predicate, context.planTranslatorContext);
- context.filterItem = new MVColumnItem(predicate.toSql(), defineExpr);
+ context.filterItem = new MVColumnItem(defineExpr.toSqlWithoutTbl(), defineExpr);
} catch (Exception ex) {
throw new AnalysisException(ex.getMessage());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java
index 4291ba32732..9369acd8d6f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java
@@ -17,33 +17,19 @@
package org.apache.doris.nereids.trees.plans.distribute.worker.job;
-import org.apache.doris.analysis.Expr;
import org.apache.doris.nereids.trees.plans.distribute.DistributeContext;
import org.apache.doris.nereids.trees.plans.distribute.worker.BackendDistributedPlanWorkerManager;
-import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
import org.apache.doris.planner.ExchangeNode;
-import org.apache.doris.planner.PlanFragment;
import org.apache.doris.planner.PlanFragmentId;
-import org.apache.doris.planner.RecursiveCteNode;
-import org.apache.doris.planner.RecursiveCteScanNode;
import org.apache.doris.thrift.TExplainLevel;
-import org.apache.doris.thrift.TExpr;
-import org.apache.doris.thrift.TNetworkAddress;
-import org.apache.doris.thrift.TRecCTENode;
-import org.apache.doris.thrift.TRecCTEResetInfo;
-import org.apache.doris.thrift.TRecCTETarget;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ListMultimap;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
/** AssignedJobBuilder */
public class AssignedJobBuilder {
@@ -53,8 +39,6 @@ public class AssignedJobBuilder {
boolean isLoadJob) {
DistributeContext distributeContext = new DistributeContext(workerManager, isLoadJob);
ListMultimap<PlanFragmentId, AssignedJob> allAssignedJobs = ArrayListMultimap.create();
- Map<PlanFragmentId, TRecCTETarget> fragmentIdToRecCteTargetMap = new TreeMap<>();
- Map<PlanFragmentId, Set<TNetworkAddress>> fragmentIdToNetworkAddressMap = new TreeMap<>();
for (Entry<PlanFragmentId, UnassignedJob> kv : unassignedJobs.entrySet()) {
PlanFragmentId fragmentId = kv.getKey();
UnassignedJob unassignedJob = kv.getValue();
@@ -71,83 +55,6 @@ public class AssignedJobBuilder {
+ ", fragment: " + unassignedJob.getFragment().getExplainString(TExplainLevel.VERBOSE));
}
allAssignedJobs.putAll(fragmentId, fragmentAssignedJobs);
-
- Set<TNetworkAddress> networkAddresses = new TreeSet<>();
- for (AssignedJob assignedJob : fragmentAssignedJobs) {
- DistributedPlanWorker distributedPlanWorker = assignedJob.getAssignedWorker();
- networkAddresses.add(new TNetworkAddress(distributedPlanWorker.host(), distributedPlanWorker.port()));
- }
- fragmentIdToNetworkAddressMap.put(fragmentId, networkAddresses);
-
- PlanFragment planFragment = unassignedJob.getFragment();
- List<RecursiveCteScanNode> recursiveCteScanNodes = planFragment.getPlanRoot()
- .collectInCurrentFragment(RecursiveCteScanNode.class::isInstance);
- if (!recursiveCteScanNodes.isEmpty()) {
- if (recursiveCteScanNodes.size() != 1) {
- throw new IllegalStateException(
- String.format("one fragment can only have 1 recursive cte scan node, but there is %d",
- recursiveCteScanNodes.size()));
- }
- if (fragmentAssignedJobs.size() != 1) {
- throw new IllegalStateException(String.format(
- "fragmentAssignedJobs's size must be 1 for recursive cte scan node, but it is %d",
- fragmentAssignedJobs.size()));
- }
- TRecCTETarget tRecCTETarget = new TRecCTETarget();
- DistributedPlanWorker distributedPlanWorker = fragmentAssignedJobs.get(0).getAssignedWorker();
- tRecCTETarget.setAddr(new TNetworkAddress(distributedPlanWorker.host(), distributedPlanWorker.port()));
- tRecCTETarget.setFragmentInstanceId(fragmentAssignedJobs.get(0).instanceId());
- tRecCTETarget.setNodeId(recursiveCteScanNodes.get(0).getId().asInt());
- fragmentIdToRecCteTargetMap.put(fragmentId, tRecCTETarget);
- }
-
- List<RecursiveCteNode> recursiveCteNodes = planFragment.getPlanRoot()
- .collectInCurrentFragment(RecursiveCteNode.class::isInstance);
- if (!recursiveCteNodes.isEmpty()) {
- if (recursiveCteNodes.size() != 1) {
- throw new IllegalStateException(
- String.format("one fragment can only have 1 recursive cte node, but there is %d",
- recursiveCteNodes.size()));
- }
-
- List<TRecCTETarget> targets = new ArrayList<>();
- List<TRecCTEResetInfo> fragmentsToReset = new ArrayList<>();
- // PhysicalPlanTranslator will swap recursiveCteNodes's child fragment,
- // so we get recursive one by 1st child
- List<PlanFragment> childFragments = new ArrayList<>();
- planFragment.getChild(0).collectAll(PlanFragment.class::isInstance, childFragments);
- for (PlanFragment child : childFragments) {
- PlanFragmentId childFragmentId = child.getFragmentId();
- TRecCTETarget tRecCTETarget = fragmentIdToRecCteTargetMap.getOrDefault(childFragmentId, null);
- if (tRecCTETarget != null) {
- targets.add(tRecCTETarget);
- }
- Set<TNetworkAddress> tNetworkAddresses = fragmentIdToNetworkAddressMap.get(childFragmentId);
- if (tNetworkAddresses == null) {
- throw new IllegalStateException(
- String.format("can't find TNetworkAddress for fragment %d", childFragmentId));
- }
- for (TNetworkAddress address : tNetworkAddresses) {
- TRecCTEResetInfo tRecCTEResetInfo = new TRecCTEResetInfo();
- tRecCTEResetInfo.setFragmentId(childFragmentId.asInt());
- tRecCTEResetInfo.setAddr(address);
- fragmentsToReset.add(tRecCTEResetInfo);
- }
- }
-
- RecursiveCteNode recursiveCteNode = recursiveCteNodes.get(0);
- List<List<Expr>> materializedResultExprLists = recursiveCteNode.getMaterializedResultExprLists();
- List<List<TExpr>> texprLists = new ArrayList<>(materializedResultExprLists.size());
- for (List<Expr> exprList : materializedResultExprLists) {
- texprLists.add(Expr.treesToThrift(exprList));
- }
- TRecCTENode tRecCTENode = new TRecCTENode();
- tRecCTENode.setIsUnionAll(recursiveCteNode.isUnionAll());
- tRecCTENode.setTargets(targets);
- tRecCTENode.setFragmentsToReset(fragmentsToReset);
- tRecCTENode.setResultExprLists(texprLists);
- recursiveCteNode.settRecCTENode(tRecCTENode);
- }
}
return allAssignedJobs;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedRecursiveCteScanJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedRecursiveCteScanJob.java
index 7fbb3f88e21..cfd3ebe7b2b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedRecursiveCteScanJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedRecursiveCteScanJob.java
@@ -26,6 +26,7 @@ import org.apache.doris.planner.ExchangeNode;
import org.apache.doris.planner.PlanFragment;
import org.apache.doris.planner.ScanNode;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ListMultimap;
@@ -57,6 +58,8 @@ public class UnassignedRecursiveCteScanJob extends AbstractUnassignedScanJob {
@Override
protected List<AssignedJob> fillUpAssignedJobs(List<AssignedJob> assignedJobs,
DistributedPlanWorkerManager workerManager, ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
- return fillUpSingleEmptyInstance(workerManager);
+ Preconditions.checkArgument(!assignedJobs.isEmpty(),
+ "assignedJobs is empty for UnassignedRecursiveCteScanJob");
+ return assignedJobs;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java
index cfdf317c620..3958d5b4df1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java
@@ -19,19 +19,24 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.DataTrait;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Cast;
+import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
-import org.apache.doris.nereids.trees.plans.algebra.Union;
+import org.apache.doris.nereids.trees.plans.algebra.RecursiveCte;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.util.ExpressionUtils;
@@ -56,68 +61,126 @@ import java.util.Set;
/**
* LogicalRecursiveCte is basically like LogicalUnion
*/
-public class LogicalRecursiveCte extends LogicalSetOperation implements Union, OutputPrunable {
-
- // in doris, we use union node to present one row relation
- private final List<List<NamedExpression>> constantExprsList;
- // When there is an agg on the union and there is a filter on the agg,
- // it is necessary to keep the filter on the agg and push the filter down to each child of the union.
- private final boolean hasPushedFilter;
+public class LogicalRecursiveCte extends AbstractLogicalPlan implements RecursiveCte, OutputPrunable {
+ protected final List<NamedExpression> outputs;
+ protected final List<List<SlotReference>> regularChildrenOutputs;
+ private final boolean isUnionAll;
/** LogicalRecursiveCte */
- public LogicalRecursiveCte(Qualifier qualifier, List<Plan> children) {
- this(qualifier, ImmutableList.of(), children);
+ public LogicalRecursiveCte(boolean isUnionAll, List<Plan> children) {
+ this(isUnionAll, ImmutableList.of(), ImmutableList.of(), children);
}
/** LogicalRecursiveCte */
- public LogicalRecursiveCte(Qualifier qualifier, List<List<NamedExpression>> constantExprsList,
- List<Plan> children) {
- this(qualifier, ImmutableList.of(), ImmutableList.of(), constantExprsList, false, children);
- }
-
- /** LogicalRecursiveCte */
- public LogicalRecursiveCte(Qualifier qualifier, List<NamedExpression> outputs,
- List<List<SlotReference>> childrenOutputs,
- List<List<NamedExpression>> constantExprsList, boolean hasPushedFilter, List<Plan> children) {
- this(qualifier, outputs, childrenOutputs, constantExprsList, hasPushedFilter, Optional.empty(),
+ public LogicalRecursiveCte(boolean isUnionAll, List<NamedExpression> outputs,
+ List<List<SlotReference>> childrenOutputs, List<Plan> children) {
+ this(isUnionAll, outputs, childrenOutputs, Optional.empty(),
Optional.empty(),
children);
}
/** LogicalRecursiveCte */
- public LogicalRecursiveCte(Qualifier qualifier, List<NamedExpression> outputs,
+ public LogicalRecursiveCte(boolean isUnionAll, List<NamedExpression> outputs,
List<List<SlotReference>> childrenOutputs,
- List<List<NamedExpression>> constantExprsList, boolean hasPushedFilter,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
List<Plan> children) {
- super(PlanType.LOGICAL_RECURSIVE_CTE, qualifier, outputs, childrenOutputs,
- groupExpression, logicalProperties, children);
- this.hasPushedFilter = hasPushedFilter;
- this.constantExprsList = Utils.fastToImmutableList(
- Objects.requireNonNull(constantExprsList, "constantExprsList should not be null"));
+ super(PlanType.LOGICAL_RECURSIVE_CTE, groupExpression, logicalProperties, children);
+ this.isUnionAll = isUnionAll;
+ this.outputs = ImmutableList.copyOf(outputs);
+ this.regularChildrenOutputs = ImmutableList.copyOf(childrenOutputs);
}
- public boolean hasPushedFilter() {
- return hasPushedFilter;
+ @Override
+ public boolean isUnionAll() {
+ return isUnionAll;
}
- public List<List<NamedExpression>> getConstantExprsList() {
- return constantExprsList;
+ @Override
+ public List<SlotReference> getRegularChildOutput(int i) {
+ return regularChildrenOutputs.get(i);
}
@Override
- public List<? extends Expression> getExpressions() {
- return constantExprsList.stream().flatMap(List::stream).collect(ImmutableList.toImmutableList());
+ public List<List<SlotReference>> getRegularChildrenOutputs() {
+ return regularChildrenOutputs;
+ }
+
+ public List<List<NamedExpression>> collectChildrenProjections() {
+ return castCommonDataTypeOutputs();
+ }
+
+ private List<List<NamedExpression>> castCommonDataTypeOutputs() {
+ int childOutputSize = child(0).getOutput().size();
+ ImmutableList.Builder<NamedExpression> newLeftOutputs = ImmutableList.builderWithExpectedSize(
+ childOutputSize);
+ ImmutableList.Builder<NamedExpression> newRightOutputs = ImmutableList.builderWithExpectedSize(
+ childOutputSize
+ );
+ // Ensure that the output types of the left and right children are consistent and expand upward.
+ for (int i = 0; i < childOutputSize; ++i) {
+ Slot left = child(0).getOutput().get(i);
+ Slot right = child(1).getOutput().get(i);
+ DataType compatibleType;
+ try {
+ compatibleType = LogicalSetOperation.getAssignmentCompatibleType(left.getDataType(),
+ right.getDataType());
+ } catch (Exception e) {
+ throw new AnalysisException(
+ "Can not find compatible type for " + left + " and " + right + ", " + e.getMessage());
+ }
+ Expression newLeft = TypeCoercionUtils.castIfNotSameTypeStrict(left, compatibleType);
+ Expression newRight = TypeCoercionUtils.castIfNotSameTypeStrict(right, compatibleType);
+ if (newLeft instanceof Cast) {
+ newLeft = new Alias(newLeft, left.getName());
+ }
+ if (newRight instanceof Cast) {
+ newRight = new Alias(newRight, right.getName());
+ }
+ newLeftOutputs.add((NamedExpression) newLeft);
+ newRightOutputs.add((NamedExpression) newRight);
+ }
+
+ return ImmutableList.of(newLeftOutputs.build(), newRightOutputs.build());
+ }
+
+ /**
+ * Generate new output for Recursive Cte.
+ */
+ public List<NamedExpression> buildNewOutputs() {
+ List<Slot> slots = resetNullableForLeftOutputs();
+ ImmutableList.Builder<NamedExpression> newOutputs = ImmutableList.builderWithExpectedSize(slots.size());
+
+ for (int i = 0; i < slots.size(); i++) {
+ Slot slot = slots.get(i);
+ ExprId exprId = i < outputs.size() ? outputs.get(i).getExprId() : StatementScopeIdGenerator.newExprId();
+ newOutputs.add(
+ new SlotReference(exprId, slot.toSql(), slot.getDataType(), slot.nullable(), ImmutableList.of())
+ );
+ }
+ return newOutputs.build();
+ }
+
+ // If the right child is nullable, need to ensure that the left child is also nullable
+ private List<Slot> resetNullableForLeftOutputs() {
+ int rightChildOutputSize = child(1).getOutput().size();
+ ImmutableList.Builder<Slot> resetNullableForLeftOutputs
+ = ImmutableList.builderWithExpectedSize(rightChildOutputSize);
+ for (int i = 0; i < rightChildOutputSize; ++i) {
+ if (child(1).getOutput().get(i).nullable() && !child(0).getOutput().get(i).nullable()) {
+ resetNullableForLeftOutputs.add(child(0).getOutput().get(i).withNullable(true));
+ } else {
+ resetNullableForLeftOutputs.add(child(0).getOutput().get(i));
+ }
+ }
+ return resetNullableForLeftOutputs.build();
}
@Override
public String toString() {
return Utils.toSqlStringSkipNull("LogicalRecursiveCte",
- "qualifier", qualifier,
+ "isUnionAll", isUnionAll,
"outputs", outputs,
"regularChildrenOutputs", regularChildrenOutputs,
- "constantExprsList", constantExprsList,
- "hasPushedFilter", hasPushedFilter,
"stats", statistics);
}
@@ -130,13 +193,13 @@ public class LogicalRecursiveCte extends LogicalSetOperation implements Union, O
return false;
}
LogicalRecursiveCte that = (LogicalRecursiveCte) o;
- return super.equals(that) && hasPushedFilter == that.hasPushedFilter
- && Objects.equals(constantExprsList, that.constantExprsList);
+ return isUnionAll == that.isUnionAll && Objects.equals(outputs, that.outputs)
+ && Objects.equals(regularChildrenOutputs, that.regularChildrenOutputs);
}
@Override
public int hashCode() {
- return Objects.hash(super.hashCode(), hasPushedFilter, constantExprsList);
+ return Objects.hash(isUnionAll, outputs, regularChildrenOutputs);
}
@Override
@@ -145,64 +208,58 @@ public class LogicalRecursiveCte extends LogicalSetOperation implements Union, O
}
@Override
- public LogicalRecursiveCte withChildren(List<Plan> children) {
- return new LogicalRecursiveCte(qualifier, outputs, regularChildrenOutputs,
- constantExprsList, hasPushedFilter, children);
+ public List<? extends Expression> getExpressions() {
+ return regularChildrenOutputs.stream().flatMap(List::stream).collect(ImmutableList.toImmutableList());
}
@Override
- public LogicalSetOperation withChildrenAndTheirOutputs(List<Plan> children,
+ public List<Slot> computeOutput() {
+ return outputs.stream()
+ .map(NamedExpression::toSlot)
+ .collect(ImmutableList.toImmutableList());
+ }
+
+ @Override
+ public LogicalRecursiveCte withChildren(List<Plan> children) {
+ return new LogicalRecursiveCte(isUnionAll, outputs, regularChildrenOutputs, children);
+ }
+
+ public LogicalRecursiveCte withChildrenAndTheirOutputs(List<Plan> children,
List<List<SlotReference>> childrenOutputs) {
Preconditions.checkArgument(children.size() == childrenOutputs.size(),
"children size %s is not equals with children outputs size %s",
children.size(), childrenOutputs.size());
- return new LogicalRecursiveCte(qualifier, outputs, childrenOutputs, constantExprsList, hasPushedFilter,
- children);
+ return new LogicalRecursiveCte(isUnionAll, outputs, childrenOutputs, children);
}
@Override
public LogicalRecursiveCte withGroupExpression(Optional<GroupExpression> groupExpression) {
- return new LogicalRecursiveCte(qualifier, outputs, regularChildrenOutputs, constantExprsList, hasPushedFilter,
+ return new LogicalRecursiveCte(isUnionAll, outputs, regularChildrenOutputs,
groupExpression, Optional.of(getLogicalProperties()), children);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
- return new LogicalRecursiveCte(qualifier, outputs, regularChildrenOutputs, constantExprsList, hasPushedFilter,
+ return new LogicalRecursiveCte(isUnionAll, outputs, regularChildrenOutputs,
groupExpression, logicalProperties, children);
}
- @Override
public LogicalRecursiveCte withNewOutputs(List<NamedExpression> newOutputs) {
- return new LogicalRecursiveCte(qualifier, newOutputs, regularChildrenOutputs, constantExprsList,
- hasPushedFilter, Optional.empty(), Optional.empty(), children);
- }
-
- public LogicalRecursiveCte withNewOutputsAndConstExprsList(List<NamedExpression> newOutputs,
- List<List<NamedExpression>> constantExprsList) {
- return new LogicalRecursiveCte(qualifier, newOutputs, regularChildrenOutputs, constantExprsList,
- hasPushedFilter, Optional.empty(), Optional.empty(), children);
- }
-
- public LogicalRecursiveCte withChildrenAndConstExprsList(List<Plan> children,
- List<List<SlotReference>> childrenOutputs, List<List<NamedExpression>> constantExprsList) {
- return new LogicalRecursiveCte(qualifier, outputs, childrenOutputs, constantExprsList, hasPushedFilter,
- children);
+ return new LogicalRecursiveCte(isUnionAll, newOutputs, regularChildrenOutputs,
+ Optional.empty(), Optional.empty(), children);
}
- public LogicalRecursiveCte withNewOutputsChildrenAndConstExprsList(List<NamedExpression> newOutputs,
- List<Plan> children,
- List<List<SlotReference>> childrenOutputs,
- List<List<NamedExpression>> constantExprsList) {
- return new LogicalRecursiveCte(qualifier, newOutputs, childrenOutputs, constantExprsList,
- hasPushedFilter, Optional.empty(), Optional.empty(), children);
+ public LogicalRecursiveCte withNewOutputsAndChildren(List<NamedExpression> newOutputs,
+ List<Plan> children,
+ List<List<SlotReference>> childrenOutputs) {
+ return new LogicalRecursiveCte(isUnionAll, newOutputs, childrenOutputs,
+ Optional.empty(), Optional.empty(), children);
}
- public LogicalRecursiveCte withAllQualifier() {
- return new LogicalRecursiveCte(Qualifier.ALL, outputs, regularChildrenOutputs, constantExprsList,
- hasPushedFilter,
- Optional.empty(), Optional.empty(), children);
+ @Override
+ public List<NamedExpression> getOutputs() {
+ return outputs;
}
@Override
@@ -212,7 +269,7 @@ public class LogicalRecursiveCte extends LogicalSetOperation implements Union, O
@Override
public void computeUnique(DataTrait.Builder builder) {
- if (qualifier == Qualifier.DISTINCT) {
+ if (!isUnionAll) {
builder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
}
}
@@ -224,19 +281,6 @@ public class LogicalRecursiveCte extends LogicalSetOperation implements Union, O
ConnectContext.get().getStatementContext(), this, PhysicalProperties.ANY)));
for (int i = 0; i < getOutputs().size(); i++) {
Optional<Literal> value = Optional.empty();
- if (!constantExprsList.isEmpty()) {
- value = ExpressionUtils.checkConstantExpr(constantExprsList.get(0).get(i), context);
- if (!value.isPresent()) {
- continue;
- }
- final int fi = i;
- Literal literal = value.get();
- if (constantExprsList.stream()
- .map(exprs -> ExpressionUtils.checkConstantExpr(exprs.get(fi), context))
- .anyMatch(val -> !val.isPresent() || !val.get().equals(literal))) {
- continue;
- }
- }
for (int childIdx = 0; childIdx < children.size(); childIdx++) {
// TODO: use originOutputs = child(childIdx).getOutput() ?
List<? extends Slot> originOutputs = regularChildrenOutputs.get(childIdx);
@@ -267,10 +311,7 @@ public class LogicalRecursiveCte extends LogicalSetOperation implements Union, O
@Override
public boolean hasUnboundExpression() {
- if (!constantExprsList.isEmpty() && children.isEmpty()) {
- return false;
- }
- return super.hasUnboundExpression();
+ return outputs.isEmpty();
}
private List<BitSet> mapSlotToIndex(Plan plan, List<Set<Slot>> equalSlotsList) {
@@ -384,7 +425,7 @@ public class LogicalRecursiveCte extends LogicalSetOperation implements Union, O
Expression otherConstant = namedExpression.child(0);
nullable |= otherConstant.nullable();
DataType otherDataType = otherConstant.getDataType();
- commonDataType = getAssignmentCompatibleType(commonDataType, otherDataType);
+ commonDataType = LogicalSetOperation.getAssignmentCompatibleType(commonDataType, otherDataType);
}
return Pair.of(commonDataType, nullable);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCte.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCte.java
index ef0cc98de0a..c0a7f0222d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCte.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCte.java
@@ -27,7 +27,7 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
-import org.apache.doris.nereids.trees.plans.algebra.Union;
+import org.apache.doris.nereids.trees.plans.algebra.RecursiveCte;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
@@ -44,55 +44,91 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.stream.Collectors;
/**
* PhysicalRecursiveCte is basically like PhysicalUnion
*/
-public class PhysicalRecursiveCte extends PhysicalSetOperation implements Union {
+public class PhysicalRecursiveCte extends AbstractPhysicalPlan implements RecursiveCte {
- // in doris, we use union node to present one row relation
- private final List<List<NamedExpression>> constantExprsList;
+ protected final List<NamedExpression> outputs;
+ protected final List<List<SlotReference>> regularChildrenOutputs;
+ private final boolean isUnionAll;
/** PhysicalRecursiveCte */
- public PhysicalRecursiveCte(Qualifier qualifier,
+ public PhysicalRecursiveCte(boolean isUnionAll,
List<NamedExpression> outputs,
List<List<SlotReference>> childrenOutputs,
- List<List<NamedExpression>> constantExprsList,
LogicalProperties logicalProperties,
List<Plan> children) {
- super(PlanType.PHYSICAL_RECURSIVE_CTE, qualifier, outputs, childrenOutputs, logicalProperties, children);
- this.constantExprsList = ImmutableList.copyOf(
- Objects.requireNonNull(constantExprsList, "constantExprsList should not be null"));
+ this(isUnionAll, outputs, childrenOutputs, Optional.empty(), logicalProperties, children);
}
/** PhysicalRecursiveCte */
- public PhysicalRecursiveCte(Qualifier qualifier,
+ public PhysicalRecursiveCte(boolean isUnionAll,
List<NamedExpression> outputs,
List<List<SlotReference>> childrenOutputs,
- List<List<NamedExpression>> constantExprsList,
Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties,
List<Plan> children) {
- super(PlanType.PHYSICAL_RECURSIVE_CTE, qualifier, outputs, childrenOutputs,
- groupExpression, logicalProperties, children);
- this.constantExprsList = ImmutableList.copyOf(
- Objects.requireNonNull(constantExprsList, "constantExprsList should not be null"));
+ this(isUnionAll, outputs, childrenOutputs, groupExpression, logicalProperties,
+ PhysicalProperties.ANY, null, children);
}
/** PhysicalRecursiveCte */
- public PhysicalRecursiveCte(Qualifier qualifier, List<NamedExpression> outputs,
- List<List<SlotReference>> childrenOutputs, List<List<NamedExpression>> constantExprsList,
+ public PhysicalRecursiveCte(boolean isUnionAll, List<NamedExpression> outputs,
+ List<List<SlotReference>> childrenOutputs,
Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties,
- PhysicalProperties physicalProperties, Statistics statistics, List<Plan> inputs) {
- super(PlanType.PHYSICAL_RECURSIVE_CTE, qualifier, outputs, childrenOutputs,
- groupExpression, logicalProperties, physicalProperties, statistics, inputs);
- this.constantExprsList = ImmutableList.copyOf(
- Objects.requireNonNull(constantExprsList, "constantExprsList should not be null"));
+ PhysicalProperties physicalProperties, Statistics statistics, List<Plan> children) {
+ super(PlanType.PHYSICAL_RECURSIVE_CTE, groupExpression, logicalProperties, physicalProperties,
+ statistics, children.toArray(new Plan[0]));
+ this.isUnionAll = isUnionAll;
+ this.outputs = ImmutableList.copyOf(outputs);
+ this.regularChildrenOutputs = ImmutableList.copyOf(childrenOutputs);
}
- public List<List<NamedExpression>> getConstantExprsList() {
- return constantExprsList;
+ @Override
+ public boolean isUnionAll() {
+ return isUnionAll;
+ }
+
+ @Override
+ public List<SlotReference> getRegularChildOutput(int i) {
+ return regularChildrenOutputs.get(i);
+ }
+
+ @Override
+ public List<NamedExpression> getOutputs() {
+ return outputs;
+ }
+
+ @Override
+ public List<Slot> computeOutput() {
+ return outputs.stream()
+ .map(NamedExpression::toSlot)
+ .collect(ImmutableList.toImmutableList());
+ }
+
+ @Override
+ public List<List<SlotReference>> getRegularChildrenOutputs() {
+ return regularChildrenOutputs;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PhysicalRecursiveCte that = (PhysicalRecursiveCte) o;
+ return isUnionAll == that.isUnionAll && Objects.equals(outputs, that.outputs) && Objects.equals(
+ regularChildrenOutputs, that.regularChildrenOutputs);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(isUnionAll, outputs, regularChildrenOutputs);
}
@Override
@@ -100,14 +136,18 @@ public class PhysicalRecursiveCte extends PhysicalSetOperation implements Union
return visitor.visitPhysicalRecursiveCte(this, context);
}
+ @Override
+ public List<? extends Expression> getExpressions() {
+ return regularChildrenOutputs.stream().flatMap(List::stream).collect(ImmutableList.toImmutableList());
+ }
+
@Override
public String toString() {
return Utils.toSqlString("PhysicalRecursiveCte" + "[" + id.asInt() + "]" + getGroupIdWithPrefix(),
"stats", statistics,
- "qualifier", qualifier,
+ "isUnionAll", isUnionAll,
"outputs", outputs,
- "regularChildrenOutputs", regularChildrenOutputs,
- "constantExprsList", constantExprsList);
+ "regularChildrenOutputs", regularChildrenOutputs);
}
@Override
@@ -117,11 +157,6 @@ public class PhysicalRecursiveCte extends PhysicalSetOperation implements Union
&& context.getSessionVariable().getDetailShapePlanNodesSet().contains(getClass().getSimpleName())) {
StringBuilder builder = new StringBuilder();
builder.append(getClass().getSimpleName());
- builder.append("(constantExprsList=");
- builder.append(constantExprsList.stream()
- .map(exprs -> exprs.stream().map(Expression::shapeInfo)
- .collect(Collectors.joining(", ", "[", "]")))
- .collect(Collectors.joining(", ", "[", "]")));
builder.append(")");
return builder.toString();
} else {
@@ -131,39 +166,39 @@ public class PhysicalRecursiveCte extends PhysicalSetOperation implements Union
@Override
public PhysicalRecursiveCte withChildren(List<Plan> children) {
- return new PhysicalRecursiveCte(qualifier, outputs, regularChildrenOutputs, constantExprsList, groupExpression,
+ return new PhysicalRecursiveCte(isUnionAll, outputs, regularChildrenOutputs, groupExpression,
getLogicalProperties(), children);
}
@Override
public PhysicalRecursiveCte withGroupExpression(Optional<GroupExpression> groupExpression) {
- return new PhysicalRecursiveCte(qualifier, outputs, regularChildrenOutputs, constantExprsList,
+ return new PhysicalRecursiveCte(isUnionAll, outputs, regularChildrenOutputs,
groupExpression, getLogicalProperties(), children);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
- return new PhysicalRecursiveCte(qualifier, outputs, regularChildrenOutputs, constantExprsList,
+ return new PhysicalRecursiveCte(isUnionAll, outputs, regularChildrenOutputs,
groupExpression, logicalProperties.get(), children);
}
@Override
public PhysicalRecursiveCte withPhysicalPropertiesAndStats(
PhysicalProperties physicalProperties, Statistics statistics) {
- return new PhysicalRecursiveCte(qualifier, outputs, regularChildrenOutputs, constantExprsList,
+ return new PhysicalRecursiveCte(isUnionAll, outputs, regularChildrenOutputs,
groupExpression, getLogicalProperties(), physicalProperties, statistics, children);
}
@Override
public PhysicalRecursiveCte resetLogicalProperties() {
- return new PhysicalRecursiveCte(qualifier, outputs, regularChildrenOutputs, constantExprsList,
+ return new PhysicalRecursiveCte(isUnionAll, outputs, regularChildrenOutputs,
Optional.empty(), null, physicalProperties, statistics, children);
}
@Override
public void computeUnique(DataTrait.Builder builder) {
- if (qualifier == Qualifier.DISTINCT) {
+ if (!isUnionAll) {
builder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java
index d34de256608..66918845e00 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java
@@ -383,7 +383,7 @@ public abstract class PlanVisitor<R, C> implements CommandVisitor<R, C>, Relatio
}
public R visitPhysicalRecursiveCte(PhysicalRecursiveCte recursiveCte, C context) {
- return visitPhysicalSetOperation(recursiveCte, context);
+ return visit(recursiveCte, context);
}
public R visitAbstractPhysicalSort(AbstractPhysicalSort<? extends Plan> sort, C context) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
index 103abd8b790..35065314bab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
@@ -17,12 +17,12 @@
package org.apache.doris.planner;
+import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.system.Backend;
-import org.apache.doris.thrift.TDataGenScanRange;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TPlanNode;
@@ -63,18 +63,13 @@ public class RecursiveCteScanNode extends ScanNode {
Collections.shuffle(backendList);
Backend selectedBackend = backendList.get(0);
- // create a dummy scan range
- TScanRange scanRange = new TScanRange();
- TDataGenScanRange dataGenScanRange = new TDataGenScanRange();
- scanRange.setDataGenScanRange(dataGenScanRange);
-
// create scan range locations
- TScanRangeLocations locations = new TScanRangeLocations();
TScanRangeLocation location = new TScanRangeLocation();
location.setBackendId(selectedBackend.getId());
location.setServer(new TNetworkAddress(selectedBackend.getHost(), selectedBackend.getBePort()));
+ TScanRangeLocations locations = new TScanRangeLocations();
locations.addToLocations(location);
- locations.setScanRange(scanRange);
+ locations.setScanRange(new TScanRange());
scanRangeLocations.add(locations);
}
@@ -96,8 +91,11 @@ public class RecursiveCteScanNode extends ScanNode {
@Override
public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
StringBuilder output = new StringBuilder();
- output.append(prefix).append("Recursive Cte: ").append(getTableIf().getName());
- output.append("\n");
+ output.append(prefix).append("Recursive Cte: ").append(getTableIf().getName()).append("\n");
+ if (!conjuncts.isEmpty()) {
+ Expr expr = convertConjunctsToAndCompoundPredicate(conjuncts);
+ output.append(prefix).append("PREDICATES: ").append(expr.toSql()).append("\n");
+ }
return output.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
index 5fcd14fcb79..f6f24aae153 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
@@ -17,6 +17,7 @@
package org.apache.doris.qe.runtime;
+import org.apache.doris.analysis.Expr;
import org.apache.doris.catalog.AIResource;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Resource;
@@ -41,6 +42,9 @@ import org.apache.doris.planner.MultiCastDataSink;
import org.apache.doris.planner.OlapScanNode;
import org.apache.doris.planner.OlapTableSink;
import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.PlanFragmentId;
+import org.apache.doris.planner.RecursiveCteNode;
+import org.apache.doris.planner.RecursiveCteScanNode;
import org.apache.doris.planner.ScanNode;
import org.apache.doris.planner.SortNode;
import org.apache.doris.qe.ConnectContext;
@@ -48,6 +52,7 @@ import org.apache.doris.qe.CoordinatorContext;
import org.apache.doris.thrift.PaloInternalServiceVersion;
import org.apache.doris.thrift.TAIResource;
import org.apache.doris.thrift.TDataSinkType;
+import org.apache.doris.thrift.TExpr;
import org.apache.doris.thrift.TFileScanRangeParams;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TPipelineFragmentParams;
@@ -56,6 +61,9 @@ import org.apache.doris.thrift.TPipelineInstanceParams;
import org.apache.doris.thrift.TPlanFragment;
import org.apache.doris.thrift.TPlanFragmentDestination;
import org.apache.doris.thrift.TQueryOptions;
+import org.apache.doris.thrift.TRecCTENode;
+import org.apache.doris.thrift.TRecCTEResetInfo;
+import org.apache.doris.thrift.TRecCTETarget;
import org.apache.doris.thrift.TRuntimeFilterInfo;
import org.apache.doris.thrift.TRuntimeFilterParams;
import org.apache.doris.thrift.TScanRangeParams;
@@ -79,6 +87,8 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
import java.util.function.BiConsumer;
import java.util.function.Supplier;
@@ -89,6 +99,7 @@ public class ThriftPlansBuilder {
CoordinatorContext coordinatorContext) {
List<PipelineDistributedPlan> distributedPlans = coordinatorContext.distributedPlans;
+ setParamsForRecursiveCteNode(distributedPlans);
// we should set runtime predicate first, then we can use heap sort and to thrift
setRuntimePredicateIfNeed(coordinatorContext.scanNodes);
@@ -571,6 +582,91 @@ public class ThriftPlansBuilder {
}
}
+ private static void setParamsForRecursiveCteNode(List<PipelineDistributedPlan> distributedPlans) {
+ Map<PlanFragmentId, TRecCTETarget> fragmentIdToRecCteTargetMap = new TreeMap<>();
+ Map<PlanFragmentId, Set<TNetworkAddress>> fragmentIdToNetworkAddressMap = new TreeMap<>();
+ for (PipelineDistributedPlan plan : distributedPlans) {
+ List<AssignedJob> fragmentAssignedJobs = plan.getInstanceJobs();
+ Set<TNetworkAddress> networkAddresses = new TreeSet<>();
+ for (AssignedJob assignedJob : fragmentAssignedJobs) {
+ DistributedPlanWorker distributedPlanWorker = assignedJob.getAssignedWorker();
+ networkAddresses.add(new TNetworkAddress(distributedPlanWorker.host(),
+ distributedPlanWorker.brpcPort()));
+ }
+ PlanFragment planFragment = plan.getFragmentJob().getFragment();
+ fragmentIdToNetworkAddressMap.put(planFragment.getFragmentId(), networkAddresses);
+
+ List<RecursiveCteScanNode> recursiveCteScanNodes = planFragment.getPlanRoot()
+ .collectInCurrentFragment(RecursiveCteScanNode.class::isInstance);
+ if (!recursiveCteScanNodes.isEmpty()) {
+ if (recursiveCteScanNodes.size() != 1) {
+ throw new IllegalStateException(
+ String.format("one fragment can only have 1 recursive cte scan node, but there is %d",
+ recursiveCteScanNodes.size()));
+ }
+ if (fragmentAssignedJobs.isEmpty()) {
+ throw new IllegalStateException(
+ "fragmentAssignedJobs is empty for recursive cte scan node");
+ }
+ TRecCTETarget tRecCTETarget = new TRecCTETarget();
+ DistributedPlanWorker distributedPlanWorker = fragmentAssignedJobs.get(0).getAssignedWorker();
+ tRecCTETarget.setAddr(new TNetworkAddress(distributedPlanWorker.host(),
+ distributedPlanWorker.brpcPort()));
+ tRecCTETarget.setFragmentInstanceId(fragmentAssignedJobs.get(0).instanceId());
+ tRecCTETarget.setNodeId(recursiveCteScanNodes.get(0).getId().asInt());
+ fragmentIdToRecCteTargetMap.put(planFragment.getFragmentId(), tRecCTETarget);
+ }
+
+ List<RecursiveCteNode> recursiveCteNodes = planFragment.getPlanRoot()
+ .collectInCurrentFragment(RecursiveCteNode.class::isInstance);
+ if (!recursiveCteNodes.isEmpty()) {
+ if (recursiveCteNodes.size() != 1) {
+ throw new IllegalStateException(
+ String.format("one fragment can only have 1 recursive cte node, but there is %d",
+ recursiveCteNodes.size()));
+ }
+
+ List<TRecCTETarget> targets = new ArrayList<>();
+ List<TRecCTEResetInfo> fragmentsToReset = new ArrayList<>();
+ // PhysicalPlanTranslator will swap recursiveCteNodes's child fragment,
+ // so we get recursive one by 1st child
+ List<PlanFragment> childFragments = new ArrayList<>();
+ planFragment.getChild(0).collectAll(PlanFragment.class::isInstance, childFragments);
+ for (PlanFragment child : childFragments) {
+ PlanFragmentId childFragmentId = child.getFragmentId();
+ TRecCTETarget tRecCTETarget = fragmentIdToRecCteTargetMap.getOrDefault(childFragmentId, null);
+ if (tRecCTETarget != null) {
+ targets.add(tRecCTETarget);
+ }
+ Set<TNetworkAddress> tNetworkAddresses = fragmentIdToNetworkAddressMap.get(childFragmentId);
+ if (tNetworkAddresses == null) {
+ throw new IllegalStateException(
+ String.format("can't find TNetworkAddress for fragment %d", childFragmentId));
+ }
+ for (TNetworkAddress address : tNetworkAddresses) {
+ TRecCTEResetInfo tRecCTEResetInfo = new TRecCTEResetInfo();
+ tRecCTEResetInfo.setFragmentId(childFragmentId.asInt());
+ tRecCTEResetInfo.setAddr(address);
+ fragmentsToReset.add(tRecCTEResetInfo);
+ }
+ }
+
+ RecursiveCteNode recursiveCteNode = recursiveCteNodes.get(0);
+ List<List<Expr>> materializedResultExprLists = recursiveCteNode.getMaterializedResultExprLists();
+ List<List<TExpr>> texprLists = new ArrayList<>(materializedResultExprLists.size());
+ for (List<Expr> exprList : materializedResultExprLists) {
+ texprLists.add(Expr.treesToThrift(exprList));
+ }
+ TRecCTENode tRecCTENode = new TRecCTENode();
+ tRecCTENode.setIsUnionAll(recursiveCteNode.isUnionAll());
+ tRecCTENode.setTargets(targets);
+ tRecCTENode.setFragmentsToReset(fragmentsToReset);
+ tRecCTENode.setResultExprLists(texprLists);
+ recursiveCteNode.settRecCTENode(tRecCTENode);
+ }
+ }
+ }
+
private static class PerNodeScanParams {
Map<Integer, List<TScanRangeParams>> perNodeScanRanges;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/09: update thrift
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch dev_rec
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ddb9501dabc9c64c6d65ba0374d2c0238fefe950
Author: BiteTheDDDDt <xl...@selectdb.com>
AuthorDate: Tue Sep 23 13:53:14 2025 +0800
update thrift
---
gensrc/thrift/PlanNodes.thrift | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index d2eeeb49194..d792410f2be 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -702,7 +702,7 @@ struct TRecCTETarget {
struct TRecCTEResetInfo {
1: optional Types.TNetworkAddress addr
- 2: optional Types.TUniqueId fragment_id
+ 2: optional i32 fragment_id
}
struct TRecCTENode {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 08/09: fix fe plan
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch dev_rec
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c94b717b92e23cafb7f7616b31546814d208d988
Author: lichi <li...@selectdb.com>
AuthorDate: Mon Oct 13 16:03:36 2025 +0800
fix fe plan
---
.../glue/translator/PhysicalPlanTranslator.java | 13 ++-
.../properties/ChildOutputPropertyDeriver.java | 10 +-
.../org/apache/doris/nereids/rules/RuleSet.java | 2 +
.../org/apache/doris/nereids/rules/RuleType.java | 1 +
.../doris/nereids/rules/analysis/AnalyzeCTE.java | 31 +++++-
...eChildToPhysicalRecursiveCteRecursiveChild.java | 36 +++++++
.../doris/nereids/stats/StatsCalculator.java | 14 +++
.../trees/copier/LogicalPlanDeepCopier.java | 8 ++
.../apache/doris/nereids/trees/plans/PlanType.java | 2 +
.../logical/LogicalRecursiveCteRecursiveChild.java | 103 +++++++++++++++++++
.../PhysicalRecursiveCteRecursiveChild.java | 110 +++++++++++++++++++++
.../nereids/trees/plans/visitor/PlanVisitor.java | 12 +++
.../apache/doris/planner/RecursiveCteScanNode.java | 6 +-
.../doris/qe/runtime/ThriftPlansBuilder.java | 2 +-
14 files changed, 343 insertions(+), 7 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index a27268a5af7..3c0007a12c0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -157,6 +157,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteRecursiveChild;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
@@ -1079,9 +1080,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
PlanTranslatorContext context) {
TableIf table = recursiveCteScan.getTable();
List<Slot> slots = ImmutableList.copyOf(recursiveCteScan.getOutput());
- TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context);
+ TupleDescriptor tupleDescriptor = generateTupleDesc(slots, null, context);
- RecursiveCteScanNode scanNode = new RecursiveCteScanNode(context.nextPlanNodeId(), tupleDescriptor);
+ RecursiveCteScanNode scanNode = new RecursiveCteScanNode(table != null ? table.getName() : "",
+ context.nextPlanNodeId(), tupleDescriptor);
scanNode.setNereidsId(recursiveCteScan.getId());
context.getNereidsIdToPlanNodeIdMap().put(recursiveCteScan.getId(), scanNode.getId());
Utils.execWithUncheckedException(scanNode::initScanRangeLocations);
@@ -2341,6 +2343,13 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return recursiveCteFragment;
}
+ @Override
+ public PlanFragment visitPhysicalRecursiveCteRecursiveChild(
+ PhysicalRecursiveCteRecursiveChild<? extends Plan> recursiveChild,
+ PlanTranslatorContext context) {
+ return recursiveChild.child().accept(this, context);
+ }
+
/**
* Returns a new fragment with a UnionNode as its root. The data partition of the
* returned fragment and how the data of the child fragments is consumed depends on the
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
index 0751f14b715..56433c82d33 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
@@ -53,6 +53,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteRecursiveChild;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
@@ -151,7 +152,7 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
@Override
public PhysicalProperties visitPhysicalRecursiveCteScan(PhysicalRecursiveCteScan cteScan, PlanContext context) {
- return PhysicalProperties.MUST_SHUFFLE;
+ return PhysicalProperties.ANY;
}
/**
@@ -475,6 +476,13 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
return PhysicalProperties.GATHER;
}
+ @Override
+ public PhysicalProperties visitPhysicalRecursiveCteRecursiveChild(
+ PhysicalRecursiveCteRecursiveChild<? extends Plan> recursiveChild,
+ PlanContext context) {
+ return PhysicalProperties.MUST_SHUFFLE;
+ }
+
@Override
public PhysicalProperties visitPhysicalUnion(PhysicalUnion union, PlanContext context) {
if (union.getConstantExprsList().isEmpty()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 122dbee03ef..62a5d7702c3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -80,6 +80,7 @@ import org.apache.doris.nereids.rules.implementation.LogicalOlapTableSinkToPhysi
import org.apache.doris.nereids.rules.implementation.LogicalOneRowRelationToPhysicalOneRowRelation;
import org.apache.doris.nereids.rules.implementation.LogicalPartitionTopNToPhysicalPartitionTopN;
import org.apache.doris.nereids.rules.implementation.LogicalProjectToPhysicalProject;
+import org.apache.doris.nereids.rules.implementation.LogicalRecursiveCteRecursiveChildToPhysicalRecursiveCteRecursiveChild;
import org.apache.doris.nereids.rules.implementation.LogicalRecursiveCteScanToPhysicalRecursiveCteScan;
import org.apache.doris.nereids.rules.implementation.LogicalRecursiveCteToPhysicalRecursiveCte;
import org.apache.doris.nereids.rules.implementation.LogicalRepeatToPhysicalRepeat;
@@ -213,6 +214,7 @@ public class RuleSet {
.add(SplitAggMultiPhase.INSTANCE)
.add(SplitAggMultiPhaseWithoutGbyKey.INSTANCE)
.add(new LogicalRecursiveCteToPhysicalRecursiveCte())
+ .add(new LogicalRecursiveCteRecursiveChildToPhysicalRecursiveCteRecursiveChild())
.add(new LogicalUnionToPhysicalUnion())
.add(new LogicalExceptToPhysicalExcept())
.add(new LogicalIntersectToPhysicalIntersect())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index c13607a838d..75550513068 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -519,6 +519,7 @@ public enum RuleType {
TWO_PHASE_AGGREGATE_WITH_DISTINCT(RuleTypeClass.IMPLEMENTATION),
LOGICAL_UNION_TO_PHYSICAL_UNION(RuleTypeClass.IMPLEMENTATION),
LOGICAL_RECURSIVE_CTE_TO_PHYSICAL_RECURSIVE_CTE(RuleTypeClass.IMPLEMENTATION),
+ LOGICAL_RECURSIVE_CTE_RECURSIVE_CHILD_TO_PHYSICAL_RECURSIVE_CTE_RECURSIVE_CHILD(RuleTypeClass.IMPLEMENTATION),
LOGICAL_EXCEPT_TO_PHYSICAL_EXCEPT(RuleTypeClass.IMPLEMENTATION),
LOGICAL_INTERSECT_TO_PHYSICAL_INTERSECT(RuleTypeClass.IMPLEMENTATION),
LOGICAL_GENERATE_TO_PHYSICAL_GENERATE(RuleTypeClass.IMPLEMENTATION),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
index a6259f0b24c..906d572c6c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
@@ -23,11 +23,13 @@ import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.CTEId;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
@@ -36,9 +38,11 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteRecursiveChild;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.trees.plans.logical.ProjectProcessor;
+import org.apache.doris.nereids.types.DataType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -146,6 +150,11 @@ public class AnalyzeCTE extends OneAnalysisRuleFactory {
});
cascadesContext.addPlanProcesses(innerAnchorCascadesCtx.getPlanProcesses());
LogicalPlan analyzedAnchorChild = (LogicalPlan) innerAnchorCascadesCtx.getRewritePlan();
+ List<NamedExpression> anchorNullableOutputs = new ArrayList<>(analyzedAnchorChild.getOutput().size());
+ for (Slot slot : analyzedAnchorChild.getOutput()) {
+ anchorNullableOutputs.add(new Alias(new Nullable(slot), slot.getName()));
+ }
+ analyzedAnchorChild = new LogicalProject<>(anchorNullableOutputs, analyzedAnchorChild);
checkColumnAlias(aliasQuery, analyzedAnchorChild.getOutput());
// analyze recursive child
@@ -158,9 +167,29 @@ public class AnalyzeCTE extends OneAnalysisRuleFactory {
});
cascadesContext.addPlanProcesses(innerRecursiveCascadesCtx.getPlanProcesses());
LogicalPlan analyzedRecursiveChild = (LogicalPlan) innerRecursiveCascadesCtx.getRewritePlan();
- LogicalUnion logicalUnion = (LogicalUnion) parsedCtePlan;
+ List<DataType> anchorChildOutputTypes = new ArrayList<>(analyzedAnchorChild.getOutput().size());
+ for (Slot slot : analyzedAnchorChild.getOutput()) {
+ anchorChildOutputTypes.add(slot.getDataType());
+ }
+ List<Slot> recursiveChildOutputs = analyzedRecursiveChild.getOutput();
+ for (int i = 0; i < recursiveChildOutputs.size(); ++i) {
+ if (recursiveChildOutputs.get(i).getDataType() != anchorChildOutputTypes.get(i)) {
+ throw new AnalysisException(String.format("recursive child's %d column's datatype in select list %s "
+ + "is different from anchor child's output datatype %s, please add cast manually "
+ + "to get expect datatype",
+ i + 1, recursiveChildOutputs.get(i).getDataType(), anchorChildOutputTypes.get(i)));
+ }
+ }
+
+ List<NamedExpression> recursiveNullableOutputs = new ArrayList<>(analyzedRecursiveChild.getOutput().size());
+ for (Slot slot : analyzedRecursiveChild.getOutput()) {
+ recursiveNullableOutputs.add(new Alias(new Nullable(slot), slot.getName()));
+ }
+ analyzedRecursiveChild = new LogicalProject<>(recursiveNullableOutputs, analyzedRecursiveChild);
+ analyzedRecursiveChild = new LogicalRecursiveCteRecursiveChild<>(analyzedRecursiveChild);
// create LogicalRecursiveCte
+ LogicalUnion logicalUnion = (LogicalUnion) parsedCtePlan;
LogicalRecursiveCte analyzedCtePlan = new LogicalRecursiveCte(
logicalUnion.getQualifier() == SetOperation.Qualifier.ALL,
ImmutableList.of(analyzedAnchorChild, analyzedRecursiveChild));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteRecursiveChildToPhysicalRecursiveCteRecursiveChild.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteRecursiveChildToPhysicalRecursiveCteRecursiveChild.java
new file mode 100644
index 00000000000..2923924a6ad
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteRecursiveChildToPhysicalRecursiveCteRecursiveChild.java
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.implementation;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteRecursiveChild;
+
+/**
+ * Implementation rule that convert logical recursive cte's recursive child to physical recursive child.
+ */
+public class LogicalRecursiveCteRecursiveChildToPhysicalRecursiveCteRecursiveChild
+ extends OneImplementationRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalRecursiveCteRecursiveChild().then(recursiveCte -> new PhysicalRecursiveCteRecursiveChild(
+ recursiveCte.getLogicalProperties(),
+ recursiveCte.child()))
+ .toRule(RuleType.LOGICAL_RECURSIVE_CTE_RECURSIVE_CHILD_TO_PHYSICAL_RECURSIVE_CTE_RECURSIVE_CHILD);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index 9ddcc7e2e40..930674b873b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -92,6 +92,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteRecursiveChild;
import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
@@ -127,6 +128,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteRecursiveChild;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
@@ -931,6 +933,12 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
.stream().map(Group::getStatistics).collect(Collectors.toList()));
}
+ @Override
+ public Statistics visitLogicalRecursiveCteRecursiveChild(LogicalRecursiveCteRecursiveChild recursiveChild,
+ Void context) {
+ return groupExpression.childStatistics(0);
+ }
+
@Override
public Statistics visitLogicalUnion(
LogicalUnion union, Void context) {
@@ -1113,6 +1121,12 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
.stream().map(Group::getStatistics).collect(Collectors.toList()));
}
+ @Override
+ public Statistics visitPhysicalRecursiveCteRecursiveChild(PhysicalRecursiveCteRecursiveChild recursiveChild,
+ Void context) {
+ return groupExpression.childStatistics(0);
+ }
+
@Override
public Statistics visitPhysicalUnion(PhysicalUnion union, Void context) {
return computeUnion(union, groupExpression.children()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
index 739047f4a77..98c3ec2ef41 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
@@ -54,6 +54,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteRecursiveChild;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
@@ -371,6 +372,13 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
return new LogicalRecursiveCte(recursiveCte.isUnionAll(), outputs, childrenOutputs, children);
}
+ @Override
+ public Plan visitLogicalRecursiveCteRecursiveChild(LogicalRecursiveCteRecursiveChild<? extends Plan> recursiveChild,
+ DeepCopierContext context) {
+ Plan child = recursiveChild.child().accept(this, context);
+ return new LogicalRecursiveCteRecursiveChild<>(child);
+ }
+
@Override
public Plan visitLogicalExcept(LogicalExcept except, DeepCopierContext context) {
List<Plan> children = except.children().stream()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index fe7661dbc4d..03df1c26fb7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -84,6 +84,7 @@ public enum PlanType {
LOGICAL_PROJECT,
LOGICAL_QUALIFY,
LOGICAL_RECURSIVE_CTE,
+ LOGICAL_RECURSIVE_CTE_RECURSIVE_CHILD,
LOGICAL_RECURSIVE_CTE_SCAN,
LOGICAL_REPEAT,
LOGICAL_SELECT_HINT,
@@ -127,6 +128,7 @@ public enum PlanType {
PHYSICAL_CTE_PRODUCER,
PHYSICAL_CTE_ANCHOR,
PHYSICAL_RECURSIVE_CTE,
+ PHYSICAL_RECURSIVE_CTE_RECURSIVE_CHILD,
PHYSICAL_DISTRIBUTE,
PHYSICAL_EXCEPT,
PHYSICAL_FILTER,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteRecursiveChild.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteRecursiveChild.java
new file mode 100644
index 00000000000..f73323b50bb
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteRecursiveChild.java
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * LogicalRecursiveCteRecursiveChild is sentinel plan for must_shuffle
+ */
+public class LogicalRecursiveCteRecursiveChild<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE> {
+
+ public LogicalRecursiveCteRecursiveChild(CHILD_TYPE child) {
+ this(Optional.empty(), Optional.empty(), child);
+ }
+
+ public LogicalRecursiveCteRecursiveChild(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, CHILD_TYPE child) {
+ this(groupExpression, logicalProperties, ImmutableList.of(child));
+ }
+
+ public LogicalRecursiveCteRecursiveChild(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> child) {
+ super(PlanType.LOGICAL_RECURSIVE_CTE_RECURSIVE_CHILD, groupExpression, logicalProperties, child);
+ }
+
+ @Override
+ public Plan withChildren(List<Plan> children) {
+ return new LogicalRecursiveCteRecursiveChild<>(Optional.empty(), Optional.empty(), children);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitLogicalRecursiveCteRecursiveChild(this, context);
+ }
+
+ @Override
+ public List<? extends Expression> getExpressions() {
+ return ImmutableList.of();
+ }
+
+ @Override
+ public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new LogicalRecursiveCteRecursiveChild<>(groupExpression, Optional.of(getLogicalProperties()), children);
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ return new LogicalRecursiveCteRecursiveChild<>(groupExpression, logicalProperties, children);
+ }
+
+ @Override
+ public void computeUnique(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public void computeUniform(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public void computeEqualSet(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public void computeFd(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public List<Slot> computeOutput() {
+ return child().getOutput();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteRecursiveChild.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteRecursiveChild.java
new file mode 100644
index 00000000000..5f598d8feac
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteRecursiveChild.java
@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.physical;
+
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.statistics.Statistics;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * PhysicalRecursiveCteRecursiveChild is sentinel plan for must_shuffle
+ */
+public class PhysicalRecursiveCteRecursiveChild<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD_TYPE> {
+ public PhysicalRecursiveCteRecursiveChild(LogicalProperties logicalProperties, CHILD_TYPE child) {
+ this(Optional.empty(), logicalProperties, child);
+ }
+
+ public PhysicalRecursiveCteRecursiveChild(Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties, CHILD_TYPE child) {
+ this(groupExpression, logicalProperties, PhysicalProperties.ANY, null, child);
+ }
+
+ public PhysicalRecursiveCteRecursiveChild(Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties, @Nullable PhysicalProperties physicalProperties, Statistics statistics,
+ CHILD_TYPE child) {
+ super(PlanType.PHYSICAL_RECURSIVE_CTE_RECURSIVE_CHILD, groupExpression, logicalProperties, physicalProperties,
+ statistics, child);
+ }
+
+ @Override
+ public Plan withChildren(List<Plan> children) {
+ Preconditions.checkArgument(children.size() == 1);
+ return new PhysicalRecursiveCteRecursiveChild<>(groupExpression, getLogicalProperties(), children.get(0));
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitPhysicalRecursiveCteRecursiveChild(this, context);
+ }
+
+ @Override
+ public List<? extends Expression> getExpressions() {
+ return ImmutableList.of();
+ }
+
+ @Override
+ public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new PhysicalRecursiveCteRecursiveChild<>(groupExpression, getLogicalProperties(), child());
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ Preconditions.checkArgument(children.size() == 1);
+ return new PhysicalRecursiveCteRecursiveChild<>(groupExpression, logicalProperties.get(), child());
+ }
+
+ @Override
+ public void computeUnique(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public void computeUniform(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public void computeEqualSet(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public void computeFd(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) {
+ return new PhysicalRecursiveCteRecursiveChild<>(groupExpression, getLogicalProperties(), physicalProperties,
+ statistics, child());
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java
index 66918845e00..5b0067bbd7a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java
@@ -45,6 +45,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPreFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalQualify;
import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteRecursiveChild;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint;
@@ -80,6 +81,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteRecursiveChild;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
@@ -217,6 +219,11 @@ public abstract class PlanVisitor<R, C> implements CommandVisitor<R, C>, Relatio
return visit(recursiveCte, context);
}
+ public R visitLogicalRecursiveCteRecursiveChild(LogicalRecursiveCteRecursiveChild<? extends Plan> recursiveChild,
+ C context) {
+ return visit(recursiveChild, context);
+ }
+
public R visitLogicalLimit(LogicalLimit<? extends Plan> limit, C context) {
return visit(limit, context);
}
@@ -386,6 +393,11 @@ public abstract class PlanVisitor<R, C> implements CommandVisitor<R, C>, Relatio
return visit(recursiveCte, context);
}
+ public R visitPhysicalRecursiveCteRecursiveChild(PhysicalRecursiveCteRecursiveChild<? extends Plan> recursiveChild,
+ C context) {
+ return visit(recursiveChild, context);
+ }
+
public R visitAbstractPhysicalSort(AbstractPhysicalSort<? extends Plan> sort, C context) {
return visit(sort, context);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
index 35065314bab..eb63ba6fcb7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
@@ -38,9 +38,11 @@ import java.util.List;
// Full scan of recursive cte temp table
public class RecursiveCteScanNode extends ScanNode {
+ private final String recursiveCteName;
- public RecursiveCteScanNode(PlanNodeId id, TupleDescriptor desc) {
+ public RecursiveCteScanNode(String recursiveCteName, PlanNodeId id, TupleDescriptor desc) {
super(id, desc, "RECURSIVE_CTE_SCAN", StatisticalType.CTE_SCAN_NODE);
+ this.recursiveCteName = recursiveCteName;
}
public void initScanRangeLocations() throws UserException {
@@ -91,7 +93,7 @@ public class RecursiveCteScanNode extends ScanNode {
@Override
public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
StringBuilder output = new StringBuilder();
- output.append(prefix).append("Recursive Cte: ").append(getTableIf().getName()).append("\n");
+ output.append(prefix).append("Recursive Cte: ").append(recursiveCteName).append("\n");
if (!conjuncts.isEmpty()) {
Expr expr = convertConjunctsToAndCompoundPredicate(conjuncts);
output.append(prefix).append("PREDICATES: ").append(expr.toSql()).append("\n");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
index f6f24aae153..df71172baad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
@@ -628,7 +628,7 @@ public class ThriftPlansBuilder {
List<TRecCTETarget> targets = new ArrayList<>();
List<TRecCTEResetInfo> fragmentsToReset = new ArrayList<>();
- // PhysicalPlanTranslator will swap recursiveCteNodes's child fragment,
+ // PhysicalPlanTranslator will swap recursiveCteNode's child fragment,
// so we get recursive one by 1st child
List<PlanFragment> childFragments = new ArrayList<>();
planFragment.getChild(0).collectAll(PlanFragment.class::isInstance, childFragments);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/09: add rec cte thrift struct
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch dev_rec
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5d0a8bffda527408beb3bc666518af37ecc8655d
Author: BiteTheDDDDt <xl...@selectdb.com>
AuthorDate: Thu Sep 18 21:02:33 2025 +0800
add rec cte thrift struct
update
---
gensrc/thrift/PlanNodes.thrift | 21 ++++++++++++++++++++-
1 file changed, 20 insertions(+), 1 deletion(-)
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index e99f93b3ff1..3cae7f63cf9 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -60,7 +60,9 @@ enum TPlanNodeType {
TEST_EXTERNAL_SCAN_NODE = 31,
PARTITION_SORT_NODE = 32,
GROUP_COMMIT_SCAN_NODE = 33,
- MATERIALIZATION_NODE = 34
+ MATERIALIZATION_NODE = 34,
+ REC_CTE_NODE = 35,
+ REC_CTE_SCAN_NODE = 36
}
struct TKeyRange {
@@ -692,6 +694,23 @@ struct TFileScanNode {
2: optional string table_name
}
+struct TRecCTETarget {
+ 1: optional Types.TNetworkAddress addr
+ 2: optional Types.TUniqueId fragment_instance_id
+ 3: optional i32 node_id
+}
+
+struct TRecCTEResetInfo {
+ 1: optional Types.TNetworkAddress addr
+ 2: optional Types.TUniqueId fragment_id
+}
+
+struct TRecCTENode {
+ 1: optional bool is_union_all
+ 2: optional list<TRecCTETarget> targets
+ 3: optional list<TRecCTEResetInfo> fragments_to_reset
+}
+
struct TEsScanNode {
1: required Types.TTupleId tuple_id
2: optional map<string,string> properties
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/09: update thrift
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch dev_rec
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9b4c89046295183461b72e3b69d5f99f933f5af4
Author: BiteTheDDDDt <xl...@selectdb.com>
AuthorDate: Mon Sep 22 16:06:13 2025 +0800
update thrift
---
gensrc/thrift/PlanNodes.thrift | 1 +
1 file changed, 1 insertion(+)
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 3cae7f63cf9..d2eeeb49194 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -1446,6 +1446,7 @@ struct TPlanNode {
36: optional list<TRuntimeFilterDesc> runtime_filters
37: optional TGroupCommitScanNode group_commit_scan_node
38: optional TMaterializationNode materialization_node
+ 39: optional TRecCTENode rec_cte_node
// Use in vec exec engine
40: optional Exprs.TExpr vconjunct
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 06/09: recursive cte fe part
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch dev_rec
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 379f7c7a6bee715583f05cf212995cc0b56714fc
Author: lichi <li...@selectdb.com>
AuthorDate: Thu Oct 9 15:14:12 2025 +0800
recursive cte fe part
---
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 2 +-
.../doris/catalog/RecursiveCteTempTable.java | 26 ++
.../java/org/apache/doris/catalog/TableIf.java | 6 +-
.../org/apache/doris/nereids/CascadesContext.java | 37 +-
.../glue/translator/PhysicalPlanTranslator.java | 29 ++
.../doris/nereids/parser/LogicalPlanBuilder.java | 3 +-
.../apache/doris/nereids/pattern/MemoPatterns.java | 24 ++
.../properties/ChildOutputPropertyDeriver.java | 12 +
.../org/apache/doris/nereids/rules/RuleSet.java | 4 +
.../org/apache/doris/nereids/rules/RuleType.java | 2 +
.../doris/nereids/rules/analysis/AnalyzeCTE.java | 111 +++++-
.../doris/nereids/rules/analysis/BindRelation.java | 27 +-
.../nereids/rules/analysis/CollectRelation.java | 11 +-
.../nereids/rules/analysis/SubExprAnalyzer.java | 4 +-
...RecursiveCteScanToPhysicalRecursiveCteScan.java | 42 +++
.../LogicalRecursiveCteToPhysicalRecursiveCte.java | 39 ++
.../nereids/rules/rewrite/AdjustNullable.java | 17 +
.../doris/nereids/rules/rewrite/CTEInline.java | 31 +-
.../doris/nereids/rules/rewrite/ColumnPruning.java | 106 ++++++
.../doris/nereids/stats/StatsCalculator.java | 29 ++
.../trees/copier/LogicalPlanDeepCopier.java | 23 ++
.../apache/doris/nereids/trees/plans/PlanType.java | 4 +
.../commands/CreateMaterializedViewCommand.java | 2 +-
.../distribute/worker/job/AssignedJobBuilder.java | 93 +++++
.../worker/job/UnassignedJobBuilder.java | 13 +
.../worker/job/UnassignedRecursiveCteScanJob.java | 62 +++
.../trees/plans/logical/LogicalRecursiveCte.java | 419 +++++++++++++++++++++
.../plans/logical/LogicalRecursiveCteScan.java | 66 ++++
.../trees/plans/logical/LogicalSubQueryAlias.java | 31 +-
.../trees/plans/physical/PhysicalRecursiveCte.java | 251 ++++++++++++
.../plans/physical/PhysicalRecursiveCteScan.java | 85 +++++
.../nereids/trees/plans/visitor/PlanVisitor.java | 10 +
.../trees/plans/visitor/RelationVisitor.java | 10 +
.../org/apache/doris/planner/RecursiveCteNode.java | 52 +++
.../apache/doris/planner/RecursiveCteScanNode.java | 113 ++++++
.../apache/doris/statistics/StatisticalType.java | 2 +
.../recursive_cte/test_recursive_cte.groovy | 102 +++++
38 files changed, 1845 insertions(+), 56 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index fa9f119258c..499b514a284 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -439,6 +439,7 @@ REAL: 'REAL';
REBALANCE: 'REBALANCE';
RECENT: 'RECENT';
RECOVER: 'RECOVER';
+RECURSIVE: 'RECURSIVE';
RECYCLE: 'RECYCLE';
REFRESH: 'REFRESH';
REFERENCES: 'REFERENCES';
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index f4e0294edab..162c6f21688 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -1204,7 +1204,7 @@ cte
;
aliasQuery
- : identifier columnAliases? AS LEFT_PAREN query RIGHT_PAREN
+ : RECURSIVE? identifier columnAliases? AS LEFT_PAREN query RIGHT_PAREN
;
columnAliases
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RecursiveCteTempTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RecursiveCteTempTable.java
new file mode 100644
index 00000000000..9f36b04dfc4
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RecursiveCteTempTable.java
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import java.util.List;
+
+public class RecursiveCteTempTable extends Table {
+ public RecursiveCteTempTable(String tableName, List<Column> fullSchema) {
+ super(-1, tableName, TableType.RECURSIVE_CTE_TEMP_TABLE, fullSchema);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
index d15fabd6b75..5aa7f9ead76 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
@@ -398,7 +398,8 @@ public interface TableIf {
@Deprecated ICEBERG, @Deprecated HUDI, JDBC,
TABLE_VALUED_FUNCTION, HMS_EXTERNAL_TABLE, ES_EXTERNAL_TABLE, MATERIALIZED_VIEW, JDBC_EXTERNAL_TABLE,
ICEBERG_EXTERNAL_TABLE, TEST_EXTERNAL_TABLE, PAIMON_EXTERNAL_TABLE, MAX_COMPUTE_EXTERNAL_TABLE,
- HUDI_EXTERNAL_TABLE, TRINO_CONNECTOR_EXTERNAL_TABLE, LAKESOUl_EXTERNAL_TABLE, DICTIONARY;
+ HUDI_EXTERNAL_TABLE, TRINO_CONNECTOR_EXTERNAL_TABLE, LAKESOUl_EXTERNAL_TABLE, DICTIONARY,
+ RECURSIVE_CTE_TEMP_TABLE;
public String toEngineName() {
switch (this) {
@@ -437,6 +438,8 @@ public interface TableIf {
return "iceberg";
case DICTIONARY:
return "dictionary";
+ case RECURSIVE_CTE_TEMP_TABLE:
+ return "RecursiveCteTempTable";
default:
return null;
}
@@ -475,6 +478,7 @@ public interface TableIf {
case PAIMON_EXTERNAL_TABLE:
case MATERIALIZED_VIEW:
case TRINO_CONNECTOR_EXTERNAL_TABLE:
+ case RECURSIVE_CTE_TEMP_TABLE:
return "BASE TABLE";
default:
return null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
index efb53b66822..96b53b0a78d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
@@ -133,6 +133,8 @@ public class CascadesContext implements ScheduleContext {
private final boolean isEnableExprTrace;
private int groupExpressionCount = 0;
+ private Optional<String> currentRecursiveCteName;
+ private List<Slot> recursiveCteOutputs;
/**
* Constructor of OptimizerContext.
@@ -142,7 +144,8 @@ public class CascadesContext implements ScheduleContext {
*/
private CascadesContext(Optional<CascadesContext> parent, Optional<CTEId> currentTree,
StatementContext statementContext, Plan plan, Memo memo,
- CTEContext cteContext, PhysicalProperties requireProperties, boolean isLeadingDisableJoinReorder) {
+ CTEContext cteContext, PhysicalProperties requireProperties, boolean isLeadingDisableJoinReorder,
+ Optional<String> currentRecursiveCteName, List<Slot> recursiveCteOutputs) {
this.parent = Objects.requireNonNull(parent, "parent should not null");
this.currentTree = Objects.requireNonNull(currentTree, "currentTree should not null");
this.statementContext = Objects.requireNonNull(statementContext, "statementContext should not null");
@@ -167,6 +170,8 @@ public class CascadesContext implements ScheduleContext {
this.isEnableExprTrace = false;
}
this.isLeadingDisableJoinReorder = isLeadingDisableJoinReorder;
+ this.currentRecursiveCteName = currentRecursiveCteName;
+ this.recursiveCteOutputs = recursiveCteOutputs;
}
/** init a temporary context to rewrite expression */
@@ -181,7 +186,7 @@ public class CascadesContext implements ScheduleContext {
}
return newContext(Optional.empty(), Optional.empty(),
statementContext, DUMMY_PLAN,
- new CTEContext(), PhysicalProperties.ANY, false);
+ new CTEContext(), PhysicalProperties.ANY, false, Optional.empty(), ImmutableList.of());
}
/**
@@ -190,24 +195,25 @@ public class CascadesContext implements ScheduleContext {
public static CascadesContext initContext(StatementContext statementContext,
Plan initPlan, PhysicalProperties requireProperties) {
return newContext(Optional.empty(), Optional.empty(), statementContext,
- initPlan, new CTEContext(), requireProperties, false);
+ initPlan, new CTEContext(), requireProperties, false, Optional.empty(), ImmutableList.of());
}
/**
* use for analyze cte. we must pass CteContext from outer since we need to get right scope of cte
*/
public static CascadesContext newContextWithCteContext(CascadesContext cascadesContext,
- Plan initPlan, CTEContext cteContext) {
+ Plan initPlan, CTEContext cteContext, Optional<String> currentRecursiveCteName,
+ List<Slot> recursiveCteOutputs) {
return newContext(Optional.of(cascadesContext), Optional.empty(),
cascadesContext.getStatementContext(), initPlan, cteContext, PhysicalProperties.ANY,
- cascadesContext.isLeadingDisableJoinReorder
- );
+ cascadesContext.isLeadingDisableJoinReorder, currentRecursiveCteName, recursiveCteOutputs);
}
public static CascadesContext newCurrentTreeContext(CascadesContext context) {
return CascadesContext.newContext(context.getParent(), context.getCurrentTree(), context.getStatementContext(),
context.getRewritePlan(), context.getCteContext(),
- context.getCurrentJobContext().getRequiredProperties(), context.isLeadingDisableJoinReorder);
+ context.getCurrentJobContext().getRequiredProperties(), context.isLeadingDisableJoinReorder,
+ Optional.empty(), ImmutableList.of());
}
/**
@@ -216,14 +222,17 @@ public class CascadesContext implements ScheduleContext {
public static CascadesContext newSubtreeContext(Optional<CTEId> subtree, CascadesContext context,
Plan plan, PhysicalProperties requireProperties) {
return CascadesContext.newContext(Optional.of(context), subtree, context.getStatementContext(),
- plan, context.getCteContext(), requireProperties, context.isLeadingDisableJoinReorder);
+ plan, context.getCteContext(), requireProperties, context.isLeadingDisableJoinReorder, Optional.empty(),
+ ImmutableList.of());
}
private static CascadesContext newContext(Optional<CascadesContext> parent, Optional<CTEId> subtree,
StatementContext statementContext, Plan initPlan, CTEContext cteContext,
- PhysicalProperties requireProperties, boolean isLeadingDisableJoinReorder) {
+ PhysicalProperties requireProperties, boolean isLeadingDisableJoinReorder,
+ Optional<String> currentRecursiveCteName, List<Slot> recursiveCteOutputs) {
return new CascadesContext(parent, subtree, statementContext, initPlan, null,
- cteContext, requireProperties, isLeadingDisableJoinReorder);
+ cteContext, requireProperties, isLeadingDisableJoinReorder, currentRecursiveCteName,
+ recursiveCteOutputs);
}
public CascadesContext getRoot() {
@@ -250,6 +259,14 @@ public class CascadesContext implements ScheduleContext {
return isTimeout;
}
+ public Optional<String> getCurrentRecursiveCteName() {
+ return currentRecursiveCteName;
+ }
+
+ public List<Slot> getRecursiveCteOutputs() {
+ return recursiveCteOutputs;
+ }
+
/**
* Init memo with plan
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index a6af723fff8..21f9bbf2126 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -116,6 +116,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PreAggStatus;
import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
import org.apache.doris.nereids.trees.plans.algebra.Relation;
+import org.apache.doris.nereids.trees.plans.algebra.SetOperation;
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin;
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort;
import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows;
@@ -156,6 +157,8 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink;
@@ -205,6 +208,8 @@ import org.apache.doris.planner.OlapTableSink;
import org.apache.doris.planner.PartitionSortNode;
import org.apache.doris.planner.PlanFragment;
import org.apache.doris.planner.PlanNode;
+import org.apache.doris.planner.RecursiveCteNode;
+import org.apache.doris.planner.RecursiveCteScanNode;
import org.apache.doris.planner.RepeatNode;
import org.apache.doris.planner.ResultFileSink;
import org.apache.doris.planner.ResultSink;
@@ -1070,6 +1075,27 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return planFragment;
}
+ @Override
+ public PlanFragment visitPhysicalRecursiveCteScan(PhysicalRecursiveCteScan recursiveCteScan,
+ PlanTranslatorContext context) {
+ TableIf table = recursiveCteScan.getTable();
+ List<Slot> slots = ImmutableList.copyOf(recursiveCteScan.getOutput());
+ TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context);
+
+ RecursiveCteScanNode scanNode = new RecursiveCteScanNode(context.nextPlanNodeId(), tupleDescriptor);
+ scanNode.setNereidsId(recursiveCteScan.getId());
+ context.getNereidsIdToPlanNodeIdMap().put(recursiveCteScan.getId(), scanNode.getId());
+ Utils.execWithUncheckedException(scanNode::initScanRangeLocations);
+
+ translateRuntimeFilter(recursiveCteScan, scanNode, context);
+
+ context.addScanNode(scanNode, recursiveCteScan);
+ PlanFragment planFragment = createPlanFragment(scanNode, DataPartition.RANDOM, recursiveCteScan);
+ context.addPlanFragment(planFragment);
+ updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), recursiveCteScan);
+ return planFragment;
+ }
+
private List<Expr> translateToExprs(List<Expression> expressions, PlanTranslatorContext context) {
List<Expr> exprs = Lists.newArrayListWithCapacity(expressions.size());
for (Expression expression : expressions) {
@@ -2274,6 +2300,9 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
setOperationNode = new ExceptNode(context.nextPlanNodeId(), setTuple.getId());
} else if (setOperation instanceof PhysicalIntersect) {
setOperationNode = new IntersectNode(context.nextPlanNodeId(), setTuple.getId());
+ } else if (setOperation instanceof PhysicalRecursiveCte) {
+ setOperationNode = new RecursiveCteNode(context.nextPlanNodeId(), setTuple.getId(),
+ setOperation.getQualifier().equals(SetOperation.Qualifier.ALL));
} else {
throw new RuntimeException("not support set operation type " + setOperation);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 5c0a4ca2a53..3a3481132b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -2149,7 +2149,8 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
.map(RuleContext::getText)
.collect(ImmutableList.toImmutableList())
);
- return new LogicalSubQueryAlias<>(ctx.identifier().getText(), columnNames, queryPlan);
+ return new LogicalSubQueryAlias<>(ctx.identifier().getText(), columnNames, ctx.RECURSIVE() != null,
+ queryPlan);
});
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java
index fa9d191f5e2..ec53deafaad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java
@@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnary;
@@ -205,6 +206,29 @@ public interface MemoPatterns extends Patterns {
defaultPromise());
}
+ /**
+ * create a LogicalRecursiveCte pattern.
+ */
+ default PatternDescriptor<LogicalRecursiveCte>
+ logicalRecursiveCte(
+ PatternDescriptor... children) {
+ return new PatternDescriptor(
+ new TypePattern(LogicalRecursiveCte.class,
+ Arrays.stream(children)
+ .map(PatternDescriptor::getPattern)
+ .toArray(Pattern[]::new)),
+ defaultPromise());
+ }
+
+ /**
+ * create a logicalRecursiveCte group.
+ */
+ default PatternDescriptor<LogicalRecursiveCte> logicalRecursiveCte() {
+ return new PatternDescriptor(
+ new TypePattern(LogicalRecursiveCte.class, multiGroup().pattern),
+ defaultPromise());
+ }
+
/**
* create a logicalExcept pattern.
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
index d1c8bccb421..d3592388dae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
@@ -52,6 +52,8 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSink;
@@ -147,6 +149,11 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
return PhysicalProperties.STORAGE_ANY;
}
+ @Override
+ public PhysicalProperties visitPhysicalRecursiveCteScan(PhysicalRecursiveCteScan cteScan, PlanContext context) {
+ return PhysicalProperties.GATHER;
+ }
+
/**
* TODO return ANY after refactor coordinator
* return STORAGE_ANY not ANY, in order to generate distribute on jdbc scan.
@@ -463,6 +470,11 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
return PhysicalProperties.createHash(request, firstType);
}
+ @Override
+ public PhysicalProperties visitPhysicalRecursiveCte(PhysicalRecursiveCte recursiveCte, PlanContext context) {
+ return PhysicalProperties.GATHER;
+ }
+
@Override
public PhysicalProperties visitPhysicalUnion(PhysicalUnion union, PlanContext context) {
if (union.getConstantExprsList().isEmpty()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 45cb22e6b52..122dbee03ef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -80,6 +80,8 @@ import org.apache.doris.nereids.rules.implementation.LogicalOlapTableSinkToPhysi
import org.apache.doris.nereids.rules.implementation.LogicalOneRowRelationToPhysicalOneRowRelation;
import org.apache.doris.nereids.rules.implementation.LogicalPartitionTopNToPhysicalPartitionTopN;
import org.apache.doris.nereids.rules.implementation.LogicalProjectToPhysicalProject;
+import org.apache.doris.nereids.rules.implementation.LogicalRecursiveCteScanToPhysicalRecursiveCteScan;
+import org.apache.doris.nereids.rules.implementation.LogicalRecursiveCteToPhysicalRecursiveCte;
import org.apache.doris.nereids.rules.implementation.LogicalRepeatToPhysicalRepeat;
import org.apache.doris.nereids.rules.implementation.LogicalResultSinkToPhysicalResultSink;
import org.apache.doris.nereids.rules.implementation.LogicalSchemaScanToPhysicalSchemaScan;
@@ -194,6 +196,7 @@ public class RuleSet {
.add(new LogicalJdbcScanToPhysicalJdbcScan())
.add(new LogicalOdbcScanToPhysicalOdbcScan())
.add(new LogicalEsScanToPhysicalEsScan())
+ .add(new LogicalRecursiveCteScanToPhysicalRecursiveCteScan())
.add(new LogicalProjectToPhysicalProject())
.add(new LogicalLimitToPhysicalLimit())
.add(new LogicalWindowToPhysicalWindow())
@@ -209,6 +212,7 @@ public class RuleSet {
.add(SplitAggWithoutDistinct.INSTANCE)
.add(SplitAggMultiPhase.INSTANCE)
.add(SplitAggMultiPhaseWithoutGbyKey.INSTANCE)
+ .add(new LogicalRecursiveCteToPhysicalRecursiveCte())
.add(new LogicalUnionToPhysicalUnion())
.add(new LogicalExceptToPhysicalExcept())
.add(new LogicalIntersectToPhysicalIntersect())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index a8bc2052348..c13607a838d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -496,6 +496,7 @@ public enum RuleType {
LOGICAL_JDBC_SCAN_TO_PHYSICAL_JDBC_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_ODBC_SCAN_TO_PHYSICAL_ODBC_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_ES_SCAN_TO_PHYSICAL_ES_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
+ LOGICAL_RECURSIVE_CTE_SCAN_TO_PHYSICAL_RECUSIVE_CTE_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_BLACKHOLE_SINK_TO_PHYSICAL_BLACKHOLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_OLAP_TABLE_SINK_TO_PHYSICAL_OLAP_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_HIVE_TABLE_SINK_TO_PHYSICAL_HIVE_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
@@ -517,6 +518,7 @@ public enum RuleType {
COUNT_ON_INDEX_WITHOUT_PROJECT(RuleTypeClass.IMPLEMENTATION),
TWO_PHASE_AGGREGATE_WITH_DISTINCT(RuleTypeClass.IMPLEMENTATION),
LOGICAL_UNION_TO_PHYSICAL_UNION(RuleTypeClass.IMPLEMENTATION),
+ LOGICAL_RECURSIVE_CTE_TO_PHYSICAL_RECURSIVE_CTE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_EXCEPT_TO_PHYSICAL_EXCEPT(RuleTypeClass.IMPLEMENTATION),
LOGICAL_INTERSECT_TO_PHYSICAL_INTERSECT(RuleTypeClass.IMPLEMENTATION),
LOGICAL_GENERATE_TO_PHYSICAL_GENERATE(RuleTypeClass.IMPLEMENTATION),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
index 4287e8ca7de..8ac4234e846 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
@@ -24,21 +24,29 @@ import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.CTEId;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+import org.apache.doris.nereids.trees.plans.logical.ProjectProcessor;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -70,7 +78,7 @@ public class AnalyzeCTE extends OneAnalysisRuleFactory {
// step 1. analyzed all cte plan
Pair<CTEContext, List<LogicalCTEProducer<Plan>>> result = analyzeCte(logicalCTE, ctx.cascadesContext);
CascadesContext outerCascadesCtx = CascadesContext.newContextWithCteContext(
- ctx.cascadesContext, logicalCTE.child(), result.first);
+ ctx.cascadesContext, logicalCTE.child(), result.first, Optional.empty(), ImmutableList.of());
outerCascadesCtx.withPlanProcess(ctx.cascadesContext.showPlanProcess(), () -> {
outerCascadesCtx.newAnalyzer().analyze();
});
@@ -95,25 +103,96 @@ public class AnalyzeCTE extends OneAnalysisRuleFactory {
List<LogicalCTEProducer<Plan>> cteProducerPlans = new ArrayList<>();
for (LogicalSubQueryAlias<Plan> aliasQuery : aliasQueries) {
// we should use a chain to ensure visible of cte
- LogicalPlan parsedCtePlan = (LogicalPlan) aliasQuery.child();
- CascadesContext innerCascadesCtx = CascadesContext.newContextWithCteContext(
- cascadesContext, parsedCtePlan, outerCteCtx);
- innerCascadesCtx.withPlanProcess(cascadesContext.showPlanProcess(), () -> {
- innerCascadesCtx.newAnalyzer().analyze();
- });
- cascadesContext.addPlanProcesses(innerCascadesCtx.getPlanProcesses());
- LogicalPlan analyzedCtePlan = (LogicalPlan) innerCascadesCtx.getRewritePlan();
- checkColumnAlias(aliasQuery, analyzedCtePlan.getOutput());
- CTEId cteId = StatementScopeIdGenerator.newCTEId();
- LogicalSubQueryAlias<Plan> logicalSubQueryAlias =
- aliasQuery.withChildren(ImmutableList.of(analyzedCtePlan));
- outerCteCtx = new CTEContext(cteId, logicalSubQueryAlias, outerCteCtx);
- outerCteCtx.setAnalyzedPlan(logicalSubQueryAlias);
- cteProducerPlans.add(new LogicalCTEProducer<>(cteId, logicalSubQueryAlias));
+ if (aliasQuery.isRecursiveCte()) {
+ Pair<CTEContext, LogicalCTEProducer<Plan>> result = analyzeRecursiveCte(aliasQuery, outerCteCtx,
+ cascadesContext);
+ outerCteCtx = result.first;
+ cteProducerPlans.add(result.second);
+ } else {
+ LogicalPlan parsedCtePlan = (LogicalPlan) aliasQuery.child();
+ CascadesContext innerCascadesCtx = CascadesContext.newContextWithCteContext(
+ cascadesContext, parsedCtePlan, outerCteCtx, Optional.empty(), ImmutableList.of());
+ innerCascadesCtx.withPlanProcess(cascadesContext.showPlanProcess(), () -> {
+ innerCascadesCtx.newAnalyzer().analyze();
+ });
+ cascadesContext.addPlanProcesses(innerCascadesCtx.getPlanProcesses());
+ LogicalPlan analyzedCtePlan = (LogicalPlan) innerCascadesCtx.getRewritePlan();
+ checkColumnAlias(aliasQuery, analyzedCtePlan.getOutput());
+ CTEId cteId = StatementScopeIdGenerator.newCTEId();
+ LogicalSubQueryAlias<Plan> logicalSubQueryAlias = aliasQuery
+ .withChildren(ImmutableList.of(analyzedCtePlan));
+ outerCteCtx = new CTEContext(cteId, logicalSubQueryAlias, outerCteCtx);
+ outerCteCtx.setAnalyzedPlan(logicalSubQueryAlias);
+ cteProducerPlans.add(new LogicalCTEProducer<>(cteId, logicalSubQueryAlias));
+ }
}
return Pair.of(outerCteCtx, cteProducerPlans);
}
+ private Pair<CTEContext, LogicalCTEProducer<Plan>> analyzeRecursiveCte(LogicalSubQueryAlias<Plan> aliasQuery,
+ CTEContext outerCteCtx, CascadesContext cascadesContext) {
+ Preconditions.checkArgument(aliasQuery.isRecursiveCte(), "alias query must be recursive cte");
+ LogicalPlan parsedCtePlan = (LogicalPlan) aliasQuery.child();
+ if (!(parsedCtePlan instanceof LogicalUnion) || parsedCtePlan.children().size() != 2) {
+ throw new AnalysisException("recursive cte must be union");
+ }
+ // analyze anchor child, its output list will be recursive cte temp table's schema
+ LogicalPlan anchorChild = (LogicalPlan) parsedCtePlan.child(0);
+ CascadesContext innerAnchorCascadesCtx = CascadesContext.newContextWithCteContext(
+ cascadesContext, anchorChild, outerCteCtx, Optional.of(aliasQuery.getAlias()), ImmutableList.of());
+ innerAnchorCascadesCtx.withPlanProcess(cascadesContext.showPlanProcess(), () -> {
+ innerAnchorCascadesCtx.newAnalyzer().analyze();
+ });
+ cascadesContext.addPlanProcesses(innerAnchorCascadesCtx.getPlanProcesses());
+ LogicalPlan analyzedAnchorChild = (LogicalPlan) innerAnchorCascadesCtx.getRewritePlan();
+ checkColumnAlias(aliasQuery, analyzedAnchorChild.getOutput());
+
+ // analyze recursive child
+ LogicalPlan recursiveChild = (LogicalPlan) parsedCtePlan.child(1);
+ CascadesContext innerRecursiveCascadesCtx = CascadesContext.newContextWithCteContext(
+ cascadesContext, recursiveChild, outerCteCtx, Optional.of(aliasQuery.getAlias()),
+ analyzedAnchorChild.getOutput());
+ innerRecursiveCascadesCtx.withPlanProcess(cascadesContext.showPlanProcess(), () -> {
+ innerRecursiveCascadesCtx.newAnalyzer().analyze();
+ });
+ cascadesContext.addPlanProcesses(innerRecursiveCascadesCtx.getPlanProcesses());
+ LogicalPlan analyzedRecursiveChild = (LogicalPlan) innerRecursiveCascadesCtx.getRewritePlan();
+ LogicalUnion logicalUnion = (LogicalUnion) parsedCtePlan;
+
+ // manually bind LogicalRecursiveCte, see bindSetOperation in BindExpression.java
+ LogicalRecursiveCte analyzedCtePlan = new LogicalRecursiveCte(
+ logicalUnion.getQualifier(), ImmutableList.of(analyzedAnchorChild, analyzedRecursiveChild));
+ List<List<NamedExpression>> childrenProjections = analyzedCtePlan.collectChildrenProjections();
+ int childrenProjectionSize = childrenProjections.size();
+ ImmutableList.Builder<List<SlotReference>> childrenOutputs = ImmutableList
+ .builderWithExpectedSize(childrenProjectionSize);
+ ImmutableList.Builder<Plan> newChildren = ImmutableList.builderWithExpectedSize(childrenProjectionSize);
+ for (int i = 0; i < childrenProjectionSize; i++) {
+ Plan newChild;
+ Plan child = analyzedCtePlan.child(i);
+ if (childrenProjections.get(i).stream().allMatch(SlotReference.class::isInstance)) {
+ newChild = child;
+ } else {
+ List<NamedExpression> parentProject = childrenProjections.get(i);
+ newChild = ProjectProcessor.tryProcessProject(parentProject, child)
+ .orElseGet(() -> new LogicalProject<>(parentProject, child));
+ }
+ newChildren.add(newChild);
+ childrenOutputs.add((List<SlotReference>) (List) newChild.getOutput());
+ }
+ analyzedCtePlan = (LogicalRecursiveCte) analyzedCtePlan.withChildrenAndTheirOutputs(newChildren.build(),
+ childrenOutputs.build());
+ List<NamedExpression> newOutputs = analyzedCtePlan.buildNewOutputs();
+ analyzedCtePlan = analyzedCtePlan.withNewOutputs(newOutputs);
+
+ CTEId cteId = StatementScopeIdGenerator.newCTEId();
+ LogicalSubQueryAlias<Plan> logicalSubQueryAlias = aliasQuery.withChildren(ImmutableList.of(analyzedCtePlan));
+ outerCteCtx = new CTEContext(cteId, logicalSubQueryAlias, outerCteCtx);
+ outerCteCtx.setAnalyzedPlan(logicalSubQueryAlias);
+ LogicalCTEProducer<Plan> cteProducer = new LogicalCTEProducer<>(cteId, logicalSubQueryAlias);
+ return Pair.of(outerCteCtx, cteProducer);
+ }
+
/**
* check columnAliases' size and name
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index 84652b9eed0..4e6be99f290 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -27,6 +27,7 @@ import org.apache.doris.catalog.FunctionRegistry;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.RecursiveCteTempTable;
import org.apache.doris.catalog.SchemaTable;
import org.apache.doris.catalog.SchemaTable.SchemaColumn;
import org.apache.doris.catalog.TableIf;
@@ -89,6 +90,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
@@ -168,16 +170,27 @@ public class BindRelation extends OneAnalysisRuleFactory {
return consumer;
}
}
+ LogicalPlan scan;
List<String> tableQualifier = RelationUtil.getQualifierName(
cascadesContext.getConnectContext(), unboundRelation.getNameParts());
- TableIf table = cascadesContext.getStatementContext().getAndCacheTable(tableQualifier, TableFrom.QUERY,
- Optional.of(unboundRelation));
+ if (tableName.equalsIgnoreCase(cascadesContext.getCurrentRecursiveCteName().orElse(""))) {
+ ImmutableList.Builder<Column> schema = new ImmutableList.Builder<>();
+ for (Slot slot : cascadesContext.getRecursiveCteOutputs()) {
+ schema.add(new Column(slot.getName(), slot.getDataType().toCatalogDataType(), slot.nullable()));
+ }
+ RecursiveCteTempTable cteTempTable = new RecursiveCteTempTable(tableName, schema.build());
+ scan = new LogicalRecursiveCteScan(cascadesContext.getStatementContext().getNextRelationId(),
+ cteTempTable, tableQualifier);
+ } else {
+ TableIf table = cascadesContext.getStatementContext().getAndCacheTable(tableQualifier, TableFrom.QUERY,
+ Optional.of(unboundRelation));
- LogicalPlan scan = getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
- if (cascadesContext.isLeadingJoin()) {
- LeadingHint leading = (LeadingHint) cascadesContext.getHintMap().get("Leading");
- leading.putRelationIdAndTableName(Pair.of(unboundRelation.getRelationId(), tableName));
- leading.getRelationIdToScanMap().put(unboundRelation.getRelationId(), scan);
+ scan = getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
+ if (cascadesContext.isLeadingJoin()) {
+ LeadingHint leading = (LeadingHint) cascadesContext.getHintMap().get("Leading");
+ leading.putRelationIdAndTableName(Pair.of(unboundRelation.getRelationId(), tableName));
+ leading.getRelationIdToScanMap().put(unboundRelation.getRelationId(), scan);
+ }
}
return scan;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
index 53650234610..468bc8c7d7c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
@@ -102,8 +102,10 @@ public class CollectRelation implements AnalysisRuleFactory {
for (LogicalSubQueryAlias<Plan> aliasQuery : aliasQueries) {
// we should use a chain to ensure visible of cte
LogicalPlan parsedCtePlan = (LogicalPlan) aliasQuery.child();
+ // 看起来需要在CascadesContext中添加当前CTE的name，以便判断自引用
CascadesContext innerCascadesCtx = CascadesContext.newContextWithCteContext(
- cascadesContext, parsedCtePlan, outerCteCtx);
+ cascadesContext, parsedCtePlan, outerCteCtx, aliasQuery.isRecursiveCte()
+ ? Optional.of(aliasQuery.getAlias()) : Optional.empty(), ImmutableList.of());
innerCascadesCtx.newTableCollector().collect();
LogicalPlan analyzedCtePlan = (LogicalPlan) innerCascadesCtx.getRewritePlan();
// cteId is not used in CollectTable stage
@@ -122,7 +124,8 @@ public class CollectRelation implements AnalysisRuleFactory {
if (e instanceof SubqueryExpr) {
SubqueryExpr subqueryExpr = (SubqueryExpr) e;
CascadesContext subqueryContext = CascadesContext.newContextWithCteContext(
- ctx.cascadesContext, subqueryExpr.getQueryPlan(), ctx.cteContext);
+ ctx.cascadesContext, subqueryExpr.getQueryPlan(), ctx.cteContext, Optional.empty(),
+ ImmutableList.of());
subqueryContext.keepOrShowPlanProcess(ctx.cascadesContext.showPlanProcess(),
() -> subqueryContext.newTableCollector().collect());
ctx.cascadesContext.addPlanProcesses(subqueryContext.getPlanProcesses());
@@ -174,6 +177,10 @@ public class CollectRelation implements AnalysisRuleFactory {
List<String> nameParts, TableFrom tableFrom, Optional<UnboundRelation> unboundRelation) {
if (nameParts.size() == 1) {
String tableName = nameParts.get(0);
+ if (cascadesContext.getCurrentRecursiveCteName().isPresent()
+ && tableName.equalsIgnoreCase(cascadesContext.getCurrentRecursiveCteName().get())) {
+ return;
+ }
// check if it is a CTE's name
CTEContext cteContext = cascadesContext.getCteContext().findCTEContext(tableName).orElse(null);
if (cteContext != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java
index 69d79934992..05a35dc6ed8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java
@@ -52,6 +52,7 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
/**
@@ -227,7 +228,8 @@ class SubExprAnalyzer<T> extends DefaultExpressionRewriter<T> {
throw new IllegalStateException("Missing CascadesContext");
}
CascadesContext subqueryContext = CascadesContext.newContextWithCteContext(
- cascadesContext, expr.getQueryPlan(), cascadesContext.getCteContext());
+ cascadesContext, expr.getQueryPlan(), cascadesContext.getCteContext(), Optional.empty(),
+ ImmutableList.of());
// don't use `getScope()` because we only need `getScope().getOuterScope()` and `getScope().getSlots()`
// otherwise unexpected errors may occur
Scope subqueryScope = new Scope(getScope().getOuterScope(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteScanToPhysicalRecursiveCteScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteScanToPhysicalRecursiveCteScan.java
new file mode 100644
index 00000000000..8714c280bb9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteScanToPhysicalRecursiveCteScan.java
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.implementation;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteScan;
+
+import java.util.Optional;
+
+/**
+ * Implementation rule that convert logical Recursive Cte Scan to physical Recursive Cte Scan.
+ */
+public class LogicalRecursiveCteScanToPhysicalRecursiveCteScan extends OneImplementationRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalRecursiveCteScan().then(recursiveCteScan ->
+ new PhysicalRecursiveCteScan(
+ recursiveCteScan.getRelationId(),
+ recursiveCteScan.getTable(),
+ recursiveCteScan.getQualifier(),
+ Optional.empty(),
+ recursiveCteScan.getLogicalProperties(),
+ recursiveCteScan.getOperativeSlots())
+ ).toRule(RuleType.LOGICAL_RECURSIVE_CTE_SCAN_TO_PHYSICAL_RECUSIVE_CTE_SCAN_RULE);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteToPhysicalRecursiveCte.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteToPhysicalRecursiveCte.java
new file mode 100644
index 00000000000..c8960aa9e51
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteToPhysicalRecursiveCte.java
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.implementation;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
+
+/**
+ * Implementation rule that convert logical Recursive Cte to Physical Recursive Cte.
+ */
+public class LogicalRecursiveCteToPhysicalRecursiveCte extends OneImplementationRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalRecursiveCte().then(recursiveCte ->
+ new PhysicalRecursiveCte(recursiveCte.getQualifier(),
+ recursiveCte.getOutputs(),
+ recursiveCte.getRegularChildrenOutputs(),
+ recursiveCte.getConstantExprsList(),
+ recursiveCte.getLogicalProperties(),
+ recursiveCte.children())
+ ).toRule(RuleType.LOGICAL_RECURSIVE_CTE_TO_PHYSICAL_RECURSIVE_CTE);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
index 8592d3f22a0..63d63f32b6f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
@@ -41,6 +41,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
@@ -330,6 +331,22 @@ public class AdjustNullable extends DefaultPlanRewriter<Map<ExprId, Slot>> imple
inputNullable.set(j, inputNullable.get(j) || constantExprs.get(j).nullable());
}
}
+ } else if (setOperation instanceof LogicalRecursiveCte) {
+ // LogicalRecursiveCte is basically like LogicalUnion, so just do same as LogicalUnion
+ LogicalRecursiveCte logicalRecursiveCte = (LogicalRecursiveCte) setOperation;
+ if (!logicalRecursiveCte.getConstantExprsList().isEmpty() && setOperation.children().isEmpty()) {
+ int outputSize = logicalRecursiveCte.getConstantExprsList().get(0).size();
+ // create the inputNullable list and fill it with all FALSE values
+ inputNullable = Lists.newArrayListWithCapacity(outputSize);
+ for (int i = 0; i < outputSize; i++) {
+ inputNullable.add(false);
+ }
+ }
+ for (List<NamedExpression> constantExprs : logicalRecursiveCte.getConstantExprsList()) {
+ for (int j = 0; j < constantExprs.size(); j++) {
+ inputNullable.set(j, inputNullable.get(j) || constantExprs.get(j).nullable());
+ }
+ }
}
if (inputNullable == null) {
// this is a fail-safe
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java
index 22ec72c99c5..30970dee649 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java
@@ -31,15 +31,19 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
/**
* pull up LogicalCteAnchor to the top of plan to avoid CteAnchor break other rewrite rules pattern
@@ -48,9 +52,15 @@ import java.util.List;
* and put all of them to the top of plan depends on dependency tree of them.
*/
public class CTEInline extends DefaultPlanRewriter<LogicalCTEProducer<?>> implements CustomRewriter {
+ Set<LogicalCTEConsumer> mustInlineCteConsumers = new HashSet<>();
@Override
public Plan rewriteRoot(Plan plan, JobContext jobContext) {
+ List<LogicalRecursiveCte> recursiveCteList = plan.collectToList(LogicalRecursiveCte.class::isInstance);
+ for (LogicalRecursiveCte recursiveCte : recursiveCteList) {
+ mustInlineCteConsumers.addAll(recursiveCte.collect(LogicalCTEConsumer.class::isInstance));
+ }
+
Plan root = plan.accept(this, null);
// collect cte id to consumer
root.foreach(p -> {
@@ -78,17 +88,24 @@ public class CTEInline extends DefaultPlanRewriter<LogicalCTEProducer<?>> implem
}
return false;
});
- ConnectContext connectContext = ConnectContext.get();
- if (connectContext.getSessionVariable().enableCTEMaterialize
- && consumers.size() > connectContext.getSessionVariable().inlineCTEReferencedThreshold) {
- // not inline
- Plan right = cteAnchor.right().accept(this, null);
- return cteAnchor.withChildren(cteAnchor.left(), right);
- } else {
+ if (!Sets.intersection(mustInlineCteConsumers, Sets.newHashSet(consumers)).isEmpty()) {
// should inline
Plan root = cteAnchor.right().accept(this, (LogicalCTEProducer<?>) cteAnchor.left());
// process child
return root.accept(this, null);
+ } else {
+ ConnectContext connectContext = ConnectContext.get();
+ if (connectContext.getSessionVariable().enableCTEMaterialize
+ && consumers.size() > connectContext.getSessionVariable().inlineCTEReferencedThreshold) {
+ // not inline
+ Plan right = cteAnchor.right().accept(this, null);
+ return cteAnchor.withChildren(cteAnchor.left(), right);
+ } else {
+ // should inline
+ Plan root = cteAnchor.right().accept(this, (LogicalCTEProducer<?>) cteAnchor.left());
+ // process child
+ return root.accept(this, null);
+ }
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
index 90e94d578ba..96daceedfe4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
@@ -44,6 +44,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
@@ -213,6 +214,47 @@ public class ColumnPruning extends DefaultPlanRewriter<PruneContext> implements
return pruneChildren(plan, new RoaringBitmap());
}
+ @Override
+ public Plan visitLogicalRecursiveCte(LogicalRecursiveCte recursiveCte, PruneContext context) {
+ // LogicalRecursiveCte is basically like LogicalUnion, so just do same as LogicalUnion
+ if (recursiveCte.getQualifier() == Qualifier.DISTINCT) {
+ return skipPruneThisAndFirstLevelChildren(recursiveCte);
+ }
+ LogicalRecursiveCte prunedOutputRecursiveCte = pruneRecursiveCteOutput(recursiveCte, context);
+ // start prune children of recursiveCte
+ List<Slot> originOutput = recursiveCte.getOutput();
+ Set<Slot> prunedOutput = prunedOutputRecursiveCte.getOutputSet();
+ List<Integer> prunedOutputIndexes = IntStream.range(0, originOutput.size())
+ .filter(index -> prunedOutput.contains(originOutput.get(index)))
+ .boxed()
+ .collect(ImmutableList.toImmutableList());
+
+ ImmutableList.Builder<Plan> prunedChildren = ImmutableList.builder();
+ ImmutableList.Builder<List<SlotReference>> prunedChildrenOutputs = ImmutableList.builder();
+ for (int i = 0; i < prunedOutputRecursiveCte.arity(); i++) {
+ List<SlotReference> regularChildOutputs = prunedOutputRecursiveCte.getRegularChildOutput(i);
+
+ RoaringBitmap prunedChildOutputExprIds = new RoaringBitmap();
+ Builder<SlotReference> prunedChildOutputBuilder
+ = ImmutableList.builderWithExpectedSize(regularChildOutputs.size());
+ for (Integer index : prunedOutputIndexes) {
+ SlotReference slot = regularChildOutputs.get(index);
+ prunedChildOutputBuilder.add(slot);
+ prunedChildOutputExprIds.add(slot.getExprId().asInt());
+ }
+
+ List<SlotReference> prunedChildOutput = prunedChildOutputBuilder.build();
+ Plan prunedChild = doPruneChild(
+ prunedOutputRecursiveCte, prunedOutputRecursiveCte.child(i), prunedChildOutputExprIds,
+ prunedChildOutput, true
+ );
+ prunedChildrenOutputs.add(prunedChildOutput);
+ prunedChildren.add(prunedChild);
+ }
+ return prunedOutputRecursiveCte.withChildrenAndTheirOutputs(prunedChildren.build(),
+ prunedChildrenOutputs.build());
+ }
+
// union can not prune children by the common logic, we must override visit method to write special code.
@Override
public Plan visitLogicalUnion(LogicalUnion union, PruneContext context) {
@@ -403,6 +445,70 @@ public class ColumnPruning extends DefaultPlanRewriter<PruneContext> implements
}
}
+ private LogicalRecursiveCte pruneRecursiveCteOutput(LogicalRecursiveCte recursiveCte, PruneContext context) {
+ List<NamedExpression> originOutput = recursiveCte.getOutputs();
+ if (originOutput.isEmpty()) {
+ return recursiveCte;
+ }
+ List<NamedExpression> prunedOutputs = Lists.newArrayList();
+ List<List<NamedExpression>> constantExprsList = recursiveCte.getConstantExprsList();
+ List<List<SlotReference>> regularChildrenOutputs = recursiveCte.getRegularChildrenOutputs();
+ List<Plan> children = recursiveCte.children();
+ List<Integer> extractColumnIndex = Lists.newArrayList();
+ for (int i = 0; i < originOutput.size(); i++) {
+ NamedExpression output = originOutput.get(i);
+ if (context.requiredSlotsIds.contains(output.getExprId().asInt())) {
+ prunedOutputs.add(output);
+ extractColumnIndex.add(i);
+ }
+ }
+
+ ImmutableList.Builder<List<NamedExpression>> prunedConstantExprsList
+ = ImmutableList.builderWithExpectedSize(constantExprsList.size());
+ if (prunedOutputs.isEmpty()) {
+ // process prune all columns
+ NamedExpression originSlot = originOutput.get(0);
+ prunedOutputs = ImmutableList.of(new SlotReference(originSlot.getExprId(), originSlot.getName(),
+ TinyIntType.INSTANCE, false, originSlot.getQualifier()));
+ regularChildrenOutputs = Lists.newArrayListWithCapacity(regularChildrenOutputs.size());
+ children = Lists.newArrayListWithCapacity(children.size());
+ for (int i = 0; i < recursiveCte.getArity(); i++) {
+ Plan child = recursiveCte.child(i);
+ List<NamedExpression> newProjectOutput = ImmutableList.of(new Alias(new TinyIntLiteral((byte) 1)));
+ LogicalProject<?> project;
+ if (child instanceof LogicalProject) {
+ LogicalProject<Plan> childProject = (LogicalProject<Plan>) child;
+ List<NamedExpression> mergeProjections = PlanUtils.mergeProjections(
+ childProject.getProjects(), newProjectOutput);
+ project = new LogicalProject<>(mergeProjections, childProject.child());
+ } else {
+ project = new LogicalProject<>(newProjectOutput, child);
+ }
+ regularChildrenOutputs.add((List) project.getOutput());
+ children.add(project);
+ }
+ for (int i = 0; i < constantExprsList.size(); i++) {
+ prunedConstantExprsList.add(ImmutableList.of(new Alias(new TinyIntLiteral((byte) 1))));
+ }
+ } else {
+ int len = extractColumnIndex.size();
+ for (List<NamedExpression> row : constantExprsList) {
+ ImmutableList.Builder<NamedExpression> newRow = ImmutableList.builderWithExpectedSize(len);
+ for (int idx : extractColumnIndex) {
+ newRow.add(row.get(idx));
+ }
+ prunedConstantExprsList.add(newRow.build());
+ }
+ }
+
+ if (prunedOutputs.equals(originOutput) && !context.requiredSlotsIds.isEmpty()) {
+ return recursiveCte;
+ } else {
+ return recursiveCte.withNewOutputsChildrenAndConstExprsList(prunedOutputs, children,
+ regularChildrenOutputs, prunedConstantExprsList.build());
+ }
+ }
+
private LogicalUnion pruneUnionOutput(LogicalUnion union, PruneContext context) {
List<NamedExpression> originOutput = union.getOutputs();
if (originOutput.isEmpty()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index f0ca1f1e6ba..a14c2dfc9cb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -90,6 +90,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
@@ -123,6 +125,8 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSchemaScan;
@@ -869,6 +873,12 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
return computeCatalogRelation(esScan);
}
+ @Override
+ public Statistics visitLogicalRecursiveCteScan(LogicalRecursiveCteScan recursiveCteScan, Void context) {
+ recursiveCteScan.getExpressions();
+ return computeCatalogRelation(recursiveCteScan);
+ }
+
@Override
public Statistics visitLogicalProject(LogicalProject<? extends Plan> project, Void context) {
return computeProject(project, groupExpression.childStatistics(0));
@@ -912,6 +922,14 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
return computeAssertNumRows(assertNumRows.getAssertNumRowsElement(), groupExpression.childStatistics(0));
}
+ @Override
+ public Statistics visitLogicalRecursiveCte(
+ LogicalRecursiveCte recursiveCte, Void context) {
+ return computeUnion(recursiveCte,
+ groupExpression.children()
+ .stream().map(Group::getStatistics).collect(Collectors.toList()));
+ }
+
@Override
public Statistics visitLogicalUnion(
LogicalUnion union, Void context) {
@@ -997,6 +1015,11 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
return computeCatalogRelation(schemaScan);
}
+ @Override
+ public Statistics visitPhysicalRecursiveCteScan(PhysicalRecursiveCteScan recursiveCteScan, Void context) {
+ return computeCatalogRelation(recursiveCteScan);
+ }
+
@Override
public Statistics visitPhysicalFileScan(PhysicalFileScan fileScan, Void context) {
return computeCatalogRelation(fileScan);
@@ -1083,6 +1106,12 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
return computeAssertNumRows(assertNumRows.getAssertNumRowsElement(), groupExpression.childStatistics(0));
}
+ @Override
+ public Statistics visitPhysicalRecursiveCte(PhysicalRecursiveCte recursiveCte, Void context) {
+ return computeUnion(recursiveCte, groupExpression.children()
+ .stream().map(Group::getStatistics).collect(Collectors.toList()));
+ }
+
@Override
public Statistics visitPhysicalUnion(PhysicalUnion union, Void context) {
return computeUnion(union, groupExpression.children()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
index e85c6eb8dae..c9cf877f709 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
@@ -53,6 +53,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
@@ -354,6 +355,28 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
constantExprsList, union.hasPushedFilter(), children);
}
+ @Override
+ public Plan visitLogicalRecursiveCte(LogicalRecursiveCte recursiveCte, DeepCopierContext context) {
+ List<Plan> children = recursiveCte.children().stream()
+ .map(c -> c.accept(this, context))
+ .collect(ImmutableList.toImmutableList());
+ List<List<NamedExpression>> constantExprsList = recursiveCte.getConstantExprsList().stream()
+ .map(l -> l.stream()
+ .map(e -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(e, context))
+ .collect(ImmutableList.toImmutableList()))
+ .collect(ImmutableList.toImmutableList());
+ List<NamedExpression> outputs = recursiveCte.getOutputs().stream()
+ .map(o -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(o, context))
+ .collect(ImmutableList.toImmutableList());
+ List<List<SlotReference>> childrenOutputs = recursiveCte.getRegularChildrenOutputs().stream()
+ .map(childOutputs -> childOutputs.stream()
+ .map(o -> (SlotReference) ExpressionDeepCopier.INSTANCE.deepCopy(o, context))
+ .collect(ImmutableList.toImmutableList()))
+ .collect(ImmutableList.toImmutableList());
+ return new LogicalRecursiveCte(recursiveCte.getQualifier(), outputs, childrenOutputs,
+ constantExprsList, recursiveCte.hasPushedFilter(), children);
+ }
+
@Override
public Plan visitLogicalExcept(LogicalExcept except, DeepCopierContext context) {
List<Plan> children = except.children().stream()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 4274e8ca5b9..fe7661dbc4d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -83,6 +83,8 @@ public enum PlanType {
LOGICAL_PARTITION_TOP_N,
LOGICAL_PROJECT,
LOGICAL_QUALIFY,
+ LOGICAL_RECURSIVE_CTE,
+ LOGICAL_RECURSIVE_CTE_SCAN,
LOGICAL_REPEAT,
LOGICAL_SELECT_HINT,
LOGICAL_SUBQUERY_ALIAS,
@@ -107,6 +109,7 @@ public enum PlanType {
PHYSICAL_OLAP_SCAN,
PHYSICAL_SCHEMA_SCAN,
PHYSICAL_TVF_RELATION,
+ PHYSICAL_RECURSIVE_CTE_SCAN,
// physical sinks
PHYSICAL_FILE_SINK,
@@ -123,6 +126,7 @@ public enum PlanType {
PHYSICAL_ASSERT_NUM_ROWS,
PHYSICAL_CTE_PRODUCER,
PHYSICAL_CTE_ANCHOR,
+ PHYSICAL_RECURSIVE_CTE,
PHYSICAL_DISTRIBUTE,
PHYSICAL_EXCEPT,
PHYSICAL_FILTER,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java
index a7f99e101ae..e8387d675dc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMaterializedViewCommand.java
@@ -326,7 +326,7 @@ public class CreateMaterializedViewCommand extends Command implements ForwardWit
}
try {
Expr defineExpr = translateToLegacyExpr(predicate, context.planTranslatorContext);
- context.filterItem = new MVColumnItem(defineExpr.toSqlWithoutTbl(), defineExpr);
+ context.filterItem = new MVColumnItem(predicate.toSql(), defineExpr);
} catch (Exception ex) {
throw new AnalysisException(ex.getMessage());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java
index 9369acd8d6f..4291ba32732 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java
@@ -17,19 +17,33 @@
package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+import org.apache.doris.analysis.Expr;
import org.apache.doris.nereids.trees.plans.distribute.DistributeContext;
import org.apache.doris.nereids.trees.plans.distribute.worker.BackendDistributedPlanWorkerManager;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.PlanFragment;
import org.apache.doris.planner.PlanFragmentId;
+import org.apache.doris.planner.RecursiveCteNode;
+import org.apache.doris.planner.RecursiveCteScanNode;
import org.apache.doris.thrift.TExplainLevel;
+import org.apache.doris.thrift.TExpr;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TRecCTENode;
+import org.apache.doris.thrift.TRecCTEResetInfo;
+import org.apache.doris.thrift.TRecCTETarget;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ListMultimap;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
/** AssignedJobBuilder */
public class AssignedJobBuilder {
@@ -39,6 +53,8 @@ public class AssignedJobBuilder {
boolean isLoadJob) {
DistributeContext distributeContext = new DistributeContext(workerManager, isLoadJob);
ListMultimap<PlanFragmentId, AssignedJob> allAssignedJobs = ArrayListMultimap.create();
+ Map<PlanFragmentId, TRecCTETarget> fragmentIdToRecCteTargetMap = new TreeMap<>();
+ Map<PlanFragmentId, Set<TNetworkAddress>> fragmentIdToNetworkAddressMap = new TreeMap<>();
for (Entry<PlanFragmentId, UnassignedJob> kv : unassignedJobs.entrySet()) {
PlanFragmentId fragmentId = kv.getKey();
UnassignedJob unassignedJob = kv.getValue();
@@ -55,6 +71,83 @@ public class AssignedJobBuilder {
+ ", fragment: " + unassignedJob.getFragment().getExplainString(TExplainLevel.VERBOSE));
}
allAssignedJobs.putAll(fragmentId, fragmentAssignedJobs);
+
+ Set<TNetworkAddress> networkAddresses = new TreeSet<>();
+ for (AssignedJob assignedJob : fragmentAssignedJobs) {
+ DistributedPlanWorker distributedPlanWorker = assignedJob.getAssignedWorker();
+ networkAddresses.add(new TNetworkAddress(distributedPlanWorker.host(), distributedPlanWorker.port()));
+ }
+ fragmentIdToNetworkAddressMap.put(fragmentId, networkAddresses);
+
+ PlanFragment planFragment = unassignedJob.getFragment();
+ List<RecursiveCteScanNode> recursiveCteScanNodes = planFragment.getPlanRoot()
+ .collectInCurrentFragment(RecursiveCteScanNode.class::isInstance);
+ if (!recursiveCteScanNodes.isEmpty()) {
+ if (recursiveCteScanNodes.size() != 1) {
+ throw new IllegalStateException(
+ String.format("one fragment can only have 1 recursive cte scan node, but there is %d",
+ recursiveCteScanNodes.size()));
+ }
+ if (fragmentAssignedJobs.size() != 1) {
+ throw new IllegalStateException(String.format(
+ "fragmentAssignedJobs's size must be 1 for recursive cte scan node, but it is %d",
+ fragmentAssignedJobs.size()));
+ }
+ TRecCTETarget tRecCTETarget = new TRecCTETarget();
+ DistributedPlanWorker distributedPlanWorker = fragmentAssignedJobs.get(0).getAssignedWorker();
+ tRecCTETarget.setAddr(new TNetworkAddress(distributedPlanWorker.host(), distributedPlanWorker.port()));
+ tRecCTETarget.setFragmentInstanceId(fragmentAssignedJobs.get(0).instanceId());
+ tRecCTETarget.setNodeId(recursiveCteScanNodes.get(0).getId().asInt());
+ fragmentIdToRecCteTargetMap.put(fragmentId, tRecCTETarget);
+ }
+
+ List<RecursiveCteNode> recursiveCteNodes = planFragment.getPlanRoot()
+ .collectInCurrentFragment(RecursiveCteNode.class::isInstance);
+ if (!recursiveCteNodes.isEmpty()) {
+ if (recursiveCteNodes.size() != 1) {
+ throw new IllegalStateException(
+ String.format("one fragment can only have 1 recursive cte node, but there is %d",
+ recursiveCteNodes.size()));
+ }
+
+ List<TRecCTETarget> targets = new ArrayList<>();
+ List<TRecCTEResetInfo> fragmentsToReset = new ArrayList<>();
+ // PhysicalPlanTranslator will swap recursiveCteNodes's child fragment,
+ // so we get recursive one by 1st child
+ List<PlanFragment> childFragments = new ArrayList<>();
+ planFragment.getChild(0).collectAll(PlanFragment.class::isInstance, childFragments);
+ for (PlanFragment child : childFragments) {
+ PlanFragmentId childFragmentId = child.getFragmentId();
+ TRecCTETarget tRecCTETarget = fragmentIdToRecCteTargetMap.getOrDefault(childFragmentId, null);
+ if (tRecCTETarget != null) {
+ targets.add(tRecCTETarget);
+ }
+ Set<TNetworkAddress> tNetworkAddresses = fragmentIdToNetworkAddressMap.get(childFragmentId);
+ if (tNetworkAddresses == null) {
+ throw new IllegalStateException(
+ String.format("can't find TNetworkAddress for fragment %d", childFragmentId));
+ }
+ for (TNetworkAddress address : tNetworkAddresses) {
+ TRecCTEResetInfo tRecCTEResetInfo = new TRecCTEResetInfo();
+ tRecCTEResetInfo.setFragmentId(childFragmentId.asInt());
+ tRecCTEResetInfo.setAddr(address);
+ fragmentsToReset.add(tRecCTEResetInfo);
+ }
+ }
+
+ RecursiveCteNode recursiveCteNode = recursiveCteNodes.get(0);
+ List<List<Expr>> materializedResultExprLists = recursiveCteNode.getMaterializedResultExprLists();
+ List<List<TExpr>> texprLists = new ArrayList<>(materializedResultExprLists.size());
+ for (List<Expr> exprList : materializedResultExprLists) {
+ texprLists.add(Expr.treesToThrift(exprList));
+ }
+ TRecCTENode tRecCTENode = new TRecCTENode();
+ tRecCTENode.setIsUnionAll(recursiveCteNode.isUnionAll());
+ tRecCTENode.setTargets(targets);
+ tRecCTENode.setFragmentsToReset(fragmentsToReset);
+ tRecCTENode.setResultExprLists(texprLists);
+ recursiveCteNode.settRecCTENode(tRecCTENode);
+ }
}
return allAssignedJobs;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java
index bc20d3efa17..1229bc59455 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java
@@ -29,6 +29,7 @@ import org.apache.doris.planner.OlapScanNode;
import org.apache.doris.planner.PlanFragment;
import org.apache.doris.planner.PlanFragmentId;
import org.apache.doris.planner.PlanNodeId;
+import org.apache.doris.planner.RecursiveCteScanNode;
import org.apache.doris.planner.ScanNode;
import org.apache.doris.planner.SchemaScanNode;
import org.apache.doris.thrift.TExplainLevel;
@@ -126,6 +127,10 @@ public class UnassignedJobBuilder {
unassignedJob = buildScanMetadataJob(
statementContext, planFragment, (SchemaScanNode) scanNode, scanWorkerSelector
);
+ } else if (scanNode instanceof RecursiveCteScanNode) {
+ unassignedJob = buildScanRecursiveCteJob(
+ statementContext, planFragment, (RecursiveCteScanNode) scanNode, inputJobs, scanWorkerSelector
+ );
} else {
// only scan external tables or cloud tables or table valued functions
// e,g. select * from numbers('number'='100')
@@ -196,6 +201,14 @@ public class UnassignedJobBuilder {
return new UnassignedScanMetadataJob(statementContext, fragment, schemaScanNode, scanWorkerSelector);
}
+ private UnassignedJob buildScanRecursiveCteJob(
+ StatementContext statementContext, PlanFragment fragment,
+ RecursiveCteScanNode recursiveCteScanNode,
+ ListMultimap<ExchangeNode, UnassignedJob> inputJobs, ScanWorkerSelector scanWorkerSelector) {
+ return new UnassignedRecursiveCteScanJob(statementContext, fragment, recursiveCteScanNode,
+ inputJobs, scanWorkerSelector);
+ }
+
private UnassignedJob buildScanRemoteTableJob(
StatementContext statementContext, PlanFragment planFragment, List<ScanNode> scanNodes,
ListMultimap<ExchangeNode, UnassignedJob> inputJobs,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedRecursiveCteScanJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedRecursiveCteScanJob.java
new file mode 100644
index 00000000000..7fbb3f88e21
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedRecursiveCteScanJob.java
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.trees.plans.distribute.DistributeContext;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager;
+import org.apache.doris.nereids.trees.plans.distribute.worker.ScanWorkerSelector;
+import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.ScanNode;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ListMultimap;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * UnassignedRecursiveCteScanJob
+ */
+public class UnassignedRecursiveCteScanJob extends AbstractUnassignedScanJob {
+ private final ScanWorkerSelector scanWorkerSelector;
+
+ public UnassignedRecursiveCteScanJob(
+ StatementContext statementContext, PlanFragment fragment, ScanNode scanNode,
+ ListMultimap<ExchangeNode, UnassignedJob> exchangeToChildJob, ScanWorkerSelector scanWorkerSelector) {
+ super(statementContext, fragment, ImmutableList.of(scanNode), exchangeToChildJob);
+ this.scanWorkerSelector = Objects.requireNonNull(scanWorkerSelector, "scanWorkerSelector is not null");
+ }
+
+ @Override
+ protected Map<DistributedPlanWorker, UninstancedScanSource> multipleMachinesParallelization(
+ DistributeContext distributeContext, ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+ return scanWorkerSelector.selectReplicaAndWorkerWithoutBucket(
+ scanNodes.get(0), statementContext.getConnectContext()
+ );
+ }
+
+ @Override
+ protected List<AssignedJob> fillUpAssignedJobs(List<AssignedJob> assignedJobs,
+ DistributedPlanWorkerManager workerManager, ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+ return fillUpSingleEmptyInstance(workerManager);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java
new file mode 100644
index 00000000000..cfdf317c620
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java
@@ -0,0 +1,419 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.algebra.Union;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.nereids.util.TypeCoercionUtils;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * LogicalRecursiveCte is basically like LogicalUnion
+ */
+public class LogicalRecursiveCte extends LogicalSetOperation implements Union, OutputPrunable {
+
+ // in doris, we use union node to present one row relation
+ private final List<List<NamedExpression>> constantExprsList;
+ // When there is an agg on the union and there is a filter on the agg,
+ // it is necessary to keep the filter on the agg and push the filter down to each child of the union.
+ private final boolean hasPushedFilter;
+
+ /** LogicalRecursiveCte */
+ public LogicalRecursiveCte(Qualifier qualifier, List<Plan> children) {
+ this(qualifier, ImmutableList.of(), children);
+ }
+
+ /** LogicalRecursiveCte */
+ public LogicalRecursiveCte(Qualifier qualifier, List<List<NamedExpression>> constantExprsList,
+ List<Plan> children) {
+ this(qualifier, ImmutableList.of(), ImmutableList.of(), constantExprsList, false, children);
+ }
+
+ /** LogicalRecursiveCte */
+ public LogicalRecursiveCte(Qualifier qualifier, List<NamedExpression> outputs,
+ List<List<SlotReference>> childrenOutputs,
+ List<List<NamedExpression>> constantExprsList, boolean hasPushedFilter, List<Plan> children) {
+ this(qualifier, outputs, childrenOutputs, constantExprsList, hasPushedFilter, Optional.empty(),
+ Optional.empty(),
+ children);
+ }
+
+ /** LogicalRecursiveCte */
+ public LogicalRecursiveCte(Qualifier qualifier, List<NamedExpression> outputs,
+ List<List<SlotReference>> childrenOutputs,
+ List<List<NamedExpression>> constantExprsList, boolean hasPushedFilter,
+ Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
+ List<Plan> children) {
+ super(PlanType.LOGICAL_RECURSIVE_CTE, qualifier, outputs, childrenOutputs,
+ groupExpression, logicalProperties, children);
+ this.hasPushedFilter = hasPushedFilter;
+ this.constantExprsList = Utils.fastToImmutableList(
+ Objects.requireNonNull(constantExprsList, "constantExprsList should not be null"));
+ }
+
+ public boolean hasPushedFilter() {
+ return hasPushedFilter;
+ }
+
+ public List<List<NamedExpression>> getConstantExprsList() {
+ return constantExprsList;
+ }
+
+ @Override
+ public List<? extends Expression> getExpressions() {
+ return constantExprsList.stream().flatMap(List::stream).collect(ImmutableList.toImmutableList());
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlStringSkipNull("LogicalRecursiveCte",
+ "qualifier", qualifier,
+ "outputs", outputs,
+ "regularChildrenOutputs", regularChildrenOutputs,
+ "constantExprsList", constantExprsList,
+ "hasPushedFilter", hasPushedFilter,
+ "stats", statistics);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ LogicalRecursiveCte that = (LogicalRecursiveCte) o;
+ return super.equals(that) && hasPushedFilter == that.hasPushedFilter
+ && Objects.equals(constantExprsList, that.constantExprsList);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), hasPushedFilter, constantExprsList);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitLogicalRecursiveCte(this, context);
+ }
+
+ @Override
+ public LogicalRecursiveCte withChildren(List<Plan> children) {
+ return new LogicalRecursiveCte(qualifier, outputs, regularChildrenOutputs,
+ constantExprsList, hasPushedFilter, children);
+ }
+
+ @Override
+ public LogicalSetOperation withChildrenAndTheirOutputs(List<Plan> children,
+ List<List<SlotReference>> childrenOutputs) {
+ Preconditions.checkArgument(children.size() == childrenOutputs.size(),
+ "children size %s is not equals with children outputs size %s",
+ children.size(), childrenOutputs.size());
+ return new LogicalRecursiveCte(qualifier, outputs, childrenOutputs, constantExprsList, hasPushedFilter,
+ children);
+ }
+
+ @Override
+ public LogicalRecursiveCte withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new LogicalRecursiveCte(qualifier, outputs, regularChildrenOutputs, constantExprsList, hasPushedFilter,
+ groupExpression, Optional.of(getLogicalProperties()), children);
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ return new LogicalRecursiveCte(qualifier, outputs, regularChildrenOutputs, constantExprsList, hasPushedFilter,
+ groupExpression, logicalProperties, children);
+ }
+
+ @Override
+ public LogicalRecursiveCte withNewOutputs(List<NamedExpression> newOutputs) {
+ return new LogicalRecursiveCte(qualifier, newOutputs, regularChildrenOutputs, constantExprsList,
+ hasPushedFilter, Optional.empty(), Optional.empty(), children);
+ }
+
+ public LogicalRecursiveCte withNewOutputsAndConstExprsList(List<NamedExpression> newOutputs,
+ List<List<NamedExpression>> constantExprsList) {
+ return new LogicalRecursiveCte(qualifier, newOutputs, regularChildrenOutputs, constantExprsList,
+ hasPushedFilter, Optional.empty(), Optional.empty(), children);
+ }
+
+ public LogicalRecursiveCte withChildrenAndConstExprsList(List<Plan> children,
+ List<List<SlotReference>> childrenOutputs, List<List<NamedExpression>> constantExprsList) {
+ return new LogicalRecursiveCte(qualifier, outputs, childrenOutputs, constantExprsList, hasPushedFilter,
+ children);
+ }
+
+ public LogicalRecursiveCte withNewOutputsChildrenAndConstExprsList(List<NamedExpression> newOutputs,
+ List<Plan> children,
+ List<List<SlotReference>> childrenOutputs,
+ List<List<NamedExpression>> constantExprsList) {
+ return new LogicalRecursiveCte(qualifier, newOutputs, childrenOutputs, constantExprsList,
+ hasPushedFilter, Optional.empty(), Optional.empty(), children);
+ }
+
+ public LogicalRecursiveCte withAllQualifier() {
+ return new LogicalRecursiveCte(Qualifier.ALL, outputs, regularChildrenOutputs, constantExprsList,
+ hasPushedFilter,
+ Optional.empty(), Optional.empty(), children);
+ }
+
+ @Override
+ public LogicalRecursiveCte pruneOutputs(List<NamedExpression> prunedOutputs) {
+ return withNewOutputs(prunedOutputs);
+ }
+
+ @Override
+ public void computeUnique(DataTrait.Builder builder) {
+ if (qualifier == Qualifier.DISTINCT) {
+ builder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
+ }
+ }
+
+ @Override
+ public void computeUniform(DataTrait.Builder builder) {
+ final Optional<ExpressionRewriteContext> context = ConnectContext.get() == null ? Optional.empty()
+ : Optional.of(new ExpressionRewriteContext(CascadesContext.initContext(
+ ConnectContext.get().getStatementContext(), this, PhysicalProperties.ANY)));
+ for (int i = 0; i < getOutputs().size(); i++) {
+ Optional<Literal> value = Optional.empty();
+ if (!constantExprsList.isEmpty()) {
+ value = ExpressionUtils.checkConstantExpr(constantExprsList.get(0).get(i), context);
+ if (!value.isPresent()) {
+ continue;
+ }
+ final int fi = i;
+ Literal literal = value.get();
+ if (constantExprsList.stream()
+ .map(exprs -> ExpressionUtils.checkConstantExpr(exprs.get(fi), context))
+ .anyMatch(val -> !val.isPresent() || !val.get().equals(literal))) {
+ continue;
+ }
+ }
+ for (int childIdx = 0; childIdx < children.size(); childIdx++) {
+ // TODO: use originOutputs = child(childIdx).getOutput() ?
+ List<? extends Slot> originOutputs = regularChildrenOutputs.get(childIdx);
+ Slot slot = originOutputs.get(i);
+ Optional<Expression> childValue = child(childIdx).getLogicalProperties()
+ .getTrait().getUniformValue(slot);
+ if (childValue == null || !childValue.isPresent() || !childValue.get().isConstant()) {
+ value = Optional.empty();
+ break;
+ }
+ Optional<Literal> constExprOpt = ExpressionUtils.checkConstantExpr(childValue.get(), context);
+ if (!constExprOpt.isPresent()) {
+ value = Optional.empty();
+ break;
+ }
+ if (!value.isPresent()) {
+ value = constExprOpt;
+ } else if (!value.equals(constExprOpt)) {
+ value = Optional.empty();
+ break;
+ }
+ }
+ if (value.isPresent()) {
+ builder.addUniformSlotAndLiteral(getOutputs().get(i).toSlot(), value.get());
+ }
+ }
+ }
+
+ @Override
+ public boolean hasUnboundExpression() {
+ if (!constantExprsList.isEmpty() && children.isEmpty()) {
+ return false;
+ }
+ return super.hasUnboundExpression();
+ }
+
+ private List<BitSet> mapSlotToIndex(Plan plan, List<Set<Slot>> equalSlotsList) {
+ Map<Slot, Integer> slotToIndex = new HashMap<>();
+ for (int i = 0; i < plan.getOutput().size(); i++) {
+ slotToIndex.put(plan.getOutput().get(i), i);
+ }
+ List<BitSet> equalSlotIndicesList = new ArrayList<>();
+ for (Set<Slot> equalSlots : equalSlotsList) {
+ BitSet equalSlotIndices = new BitSet();
+ for (Slot slot : equalSlots) {
+ if (slotToIndex.containsKey(slot)) {
+ equalSlotIndices.set(slotToIndex.get(slot));
+ }
+ }
+ if (equalSlotIndices.cardinality() > 1) {
+ equalSlotIndicesList.add(equalSlotIndices);
+ }
+ }
+ return equalSlotIndicesList;
+ }
+
+ @Override
+ public void computeEqualSet(DataTrait.Builder builder) {
+ if (children.isEmpty()) {
+ return;
+ }
+
+ // Get the list of equal slot sets and their corresponding index mappings for the first child
+ List<Set<Slot>> childEqualSlotsList = child(0).getLogicalProperties()
+ .getTrait().calAllEqualSet();
+ List<BitSet> childEqualSlotsIndicesList = mapSlotToIndex(child(0), childEqualSlotsList);
+ List<BitSet> unionEqualSlotIndicesList = new ArrayList<>(childEqualSlotsIndicesList);
+
+ // Traverse all children and find the equal sets that exist in all children
+ for (int i = 1; i < children.size(); i++) {
+ Plan child = children.get(i);
+
+ // Get the equal slot sets for the current child
+ childEqualSlotsList = child.getLogicalProperties().getTrait().calAllEqualSet();
+
+ // Map slots to indices for the current child
+ childEqualSlotsIndicesList = mapSlotToIndex(child, childEqualSlotsList);
+
+ // Only keep the equal pairs that exist in all children of the union
+ // This is done by calculating the intersection of all children's equal slot indices
+ for (BitSet unionEqualSlotIndices : unionEqualSlotIndicesList) {
+ BitSet intersect = new BitSet();
+ for (BitSet childEqualSlotIndices : childEqualSlotsIndicesList) {
+ if (unionEqualSlotIndices.intersects(childEqualSlotIndices)) {
+ intersect = childEqualSlotIndices;
+ break;
+ }
+ }
+ unionEqualSlotIndices.and(intersect);
+ }
+ }
+
+ // Build the functional dependencies for the output slots
+ List<Slot> outputList = getOutput();
+ for (BitSet equalSlotIndices : unionEqualSlotIndicesList) {
+ if (equalSlotIndices.cardinality() <= 1) {
+ continue;
+ }
+ int first = equalSlotIndices.nextSetBit(0);
+ int next = equalSlotIndices.nextSetBit(first + 1);
+ while (next > 0) {
+ builder.addEqualPair(outputList.get(first), outputList.get(next));
+ next = equalSlotIndices.nextSetBit(next + 1);
+ }
+ }
+ }
+
+ @Override
+ public void computeFd(DataTrait.Builder builder) {
+ // don't generate
+ }
+
+ /** castCommonDataTypeAndNullableByConstants */
+ public static Pair<List<List<NamedExpression>>, List<Boolean>> castCommonDataTypeAndNullableByConstants(
+ List<List<NamedExpression>> constantExprsList) {
+ int columnCount = constantExprsList.isEmpty() ? 0 : constantExprsList.get(0).size();
+ Pair<List<DataType>, List<Boolean>> commonInfo = computeCommonDataTypeAndNullable(constantExprsList,
+ columnCount);
+ List<List<NamedExpression>> castedRows = castToCommonType(constantExprsList, commonInfo.key(), columnCount);
+ List<Boolean> nullables = commonInfo.second;
+ return Pair.of(castedRows, nullables);
+ }
+
+ private static Pair<List<DataType>, List<Boolean>> computeCommonDataTypeAndNullable(
+ List<List<NamedExpression>> constantExprsList, int columnCount) {
+ List<Boolean> nullables = Lists.newArrayListWithCapacity(columnCount);
+ List<DataType> commonDataTypes = Lists.newArrayListWithCapacity(columnCount);
+ List<NamedExpression> firstRow = constantExprsList.get(0);
+ for (int columnId = 0; columnId < columnCount; columnId++) {
+ Expression constant = firstRow.get(columnId).child(0);
+ Pair<DataType, Boolean> commonDataTypeAndNullable = computeCommonDataTypeAndNullable(constant, columnId,
+ constantExprsList);
+ commonDataTypes.add(commonDataTypeAndNullable.first);
+ nullables.add(commonDataTypeAndNullable.second);
+ }
+ return Pair.of(commonDataTypes, nullables);
+ }
+
+ private static Pair<DataType, Boolean> computeCommonDataTypeAndNullable(
+ Expression firstRowExpr, int columnId, List<List<NamedExpression>> constantExprsList) {
+ DataType commonDataType = firstRowExpr.getDataType();
+ boolean nullable = firstRowExpr.nullable();
+ for (int rowId = 1; rowId < constantExprsList.size(); rowId++) {
+ NamedExpression namedExpression = constantExprsList.get(rowId).get(columnId);
+ Expression otherConstant = namedExpression.child(0);
+ nullable |= otherConstant.nullable();
+ DataType otherDataType = otherConstant.getDataType();
+ commonDataType = getAssignmentCompatibleType(commonDataType, otherDataType);
+ }
+ return Pair.of(commonDataType, nullable);
+ }
+
+ private static List<List<NamedExpression>> castToCommonType(
+ List<List<NamedExpression>> rows, List<DataType> commonDataTypes, int columnCount) {
+ ImmutableList.Builder<List<NamedExpression>> castedConstants = ImmutableList
+ .builderWithExpectedSize(rows.size());
+ for (List<NamedExpression> row : rows) {
+ castedConstants.add(castToCommonType(row, commonDataTypes));
+ }
+ return castedConstants.build();
+ }
+
+ private static List<NamedExpression> castToCommonType(List<NamedExpression> row, List<DataType> commonTypes) {
+ ImmutableList.Builder<NamedExpression> castedRow = ImmutableList.builderWithExpectedSize(row.size());
+ boolean changed = false;
+ for (int columnId = 0; columnId < row.size(); columnId++) {
+ NamedExpression constantAlias = row.get(columnId);
+ Expression constant = constantAlias.child(0);
+ DataType commonType = commonTypes.get(columnId);
+ if (commonType.equals(constant.getDataType())) {
+ castedRow.add(constantAlias);
+ } else {
+ changed = true;
+ Expression expression = TypeCoercionUtils.castIfNotSameTypeStrict(constant, commonType);
+ castedRow.add((NamedExpression) constantAlias.withChildren(expression));
+ }
+ }
+ return changed ? castedRow.build() : row;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteScan.java
new file mode 100644
index 00000000000..7b1fc4862ee
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteScan.java
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * LogicalRecursiveCteScan.
+ */
+public class LogicalRecursiveCteScan extends LogicalCatalogRelation {
+ public LogicalRecursiveCteScan(RelationId relationId, TableIf table, List<String> qualifier) {
+ this(relationId, table, qualifier, Optional.empty(), Optional.empty());
+ }
+
+ private LogicalRecursiveCteScan(RelationId relationId, TableIf table, List<String> qualifier,
+ Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties) {
+ super(relationId, PlanType.LOGICAL_RECURSIVE_CTE_SCAN, table, qualifier, groupExpression, logicalProperties);
+ }
+
+ @Override
+ public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new LogicalRecursiveCteScan(relationId, table, qualifier,
+ groupExpression, Optional.ofNullable(getLogicalProperties()));
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ return new LogicalRecursiveCteScan(relationId, table, qualifier, groupExpression, logicalProperties);
+ }
+
+ @Override
+ public LogicalCatalogRelation withRelationId(RelationId relationId) {
+ return new LogicalRecursiveCteScan(relationId, table, qualifier,
+ groupExpression, Optional.ofNullable(getLogicalProperties()));
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitLogicalRecursiveCteScan(this, context);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
index 2f49dc1ce85..f7f8d718e07 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
@@ -56,28 +56,36 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
private final List<String> qualifier;
private final Optional<List<String>> columnAliases;
+ private final boolean isRecursiveCte;
+
public LogicalSubQueryAlias(String tableAlias, CHILD_TYPE child) {
- this(ImmutableList.of(tableAlias), Optional.empty(), Optional.empty(), Optional.empty(), child);
+ this(ImmutableList.of(tableAlias), Optional.empty(), false, Optional.empty(), Optional.empty(), child);
}
public LogicalSubQueryAlias(List<String> qualifier, CHILD_TYPE child) {
- this(qualifier, Optional.empty(), Optional.empty(), Optional.empty(), child);
+ this(qualifier, Optional.empty(), false, Optional.empty(), Optional.empty(), child);
}
public LogicalSubQueryAlias(String tableAlias, Optional<List<String>> columnAliases, CHILD_TYPE child) {
- this(ImmutableList.of(tableAlias), columnAliases, Optional.empty(), Optional.empty(), child);
+ this(ImmutableList.of(tableAlias), columnAliases, false, Optional.empty(), Optional.empty(), child);
+ }
+
+ public LogicalSubQueryAlias(String tableAlias, Optional<List<String>> columnAliases, boolean isRecursiveCte,
+ CHILD_TYPE child) {
+ this(ImmutableList.of(tableAlias), columnAliases, isRecursiveCte, Optional.empty(), Optional.empty(), child);
}
public LogicalSubQueryAlias(List<String> qualifier, Optional<List<String>> columnAliases, CHILD_TYPE child) {
- this(qualifier, columnAliases, Optional.empty(), Optional.empty(), child);
+ this(qualifier, columnAliases, false, Optional.empty(), Optional.empty(), child);
}
- public LogicalSubQueryAlias(List<String> qualifier, Optional<List<String>> columnAliases,
+ public LogicalSubQueryAlias(List<String> qualifier, Optional<List<String>> columnAliases, boolean isRecursiveCte,
Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, CHILD_TYPE child) {
super(PlanType.LOGICAL_SUBQUERY_ALIAS, groupExpression, logicalProperties, child);
this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier is null"));
this.columnAliases = columnAliases;
+ this.isRecursiveCte = isRecursiveCte;
}
@Override
@@ -129,10 +137,14 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
return columnAliases;
}
+ public boolean isRecursiveCte() {
+ return isRecursiveCte;
+ }
+
@Override
public String toString() {
return columnAliases.map(strings -> Utils.toSqlString("LogicalSubQueryAlias",
- "qualifier", qualifier,
+ "qualifier", qualifier, "isRecursiveCte", isRecursiveCte,
"columnAliases", StringUtils.join(strings, ",")
)).orElseGet(() -> Utils.toSqlString("LogicalSubQueryAlias",
"qualifier", qualifier
@@ -171,7 +183,8 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
@Override
public LogicalSubQueryAlias<Plan> withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
- return new LogicalSubQueryAlias<>(qualifier, columnAliases, children.get(0));
+ return new LogicalSubQueryAlias<>(qualifier, columnAliases, isRecursiveCte, Optional.empty(), Optional.empty(),
+ children.get(0));
}
@Override
@@ -186,7 +199,7 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
@Override
public LogicalSubQueryAlias<CHILD_TYPE> withGroupExpression(Optional<GroupExpression> groupExpression) {
- return new LogicalSubQueryAlias<>(qualifier, columnAliases, groupExpression,
+ return new LogicalSubQueryAlias<>(qualifier, columnAliases, isRecursiveCte, groupExpression,
Optional.of(getLogicalProperties()), child());
}
@@ -194,7 +207,7 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
- return new LogicalSubQueryAlias<>(qualifier, columnAliases, groupExpression, logicalProperties,
+ return new LogicalSubQueryAlias<>(qualifier, columnAliases, isRecursiveCte, groupExpression, logicalProperties,
children.get(0));
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCte.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCte.java
new file mode 100644
index 00000000000..ef0cc98de0a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCte.java
@@ -0,0 +1,251 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.physical;
+
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.algebra.Union;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.statistics.Statistics;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * PhysicalRecursiveCte is basically like PhysicalUnion
+ */
+public class PhysicalRecursiveCte extends PhysicalSetOperation implements Union {
+
+ // in doris, we use union node to present one row relation
+ private final List<List<NamedExpression>> constantExprsList;
+
+ /** PhysicalRecursiveCte */
+ public PhysicalRecursiveCte(Qualifier qualifier,
+ List<NamedExpression> outputs,
+ List<List<SlotReference>> childrenOutputs,
+ List<List<NamedExpression>> constantExprsList,
+ LogicalProperties logicalProperties,
+ List<Plan> children) {
+ super(PlanType.PHYSICAL_RECURSIVE_CTE, qualifier, outputs, childrenOutputs, logicalProperties, children);
+ this.constantExprsList = ImmutableList.copyOf(
+ Objects.requireNonNull(constantExprsList, "constantExprsList should not be null"));
+ }
+
+ /** PhysicalRecursiveCte */
+ public PhysicalRecursiveCte(Qualifier qualifier,
+ List<NamedExpression> outputs,
+ List<List<SlotReference>> childrenOutputs,
+ List<List<NamedExpression>> constantExprsList,
+ Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties,
+ List<Plan> children) {
+ super(PlanType.PHYSICAL_RECURSIVE_CTE, qualifier, outputs, childrenOutputs,
+ groupExpression, logicalProperties, children);
+ this.constantExprsList = ImmutableList.copyOf(
+ Objects.requireNonNull(constantExprsList, "constantExprsList should not be null"));
+ }
+
+ /** PhysicalRecursiveCte */
+ public PhysicalRecursiveCte(Qualifier qualifier, List<NamedExpression> outputs,
+ List<List<SlotReference>> childrenOutputs, List<List<NamedExpression>> constantExprsList,
+ Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties,
+ PhysicalProperties physicalProperties, Statistics statistics, List<Plan> inputs) {
+ super(PlanType.PHYSICAL_RECURSIVE_CTE, qualifier, outputs, childrenOutputs,
+ groupExpression, logicalProperties, physicalProperties, statistics, inputs);
+ this.constantExprsList = ImmutableList.copyOf(
+ Objects.requireNonNull(constantExprsList, "constantExprsList should not be null"));
+ }
+
+ public List<List<NamedExpression>> getConstantExprsList() {
+ return constantExprsList;
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitPhysicalRecursiveCte(this, context);
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlString("PhysicalRecursiveCte" + "[" + id.asInt() + "]" + getGroupIdWithPrefix(),
+ "stats", statistics,
+ "qualifier", qualifier,
+ "outputs", outputs,
+ "regularChildrenOutputs", regularChildrenOutputs,
+ "constantExprsList", constantExprsList);
+ }
+
+ @Override
+ public String shapeInfo() {
+ ConnectContext context = ConnectContext.get();
+ if (context != null
+ && context.getSessionVariable().getDetailShapePlanNodesSet().contains(getClass().getSimpleName())) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(getClass().getSimpleName());
+ builder.append("(constantExprsList=");
+ builder.append(constantExprsList.stream()
+ .map(exprs -> exprs.stream().map(Expression::shapeInfo)
+ .collect(Collectors.joining(", ", "[", "]")))
+ .collect(Collectors.joining(", ", "[", "]")));
+ builder.append(")");
+ return builder.toString();
+ } else {
+ return super.shapeInfo();
+ }
+ }
+
+ @Override
+ public PhysicalRecursiveCte withChildren(List<Plan> children) {
+ return new PhysicalRecursiveCte(qualifier, outputs, regularChildrenOutputs, constantExprsList, groupExpression,
+ getLogicalProperties(), children);
+ }
+
+ @Override
+ public PhysicalRecursiveCte withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new PhysicalRecursiveCte(qualifier, outputs, regularChildrenOutputs, constantExprsList,
+ groupExpression, getLogicalProperties(), children);
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ return new PhysicalRecursiveCte(qualifier, outputs, regularChildrenOutputs, constantExprsList,
+ groupExpression, logicalProperties.get(), children);
+ }
+
+ @Override
+ public PhysicalRecursiveCte withPhysicalPropertiesAndStats(
+ PhysicalProperties physicalProperties, Statistics statistics) {
+ return new PhysicalRecursiveCte(qualifier, outputs, regularChildrenOutputs, constantExprsList,
+ groupExpression, getLogicalProperties(), physicalProperties, statistics, children);
+ }
+
+ @Override
+ public PhysicalRecursiveCte resetLogicalProperties() {
+ return new PhysicalRecursiveCte(qualifier, outputs, regularChildrenOutputs, constantExprsList,
+ Optional.empty(), null, physicalProperties, statistics, children);
+ }
+
+ @Override
+ public void computeUnique(DataTrait.Builder builder) {
+ if (qualifier == Qualifier.DISTINCT) {
+ builder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
+ }
+ }
+
+ @Override
+ public void computeUniform(DataTrait.Builder builder) {
+ // don't propagate uniform slots
+ }
+
+ private List<BitSet> mapSlotToIndex(Plan plan, List<Set<Slot>> equalSlotsList) {
+ Map<Slot, Integer> slotToIndex = new HashMap<>();
+ for (int i = 0; i < plan.getOutput().size(); i++) {
+ slotToIndex.put(plan.getOutput().get(i), i);
+ }
+ List<BitSet> equalSlotIndicesList = new ArrayList<>();
+ for (Set<Slot> equalSlots : equalSlotsList) {
+ BitSet equalSlotIndices = new BitSet();
+ for (Slot slot : equalSlots) {
+ if (slotToIndex.containsKey(slot)) {
+ equalSlotIndices.set(slotToIndex.get(slot));
+ }
+ }
+ if (equalSlotIndices.cardinality() > 1) {
+ equalSlotIndicesList.add(equalSlotIndices);
+ }
+ }
+ return equalSlotIndicesList;
+ }
+
+ @Override
+ public void computeEqualSet(DataTrait.Builder builder) {
+ if (children.isEmpty()) {
+ return;
+ }
+
+ // Get the list of equal slot sets and their corresponding index mappings for the first child
+ List<Set<Slot>> childEqualSlotsList = child(0).getLogicalProperties()
+ .getTrait().calAllEqualSet();
+ List<BitSet> childEqualSlotsIndicesList = mapSlotToIndex(child(0), childEqualSlotsList);
+ List<BitSet> unionEqualSlotIndicesList = new ArrayList<>(childEqualSlotsIndicesList);
+
+ // Traverse all children and find the equal sets that exist in all children
+ for (int i = 1; i < children.size(); i++) {
+ Plan child = children.get(i);
+
+ // Get the equal slot sets for the current child
+ childEqualSlotsList = child.getLogicalProperties().getTrait().calAllEqualSet();
+
+ // Map slots to indices for the current child
+ childEqualSlotsIndicesList = mapSlotToIndex(child, childEqualSlotsList);
+
+ // Only keep the equal pairs that exist in all children of the union
+ // This is done by calculating the intersection of all children's equal slot indices
+ for (BitSet unionEqualSlotIndices : unionEqualSlotIndicesList) {
+ BitSet intersect = new BitSet();
+ for (BitSet childEqualSlotIndices : childEqualSlotsIndicesList) {
+ if (unionEqualSlotIndices.intersects(childEqualSlotIndices)) {
+ intersect = childEqualSlotIndices;
+ break;
+ }
+ }
+ unionEqualSlotIndices.and(intersect);
+ }
+ }
+
+ // Build the functional dependencies for the output slots
+ List<Slot> outputList = getOutput();
+ for (BitSet equalSlotIndices : unionEqualSlotIndicesList) {
+ if (equalSlotIndices.cardinality() <= 1) {
+ continue;
+ }
+ int first = equalSlotIndices.nextSetBit(0);
+ int next = equalSlotIndices.nextSetBit(first + 1);
+ while (next > 0) {
+ builder.addEqualPair(outputList.get(first), outputList.get(next));
+ next = equalSlotIndices.nextSetBit(next + 1);
+ }
+ }
+ }
+
+ @Override
+ public void computeFd(DataTrait.Builder builder) {
+ // don't generate
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteScan.java
new file mode 100644
index 00000000000..3450ae0de18
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteScan.java
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.physical;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.statistics.Statistics;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * PhysicalRecursiveCteScan.
+ */
+public class PhysicalRecursiveCteScan extends PhysicalCatalogRelation {
+ public PhysicalRecursiveCteScan(RelationId relationId, TableIf table, List<String> qualifier,
+ Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties,
+ Collection<Slot> operativeSlots) {
+ this(relationId, table, qualifier, groupExpression, logicalProperties, PhysicalProperties.ANY, null,
+ operativeSlots);
+ }
+
+ public PhysicalRecursiveCteScan(RelationId relationId, TableIf table, List<String> qualifier,
+ Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties,
+ PhysicalProperties physicalProperties, Statistics statistics, Collection<Slot> operativeSlots) {
+ super(relationId, PlanType.PHYSICAL_RECURSIVE_CTE_SCAN, table, qualifier, groupExpression, logicalProperties,
+ physicalProperties, statistics, operativeSlots);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitPhysicalRecursiveCteScan(this, context);
+ }
+
+ @Override
+ public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new PhysicalRecursiveCteScan(relationId, table, qualifier, groupExpression, getLogicalProperties(),
+ physicalProperties, statistics, operativeSlots);
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ return new PhysicalRecursiveCteScan(relationId, table, qualifier, groupExpression, getLogicalProperties(),
+ physicalProperties, statistics, operativeSlots);
+ }
+
+ @Override
+ public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) {
+ return new PhysicalRecursiveCteScan(relationId, table, qualifier, groupExpression, getLogicalProperties(),
+ physicalProperties, statistics, operativeSlots);
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlString("PhysicalRecursiveCteScan[" + table.getName() + "]" + getGroupIdWithPrefix(),
+ "stats", statistics,
+ "qualified", Utils.qualifiedName(qualifier, table.getName()),
+ "operativeCols", getOperativeSlots());
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java
index 291f9ed5d80..d34de256608 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java
@@ -44,6 +44,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPreAggOnHint;
import org.apache.doris.nereids.trees.plans.logical.LogicalPreFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalQualify;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint;
@@ -78,6 +79,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
@@ -211,6 +213,10 @@ public abstract class PlanVisitor<R, C> implements CommandVisitor<R, C>, Relatio
return visit(join, context);
}
+ public R visitLogicalRecursiveCte(LogicalRecursiveCte recursiveCte, C context) {
+ return visit(recursiveCte, context);
+ }
+
public R visitLogicalLimit(LogicalLimit<? extends Plan> limit, C context) {
return visit(limit, context);
}
@@ -376,6 +382,10 @@ public abstract class PlanVisitor<R, C> implements CommandVisitor<R, C>, Relatio
return visitPhysicalSetOperation(union, context);
}
+ public R visitPhysicalRecursiveCte(PhysicalRecursiveCte recursiveCte, C context) {
+ return visitPhysicalSetOperation(recursiveCte, context);
+ }
+
public R visitAbstractPhysicalSort(AbstractPhysicalSort<? extends Plan> sort, C context) {
return visit(sort, context);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
index fef94ff52f9..b325849b1b1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
@@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
@@ -45,6 +46,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSchemaScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation;
@@ -139,6 +141,10 @@ public interface RelationVisitor<R, C> {
return visitLogicalCatalogRelation(testScan, context);
}
+ default R visitLogicalRecursiveCteScan(LogicalRecursiveCteScan recursiveCteScan, C context) {
+ return visitLogicalCatalogRelation(recursiveCteScan, context);
+ }
+
// *******************************
// physical relations
// *******************************
@@ -176,6 +182,10 @@ public interface RelationVisitor<R, C> {
return visitPhysicalCatalogRelation(deferMaterializeOlapScan, context);
}
+ default R visitPhysicalRecursiveCteScan(PhysicalRecursiveCteScan recursiveCteScan, C context) {
+ return visitPhysicalCatalogRelation(recursiveCteScan, context);
+ }
+
default R visitPhysicalOneRowRelation(PhysicalOneRowRelation oneRowRelation, C context) {
return visitPhysicalRelation(oneRowRelation, context);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java
new file mode 100644
index 00000000000..b4075d66663
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/UnionNode.java
+// and modified by Doris
+
+package org.apache.doris.planner;
+
+import org.apache.doris.analysis.TupleId;
+import org.apache.doris.statistics.StatisticalType;
+import org.apache.doris.thrift.TPlanNode;
+import org.apache.doris.thrift.TPlanNodeType;
+import org.apache.doris.thrift.TRecCTENode;
+
+public class RecursiveCteNode extends SetOperationNode {
+
+ private boolean isUnionAll;
+ private TRecCTENode tRecCTENode;
+
+ public RecursiveCteNode(PlanNodeId id, TupleId tupleId, boolean isUnionAll) {
+ super(id, tupleId, "RECURSIVE_CTE", StatisticalType.RECURSIVE_CTE_NODE);
+ this.isUnionAll = isUnionAll;
+ }
+
+ public boolean isUnionAll() {
+ return isUnionAll;
+ }
+
+ public void settRecCTENode(TRecCTENode tRecCTENode) {
+ this.tRecCTENode = tRecCTENode;
+ }
+
+ @Override
+ protected void toThrift(TPlanNode msg) {
+ msg.node_type = TPlanNodeType.REC_CTE_NODE;
+ msg.rec_cte_node = tRecCTENode;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
new file mode 100644
index 00000000000..103abd8b790
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
@@ -0,0 +1,113 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.planner;
+
+import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.UserException;
+import org.apache.doris.statistics.StatisticalType;
+import org.apache.doris.system.Backend;
+import org.apache.doris.thrift.TDataGenScanRange;
+import org.apache.doris.thrift.TExplainLevel;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TPlanNode;
+import org.apache.doris.thrift.TPlanNodeType;
+import org.apache.doris.thrift.TScanRange;
+import org.apache.doris.thrift.TScanRangeLocation;
+import org.apache.doris.thrift.TScanRangeLocations;
+
+import com.google.common.collect.Lists;
+
+import java.util.Collections;
+import java.util.List;
+
+// Full scan of recursive cte temp table
+public class RecursiveCteScanNode extends ScanNode {
+
+ public RecursiveCteScanNode(PlanNodeId id, TupleDescriptor desc) {
+ super(id, desc, "RECURSIVE_CTE_SCAN", StatisticalType.CTE_SCAN_NODE);
+ }
+
+ public void initScanRangeLocations() throws UserException {
+ createScanRangeLocations();
+ }
+
+ @Override
+ protected void createScanRangeLocations() throws UserException {
+ scanRangeLocations = Lists.newArrayList();
+ // randomly select 1 backend
+ List<Backend> backendList = Lists.newArrayList();
+ for (Backend be : Env.getCurrentSystemInfo().getBackendsByCurrentCluster().values()) {
+ if (be.isAlive()) {
+ backendList.add(be);
+ }
+ }
+ if (backendList.isEmpty()) {
+ throw new UserException("No Alive backends");
+ }
+ Collections.shuffle(backendList);
+ Backend selectedBackend = backendList.get(0);
+
+ // create a dummy scan range
+ TScanRange scanRange = new TScanRange();
+ TDataGenScanRange dataGenScanRange = new TDataGenScanRange();
+ scanRange.setDataGenScanRange(dataGenScanRange);
+
+ // create scan range locations
+ TScanRangeLocations locations = new TScanRangeLocations();
+ TScanRangeLocation location = new TScanRangeLocation();
+ location.setBackendId(selectedBackend.getId());
+ location.setServer(new TNetworkAddress(selectedBackend.getHost(), selectedBackend.getBePort()));
+ locations.addToLocations(location);
+ locations.setScanRange(scanRange);
+ scanRangeLocations.add(locations);
+ }
+
+ @Override
+ public List<TScanRangeLocations> getScanRangeLocations(long maxScanRangeLength) {
+ return scanRangeLocations;
+ }
+
+ @Override
+ public int getNumInstances() {
+ return 1;
+ }
+
+ @Override
+ public int getScanRangeNum() {
+ return 1;
+ }
+
+ @Override
+ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
+ StringBuilder output = new StringBuilder();
+ output.append(prefix).append("Recursive Cte: ").append(getTableIf().getName());
+ output.append("\n");
+ return output.toString();
+ }
+
+ @Override
+ protected void toThrift(TPlanNode msg) {
+ msg.node_type = TPlanNodeType.REC_CTE_SCAN_NODE;
+ }
+
+ @Override
+ public boolean isSerialOperator() {
+ return true;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java
index 42a930c2471..a2633b51449 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java
@@ -41,6 +41,8 @@ public enum StatisticalType {
ODBC_SCAN_NODE,
OLAP_SCAN_NODE,
PARTITION_TOPN_NODE,
+ RECURSIVE_CTE_NODE,
+ RECURSIVE_CTE_SCAN_NODE,
REPEAT_NODE,
SELECT_NODE,
SET_OPERATION_NODE,
diff --git a/regression-test/suites/nereids_p0/recursive_cte/test_recursive_cte.groovy b/regression-test/suites/nereids_p0/recursive_cte/test_recursive_cte.groovy
new file mode 100644
index 00000000000..6055f9798cd
--- /dev/null
+++ b/regression-test/suites/nereids_p0/recursive_cte/test_recursive_cte.groovy
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+suite("test_recursive_cte") {
+ sql """drop table if exists t1"""
+ sql """drop table if exists t2"""
+ sql """drop table if exists t3"""
+ sql """drop table if exists t4"""
+
+ sql """
+ create table if not exists t1 (
+ c2 int ,
+ c1 int ,
+ c3 int ,
+ c4 int ,
+ pk int
+ )
+ distributed by hash(pk) buckets 10
+ properties("replication_num" = "1");
+ """
+
+ sql """
+ create table if not exists t2 (
+ c1 int ,
+ c2 int ,
+ c3 int ,
+ c4 int ,
+ pk int
+ )
+ distributed by hash(pk) buckets 10
+ properties("replication_num" = "1");
+ """
+
+ sql """
+ create table if not exists t3 (
+ c2 int ,
+ c1 int ,
+ c3 int ,
+ c4 int ,
+ pk int
+ )
+ distributed by hash(pk) buckets 10
+ properties("replication_num" = "1");
+ """
+
+ sql """
+ create table if not exists t4 (
+ c1 int ,
+ c2 int ,
+ c3 int ,
+ c4 int ,
+ pk int
+ )
+ distributed by hash(pk) buckets 10
+ properties("replication_num" = "1");
+ """
+
+ sql """
+ insert into t1(pk,c1,c2,c3,c4) values (0,7,2,3328056,7),(1,3,5,3,3045349),(2,2130015,0,7,-7116176),(3,4411710,1203314,1,2336164),(4,4,-8001461,0,8),(5,9,3,6,2),(6,-8088092,null,-7256698,-2025142),(7,8,2,5,1),(8,4,4953685,3,null),(9,-6662413,-3845449,4,2),(10,5315281,0,5,null),(11,9,3,7,7),(12,4341905,null,null,8),(13,3,6,5,1),(14,5,9,6541164,3),(15,1,-582319,1,9),(16,5533636,4,39841,0),(17,1,1,null,7),(18,742881,-1420303,6,1),(19,281430,6753011,3,2),(20,7,1,4,-31350),(21,-5663089 [...]
+ """
+
+ sql """
+ insert into t2(pk,c1,c2,c3,c4) values (0,5,4,189864,-7663457),(1,7,null,6,1),(2,null,8,-3362640,9),(3,3,2,5,-2197130),(4,2,3,7160615,1),(5,null,-57834,420441,3),(6,0,null,2,2),(7,1,-3681539,3,4),(8,548866,3,0,5),(9,8,-2824887,0,3246956),(10,5,3,7,2),(11,8,8,6,8),(12,0,2,7,9),(13,8,6,null,null),(14,-4103729,4,5,8),(15,-3659292,2,7,5),(16,8,7,1,null),(17,2526018,4,8069607,5),(18,6,6,5,2802235),(19,9,0,6379201,null),(20,3,null,4,3),(21,0,8,-5506402,2),(22,6,4,3,1),(23,4,5225086,3,1) [...]
+ """
+
+ sql """
+ insert into t3(pk,c1,c2,c3,c4) values (0,3,2,6,-3164679),(1,-6216443,3437690,-288827,6),(2,4,-5352286,-1005469,4118240),(3,9,6795167,5,1616205),(4,8,-4659990,-4816829,6),(5,0,9,4,8),(6,-4454766,2,2510766,3),(7,7860071,-3434966,8,3),(8,null,0,2,1),(9,8031908,2,-6673194,-5981416),(10,5,6716310,8,2529959),(11,null,-3622116,1,-7891010),(12,null,3527222,7993802,null),(13,null,1,2,1),(14,2,8,7,7),(15,0,9,5,null),(16,7452083,null,-4620796,0),(17,9,9,null,6),(18,3,1,-1578776,5),(19,9,253 [...]
+ """
+
+ sql """
+ insert into t4(pk,c1,c2,c3,c4) values (0,-4263513,null,null,6),(1,1,3,4,null),(2,2460936,6,5,6299003),(3,null,7,7107446,-2366754),(4,6247611,4785035,3,-8014875),(5,0,2,5249218,3),(6,null,253825,4,3),(7,null,2,9,-350785),(8,6,null,null,4),(9,1,3,1,3422691),(10,0,-6596165,1808018,3),(11,2,752342,null,1),(12,-5220927,2676278,9,7),(13,6025864,2,1,4),(14,7,4,4,9),(15,5,9,9,849881),(16,-4253076,null,-4404479,-6365351),(17,null,6,4240023,3),(18,7,1276495,7,6),(19,null,-4459040,178194,-6 [...]
+ """
+
+ sql """
+ EXPLAIN ALL PLAN WITH XX AS (
+ SELECT C1
+ FROM t1
+ ),
+ RECURSIVE YY AS (
+ SELECT C1 + 1.0 as c2
+ FROM XX
+ UNION
+ SELECT c1
+ FROM XX t2 join YY
+ WHERE t2.c1 = YY.c2
+ )
+ SELECT * FROM YY;
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/09: add result_expr_lists to TRecCTENode
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch dev_rec
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 061083ced8758f8eca00d8188d9089816ff62e21
Author: BiteTheDDDDt <xl...@selectdb.com>
AuthorDate: Sun Sep 28 17:39:17 2025 +0800
add result_expr_lists to TRecCTENode
---
gensrc/thrift/PlanNodes.thrift | 1 +
1 file changed, 1 insertion(+)
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index d792410f2be..c41bc50add3 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -709,6 +709,7 @@ struct TRecCTENode {
1: optional bool is_union_all
2: optional list<TRecCTETarget> targets
3: optional list<TRecCTEResetInfo> fragments_to_reset
+ 4: optional list<list<Exprs.TExpr>> result_expr_lists
}
struct TEsScanNode {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org