You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by pa...@apache.org on 2025/10/27 07:20:55 UTC
(doris) branch dev_rec2 updated (3fbbac4c917 -> a0931099bfa)
This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a change to branch dev_rec2
in repository https://gitbox.apache.org/repos/asf/doris.git
omit 3fbbac4c917 add rec cte thrift struct
omit e45c7c6a71d recursive cte fe part
omit ee5f8d46f79 add rec cte thrift struct
add e3181d5cc88 [fix](nereids)ReorderJoin rule should not absorb mark join to multi join (#56837)
add 3e91ad98535 [chore](jsonb) remove unused code from jsonb_document (#56496)
add 063f3534c2e [fix](docker) modify readiness probe for ipv6 on k8s (#56917)
add 7e886ee237a [opt](fe) Add alias element_at as struct_element (#56898)
add c36027bec4e [test](sqlcache)Add sql cache with mtmv test case (#56753)
add ed674c7fe8f [test](nereids)fix prune partitions test case (#56892)
add 74109427066 [improvement](cloud) only add filecache profile for olap table in cloud deploy mode (#52118)
add 2594f8c7927 [Improve](fix) add streamjob case with create alter select delete and resume (#56849)
add fc38d1f0c65 [feature](regression-plugin) support version compare plugin with cloud (#56850)
add 16fc34e0cb2 [fix](mysql buffer) fix buffer overflow of output complex type columns (#56922)
add 7f41367e2f0 [opt](function) Make coalesce support more types (#56430)
add 807a87fe970 [fix](search) fix search result for non-tokenizer match (#56914)
add a91c067fabb [Enhancement](nereids)remove the class TableName (#56862)
add dca62cd5862 [fix](field) implement move constructor (#56803)
add ed2e880b44c [chore](code)Remove unnecessary accurate code and reduce template parameters (#56909)
add 27687930852 [feat](nereids) add table alias for catalog relation (#56802)
add d26e4953148 [Feature](func) Support function EXPORT_SET (#56798)
add 032ebaed6bd [fix](hdfs)Fix be coredump in HDFS reader during profile collection. (#56806)
add 588c37bc678 [fix](nereids)fix some show command's redirect policy (#55185)
add 483cd1c6448 [fix](nereids)forbid create sync mv based on annother sync mv (#56912)
add a5cf96230f4 [refactor](query) Clear query context once it is finished (#56911)
add fbb64e1f5ca [test](cache) add regression test cases for filecache statistics and cache eviction patterns (#54062)
add 8c1e26037fe [fix](insert) do not generate next id when fe is not master (#56959)
add 4971c1dc0b6 [feature] (nereids) generate sql digest for nereids prased stmt (#56256)
add ed6fb629e4c [fix](cloud) Fix unhandled error when get_partition_boundaries (#56935)
add 071a7e3fd30 [chore](build) Add feature list to version info (#56604)
add ac66ed0d232 [chore](checker) Add `get_all_tablet_ids` for MetaReader skeleton (#56888)
add 6ca89e31b17 [fix](schema-change) Fix distribution columns with varchar type could not increase length (#56743)
add 1d91724d5a6 [enhance](cloud)enable tablets meta url show rs_meta for cloud (#55167)
add b8688964673 [fix](iceberg) support reading iceberg partition table without partition value stored in file (#56918)
add fffbeabab16 [fix](nereids) partition topn optimization requires all window expressions are in the same order (#56622)
add 886fef76e73 [Enhancement](Snapshot) Add some snapshot interface (#56953)
add 31079ef49ca [Enhancement](nereids) remove class PartitionNames (#56947)
add c4ff082d9c3 [fix](search) fix mow support for search function (#56927)
add d1b0a19c083 [opt](inverted index) add custom analyzer support with char_filter, basic and icu tokenizer (#56243)
add 23ada5ec5c9 [fix](editlog) do not write editlog if not master (#56896)
add 996a54482d1 [opt](if) vec exec for more type in function if (#56905)
add 106e5b3f294 [refactor](insert) unified INSERT_JOB to INSERT for EtlJobType (#56780)
add 9bdc9695a3c [feature](agg) Add a knob to shuffled streaming agg (#56956)
add 7b9c3fea6cf [feat](snapshot) fix Unknown snapshot status when select snapshots (#56726)
add db0593bc591 [fix](function) Crash caused by explode function (#56991)
add e138c3e25b0 [Fix](Repository)Handle non-StorageException errors in Storage parameter validation (#56929)
add 753f3467f6e [fix](string) fix string len check of nested types (#56963)
add 3038a44b8fc [fix](inverted index) fix is null predicate for inverted index evaluate (#56964)
add f0190dbb459 [fix](ubsan) fix ub error of uninitialized variable (#56854)
add ce665f71aa3 [fix](iceberg) fix logic of getting namespaces (#56874)
add 65b2a5f0aa0 [Opt](sys table) Add a system table column_data_sizes (#56815)
add 8d7da73b0e4 [Improvement](execute) optimize for ColumnNullable's serialize_vec/deserialize_vec (#56885)
add 0a2b8d8a9af [enhance](memtable) support adaptive memtable write buffer size (#56948)
add 9a16dcd3fec [opt](case) move some time consuming case to p1 (#56986)
add 3199414db51 [chore](cloud) Add decode_*_key (#57008)
add fd47c36ceb8 [test](cloud) Add more multi cluster case for cloud (#56840)
add d57b02adb42 [chore](tde) Pick rotate open source code (#56838)
add 58203c8e278 [feat](case when) rewrite case when to compound predicate (#56756)
add c436e9224ad [chore](cloud) expose split_tablet_stats_into_load_and_compact_parts (#57009)
add 286b47f04a2 [feat](cloud) Add snapshot data migrator & chain compactor skeleton (#56939)
add ce7b17aea79 [opt](memory) remove useless replicaToTabletMap in TabletInvertedIndex (#57011)
add aad5e485bdc [refine](float) Add NormalizeFloat to handle special values of floating-point numbers. (#56937)
add 01f83399600 [test](auth)Fix backup backup of auth test case (#56988)
add 46d27a21f43 [fix](pipeline) Crashing caused by repeated spill operations (#56755)
add 456af82122a [opt](nereids)move runtime filter info from instance level to BE level (#56978)
add 9451276b524 [feature](search) add variant subcolumn suppport for search function (#56718)
add 7f908857eaa [chore](UT) Make cloud mgr test stable (#57033)
add 02cd275d7f3 [Improve](variant) optimize cast performance when variant only contains null values (#57003)
add 16c62be4598 [fix](nereids) fix rule AvgDistinctToSumDivCount need add cast (#56887)
add 302aec1316a [Enhancement](auto partition) Auto partition could use auto bucket (#56921)
add ea1b7fabe66 [refine](DECIMALV2)Only DECIMALV2 has the member variables original_precision and original_scale. (#56913)
add bdca890342a [Improve](job) add priv check for streaming job (#56965)
add 0578af22b20 [improvement](cache) (log) add logs for mode switching and optimize log output frequency (#54085)
add 1f4cd9a22ab [chore](MS)Rename HMSProperties to HiveProperties to avoid confusion with Hive/HMS configs (#57064)
add 04e5604246c [improve](case) Fix spark flink case in jdk17 (#56992)
add b31c45444ae [fix](schema_change) fix incorrect unique id in rollup after schema change (#57073)
add 7165d4ee6f6 [Fix](metrics) Fix FE's incorrect prometheus metrics format (#57059)
add 490ade94225 [Fix](startup) Skip wrong file when loading tzdata (#57061)
add 78d8cb8ff98 [fix](exception) avoid not set required field exception (#57069)
add b6d920081ca [fix](fe)show config command use readline instead of ready (#54791)
add 508b5712db8 [Feature](func) Support GET_FORAMT function (#56691)
add 7ce60da00f8 [fix](cloud) Fix /get_snapshot_property result (#57067)
add 50123ad8a83 [opt](function) Enhance the handling of a single struct-type argument in the EXPLODE function (#57050)
add 71480ff12ab [fix](cloud) Return error if migrate to versioned keys/compact snapsh… (#57066)
add 35f7fab36b4 [env](case) remove disable pipeline engine case (#57046)
add 19cf4c4a4e1 [feature](function) support json_hash function (#56962)
add af9fb9d82e4 [refine](type)use new type dispatch (#56990)
add 6b41b1ba55c [fix](cache) fix incorrectly incrementing s3_read_counter (#55805)
add b3ff40587b7 [Chore](runtime-filter) catch Exception from runtime filter rpc (#57085)
add b276af3bb68 [Fix](autoinc) Fix wrong auto inc value after FE do checkpoint and restart (#57104)
add e2cb3d6b877 [enhancement](filecache) add wait time for other downloader in profile (#57090)
add 82d5a3aac83 [fix](cloud) Fix /set_snapshot_property property names (#57099)
add bb121e30810 [chore](external catalog) deprecated lakesoul external catalog (#56843)
add 175e6ebcc38 [chore](inverted index) remove duplicate code in comparison evaluate index (#57115)
add 26aacf428d2 [opt](ann index) omp_threads_limit working better & minor change for observability (#56796)
add 1b6476e3840 [enhance](job) delay load task schedule when transaction fail (#57092)
add c1e72907efd [chore](search) remove useless params (#57117)
add 3e3efb572f6 [Fix](job) fix streaming job block when fe restart (#57103)
add e46317fdbb6 [function](mask) mask_last/first second parameter can be non-const (#57062)
add 2794fcc9c90 [fix](case) add query to force sync rowset meta (#56949)
add 5c610cd4b96 [opt](inverted index) Add built_in_analyzer parser alias and modify custom analyzer logic (#57055)
add 26645e9da84 [fix] remove useless const_cast and explain const_cast for vec (2) (#56464)
add ab1a05a4d07 [mem](jemalloc) Improve memory reuse efficiency and reduce page faults to change jemalloc conf (#57076)
add ad7fc73265f [fix](docker) readiness-modify for http code (#57113)
add f34b8c1a1b7 [enhance](job) support adaptive batch param for routine load job (#56930)
add 9e6926909ed [Enhancement](grammar) Support omit auto keyword for auto range partition (#57060)
add 017f9e49c72 [fix](catalog)Disable Hadoop FileSystem cache for multi-configurationobject storage catalogs (#57063)
add 04e333b442b [fix](parquet) incorrect skip files when returning eof (#57071)
add 72bcfd6d2fe [opt](ann index) Support product quantization (#56954)
add fa36b1c6cc2 [feature](mtmv)create mtmv can use view (#56423)
add e08709fba10 [chore](cloud) support notifing itself to refresh instance (#57107)
add f06196e9a24 [chore](regression) fix java home inconsistency (#57095)
add f27070c2d13 [fix](load) fix load plan when using WHERE clause with CompoundPredicate (#57128)
add 0dfdfcd1a01 [fix](filecache) fix flaky case populate_empty_cache_with_normal (#57091)
add 5ef4666eb13 [Bug](cloud restore) Fix npe in show restore command after fe restart (#57051)
add d24e675cb1a [Fix](column) Fix `ColumnFixedLengthObject::{insert_many_continuous_binary_data, insert_many_strings}` (#57164)
add e51d64d7ffd [chore](compile) fix macos compile be failed (#57150)
add 5f0a8c68ded [fix](inverted index) remove shadow prefix when matching index columns in schema change (#57006)
add b36f517e1a6 [Feature](varbinary) add length, from_base64_binary, to_base64_bianry, sub_binary for VarBinary type (#56648)
add 87a53626d48 [Exec](Cache) Support condition cache in Apache doris (#55534)
add 0f82adf9856 [chore](cloud) Make key info declaration more clear (#57034)
add 7be788844c8 [Enhancement] add column_uid and schema_version field for table schema open api (#56919)
add b99f8e972ed [Fix](ai) Fix the wrong check in AI_FILTER (#56960)
add 69d48801dc4 [fix](docker) update node to v14 in docker compilation image (#56029)
add 322546caf2e [fix](nereids)fix sync mv parse error if it contains sql KEYWORD (#57052)
add 7945b29db60 [fix](nullable) Fix nullable property for array (#57179)
add 33db05e5c1e [fix](nereids)set RuntimeFilterInfo only on BE which is merge node (#57108)
add 94d7f2eabca [refactor](segment iterator) rename _non_predicate_column_ids to _common_expr_column_ids (#56931)
add d29b3fe32eb [improve](job) support change offset for streaming job (#57138)
add a301e313569 [fix](job) fix streaming job repeated load after FE master change in cloud mode (#57205)
add f7876530964 [fix](olap) Need to handle floating-point Infinity values in the predicate (#57100)
add 3f87ec38d43 [fix](segcompaction) Fix segcompaction coredump when adding key column (#57162)
add 903d052924e [test](jdbc) Upgraded the mysql jdbc driver jar under test to 8.4.0 (#57142)
add 9177047d17c [feat](storage)Support Azure Blob Storage (#56861)
add 5641a274d4b [test](nereids)fix sql cache of mtmv test case (#57177)
add b7612ab0d9c [chore](cold) add a score threshold for cold data compaction (#55333)
add dc7e1d82c0e [feat](nereids) Add fold constant for nullif function (#56932)
add 8226a2a3bac [Opt](encoding) Add new encoding `BinaryPlainPageV2` (#56846)
add cb1e3bd9f8c [feat](nereids) opt condition expression in filter / join / if /case when (#57025)
add 69f82ad67db [Feature](variable) Support session variable lc_time_names (#56915)
add bd8a44f2db0 [fix](case) fix case bug (#56871)
add d9daa81c18e [chore](regression) add conf for run ccr cases in regression (#57222)
add 8f22466b6f8 [opt](inverted index) default index format uses v3 (#57140)
add 3eca43c56a3 [Chore](compile) fix compile fail coz base64_encode_part_num (#57225)
add b9bac89b66f [improve](sort) opt heap sort without runtime predicate (#57149)
add 6719ea0f144 [chore](checker) Add skeleton for mvcc meta checker (#57174)
add 5a9cb5aa255 [optimize](function) modify to_base64, from_base64, unhex to avoid a extra copy (#57182)
add 1553bc61dc8 [fix](outfile) fix analysis error when specifying parquet schema (#57232)
add dce846d5576 [Fix](function) Fix wrong nullable of floor/ceil functions (#57238)
add 4a4c2b223de [chore](cloud) Log the large value info (#57237)
add c4193bdb676 [feat](regression) Add dockers to manage multi docker clusters (#57236)
add 72385a5f364 [fix](cloud) Change CloudGlobalTransactionMgr some log to debug level (#57168)
add 6ee5dae4950 [fix](cloud) Fix cloud balance warm up lost idx file (#57114)
add 641b3861ee4 [Fix](job) add thread num config for streaming task exec (#57230)
add 8e919ba0aa1 [fix](olap) Crash caused by out-of-bounds PODArray access (#57239)
add 324fd59bd09 [opt](nereids) print slotReference in explain verbose (#57167)
add 0b20f0313ee [improvement](tvf load)add the data_{1..200}.csv wildcard in tvf load (#56705)
add d739136485c [fix](paimon)Support user-defined S3 config prefixes and unify to HDFS S3A protocol (#57116)
add 95c55bbc217 [fix](case) make case stable (#57040)
add e1884e4d323 [fix](cloud) fix correct pipeline task number calculation in cloud mode (#57129)
add 08a5143b8c1 [opt](arm) Improve BlockBloomFilter performance with NEON intrinsics (#57183)
add 75d45605d59 [opt](rowset) Remote fetch rowsets to avoid -230 error when capturing rowsets (#52995)
add 3c8cc786ba1 [External](parquet) pass non predicates column's offset index to RowGroupReader (#55795)
add 008b51bb157 [Bug](memtracker) fix wrong memtracker scope on PipelineTask destructor (#57276)
add 821aae2e769 [feat](deps) add hadoop 3.4.2 (#57181)
add fad1907e74b [chore](cloud) Add find_derived_instance_ids function (#57274)
add 890c778bacb [feature](maxcompute)support maxcompte catalog read project-schema-table. (#57012)
add 4f00ca1cbad [feature](analyzer) support * as json_object's parameter (#57256)
add c21b125e2ce [feat](load) support merge into (#57044)
add f1faa1a3fdd [opt](var) change merge_io_read_slice_size to merge_io_read_slice_size_bytes (#57272)
add ff322803ac0 [fix](case)drop test_temp_table_with_conn_timeout (#56996)
add eea9d23fdb2 [fix](ut)fix ut AlterMTMVTest (#56997)
add ebc8c679f38 [fix](ann index)Remove redundant omp_set_num_threads (#57224)
add 765c98f771a [fix](array) fix arr funcs with ret which BE should eginore precision because of deduce that the FE planner cannot infer (#57018)
add 84c1966b6d0 [chore](typo)Fix typos in thrift (#57267)
add 26392bfe032 [improvement](storage) Release unused memory earlier, rather than leaving it to the memory pool for guaranteed release (#57215)
add ef01efc780b [chore](regression) Reject getJdbcUrl request in dockers() context (#57251)
add 8df7321cc79 [fix](bin) Remove leading - of the cluster_snapshot arg (#57289)
add 002579e86b0 [fix](binlog) Avoid to issue getVersion RPC during replay journals (#57290)
add 62b00a68fb2 [fix](case) broker load cases compactibility (#56870)
add ca8c12d8f1a [fix](nereids)relase physical plan in Profile.releaseMemory() (#57257)
add b6c001756d9 [chore](cloud) Disable auto snapshot by default (#57285)
add f976ef7921a [fix](cloud)Delete debug log because it reveals too much sensitive information (#57277)
add 58c9a18dd6e [Fix](brpc) Fix wrong usage of brpc call_id (#56825)
add e08e53efb23 [refine](type)use new type dispatch for agg function (#57112)
add ef415d2990e [feature](search) add another two params for search (#57312)
add 5130ee07eec [cases](map) Update test_map_load_and_compaction.groovy for not stable out rowsets in cumative compaction (#56562)
add 77b8f787f53 [fix](regression-test) Add some test for str functions (#57315)
add d9f3044ecfa [fix] remove parse_id (#57240)
add 955d04d13a6 [fix] remove useless const_cast and explain const_cast(3) (#56441)
add e7e2b5bdc31 [fix](cast)Allow cast array, struct and map type to stringlike type. (#57326)
add 3da6abc4ba9 [cases](print) Add print date/datetime/time cases (#57332)
add 0513457e2e0 [p0](to_string) add p0 tests for casting int and decimal to string (#57335)
new 8e2fda6d44a add rec cte thrift struct
new 1e349c4dd4c recursive cte fe part
new a0931099bfa add rec cte thrift struct
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (3fbbac4c917)
\
N -- N -- N refs/heads/dev_rec2 (a0931099bfa)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/build-thirdparty.yml | 6 +
be/benchmark/benchmark_bit_pack.hpp | 8 +-
be/benchmark/benchmark_block_bloom_filter.hpp | 97 ++
be/benchmark/benchmark_main.cpp | 2 +
be/benchmark/benchmark_string.hpp | 394 ++++++
be/src/cloud/cloud_full_compaction.cpp | 5 +-
be/src/cloud/cloud_internal_service.cpp | 72 +-
be/src/cloud/cloud_schema_change_job.cpp | 18 +-
be/src/cloud/cloud_snapshot_mgr.cpp | 2 +-
be/src/cloud/cloud_tablet.cpp | 107 +-
be/src/cloud/cloud_tablet.h | 28 +-
be/src/cloud/cloud_tablet_mgr.cpp | 17 +-
be/src/cloud/cloud_tablet_mgr.h | 2 +-
be/src/common/compare.h | 12 +
be/src/common/config.cpp | 35 +-
be/src/common/config.h | 14 +-
be/src/common/version_internal.cpp | 3 +
be/src/common/version_internal.h | 2 +
be/src/exec/schema_scanner.cpp | 3 +
.../schema_cluster_snapshots_scanner.cpp | 19 +-
.../schema_column_data_sizes_scanner.cpp | 363 +++++
.../schema_column_data_sizes_scanner.h | 71 +
be/src/exprs/block_bloom_filter.hpp | 109 +-
be/src/exprs/block_bloom_filter_avx_impl.cc | 8 +-
be/src/exprs/block_bloom_filter_impl.cc | 58 +-
be/src/http/action/check_encryption_action.cpp | 80 +-
be/src/http/action/meta_action.cpp | 2 +-
be/src/io/cache/block_file_cache.cpp | 79 +-
be/src/io/cache/block_file_cache_downloader.cpp | 16 +-
be/src/io/cache/block_file_cache_profile.cpp | 3 +
be/src/io/cache/block_file_cache_profile.h | 1 +
be/src/io/cache/cached_remote_file_reader.cpp | 8 +-
be/src/io/cache/file_cache_common.h | 1 +
be/src/io/file_factory.h | 2 +
be/src/io/fs/azure_obj_storage_client.cpp | 6 +-
be/src/io/fs/hdfs_file_reader.cpp | 10 +
be/src/io/io_common.h | 1 +
be/src/olap/base_tablet.cpp | 57 +-
be/src/olap/base_tablet.h | 90 +-
be/src/olap/full_compaction.cpp | 5 +-
be/src/olap/inverted_index_parser.cpp | 16 +-
be/src/olap/inverted_index_parser.h | 2 +
be/src/olap/iterators.h | 24 +-
be/src/olap/memtable.cpp | 30 +-
be/src/olap/memtable.h | 6 +
be/src/olap/memtable_writer.cpp | 23 +-
be/src/olap/memtable_writer.h | 2 +
be/src/olap/merger.cpp | 8 +-
be/src/olap/olap_common.h | 4 +
be/src/olap/olap_server.cpp | 2 +-
be/src/olap/parallel_scanner_builder.cpp | 100 +-
be/src/olap/parallel_scanner_builder.h | 13 +-
be/src/olap/predicate_creator.h | 37 +-
be/src/olap/primary_key_index.cpp | 2 +-
be/src/olap/push_handler.cpp | 9 +-
be/src/olap/rowset/beta_rowset_reader.cpp | 16 +
be/src/olap/rowset/rowset_reader_context.h | 4 +-
be/src/olap/rowset/rowset_writer_context.h | 4 +-
.../olap/rowset/segment_v2/ann_index/ann_index.h | 14 +-
.../segment_v2/ann_index/ann_index_writer.cpp | 4 +-
.../rowset/segment_v2/ann_index/ann_index_writer.h | 2 +
.../segment_v2/ann_index/faiss_ann_index.cpp | 139 +-
.../rowset/segment_v2/ann_index/faiss_ann_index.h | 62 +-
be/src/olap/rowset/segment_v2/binary_dict_page.cpp | 79 +-
be/src/olap/rowset/segment_v2/binary_dict_page.h | 18 +-
be/src/olap/rowset/segment_v2/binary_plain_page.h | 12 +-
...{binary_plain_page.h => binary_plain_page_v2.h} | 261 ++--
.../olap/rowset/segment_v2/binary_prefix_page.cpp | 1 +
be/src/olap/rowset/segment_v2/binary_prefix_page.h | 4 +
.../olap/rowset/segment_v2/bitmap_index_writer.cpp | 4 +-
be/src/olap/rowset/segment_v2/bitshuffle_page.h | 5 +
be/src/olap/rowset/segment_v2/bloom_filter.h | 2 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 19 +-
be/src/olap/rowset/segment_v2/column_writer.cpp | 35 +-
be/src/olap/rowset/segment_v2/column_writer.h | 119 +-
be/src/olap/rowset/segment_v2/condition_cache.cpp | 52 +
be/src/olap/rowset/segment_v2/condition_cache.h | 135 ++
be/src/olap/rowset/segment_v2/encoding_info.cpp | 148 +-
be/src/olap/rowset/segment_v2/encoding_info.h | 83 +-
.../rowset/segment_v2/frame_of_reference_page.h | 5 +
.../rowset/segment_v2/indexed_column_reader.cpp | 2 +-
.../rowset/segment_v2/indexed_column_writer.cpp | 2 +-
.../inverted_index/analysis_factory_mgr.cpp | 61 +-
.../inverted_index/analysis_factory_mgr.h | 8 +-
.../inverted_index/analyzer/analyzer.cpp | 23 +-
.../segment_v2/inverted_index/analyzer/analyzer.h | 5 +-
.../inverted_index/analyzer/basic/basic_analyzer.h | 47 +-
.../inverted_index/analyzer/custom_analyzer.cpp | 51 +-
.../inverted_index/analyzer/custom_analyzer.h | 28 +-
.../analyzer/custom_analyzer_config.cpp | 10 +
.../analyzer/custom_analyzer_config.h | 4 +
.../inverted_index/analyzer/icu/icu_analyzer.h | 49 +-
.../tokenizer.h => char_filter/char_filter.h} | 41 +-
.../char_filter/char_filter_factory.h | 24 +-
.../char_filter/char_replace_char_filter.cpp | 26 +-
.../char_filter/char_replace_char_filter.h | 16 +-
.../char_filter/char_replace_char_filter_factory.h | 75 ++
.../char_filter/empty_char_filter_factory.h | 61 +
.../rowset/segment_v2/inverted_index/setting.h | 61 +-
.../empty_token_filter_factory.h} | 43 +-
.../inverted_index/token_filter/token_filter.h | 2 -
.../token_filter/word_delimiter_filter_factory.h | 2 +
.../segment_v2/inverted_index/token_stream.h | 37 +-
.../basic/basic_tokenizer.cpp | 43 +-
.../basic/basic_tokenizer.h | 26 +-
.../tokenizer/basic/basic_tokenizer_factory.h | 50 +
.../tokenizer/empty/empty_tokenizer_factory.h | 72 +
.../icu/break_iterator_wrapper.cpp | 4 +-
.../icu/break_iterator_wrapper.h | 4 +-
.../icu/composite_break_iterator.cpp | 4 +-
.../icu/composite_break_iterator.h | 4 +-
.../icu/default_icu_tokenizer_config.cpp | 4 +-
.../icu/default_icu_tokenizer_config.h | 4 +-
.../{analyzer => tokenizer}/icu/icu_common.h | 4 +-
.../{analyzer => tokenizer}/icu/icu_tokenizer.cpp | 18 +-
.../{analyzer => tokenizer}/icu/icu_tokenizer.h | 13 +-
.../icu/icu_tokenizer_config.h | 4 +-
.../icu/icu_tokenizer_factory.h} | 21 +-
.../icu/script_iterator.cpp | 4 +-
.../{analyzer => tokenizer}/icu/script_iterator.h | 4 +-
.../tokenizer/standard/standard_tokenizer_impl.h | 7 +-
.../inverted_index/tokenizer/tokenizer.h | 8 +-
.../rowset/segment_v2/inverted_index/util/reader.h | 15 +-
.../segment_v2/inverted_index_fs_directory.cpp | 22 +-
.../segment_v2/inverted_index_fs_directory.h | 8 +-
.../rowset/segment_v2/inverted_index_writer.cpp | 21 +-
.../olap/rowset/segment_v2/inverted_index_writer.h | 8 +-
be/src/olap/rowset/segment_v2/ngram_bloom_filter.h | 4 +-
be/src/olap/rowset/segment_v2/page_builder.h | 13 +
be/src/olap/rowset/segment_v2/page_decoder.h | 4 +
be/src/olap/rowset/segment_v2/page_io.cpp | 1 +
be/src/olap/rowset/segment_v2/plain_page.h | 5 +
be/src/olap/rowset/segment_v2/rle_page.h | 5 +
be/src/olap/rowset/segment_v2/row_ranges.h | 14 +
be/src/olap/rowset/segment_v2/segment.cpp | 9 +
be/src/olap/rowset/segment_v2/segment.h | 2 +
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 127 +-
be/src/olap/rowset/segment_v2/segment_iterator.h | 11 +-
be/src/olap/rowset/segment_v2/segment_writer.cpp | 18 +-
be/src/olap/rowset/segment_v2/segment_writer.h | 2 +-
.../variant/variant_column_writer_impl.cpp | 2 +-
.../rowset/segment_v2/vertical_segment_writer.cpp | 18 +-
.../rowset/segment_v2/vertical_segment_writer.h | 2 +-
be/src/olap/rowset/segment_v2/zone_map_index.cpp | 2 +-
be/src/olap/rowset_builder.cpp | 1 +
be/src/olap/rowset_version_mgr.cpp | 449 +++++++
be/src/olap/schema_change.cpp | 17 +-
be/src/olap/snapshot_manager.cpp | 21 +-
be/src/olap/tablet.cpp | 99 +-
be/src/olap/tablet.h | 13 +-
be/src/olap/tablet_meta.cpp | 57 +-
be/src/olap/tablet_meta.h | 14 +-
be/src/olap/tablet_reader.cpp | 16 +-
be/src/olap/tablet_reader.h | 18 +-
be/src/olap/tablet_schema.h | 1 +
be/src/olap/task/engine_checksum_task.cpp | 12 +-
be/src/olap/task/engine_storage_migration_task.cpp | 9 +-
be/src/pipeline/exec/aggregation_sink_operator.h | 5 +-
be/src/pipeline/exec/analytic_sink_operator.h | 2 +-
be/src/pipeline/exec/assert_num_rows_operator.h | 2 +-
.../exec/distinct_streaming_aggregation_operator.h | 4 +-
be/src/pipeline/exec/exchange_source_operator.h | 2 +-
be/src/pipeline/exec/hashjoin_build_sink.h | 2 +-
be/src/pipeline/exec/hashjoin_probe_operator.h | 2 +-
.../exec/nested_loop_join_build_operator.h | 2 +-
.../exec/nested_loop_join_probe_operator.h | 2 +-
be/src/pipeline/exec/olap_scan_operator.cpp | 23 +-
be/src/pipeline/exec/olap_scan_operator.h | 6 +-
be/src/pipeline/exec/operator.cpp | 6 +-
be/src/pipeline/exec/operator.h | 5 +-
.../pipeline/exec/partition_sort_sink_operator.h | 2 +-
.../exec/partitioned_aggregation_sink_operator.h | 4 +-
.../exec/partitioned_hash_join_probe_operator.h | 2 +-
.../exec/partitioned_hash_join_sink_operator.h | 2 +-
.../pipeline/exec/rec_cte_anchor_sink_operator.cpp | 1 +
.../pipeline/exec/rec_cte_anchor_sink_operator.h | 20 +-
be/src/pipeline/exec/rec_cte_scan_operator.h | 19 +-
be/src/pipeline/exec/rec_cte_sink_operator.h | 13 +-
be/src/pipeline/exec/rec_cte_source_operator.cpp | 21 +-
be/src/pipeline/exec/rec_cte_source_operator.h | 164 ++-
be/src/pipeline/exec/result_sink_operator.cpp | 18 +-
be/src/pipeline/exec/result_sink_operator.h | 4 +-
be/src/pipeline/exec/scan_operator.cpp | 20 +-
be/src/pipeline/exec/scan_operator.h | 15 +-
be/src/pipeline/exec/set_probe_sink_operator.h | 2 +-
be/src/pipeline/exec/set_sink_operator.h | 2 +-
be/src/pipeline/exec/sort_sink_operator.cpp | 2 +-
be/src/pipeline/exec/sort_sink_operator.h | 2 +-
be/src/pipeline/exec/spill_sort_sink_operator.h | 4 +-
be/src/pipeline/exec/spill_utils.h | 4 +-
.../exec/streaming_aggregation_operator.cpp | 20 +-
.../pipeline/exec/streaming_aggregation_operator.h | 16 +-
be/src/pipeline/exec/table_function_operator.h | 2 +-
be/src/pipeline/exec/union_sink_operator.h | 22 +-
be/src/pipeline/exec/union_source_operator.h | 2 +-
be/src/pipeline/pipeline.h | 6 +-
be/src/pipeline/pipeline_fragment_context.cpp | 88 +-
be/src/pipeline/pipeline_fragment_context.h | 10 +-
be/src/pipeline/pipeline_task.cpp | 101 +-
be/src/pipeline/pipeline_task.h | 40 +-
be/src/pipeline/rec_cte_shared_state.h | 72 +-
be/src/pipeline/revokable_task.h | 76 ++
be/src/pipeline/task_scheduler.cpp | 11 +-
be/src/runtime/exec_env.h | 8 +
be/src/runtime/exec_env_init.cpp | 16 +-
be/src/runtime/fragment_mgr.cpp | 48 +-
be/src/runtime/fragment_mgr.h | 9 +-
be/src/runtime/index_policy/index_policy_mgr.cpp | 84 +-
be/src/runtime/index_policy/index_policy_mgr.h | 7 +
be/src/runtime/memory/cache_policy.h | 5 +
be/src/runtime/query_context.cpp | 22 +-
be/src/runtime/query_context.h | 24 +-
be/src/runtime/runtime_predicate.h | 1 +
be/src/runtime/runtime_state.cpp | 14 +-
be/src/runtime/runtime_state.h | 15 +-
be/src/runtime_filter/runtime_filter_consumer.h | 15 +-
be/src/runtime_filter/runtime_filter_merger.h | 2 +
be/src/runtime_filter/runtime_filter_mgr.cpp | 22 +-
be/src/runtime_filter/runtime_filter_mgr.h | 26 +-
be/src/service/arrow_flight/flight_sql_service.cpp | 8 +-
be/src/service/internal_service.cpp | 122 +-
be/src/service/internal_service.h | 12 +-
be/src/tools/meta_tool.cpp | 2 +-
be/src/util/brpc_closure.h | 6 +-
be/src/util/debug_util.cpp | 4 +
be/src/util/doris_metrics.cpp | 11 +
be/src/util/doris_metrics.h | 5 +
be/src/util/jsonb_document.cpp | 20 +
be/src/util/jsonb_document.h | 23 +-
be/src/util/mysql_row_buffer.cpp | 11 +-
be/src/util/mysql_row_buffer.h | 7 +-
be/src/util/timezone_utils.cpp | 21 +-
be/src/util/uid_util.cpp | 23 -
be/src/util/uid_util.h | 5 -
.../aggregate_function_array_agg.cpp | 74 +-
.../aggregate_function_collect.cpp | 79 +-
.../aggregate_functions/aggregate_function_map.cpp | 59 +-
be/src/vec/columns/column.cpp | 41 +
be/src/vec/columns/column.h | 15 +-
be/src/vec/columns/column_array.cpp | 4 +-
be/src/vec/columns/column_array.h | 4 +-
be/src/vec/columns/column_const.h | 9 +-
be/src/vec/columns/column_decimal.cpp | 48 +-
be/src/vec/columns/column_decimal.h | 8 +-
be/src/vec/columns/column_fixed_length_object.h | 4 +-
be/src/vec/columns/column_map.cpp | 6 +-
be/src/vec/columns/column_map.h | 4 +-
be/src/vec/columns/column_nullable.cpp | 36 +-
be/src/vec/columns/column_nullable.h | 4 +-
be/src/vec/columns/column_string.cpp | 47 +-
be/src/vec/columns/column_string.h | 9 +-
be/src/vec/columns/column_struct.cpp | 4 +-
be/src/vec/columns/column_struct.h | 4 +-
be/src/vec/columns/column_varbinary.h | 8 +
be/src/vec/columns/column_variant.cpp | 9 +-
be/src/vec/columns/column_variant.h | 2 +-
be/src/vec/columns/column_vector.cpp | 61 +-
be/src/vec/columns/column_vector.h | 8 +-
be/src/vec/common/hash_table/hash_map_context.h | 4 +-
be/src/vec/common/pod_array.h | 5 +
be/src/vec/common/sort/heap_sorter.cpp | 64 +-
be/src/vec/common/sort/heap_sorter.h | 13 +-
be/src/vec/common/sort/sorter.cpp | 21 +-
be/src/vec/common/sort/sorter.h | 2 +-
be/src/vec/common/string_view.h | 2 +
be/src/vec/core/accurate_comparison.h | 303 +----
be/src/vec/core/call_on_type_index.h | 250 ++--
be/src/vec/core/decimal_comparison.h | 7 +-
be/src/vec/core/field.cpp | 261 +++-
be/src/vec/core/field.h | 7 +-
be/src/vec/core/sort_cursor.h | 5 +
be/src/vec/data_types/data_type_decimal.h | 195 +--
be/src/vec/data_types/data_type_factory.cpp | 4 +-
be/src/vec/exec/format/column_type_convert.cpp | 3 +-
be/src/vec/exec/format/csv/csv_reader.cpp | 9 +-
be/src/vec/exec/format/orc/vorc_reader.cpp | 16 +-
be/src/vec/exec/format/orc/vorc_reader.h | 10 +-
.../format/parquet/byte_stream_split_decoder.cpp | 2 +
.../format/parquet/fix_length_dict_decoder.hpp | 2 +
.../exec/format/parquet/fix_length_plain_decoder.h | 2 +
.../exec/format/parquet/parquet_column_convert.cpp | 3 +-
.../exec/format/parquet/parquet_column_convert.h | 8 +-
be/src/vec/exec/format/parquet/parquet_pred_cmp.h | 2 +-
be/src/vec/exec/format/parquet/schema_desc.cpp | 2 +-
be/src/vec/exec/format/parquet/schema_desc.h | 4 +-
.../parquet/vparquet_column_chunk_reader.cpp | 2 +-
.../format/parquet/vparquet_column_chunk_reader.h | 2 +-
.../exec/format/parquet/vparquet_column_reader.cpp | 35 +-
.../exec/format/parquet/vparquet_column_reader.h | 24 +-
.../exec/format/parquet/vparquet_file_metadata.h | 1 +
.../exec/format/parquet/vparquet_group_reader.cpp | 7 +-
.../exec/format/parquet/vparquet_group_reader.h | 4 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 36 +-
be/src/vec/exec/format/parquet/vparquet_reader.h | 4 +-
be/src/vec/exec/format/table/iceberg_reader.cpp | 14 +-
be/src/vec/exec/format/table/iceberg_reader.h | 3 +-
.../vec/exec/format/table/lakesoul_jni_reader.cpp | 3 +
be/src/vec/exec/format/table/lakesoul_jni_reader.h | 3 +
be/src/vec/exec/jni_connector.cpp | 28 +-
be/src/vec/exec/jni_connector.h | 8 +-
be/src/vec/exec/scan/file_scanner.cpp | 25 +-
be/src/vec/exec/scan/file_scanner.h | 1 +
be/src/vec/exec/scan/olap_scanner.cpp | 52 +-
be/src/vec/exec/scan/olap_scanner.h | 2 +-
be/src/vec/exprs/table_function/vexplode_v2.cpp | 75 +-
be/src/vec/exprs/vbitmap_predicate.h | 3 +
be/src/vec/exprs/vbloom_predicate.cpp | 7 +
be/src/vec/exprs/vbloom_predicate.h | 2 +
be/src/vec/exprs/vdirect_in_predicate.h | 2 +
be/src/vec/exprs/vexpr.cpp | 19 +
be/src/vec/exprs/vexpr.h | 2 +
be/src/vec/exprs/vexpr_context.cpp | 5 +-
be/src/vec/exprs/vexpr_context.h | 2 +
be/src/vec/exprs/vin_predicate.h | 2 +
be/src/vec/exprs/vlambda_function_call_expr.h | 2 +
be/src/vec/exprs/vlambda_function_expr.h | 2 +
be/src/vec/exprs/vliteral.cpp | 5 +
be/src/vec/exprs/vliteral.h | 2 +
be/src/vec/exprs/vruntimefilter_wrapper.h | 2 +
be/src/vec/exprs/vsearch.cpp | 87 +-
be/src/vec/exprs/vsearch.h | 2 +
be/src/vec/exprs/vslot_ref.cpp | 9 +
be/src/vec/exprs/vslot_ref.h | 3 +
be/src/vec/functions/ai/ai_functions.h | 6 +-
be/src/vec/functions/array/function_array_binary.h | 2 +-
.../array/function_array_contains_all.cpp | 133 +-
.../vec/functions/array/function_array_distinct.h | 76 +-
.../vec/functions/array/function_array_element.h | 104 +-
.../array/function_array_enumerate_uniq.cpp | 78 +-
be/src/vec/functions/array/function_array_index.h | 76 +-
be/src/vec/functions/array/function_array_remove.h | 76 +-
.../vec/functions/array/function_arrays_overlap.h | 122 +-
be/src/vec/functions/cast/cast_to_variant.h | 3 +-
be/src/vec/functions/date_time_transforms.h | 15 +-
be/src/vec/functions/function_coalesce.cpp | 94 +-
.../function_date_or_datetime_computation.cpp | 1 +
.../function_date_or_datetime_computation.h | 111 ++
.../function_date_or_datetime_to_string.cpp | 78 +-
.../vec/functions/function_datetime_floor_ceil.cpp | 2 +-
be/src/vec/functions/function_fake.cpp | 6 +-
be/src/vec/functions/function_json_hash.cpp | 104 ++
be/src/vec/functions/function_jsonb_transform.cpp | 8 +-
be/src/vec/functions/function_map.cpp | 128 +-
.../vec/functions/function_other_types_to_date.cpp | 2 +-
be/src/vec/functions/function_search.cpp | 85 +-
be/src/vec/functions/function_search.h | 24 +-
be/src/vec/functions/function_string.cpp | 145 +-
be/src/vec/functions/function_string.h | 171 ++-
be/src/vec/functions/function_tokenize.cpp | 4 +-
be/src/vec/functions/function_totype.h | 81 +-
be/src/vec/functions/function_varbinary.cpp | 136 +-
be/src/vec/functions/function_varbinary.h | 53 +
be/src/vec/functions/functions_comparison.h | 105 +-
be/src/vec/functions/if.cpp | 68 +-
be/src/vec/functions/if.h | 39 +-
be/src/vec/functions/is_null.h | 23 +-
be/src/vec/functions/least_greast.cpp | 241 +---
be/src/vec/functions/match.cpp | 6 +-
be/src/vec/functions/simple_function_factory.h | 2 +
be/src/vec/olap/vcollect_iterator.cpp | 2 +-
be/src/vec/runtime/vdatetime_value.cpp | 34 +
be/src/vec/runtime/vdatetime_value.h | 4 +
be/src/vec/sink/vtablet_block_convertor.cpp | 156 ++-
be/src/vec/utils/varbinaryop_subbinary.h | 119 ++
be/test/CMakeLists.txt | 1 -
be/test/cloud/cloud_meta_mgr_test.cpp | 135 +-
.../cloud/cloud_tablet_query_prefer_cache_test.cpp | 14 +-
.../cloud_tablet_query_with_tolerance_test.cpp | 12 +-
.../common/compare_test.cpp} | 43 +-
be/test/io/cache/block_file_cache_test.cpp | 2 +-
be/test/olap/delete_handler_test.cpp | 9 +-
be/test/olap/index_builder_test.cpp | 6 +-
.../rowset/segment_v2/binary_dict_page_test.cpp | 868 ++++++++++--
.../segment_v2/binary_plain_page_v2_test.cpp | 576 ++++++++
.../rowset/segment_v2/block_bloom_filter_test.cpp | 2 +-
.../olap/rowset/segment_v2/encoding_info_test.cpp | 76 +-
.../inverted_index/analysis_factory_mgr_test.cpp | 249 ++++
.../inverted_index/analyzer/icu_analyzer_test.cpp | 59 +-
.../analyzer/simple_analyzer_test.cpp | 8 +-
.../ananlyzer/custom_analyzer_test.cpp | 328 +++--
.../char_filter/char_filter_test.cpp | 76 ++
.../char_replace_char_filter_factory_test.cpp | 193 +++
.../inverted_index/query_v2/boolean_query_test.cpp | 3 +-
.../segment_v2/inverted_index/setting_test.cpp | 95 +-
.../ascii_folding_filter_factory_test.cpp | 6 +-
.../lower_case_filter_factory_test.cpp | 6 +-
.../token_filter/word_delimiter_filter_test.cpp | 6 +-
.../tokenizer/basic_tokenizer_factory_test.cpp | 235 ++++
.../char_group_tokenizer_factory_test.cpp | 6 +-
.../tokenizer/edge_ngram_tokenizer_test.cpp | 6 +-
.../tokenizer/icu_tokenizer_factory_test.cpp | 212 +++
.../tokenizer/keyword_analyzer_test.cpp | 6 +-
.../tokenizer/ngram_tokenizer_test.cpp | 6 +-
.../tokenizer/standard_tokenizer_factory_test.cpp | 6 +-
.../segment_v2/inverted_index/util/reader_test.cpp | 92 ++
be/test/olap/segcompaction_mow_test.cpp | 4 +-
be/test/olap/segcompaction_test.cpp | 1 +
be/test/olap/tablet_meta_test.cpp | 10 +-
.../operator/hashjoin_probe_operator_test.cpp | 6 +-
.../operator/spill_sort_sink_operator_test.cpp | 5 +-
be/test/pipeline/pipeline_test.cpp | 46 +-
.../runtime_filter_consumer_test.cpp | 20 +-
be/test/runtime_filter/runtime_filter_mgr_test.cpp | 2 +-
be/test/util/mysql_row_buffer_test.cpp | 29 +
.../columns/column_fixed_length_object_test.cpp | 22 +-
.../column_nullable_seriazlization_test.cpp | 72 +-
be/test/vec/columns/column_variant_test.cpp | 11 +-
be/test/vec/columns/common_column_test.h | 24 +-
be/test/vec/core/accurate_comparison_test.cpp | 63 +-
be/test/vec/core/field_test.cpp | 88 ++
be/test/vec/function/function_hash_test.cpp | 23 +-
be/test/vec/function/function_is_null_test.cpp | 147 ++
be/test/vec/function/function_num_if_test.cpp | 6 +-
be/test/vec/function/function_string_test.cpp | 87 +-
be/test/vec/function/function_test_util.cpp | 4 +-
be/test/vec/function/function_test_util.h | 32 +-
be/test/vec/function/function_varbinary_test.cpp | 426 ++++++
be/test/vec/function/table_function_test.cpp | 46 +-
bin/start_fe.sh | 2 +-
build.sh | 17 +-
cloud/src/common/config.h | 6 +
cloud/src/common/stopwatch.h | 4 +
cloud/src/main.cpp | 6 +-
cloud/src/meta-service/meta_service_helper.h | 2 +-
cloud/src/meta-service/meta_service_http.cpp | 34 +-
cloud/src/meta-service/meta_service_job.cpp | 23 -
cloud/src/meta-service/meta_service_resource.cpp | 37 +-
.../src/meta-service/meta_service_tablet_stats.cpp | 23 +
cloud/src/meta-service/meta_service_tablet_stats.h | 3 +
cloud/src/meta-store/keys.cpp | 332 ++++-
cloud/src/meta-store/keys.h | 163 ++-
cloud/src/meta-store/meta_reader.cpp | 197 +--
cloud/src/meta-store/meta_reader.h | 27 +-
cloud/src/meta-store/txn_kv.cpp | 51 +-
cloud/src/recycler/azure_obj_client.cpp | 7 +-
cloud/src/recycler/checker.cpp | 2 +-
cloud/src/recycler/checker.h | 4 +-
cloud/src/recycler/checker_snapshot.cpp | 2 +-
cloud/src/recycler/meta_checker.cpp | 526 ++++----
cloud/src/recycler/meta_checker.h | 81 +-
cloud/src/recycler/recycler.cpp | 31 +-
cloud/src/recycler/recycler.h | 4 +
cloud/src/recycler/recycler_service.cpp | 5 +-
cloud/src/recycler/snapshot_chain_compactor.cpp | 323 +++++
cloud/src/recycler/snapshot_chain_compactor.h | 111 ++
cloud/src/recycler/snapshot_data_migrator.cpp | 362 +++++
cloud/src/recycler/snapshot_data_migrator.h | 132 ++
cloud/src/snapshot/snapshot_manager.cpp | 14 +
cloud/src/snapshot/snapshot_manager.h | 13 +
cloud/test/keys_test.cpp | 350 ++++-
cloud/test/meta_reader_test.cpp | 171 +++
cloud/test/meta_server_test.cpp | 6 +-
cloud/test/meta_service_test.cpp | 2 +-
conf/be.conf | 2 +-
docker/compilation/Dockerfile | 10 +-
.../runtime/be/resource/be_disaggregated_probe.sh | 37 +-
docker/runtime/doris-compose/Readme.md | 56 +
docker/runtime/doris-compose/cluster.py | 86 +-
docker/runtime/doris-compose/command.py | 116 +-
docker/runtime/doris-compose/resource/common.sh | 37 +
docker/runtime/doris-compose/resource/init_be.sh | 6 +-
.../runtime/doris-compose/resource/init_cloud.sh | 35 +-
docker/runtime/doris-compose/resource/init_fe.sh | 17 +-
docker/runtime/doris-compose/utils.py | 6 +-
.../runtime/fe/resource/fe_disaggregated_probe.sh | 4 +-
fe/be-java-extensions/lakesoul-scanner/pom.xml | 1 +
.../apache/doris/lakesoul/LakeSoulJniScanner.java | 4 +
.../org/apache/doris/lakesoul/LakeSoulUtils.java | 4 +
.../apache/doris/lakesoul/arrow/ArrowUtils.java | 4 +
.../lakesoul/arrow/LakeSoulArrowJniScanner.java | 4 +
fe/be-java-extensions/pom.xml | 3 +-
.../main/java/org/apache/doris/common/Config.java | 17 +-
fe/fe-core/pom.xml | 9 +-
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 2 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 27 +-
.../org/apache/doris/nereids/search/SearchLexer.g4 | 1 +
.../apache/doris/nereids/search/SearchParser.g4 | 7 +-
.../src/main/java/org/apache/doris/DorisFE.java | 5 +
.../main/java/org/apache/doris/alter/Alter.java | 9 +-
.../apache/doris/alter/SchemaChangeHandler.java | 1 +
.../org/apache/doris/analysis/AddColumnClause.java | 8 +-
.../apache/doris/analysis/AlterTableClause.java | 7 +-
.../doris/analysis/AnnIndexPropertiesChecker.java | 54 +-
.../java/org/apache/doris/analysis/BrokerDesc.java | 2 +-
.../apache/doris/analysis/BuildIndexClause.java | 10 +-
.../apache/doris/analysis/ChannelDescription.java | 4 +-
.../org/apache/doris/analysis/CopyFromParam.java | 15 +-
.../apache/doris/analysis/CreateIndexClause.java | 17 +-
.../org/apache/doris/analysis/DataDescription.java | 44 +-
.../org/apache/doris/analysis/DropIndexClause.java | 13 +-
.../java/org/apache/doris/analysis/IndexDef.java | 5 +-
.../org/apache/doris/analysis/InsertTarget.java | 17 +-
.../apache/doris/analysis/InvertedIndexUtil.java | 21 +-
.../java/org/apache/doris/analysis/LockTable.java | 10 +-
.../apache/doris/analysis/ModifyColumnClause.java | 8 +-
.../analysis/ModifyTablePropertiesClause.java | 14 +-
.../org/apache/doris/analysis/OutFileClause.java | 17 +-
.../apache/doris/analysis/PartitionExprUtil.java | 28 +-
.../org/apache/doris/analysis/PartitionNames.java | 142 --
.../org/apache/doris/analysis/SchemaTableType.java | 4 +-
.../org/apache/doris/analysis/SearchPredicate.java | 24 +-
.../org/apache/doris/analysis/SlotDescriptor.java | 41 +-
.../java/org/apache/doris/analysis/SlotRef.java | 53 +-
.../java/org/apache/doris/analysis/StmtType.java | 1 +
.../org/apache/doris/analysis/StorageBackend.java | 25 +-
.../java/org/apache/doris/analysis/TableName.java | 195 ---
.../org/apache/doris/analysis/TableSnapshot.java | 8 +
.../org/apache/doris/analysis/TupleDescriptor.java | 5 +-
.../java/org/apache/doris/backup/Repository.java | 17 +-
.../apache/doris/binlog/AddPartitionRecord.java | 4 +-
.../doris/catalog/AutoIncrementGenerator.java | 2 +-
.../doris/catalog/BuiltinScalarFunctions.java | 53 +-
.../main/java/org/apache/doris/catalog/Column.java | 4 +
.../main/java/org/apache/doris/catalog/Env.java | 4 +-
.../main/java/org/apache/doris/catalog/Index.java | 9 +-
.../main/java/org/apache/doris/catalog/MTMV.java | 42 +-
.../org/apache/doris/catalog/MetadataViewer.java | 179 ---
.../java/org/apache/doris/catalog/OlapTable.java | 4 +
.../java/org/apache/doris/catalog/SchemaTable.java | 16 +
.../apache/doris/catalog/TabletInvertedIndex.java | 34 -
.../doris/clone/DynamicPartitionScheduler.java | 42 +-
.../apache/doris/cloud/backup/CloudRestoreJob.java | 1 +
.../cloud/catalog/CloudInstanceStatusChecker.java | 3 -
.../apache/doris/cloud/catalog/CloudReplica.java | 16 +
.../cloud/datasource/CloudInternalCatalog.java | 6 +-
.../apache/doris/cloud/storage/AzureRemote.java | 50 +-
.../apache/doris/cloud/storage/DefaultRemote.java | 116 +-
.../doris/cloud/system/CloudSystemInfoService.java | 11 +-
.../transaction/CloudGlobalTransactionMgr.java | 18 +-
.../org/apache/doris/common/profile/Profile.java | 3 +
.../doris/common/util/AutoBucketCalculator.java | 205 +++
.../apache/doris/common/util/AutoBucketUtils.java | 10 +-
.../org/apache/doris/common/util/LocationPath.java | 15 +
.../apache/doris/common/util/PropertyAnalyzer.java | 12 +-
.../java/org/apache/doris/common/util/S3Util.java | 88 ++
.../apache/doris/datasource/CatalogFactory.java | 4 +-
.../org/apache/doris/datasource/CatalogIf.java | 4 +-
.../apache/doris/datasource/ExternalCatalog.java | 4 +-
.../apache/doris/datasource/InternalCatalog.java | 15 +-
.../doris/datasource/hive/HMSCachedClient.java | 4 +-
.../doris/datasource/hive/HMSExternalCatalog.java | 10 +-
.../doris/datasource/hive/HMSTransaction.java | 61 +-
.../doris/datasource/hive/HiveMetaStoreCache.java | 3 +-
.../doris/datasource/hive/HiveTransaction.java | 4 +-
.../datasource/hive/ThriftHMSCachedClient.java | 8 +-
.../datasource/iceberg/IcebergMetadataOps.java | 28 +-
.../doris/datasource/jdbc/source/JdbcScanNode.java | 2 +-
.../lakesoul/LakeSoulExternalCatalog.java | 40 +-
.../lakesoul/LakeSoulExternalDatabase.java | 4 +
.../datasource/lakesoul/LakeSoulExternalTable.java | 4 +
.../doris/datasource/lakesoul/LakeSoulUtils.java | 4 +
.../lakesoul/source/LakeSoulScanNode.java | 4 +
.../datasource/lakesoul/source/LakeSoulSplit.java | 4 +
.../maxcompute/MaxComputeExternalCatalog.java | 104 +-
.../maxcompute/MaxComputeExternalTable.java | 19 +-
.../maxcompute/MaxComputeSchemaCacheValue.java | 8 +-
.../datasource/maxcompute/McStructureHelper.java | 221 +++
.../maxcompute/source/MaxComputeScanNode.java | 4 +-
.../doris/datasource/odbc/source/OdbcScanNode.java | 2 +-
.../property/constants/MCProperties.java | 20 +
...Properties.java => AbstractHiveProperties.java} | 4 +-
.../metastore/AbstractPaimonProperties.java | 49 +-
.../property/metastore/HMSBaseProperties.java | 3 +
....java => HiveAliyunDLFMetaStoreProperties.java} | 4 +-
...rties.java => HiveGlueMetaStoreProperties.java} | 6 +-
.../{HMSProperties.java => HiveHMSProperties.java} | 4 +-
...tiesFactory.java => HivePropertiesFactory.java} | 18 +-
.../IcebergFileSystemMetaStoreProperties.java | 4 +-
.../metastore/IcebergHMSMetaStoreProperties.java | 3 -
.../property/metastore/IcebergRestProperties.java | 17 +-
.../property/metastore/MetastoreProperties.java | 2 +-
.../PaimonAliyunDLFMetaStoreProperties.java | 6 +-
.../PaimonFileSystemMetaStoreProperties.java | 7 +-
.../metastore/PaimonHMSMetaStoreProperties.java | 6 +-
.../metastore/PaimonRestMetaStoreProperties.java | 2 +-
.../storage/AbstractS3CompatibleProperties.java | 4 -
.../property/storage/AzureProperties.java | 60 +-
.../property/storage/AzurePropertyUtils.java | 189 +++
.../property/storage/BrokerProperties.java | 8 +
.../datasource/property/storage/COSProperties.java | 5 +
.../datasource/property/storage/GCSProperties.java | 5 +
.../property/storage/HdfsCompatibleProperties.java | 7 +
.../property/storage/LocalProperties.java | 6 +
.../property/storage/MinioProperties.java | 5 +
.../datasource/property/storage/OBSProperties.java | 5 +
.../datasource/property/storage/OSSProperties.java | 5 +
.../datasource/property/storage/S3Properties.java | 10 +-
.../property/storage/S3PropertyUtils.java | 8 +-
.../property/storage/StorageProperties.java | 63 +-
.../org/apache/doris/encryption/EncryptionKey.java | 17 +
.../apache/doris/encryption/KeyManagerStore.java | 11 +-
.../org/apache/doris/encryption/RootKeyInfo.java | 2 +-
.../java/org/apache/doris/fs/SchemaTypeMapper.java | 3 +-
.../org/apache/doris/fs/obj/AzureObjStorage.java | 97 +-
.../java/org/apache/doris/fs/obj/S3ObjStorage.java | 4 +-
.../apache/doris/fs/remote/AzureFileSystem.java | 19 +-
.../org/apache/doris/fs/remote/ObjFileSystem.java | 26 +-
.../org/apache/doris/fs/remote/S3FileSystem.java | 24 +
.../apache/doris/fs/remote/dfs/DFSFileSystem.java | 3 +-
.../httpv2/controller/HardwareInfoController.java | 1 +
.../doris/httpv2/rest/TableSchemaAction.java | 80 +-
...Validator.java => BasicTokenizerValidator.java} | 23 +-
...or.java => CharReplaceCharFilterValidator.java} | 30 +-
...erValidator.java => ICUTokenizerValidator.java} | 9 +-
.../org/apache/doris/indexpolicy/IndexPolicy.java | 8 +-
.../apache/doris/indexpolicy/IndexPolicyMgr.java | 69 +-
.../doris/indexpolicy/IndexPolicyTypeEnum.java | 3 +-
...zerValidator.java => NoOperationValidator.java} | 10 +-
.../org/apache/doris/info/PartitionNamesInfo.java | 3 +
.../java/org/apache/doris/info/TableNameInfo.java | 22 +-
.../org/apache/doris/job/common/FailureReason.java | 6 +
.../insert/streaming/StreamingInsertJob.java | 138 +-
.../insert/streaming/StreamingInsertTask.java | 2 +-
.../insert/streaming/StreamingJobProperties.java | 72 +-
.../streaming/StreamingJobSchedulerTask.java | 21 +-
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 63 +-
.../org/apache/doris/job/manager/JobManager.java | 21 +-
.../java/org/apache/doris/job/offset/Offset.java | 5 +
.../doris/job/offset/SourceOffsetProvider.java | 6 +
.../org/apache/doris/job/offset/s3/S3Offset.java | 5 +
.../job/offset/s3/S3SourceOffsetProvider.java | 42 +-
.../job/scheduler/StreamingTaskScheduler.java | 31 +-
.../org/apache/doris/load/BrokerFileGroup.java | 10 +-
.../main/java/org/apache/doris/load/DeleteJob.java | 5 -
.../java/org/apache/doris/load/EtlJobType.java | 1 +
.../main/java/org/apache/doris/load/ExportJob.java | 4 +-
.../main/java/org/apache/doris/load/ExportMgr.java | 6 +-
.../org/apache/doris/load/RoutineLoadDesc.java | 12 +-
.../apache/doris/load/loadv2/InsertLoadJob.java | 6 +-
.../org/apache/doris/load/loadv2/LoadManager.java | 5 -
.../apache/doris/load/loadv2/MysqlLoadManager.java | 6 +-
.../load/routineload/KafkaRoutineLoadJob.java | 11 +-
.../doris/load/routineload/KafkaTaskInfo.java | 21 +-
.../doris/load/routineload/RoutineLoadJob.java | 38 +-
.../load/routineload/RoutineLoadTaskInfo.java | 12 +-
.../load/routineload/RoutineLoadTaskScheduler.java | 4 +-
.../doris/metric/PrometheusMetricVisitor.java | 4 +-
.../MTMVAnalyzeQueryInfo.java} | 34 +-
.../org/apache/doris/mtmv/MTMVHookService.java | 14 +
.../java/org/apache/doris/mtmv/MTMVJobManager.java | 9 +
.../org/apache/doris/mtmv/MTMVPartitionInfo.java | 19 +
.../org/apache/doris/mtmv/MTMVPartitionUtil.java | 26 +-
.../java/org/apache/doris/mtmv/MTMVPlanUtil.java | 261 +++-
.../doris/mtmv/MTMVRefreshPartitionSnapshot.java | 6 +-
.../java/org/apache/doris/mtmv/MTMVRelation.java | 34 +-
.../org/apache/doris/mtmv/MTMVRelationManager.java | 99 +-
.../org/apache/doris/mtmv/MTMVRewriteUtil.java | 2 +-
.../java/org/apache/doris/mtmv/MTMVService.java | 26 +-
.../main/java/org/apache/doris/mtmv/MTMVUtil.java | 10 +-
.../java/org/apache/doris/mysql/MysqlChannel.java | 4 +-
.../mysql/privilege/AccessControllerManager.java | 6 -
.../org/apache/doris/nereids/CascadesContext.java | 8 +-
.../org/apache/doris/nereids/NereidsPlanner.java | 2 +-
.../org/apache/doris/nereids/StatementContext.java | 23 +
.../doris/nereids/analyzer/UnboundAlias.java | 10 +
.../doris/nereids/analyzer/UnboundFunction.java | 14 +
.../doris/nereids/analyzer/UnboundInlineTable.java | 6 +
.../nereids/analyzer/UnboundOneRowRelation.java | 12 +
.../doris/nereids/analyzer/UnboundRelation.java | 23 +
.../doris/nereids/analyzer/UnboundResultSink.java | 5 +
.../apache/doris/nereids/analyzer/UnboundSlot.java | 5 +
.../apache/doris/nereids/analyzer/UnboundStar.java | 15 +
.../doris/nereids/analyzer/UnboundTVFRelation.java | 7 +
.../doris/nereids/analyzer/UnboundTableSink.java | 11 +
.../doris/nereids/analyzer/UnboundVariable.java | 5 +
.../doris/nereids/glue/LogicalPlanAdapter.java | 2 +-
.../glue/translator/PhysicalPlanTranslator.java | 20 +-
.../glue/translator/PlanTranslatorContext.java | 6 +-
.../doris/nereids/jobs/executor/Rewriter.java | 7 +-
.../executor/TableCollectAndHookInitializer.java | 16 +-
.../jobs/rewrite/TopDownVisitorRewriteJob.java | 2 -
.../doris/nereids/load/NereidsBrokerFileGroup.java | 10 +-
.../doris/nereids/load/NereidsBrokerLoadTask.java | 12 +-
.../doris/nereids/load/NereidsDataDescription.java | 68 +-
.../nereids/load/NereidsLoadPlanInfoCollector.java | 27 +-
.../doris/nereids/load/NereidsLoadTaskInfo.java | 4 +-
.../doris/nereids/load/NereidsLoadUtils.java | 10 +-
.../nereids/load/NereidsLoadingTaskPlanner.java | 14 +-
.../nereids/load/NereidsRoutineLoadTaskInfo.java | 12 +-
.../nereids/load/NereidsStreamLoadPlanner.java | 6 +-
.../doris/nereids/load/NereidsStreamLoadTask.java | 14 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 153 ++-
.../processor/pre/PullUpSubqueryAliasToCTE.java | 8 +-
.../apache/doris/nereids/properties/OrderKey.java | 18 +
.../nereids/properties/RequestPropertyDeriver.java | 11 +
.../doris/nereids/rules/analysis/AnalyzeCTE.java | 64 +-
.../rules/analysis/AvgDistinctToSumDivCount.java | 3 +-
.../nereids/rules/analysis/BindExpression.java | 32 +-
.../doris/nereids/rules/analysis/BindRelation.java | 5 +
.../doris/nereids/rules/analysis/BindSink.java | 4 +-
.../nereids/rules/analysis/CheckSearchUsage.java | 29 +-
.../rules/analysis/CollectOneLevelRelation.java | 4 +-
.../nereids/rules/analysis/CollectRelation.java | 19 +-
.../nereids/rules/analysis/ExpressionAnalyzer.java | 67 +-
.../rules/analysis/GetFormatFunctionBinder.java | 78 ++
.../LogicalSubQueryAliasToLogicalProject.java | 20 +-
.../nereids/rules/analysis/SubqueryToApply.java | 3 +-
.../mv/InitMaterializationContextHook.java | 24 +-
.../exploration/mv/MaterializedViewUtils.java | 13 +-
.../rules/expression/ExpressionOptimization.java | 6 +-
.../rules/expression/ExpressionRewrite.java | 58 +-
.../rules/expression/ExpressionRewriteContext.java | 28 +
.../rules/expression/ExpressionRuleType.java | 4 +-
.../nereids/rules/expression/check/CheckCast.java | 29 -
.../rules/CaseWhenToCompoundPredicate.java | 152 +++
.../expression/rules/CondReplaceNullWithFalse.java | 61 +
.../rules/expression/rules/ConditionRewrite.java | 183 +++
.../expression/rules/FoldConstantRuleOnFE.java | 58 +-
.../expression/rules/NullSafeEqualToEqual.java | 49 +-
.../rules/OneListPartitionEvaluator.java | 5 +-
.../rules/ReplaceNullWithFalseForCond.java | 131 --
.../rules/SimplifyArithmeticComparisonRule.java | 2 +-
.../LogicalEsScanToPhysicalEsScan.java | 9 +-
.../LogicalJdbcScanToPhysicalJdbcScan.java | 10 +-
.../LogicalOdbcScanToPhysicalOdbcScan.java | 10 +-
.../LogicalOlapScanToPhysicalOlapScan.java | 5 +-
...eChildToPhysicalRecursiveCteRecursiveChild.java | 1 +
.../LogicalRecursiveCteToPhysicalRecursiveCte.java | 3 +-
.../doris/nereids/rules/rewrite/CTEInline.java | 25 +-
.../doris/nereids/rules/rewrite/ColumnPruning.java | 39 +-
.../nereids/rules/rewrite/ConstantPropagation.java | 78 +-
.../nereids/rules/rewrite/EliminateFilter.java | 14 +-
.../rules/rewrite/PushProjectIntoUnion.java | 2 +-
.../doris/nereids/rules/rewrite/ReorderJoin.java | 4 +
.../rules/rewrite/RewriteSearchToSlots.java | 61 +-
.../org/apache/doris/nereids/trees/TreeNode.java | 4 +
.../trees/copier/LogicalPlanDeepCopier.java | 5 +-
.../doris/nereids/trees/expressions/Alias.java | 12 +
.../nereids/trees/expressions/BinaryOperator.java | 9 +
.../doris/nereids/trees/expressions/CaseWhen.java | 14 +
.../doris/nereids/trees/expressions/Cast.java | 11 +
.../trees/expressions/CompoundPredicate.java | 10 +
.../trees/expressions/DefaultValueSlot.java | 5 +
.../doris/nereids/trees/expressions/Exists.java | 10 +
.../trees/expressions/ExpressionEvaluator.java | 3 +-
.../nereids/trees/expressions/InPredicate.java | 25 +
.../nereids/trees/expressions/InSubquery.java | 10 +
.../doris/nereids/trees/expressions/IsNull.java | 8 +
.../doris/nereids/trees/expressions/Not.java | 7 +
.../nereids/trees/expressions/OrderExpression.java | 5 +
.../nereids/trees/expressions/Placeholder.java | 4 +
.../nereids/trees/expressions/Properties.java | 5 +
.../nereids/trees/expressions/ScalarSubquery.java | 9 +
.../trees/expressions/SearchExpression.java | 8 +-
.../trees/expressions/StringRegexPredicate.java | 11 +
.../doris/nereids/trees/expressions/Subtract.java | 13 +
.../trees/expressions/TimestampArithmetic.java | 15 +
.../nereids/trees/expressions/UnaryOperator.java | 8 +
.../nereids/trees/expressions/WhenClause.java | 7 +
.../trees/expressions/WindowExpression.java | 18 +
.../nereids/trees/expressions/WindowFrame.java | 38 +
.../trees/expressions/functions/BoundFunction.java | 11 +
.../functions/agg/AggregateFunction.java | 19 +
.../trees/expressions/functions/agg/Count.java | 8 +
.../executable/DateTimeExtractAndTransform.java | 108 +-
.../functions/executable/StringArithmetic.java | 101 ++
.../expressions/functions/generator/Explode.java | 7 +-
.../functions/generator/ExplodeOuter.java | 7 +-
.../functions/generator/ExplodeVariantArray.java | 7 +-
.../generator/TableGeneratingFunction.java | 5 +
.../expressions/functions/scalar/ElementAt.java | 15 +-
.../scalar/{MaskLastN.java => ExportSet.java} | 58 +-
.../scalar/{Length.java => FromBase64Binary.java} | 30 +-
.../scalar/{Length.java => GetFormat.java} | 35 +-
.../scalar/{Length.java => JsonHash.java} | 33 +-
.../trees/expressions/functions/scalar/Lambda.java | 12 +
.../trees/expressions/functions/scalar/Length.java | 5 +-
.../expressions/functions/scalar/MaskFirstN.java | 8 -
.../expressions/functions/scalar/MaskLastN.java | 8 -
.../trees/expressions/functions/scalar/Search.java | 46 +-
.../functions/scalar/SearchDslParser.java | 376 +++++-
.../scalar/{MaskFirstN.java => SubBinary.java} | 61 +-
.../scalar/{Length.java => ToBase64Binary.java} | 23 +-
.../trees/expressions/literal/ArrayLiteral.java | 7 +
.../trees/expressions/literal/Interval.java | 9 +
.../nereids/trees/expressions/literal/Literal.java | 5 +
.../trees/expressions/literal/NullLiteral.java | 4 +-
.../expressions/visitor/ScalarFunctionVisitor.java | 30 +
.../apache/doris/nereids/trees/plans/PlanType.java | 4 +
.../trees/plans/commands/AlterJobCommand.java | 110 +-
.../plans/commands/AlterJobStatusCommand.java | 23 +-
.../trees/plans/commands/AlterTableCommand.java | 2 +-
...mand.java => CreateIndexCharFilterCommand.java} | 64 +-
.../commands/CreateMaterializedViewCommand.java | 3 +
...ommand.java => DropIndexCharFilterCommand.java} | 37 +-
.../trees/plans/commands/DropJobCommand.java | 24 +-
.../trees/plans/commands/ExplainCommand.java | 8 +
.../trees/plans/commands/ExportCommand.java | 70 +-
.../nereids/trees/plans/commands/LoadCommand.java | 4 +-
.../trees/plans/commands/PauseJobCommand.java | 7 -
.../trees/plans/commands/ResumeJobCommand.java | 7 -
.../trees/plans/commands/ShowBackupCommand.java | 6 +
.../trees/plans/commands/ShowConfigCommand.java | 20 +-
...ommand.java => ShowIndexCharFilterCommand.java} | 36 +-
.../commands/ShowQueuedAnalyzeJobsCommand.java | 9 +-
.../trees/plans/commands/ShowRestoreCommand.java | 6 +
.../plans/commands/ShowWorkloadGroupsCommand.java | 10 +
.../trees/plans/commands/UpdateCommand.java | 22 +-
.../plans/commands/UpdateMvByPartitionCommand.java | 9 +-
.../trees/plans/commands/info/AlterViewInfo.java | 5 +-
.../trees/plans/commands/info/BuildIndexOp.java | 3 +-
.../trees/plans/commands/info/CopyFromDesc.java | 6 +-
.../trees/plans/commands/info/CopyIntoInfo.java | 17 +-
.../trees/plans/commands/info/CreateIndexOp.java | 3 +-
.../trees/plans/commands/info/CreateJobInfo.java | 7 +-
.../trees/plans/commands/info/CreateMTMVInfo.java | 174 +--
.../plans/commands/info/CreateRoutineLoadInfo.java | 10 +-
.../trees/plans/commands/info/CreateTableInfo.java | 8 +
.../trees/plans/commands/info/CreateViewInfo.java | 5 +-
.../trees/plans/commands/info/DropIndexOp.java | 3 +-
.../trees/plans/commands/info/IndexDefinition.java | 5 +-
.../trees/plans/commands/info/ModifyColumnOp.java | 4 -
.../trees/plans/commands/info/SetSessionVarOp.java | 10 +
.../commands/insert/AbstractInsertExecutor.java | 26 +-
.../insert/BaseExternalTableInsertExecutor.java | 4 +-
.../insert/BatchInsertIntoTableCommand.java | 7 +
.../commands/insert/BlackholeInsertExecutor.java | 4 +-
.../commands/insert/DictionaryInsertExecutor.java | 4 +-
.../plans/commands/insert/HiveInsertExecutor.java | 4 +-
.../commands/insert/IcebergInsertExecutor.java | 4 +-
.../commands/insert/InsertIntoTableCommand.java | 31 +-
.../insert/InsertOverwriteTableCommand.java | 12 +
.../plans/commands/insert/JdbcInsertExecutor.java | 4 +-
.../plans/commands/insert/OlapInsertExecutor.java | 3 -
.../plans/commands/merge/MergeIntoCommand.java | 534 ++++++++
.../plans/commands/merge/MergeMatchedClause.java | 55 +
.../commands/merge/MergeNotMatchedClause.java | 55 +
.../plans/distribute/worker/BackendWorker.java | 2 +-
.../trees/plans/logical/LogicalAggregate.java | 68 +-
.../nereids/trees/plans/logical/LogicalCTE.java | 41 +-
.../plans/logical/LogicalCatalogRelation.java | 49 +-
.../trees/plans/logical/LogicalCheckPolicy.java | 22 +
.../logical/LogicalDeferMaterializeOlapScan.java | 14 +-
.../nereids/trees/plans/logical/LogicalEsScan.java | 39 +-
.../nereids/trees/plans/logical/LogicalExcept.java | 9 +
.../trees/plans/logical/LogicalFileScan.java | 38 +-
.../trees/plans/logical/LogicalFileSink.java | 10 +
.../nereids/trees/plans/logical/LogicalFilter.java | 12 +
.../trees/plans/logical/LogicalGenerate.java | 22 +
.../nereids/trees/plans/logical/LogicalHaving.java | 13 +
.../trees/plans/logical/LogicalIntersect.java | 9 +
.../trees/plans/logical/LogicalJdbcScan.java | 29 +-
.../nereids/trees/plans/logical/LogicalJoin.java | 18 +
.../nereids/trees/plans/logical/LogicalLimit.java | 11 +
.../trees/plans/logical/LogicalOdbcScan.java | 25 +-
.../trees/plans/logical/LogicalOlapScan.java | 63 +-
.../trees/plans/logical/LogicalProject.java | 19 +
.../trees/plans/logical/LogicalQualify.java | 10 +
.../trees/plans/logical/LogicalRecursiveCte.java | 220 +--
.../logical/LogicalRecursiveCteRecursiveChild.java | 30 +-
.../plans/logical/LogicalRecursiveCteScan.java | 27 +-
.../nereids/trees/plans/logical/LogicalRepeat.java | 45 +-
.../trees/plans/logical/LogicalSchemaScan.java | 53 +-
.../trees/plans/logical/LogicalSelectHint.java | 5 +
.../nereids/trees/plans/logical/LogicalSort.java | 12 +
.../trees/plans/logical/LogicalSubQueryAlias.java | 66 +-
.../trees/plans/logical/LogicalTestScan.java | 25 +-
.../nereids/trees/plans/logical/LogicalUnion.java | 11 +-
.../trees/plans/logical/LogicalUsingJoin.java | 16 +
.../nereids/trees/plans/logical/LogicalWindow.java | 16 +
.../plans/physical/PhysicalCatalogRelation.java | 33 +-
.../trees/plans/physical/PhysicalEsScan.java | 29 +-
.../trees/plans/physical/PhysicalJdbcScan.java | 33 +-
.../trees/plans/physical/PhysicalOdbcScan.java | 31 +-
.../trees/plans/physical/PhysicalOlapScan.java | 43 +-
.../trees/plans/physical/PhysicalRecursiveCte.java | 38 +-
.../PhysicalRecursiveCteRecursiveChild.java | 31 +-
.../trees/plans/visitor/CommandVisitor.java | 23 +
.../apache/doris/nereids/util/ExpressionUtils.java | 11 +-
.../java/org/apache/doris/nereids/util/Utils.java | 5 +-
.../org/apache/doris/planner/HiveTableSink.java | 3 +-
.../org/apache/doris/planner/IcebergTableSink.java | 5 +-
.../org/apache/doris/planner/PlanFragment.java | 2 +-
.../org/apache/doris/planner/RecursiveCteNode.java | 47 +-
.../apache/doris/planner/RecursiveCteScanNode.java | 9 +
.../java/org/apache/doris/planner/ResultSink.java | 2 +-
.../java/org/apache/doris/qe/ConnectContext.java | 2 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 1 +
.../org/apache/doris/qe/CoordinatorContext.java | 1 +
.../java/org/apache/doris/qe/SessionVariable.java | 220 ++-
.../qe/runtime/RuntimeFiltersThriftBuilder.java | 15 +-
.../doris/qe/runtime/ThriftPlansBuilder.java | 105 +-
.../apache/doris/service/FrontendServiceImpl.java | 250 ++--
.../sessions/FlightSqlConnectContext.java | 2 +-
.../apache/doris/statistics/AnalysisManager.java | 32 +-
.../doris/statistics/StatisticsAutoCollector.java | 20 +-
.../doris/statistics/StatisticsJobAppender.java | 17 +-
.../doris/statistics/util/StatisticsUtil.java | 24 +-
.../org/apache/doris/system/SystemInfoService.java | 14 +-
.../ExternalFileTableValuedFunction.java | 3 +-
.../tablefunction/HudiTableValuedFunction.java | 6 +-
.../tablefunction/IcebergTableValuedFunction.java | 6 +-
.../tablefunction/JobsTableValuedFunction.java | 7 -
.../doris/tablefunction/MetadataGenerator.java | 52 +-
.../tablefunction/PaimonTableValuedFunction.java | 6 +-
.../tablefunction/TasksTableValuedFunction.java | 7 -
.../java/org/apache/doris/task/LoadTaskInfo.java | 4 +-
.../doris/transaction/PublishVersionDaemon.java | 3 +
.../org/apache/doris/alter/CloudIndexTest.java | 12 +-
.../org/apache/doris/alter/IndexChangeJobTest.java | 25 +-
.../doris/analysis/PartitionExprUtilTest.java | 223 +++
.../doris/catalog/AutoIncrementGeneratorTest.java | 178 +++
.../doris/catalog/ListPartitionInfoTest.java | 6 +-
.../apache/doris/catalog/MetadataViewerTest.java | 4 +-
.../doris/catalog/RangePartitionInfoTest.java | 4 +-
.../cloud/system/CloudSystemInfoServiceTest.java | 645 ++++++++-
.../apache/doris/common/PropertyAnalyzerTest.java | 90 +-
.../doris/common/proc/IndexSchemaProcNodeTest.java | 6 +-
.../common/util/AutoBucketCalculatorTest.java | 106 ++
.../org/apache/doris/common/util/S3UtilTest.java | 252 ++++
.../doris/datasource/TestHMSCachedClient.java | 4 +-
.../datasource/iceberg/IcebergMetadataOpTest.java | 48 +
.../datasource/iceberg/IcebergPredicateTest.java | 20 +-
.../datasource/lakesoul/LakeSoulPredicateTest.java | 20 +-
.../metastore/AbstractPaimonPropertiesTest.java | 89 ++
.../HMSAliyunDLFMetaStorePropertiesTest.java | 4 +-
.../datasource/property/metastore/HMSGlueIT.java | 2 +-
.../metastore/HMSGlueMetaStorePropertiesTest.java | 16 +-
.../property/metastore/HMSPropertiesTest.java | 14 +-
.../metastore/IcebergRestPropertiesTest.java | 4 +-
.../PaimonRestMetaStorePropertiesTest.java | 14 +-
.../property/storage/AzurePropertiesTest.java | 28 +-
.../property/storage/AzurePropertyUtilsTest.java | 145 ++
.../property/storage/COSPropertiesTest.java | 16 +
.../property/storage/GCSPropertiesTest.java | 18 +
.../property/storage/OBSPropertyTest.java | 18 +
.../property/storage/OSSPropertiesTest.java | 18 +
.../property/storage/S3PropertiesTest.java | 17 +
.../TrinoConnectorPredicateTest.java | 36 +-
.../apache/doris/load/loadv2/ExportMgrTest.java | 4 +-
.../load/routineload/KafkaRoutineLoadJobTest.java | 6 +-
.../java/org/apache/doris/mtmv/AlterMTMVTest.java | 15 +-
.../apache/doris/mtmv/MTMVPartitionUtilTest.java | 54 +-
.../org/apache/doris/mtmv/MTMVPlanUtilTest.java | 226 +++-
.../apache/doris/mtmv/MTMVRelationManagerTest.java | 42 +-
.../org/apache/doris/mtmv/MTMVRelationTest.java | 117 ++
.../org/apache/doris/mtmv/MTMVRewriteUtilTest.java | 12 +-
.../apache/doris/mtmv/MTMVSchemaChangeTest.java | 94 ++
.../java/org/apache/doris/mtmv/MTMVTaskTest.java | 11 +-
.../test/java/org/apache/doris/mtmv/MTMVTest.java | 21 +-
.../doris/nereids/jobs/RewriteTopDownJobTest.java | 5 +
.../nereids/parser/NereidsParserDigestTest.java | 300 +++++
.../doris/nereids/parser/NereidsParserTest.java | 109 +-
.../parser/UnboundFunctionWithUnboundStarTest.java | 106 ++
.../nereids/rules/analysis/AnalyzeCTETest.java | 219 +++
.../rules/analysis/ExpressionAnalyzerTest.java | 36 +
.../expression/ExpressionRewriteTestHelper.java | 3 +-
.../nereids/rules/expression/FoldConstantTest.java | 30 +-
.../rules/expression/check/CheckCastTest.java | 48 +-
.../rules/CaseWhenToCompoundPredicateTest.java | 87 ++
.../rules/CondReplaceNullWithFalseTest.java | 145 ++
.../expression/rules/ExpressionRewriteSqlTest.java | 2 +-
.../expression/rules/NullSafeEqualToEqualTest.java | 54 +-
.../rules/ReplaceNullWithFalseForCondTest.java | 110 --
.../doris/nereids/rules/rewrite/CTEInlineTest.java | 84 ++
.../nereids/rules/rewrite/ColumnPruningTest.java | 30 +
.../rules/rewrite/ConstantPropagationTest.java | 25 +-
.../GeneratePartitionTopnFromWindowTest.java | 40 +
.../nereids/rules/rewrite/ReorderJoinTest.java | 19 +
.../functions/scalar/SearchDslParserTest.java | 327 +++++
.../plans/commands/merge/MergeIntoCommandTest.java | 956 +++++++++++++
.../org/apache/doris/nereids/util/PlanChecker.java | 6 +-
.../org/apache/doris/planner/OlapScanNodeTest.java | 8 +-
.../statistics/StatisticsAutoCollectorTest.java | 14 +-
.../statistics/StatisticsJobAppenderTest.java | 58 +-
.../apache/doris/system/SystemInfoServiceTest.java | 61 +
.../apache/doris/utframe/TestWithFeService.java | 7 +
fe/pom.xml | 18 +
gensrc/proto/cloud.proto | 30 +
gensrc/proto/internal_service.proto | 43 +-
gensrc/proto/segment_v2.proto | 5 +
gensrc/script/gen_build_version.sh | 5 +
gensrc/thrift/AgentService.thrift | 5 +-
gensrc/thrift/DataSinks.thrift | 4 +-
gensrc/thrift/Descriptors.thrift | 1 +
gensrc/thrift/Exprs.thrift | 5 +-
gensrc/thrift/PaloInternalService.thrift | 17 +-
gensrc/thrift/PlanNodes.thrift | 2 +
regression-test/conf/regression-conf.groovy | 6 +
.../product_quantization.out} | 6 +-
regression-test/data/cast_p0/datelike_print.out | 37 +
.../routine_load/test_routine_load.out | 33 +
.../test_schema_change_with_readd_cluster.out} | 6 +-
.../multi_cluster/stream_load}/all_types.csv | 0
.../multi_cluster/stream_load/stream_load.out} | 12 +-
.../multi_cluster/stream_load/stream_load_2pc.out | 25 +
.../multi_cluster/stream_load/stream_load_lb.out | 25 +
.../test_apsaradb_internal_stage.out} | 7 +-
.../data/correctness/test_compare_float.out | 8 -
.../data/correctness_p0/test_mask_function.out | 48 +-
.../data/correctness_p0/test_probe_clean.out | 6 -
.../one_level_nestedtypes_with_s3data.out | 1020 +++++++-------
.../string_len/test_string_len_complex.out | 43 +
.../sql-functions/doc_date_functions_test.out | 351 +++++
.../outfile/test_outfile_parquet_schema.out | 11 +
.../cache/test_file_cache_statistics.out | 5 +-
.../data/external_table_p0/tvf/hdfs_data_1.txt | 1 +
.../data/external_table_p0/tvf/hdfs_data_2.txt | 1 +
.../data/external_table_p0/tvf/hdfs_data_3.txt | 1 +
.../tvf/test_s3_tvf_number_range.out | 57 +
.../maxcompute/test_max_compute_schema.out | 258 ++++
.../cloud/test_cloud_version_already_merged.out | 15 +
.../test_cast_to_string_from_decimalv3.out | 73 +
.../to_string/test_cast_to_string_from_int.out | 59 +
.../analyzer/test_custom_analyzer.out | 6 +
.../analyzer/test_custom_analyzer1.out | 6 +
.../analyzer/test_custom_analyzer2.out | 16 +
.../test_storage_format_default.out} | 3 +-
.../test_streaming_insert_job_alter.out | 23 +
.../test_streaming_insert_job_crud.out | 23 +
.../test_streaming_insert_job_offset.out | 45 +
regression-test/data/jsonb_p0/test_json_hash.out | 49 +
.../data/load_p0/broker_load/test_array_load.out | 48 +
.../test_broker_load_with_partition.out | 11 +-
.../broker_load/test_broker_load_with_where.out | 4 +-
..._with_where.out => test_s3_load_with_where.out} | 4 +-
.../load_p0/insert/test_insert_move_memtable.out | 29 -
.../data/load_p0/merge_into/test_merge_into.out | 17 +
.../data/mtmv_p0/test_create_mtmv_with_view.out | 30 +
.../mtmv_p0/test_create_mtmv_with_view_alter.out | 19 +
.../test_create_mtmv_with_view_alter_table.out | 19 +
.../test_create_mtmv_with_view_commit.out} | 7 +-
.../test_create_mtmv_with_view_cte.out} | 7 +-
.../mtmv_p0/test_create_mtmv_with_view_mtmv.out | 20 +
.../mtmv_p0/test_create_mtmv_with_view_pct.out | 21 +
.../mtmv_p0/test_create_mtmv_with_view_rollup.out | 9 +
...mtmv.out => test_limit_partition_view_mtmv.out} | 0
.../data/mtmv_p0/test_view_hive_mtmv.out | 28 +
.../partition_topn/check_orderkey.out} | 3 +-
.../datetime_functions/test_date_function_v2.out | 81 ++
.../json_function/test_query_json_object.out | 16 +-
.../agg_skew_rewrite/agg_skew_rewrite.out | 10 +-
.../avg_distinct_to_sum_div_count.out} | 3 +-
.../nereids_rules_p0/eager_aggregate/basic.out | 32 +-
.../eager_aggregate/basic_one_side.out | 32 +-
...h_down_count_distinct_through_join_one_side.out | 20 +-
.../push_down_count_through_join.out | 352 ++---
.../push_down_count_through_join_one_side.out | 352 ++---
.../eager_aggregate/push_down_max_through_join.out | 196 +--
...ush_down_min_distinct_through_join_one_side.out | 20 +-
.../eager_aggregate/push_down_min_through_join.out | 196 +--
...ush_down_sum_distinct_through_join_one_side.out | 16 +-
.../eager_aggregate/push_down_sum_through_join.out | 196 +--
.../push_down_sum_through_join_one_side.out | 196 +--
.../eliminate_gby_key/eliminate_group_by.out | 84 +-
.../eliminate_group_by_key_by_uniform.out | 2 +-
.../eliminate_join_condition.out | 24 +-
.../eliminate_outer_join/eliminate_outer_join.out | 130 +-
...simplify_comparison_predicate_int_vs_double.out | 4 +-
.../extend_infer_equal_predicate.out | 302 ++---
.../infer_predicate/infer_intersect_except.out | 8 +-
.../infer_predicate/infer_unequal_predicates.out | 68 +-
.../infer_predicate/pull_up_predicate_agg.out | 4 +-
.../infer_predicate/pull_up_predicate_literal.out | 258 ++--
.../infer_predicate/pull_up_predicate_set_op.out | 84 +-
.../data/nereids_rules_p0/pkfk/eliminate_inner.out | 24 +-
.../predicate_infer/infer_predicate.out | 152 +--
.../pull_up_join_from_union.out | 82 +-
.../push_down_distinct_through_join.out | 10 +-
.../extract_from_disjunction_in_join.out | 32 +-
.../push_down_limit_distinct_through_join.out | 4 +-
.../push_down_top_n_distinct_through_join.out | 6 +-
.../push_down_top_n_distinct_through_union.out | 44 +-
.../push_down_top_n_through_union.out | 44 +-
.../data/nereids_rules_p0/salt_join/salt_join.out | 72 +-
.../data/query_p0/cache/condition_cache.out | 121 ++
.../join/eliminate_const_join_condition.out | 24 +-
.../conditional_functions/test_coalesce.out | 28 +
.../conditional_functions/test_query_in.out | 14 +
.../string_functions/test_string_all.out | 1414 ++++++++++++++++++++
.../string_functions/test_string_function.out | 217 +++
.../sql_functions/table_function/explode.out | 88 ++
.../system/test_query_sys_column_data_sizes.out | 15 +
.../data/query_p0/system/test_table_properties.out | 12 +-
.../adjust_virtual_slot_nullable.out | 4 +-
.../data/rec_cte_p0/rec_cte/rec_cte.out | 953 +++++++++++++
.../rec_cte_from_ck_doc/rec_cte_from_ck_doc.out | 29 +
.../rec_cte_from_duckdb_doc.out | 30 +
.../rec_cte_from_mysql_doc.out | 42 +
.../search/test_search_vs_match_consistency.out | 6 +
.../data/shape_check/ssb_sf100/shape/flat.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query10.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query14.out | 12 +-
.../tpcds_sf100/noStatsRfPrune/query16.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query17.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query18.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query25.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query29.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query3.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query35.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query36.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query41.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query42.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query44.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query46.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query49.out | 12 +-
.../tpcds_sf100/noStatsRfPrune/query50.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query52.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query59.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query6.out | 12 +-
.../tpcds_sf100/noStatsRfPrune/query64.out | 22 +-
.../tpcds_sf100/noStatsRfPrune/query68.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query69.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query70.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query72.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query85.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query86.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query94.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query95.out | 6 +-
.../tpcds_sf100/no_stats_shape/query10.out | 4 +-
.../tpcds_sf100/no_stats_shape/query14.out | 12 +-
.../tpcds_sf100/no_stats_shape/query16.out | 6 +-
.../tpcds_sf100/no_stats_shape/query17.out | 6 +-
.../tpcds_sf100/no_stats_shape/query18.out | 4 +-
.../tpcds_sf100/no_stats_shape/query25.out | 6 +-
.../tpcds_sf100/no_stats_shape/query29.out | 6 +-
.../tpcds_sf100/no_stats_shape/query3.out | 2 +-
.../tpcds_sf100/no_stats_shape/query35.out | 4 +-
.../tpcds_sf100/no_stats_shape/query36.out | 2 +-
.../tpcds_sf100/no_stats_shape/query41.out | 2 +-
.../tpcds_sf100/no_stats_shape/query42.out | 2 +-
.../tpcds_sf100/no_stats_shape/query44.out | 4 +-
.../tpcds_sf100/no_stats_shape/query46.out | 2 +-
.../tpcds_sf100/no_stats_shape/query49.out | 12 +-
.../tpcds_sf100/no_stats_shape/query50.out | 4 +-
.../tpcds_sf100/no_stats_shape/query52.out | 2 +-
.../tpcds_sf100/no_stats_shape/query59.out | 4 +-
.../tpcds_sf100/no_stats_shape/query6.out | 12 +-
.../tpcds_sf100/no_stats_shape/query64.out | 22 +-
.../tpcds_sf100/no_stats_shape/query68.out | 2 +-
.../tpcds_sf100/no_stats_shape/query69.out | 4 +-
.../tpcds_sf100/no_stats_shape/query70.out | 2 +-
.../tpcds_sf100/no_stats_shape/query72.out | 6 +-
.../tpcds_sf100/no_stats_shape/query85.out | 4 +-
.../tpcds_sf100/no_stats_shape/query86.out | 2 +-
.../tpcds_sf100/no_stats_shape/query94.out | 6 +-
.../tpcds_sf100/no_stats_shape/query95.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query10.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query14.out | 12 +-
.../shape_check/tpcds_sf100/rf_prune/query16.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query17.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query18.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query25.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query29.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query3.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query35.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query36.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query41.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query42.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query44.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query46.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query49.out | 12 +-
.../shape_check/tpcds_sf100/rf_prune/query50.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query52.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query59.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query6.out | 12 +-
.../shape_check/tpcds_sf100/rf_prune/query64.out | 22 +-
.../shape_check/tpcds_sf100/rf_prune/query68.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query69.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query70.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query72.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query85.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query86.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query94.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query95.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query10.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query14.out | 12 +-
.../data/shape_check/tpcds_sf100/shape/query16.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query17.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query18.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query25.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query29.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query3.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query35.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query36.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query41.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query42.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query44.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query46.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query49.out | 12 +-
.../data/shape_check/tpcds_sf100/shape/query50.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query52.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query59.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query6.out | 12 +-
.../data/shape_check/tpcds_sf100/shape/query64.out | 22 +-
.../data/shape_check/tpcds_sf100/shape/query68.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query69.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query70.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query72.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query85.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query86.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query94.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query95.out | 6 +-
.../tpcds_sf1000/bs_downgrade_shape/query44.out | 4 +-
.../tpcds_sf1000/bs_downgrade_shape/query6.out | 12 +-
.../tpcds_sf1000/bs_downgrade_shape/query68.out | 2 +-
.../tpcds_sf1000/bs_downgrade_shape/query95.out | 6 +-
.../tpcds_sf1000/eliminate_empty/query10_empty.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query10.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query14.out | 12 +-
.../data/shape_check/tpcds_sf1000/hint/query16.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query17.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query18.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query25.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query29.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query3.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query36.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query41.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query42.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query44.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query46.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query49.out | 12 +-
.../data/shape_check/tpcds_sf1000/hint/query50.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query52.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query59.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query6.out | 12 +-
.../data/shape_check/tpcds_sf1000/hint/query64.out | 22 +-
.../data/shape_check/tpcds_sf1000/hint/query68.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query69.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query70.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query72.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query85.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query86.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query94.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query95.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query10.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query14.out | 12 +-
.../shape_check/tpcds_sf1000/shape/query16.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query17.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query18.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query25.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query29.out | 6 +-
.../data/shape_check/tpcds_sf1000/shape/query3.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query35.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query36.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query41.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query42.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query44.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query46.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query49.out | 12 +-
.../shape_check/tpcds_sf1000/shape/query50.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query52.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query59.out | 4 +-
.../data/shape_check/tpcds_sf1000/shape/query6.out | 12 +-
.../shape_check/tpcds_sf1000/shape/query64.out | 22 +-
.../shape_check/tpcds_sf1000/shape/query68.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query69.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query70.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query72.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query85.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query86.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query94.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query95.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query10.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query14.out | 12 +-
.../shape_check/tpcds_sf10t_orc/shape/query16.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query17.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query18.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query25.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query29.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query3.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query35.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query36.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query41.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query42.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query44.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query46.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query49.out | 12 +-
.../shape_check/tpcds_sf10t_orc/shape/query50.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query52.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query59.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query6.out | 12 +-
.../shape_check/tpcds_sf10t_orc/shape/query64.out | 22 +-
.../shape_check/tpcds_sf10t_orc/shape/query68.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query69.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query70.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query72.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query85.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query86.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query94.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query95.out | 6 +-
.../data/shape_check/tpch_sf1000/hint/q7.out | 4 +-
.../data/shape_check/tpch_sf1000/hint/q8.out | 4 +-
.../tpch_sf1000/nostats_rf_prune/q21.out | 6 +-
.../tpch_sf1000/nostats_rf_prune/q7.out | 4 +-
.../tpch_sf1000/nostats_rf_prune/q8.out | 4 +-
.../data/shape_check/tpch_sf1000/rf_prune/q21.out | 6 +-
.../data/shape_check/tpch_sf1000/rf_prune/q7.out | 4 +-
.../data/shape_check/tpch_sf1000/rf_prune/q8.out | 4 +-
.../data/shape_check/tpch_sf1000/shape/q21.out | 6 +-
.../data/shape_check/tpch_sf1000/shape/q7.out | 4 +-
.../data/shape_check/tpch_sf1000/shape/q8.out | 4 +-
.../shape_check/tpch_sf1000/shape_no_stats/q21.out | 6 +-
.../shape_check/tpch_sf1000/shape_no_stats/q7.out | 4 +-
.../shape_check/tpch_sf1000/shape_no_stats/q8.out | 4 +-
.../test_show_create_table_and_views_nereids.out | 8 +-
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q2.1.out | 283 ----
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q2.2.out | 59 -
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q2.3.out | 10 -
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q3.1.out | 153 ---
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q3.2.out | 317 -----
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q3.3.out | 9 -
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q4.1.out | 38 -
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q4.2.out | 103 --
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q4.3.out | 205 ---
.../data/ssb_sf0.1_p1/sql/pipelinex_q1.1.out | 4 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q1.3.out | 4 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q2.1.out | 43 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q2.2.out | 11 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q2.3.out | 4 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q3.1.out | 28 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q3.2.out | 51 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q3.3.out | 4 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q4.1.out | 8 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q4.3.out | 3 -
.../sql/q01.out | 0
.../sql/q02.out | 0
.../sql/q03.out | 0
.../sql/q04.out | 0
.../sql/q05.out | 0
.../sql/q06.out | 0
.../sql/q07.out | 0
.../sql/q08.out | 0
.../sql/q09.out | 0
.../sql/q10.out | 0
.../sql/q11.out | 0
.../sql/q12.out | 0
.../sql/q13.out | 0
.../sql/q14.out | 0
.../sql/q15.out | 0
.../sql/q16.out | 0
.../sql/q17.out | 0
.../sql/q18.out | 0
.../sql/q19.out | 0
.../sql/q20.out | 0
.../sql/q21.out | 0
.../sql/q22.out | 0
.../sql/test_left_anti_join_batch_size.out | 0
.../variant_p0/test_variant_search_subcolumn.out | 32 +
.../org/apache/doris/regression/Config.groovy | 1 +
.../org/apache/doris/regression/suite/Suite.groovy | 154 ++-
.../doris/regression/suite/SuiteCluster.groovy | 43 +
.../doris/regression/suite/SuiteContext.groovy | 4 +
.../regression/util/RoutineLoadTestUtils.groovy | 152 +++
.../cloud_p0/conf/regression-conf-custom.groovy | 1 +
.../pipeline/cloud_p1/conf/be_custom.conf | 2 +
.../nonConcurrent/conf/regression-conf.groovy | 1 +
regression-test/pipeline/p0/conf/be.conf | 1 +
.../plugins/plugin_cluster_version.groovy | 46 +-
.../ann_index_p0/product_quantization.groovy | 107 ++
.../suites/ann_index_p0/scalar_quantization.groovy | 6 +-
.../suites/auth_call/test_ddl_backup_auth.groovy | 12 +-
.../suites/auth_call/test_ddl_job_auth.groovy | 22 +-
.../suites/auth_p0/test_query_tvf_auth.groovy | 2 +-
.../bloom_filter_p0/test_bloom_filter_hit.groovy | 2 +-
...est_bloom_filter_hit_with_renamed_column.groovy | 87 +-
.../suites/cast_p0/datelike_print.groovy | 44 +
.../cloud_p0/balance/test_balance_warm_up.groovy | 86 +-
.../async_copy_into/async_load.groovy | 275 ++++
.../async_copy_into}/ddl/customer.sql | 6 +-
.../async_copy_into}/ddl/customer_delete.sql | 0
.../async_copy_into}/ddl/lineitem.sql | 4 -
.../async_copy_into}/ddl/lineitem_delete.sql | 0
.../multi_cluster/async_copy_into}/ddl/nation.sql | 6 +-
.../async_copy_into}/ddl/nation_delete.sql | 0
.../multi_cluster/async_copy_into}/ddl/orders.sql | 6 +-
.../async_copy_into}/ddl/orders_delete.sql | 0
.../multi_cluster/async_copy_into}/ddl/part.sql | 6 +-
.../async_copy_into}/ddl/part_delete.sql | 0
.../async_copy_into}/ddl/partsupp.sql | 4 -
.../async_copy_into}/ddl/partsupp_delete.sql | 0
.../multi_cluster/async_copy_into}/ddl/region.sql | 6 +-
.../async_copy_into}/ddl/region_delete.sql | 0
.../async_copy_into}/ddl/revenue1.sql | 0
.../async_copy_into}/ddl/revenue1_delete.sql | 0
.../async_copy_into}/ddl/supplier.sql | 6 +-
.../async_copy_into}/ddl/supplier_delete.sql | 0
.../multi_cluster/copy_into}/ddl/customer.sql | 6 +-
.../copy_into}/ddl/customer_delete.sql | 0
.../multi_cluster/copy_into}/ddl/lineitem.sql | 4 -
.../copy_into}/ddl/lineitem_delete.sql | 0
.../multi_cluster/copy_into}/ddl/nation.sql | 6 +-
.../ddl/nation_delete.sql | 0
.../multi_cluster/copy_into}/ddl/orders.sql | 6 +-
.../multi_cluster/copy_into}/ddl/orders_delete.sql | 0
.../multi_cluster/copy_into}/ddl/part.sql | 6 +-
.../multi_cluster/copy_into}/ddl/part_delete.sql | 0
.../multi_cluster/copy_into}/ddl/partsupp.sql | 4 -
.../copy_into}/ddl/partsupp_delete.sql | 0
.../multi_cluster/copy_into}/ddl/region.sql | 6 +-
.../multi_cluster/copy_into}/ddl/region_delete.sql | 0
.../multi_cluster/copy_into}/ddl/revenue1.sql | 0
.../copy_into}/ddl/revenue1_delete.sql | 0
.../multi_cluster/copy_into}/ddl/supplier.sql | 6 +-
.../copy_into}/ddl/supplier_delete.sql | 0
.../multi_cluster/copy_into/sync_laod.groovy | 846 ++++++++++++
.../cloud_p0/multi_cluster/default_cluster.groovy | 172 +++
.../multi_cluster_s3_load}/ddl/customer.sql | 8 +-
.../multi_cluster_s3_load}/ddl/customer_delete.sql | 0
.../multi_cluster_s3_load}/ddl/customer_load.sql | 0
.../multi_cluster_s3_load}/ddl/lineitem.sql | 8 +-
.../multi_cluster_s3_load}/ddl/lineitem_delete.sql | 0
.../multi_cluster_s3_load/ddl/lineitem_load.sql | 6 +
.../multi_cluster_s3_load}/ddl/nation.sql | 7 +-
.../ddl/nation_delete.sql | 0
.../ddl/nation_load.sql | 0
.../multi_cluster_s3_load}/ddl/orders.sql | 9 +-
.../multi_cluster_s3_load}/ddl/orders_delete.sql | 0
.../multi_cluster_s3_load}/ddl/orders_load.sql | 0
.../multi_cluster_s3_load}/ddl/part.sql | 9 +-
.../multi_cluster_s3_load}/ddl/part_delete.sql | 0
.../multi_cluster_s3_load}/ddl/part_load.sql | 0
.../multi_cluster_s3_load}/ddl/partsupp.sql | 9 +-
.../multi_cluster_s3_load}/ddl/partsupp_delete.sql | 0
.../multi_cluster_s3_load}/ddl/partsupp_load.sql | 0
.../multi_cluster_s3_load}/ddl/region.sql | 7 +-
.../multi_cluster_s3_load}/ddl/region_delete.sql | 0
.../multi_cluster_s3_load}/ddl/region_load.sql | 0
.../multi_cluster_s3_load}/ddl/supplier.sql | 9 +-
.../multi_cluster_s3_load}/ddl/supplier_delete.sql | 0
.../multi_cluster_s3_load}/ddl/supplier_load.sql | 0
.../multi_cluster_s3_load/load.groovy | 166 +++
.../op/test_drop_and_add_cluster.groovy | 221 +++
.../cloud_p0/multi_cluster/rename_cluster.groovy | 115 ++
.../routine_load/test_routine_load.groovy | 189 +++
.../test_bitmap_index_with_drop_cluster.groovy | 143 ++
.../test_bitmap_index_with_readd_cluster.groovy | 154 +++
...test_materialized_view_with_drop_cluster.groovy | 123 ++
...est_materialized_view_with_readd_cluster.groovy | 155 +++
.../test_rollup_with_drop_cluster.groovy | 122 ++
.../test_rollup_with_readd_cluster.groovy | 156 +++
.../test_schema_change_with_drop_cluster.groovy | 120 ++
.../test_schema_change_with_readd_cluster.groovy | 169 +++
.../multi_cluster/smooth_upgrade/ddl/customer.sql | 15 +
.../smooth_upgrade}/ddl/customer_delete.sql | 0
.../smooth_upgrade}/ddl/customer_load.sql | 0
.../smooth_upgrade}/ddl/customer_load_sequence.sql | 0
.../smooth_upgrade}/ddl/customer_part_delete.sql | 0
.../smooth_upgrade/ddl/customer_sequence.sql | 17 +
.../multi_cluster/smooth_upgrade/ddl/lineitem.sql | 23 +
.../smooth_upgrade}/ddl/lineitem_delete.sql | 0
.../smooth_upgrade/ddl/lineitem_load.sql | 6 +
.../smooth_upgrade/ddl/lineitem_load_sequence.sql | 7 +
.../smooth_upgrade}/ddl/lineitem_part_delete.sql | 0
.../smooth_upgrade/ddl/lineitem_sequence.sql | 25 +
.../multi_cluster/smooth_upgrade/ddl/nation.sql | 11 +
.../smooth_upgrade}/ddl/nation_delete.sql | 0
.../smooth_upgrade}/ddl/nation_load.sql | 0
.../smooth_upgrade}/ddl/nation_load_sequence.sql | 0
.../smooth_upgrade}/ddl/nation_part_delete.sql | 0
.../smooth_upgrade/ddl/nation_sequence.sql | 13 +
.../multi_cluster/smooth_upgrade/ddl/orders.sql | 15 +
.../smooth_upgrade}/ddl/orders_delete.sql | 0
.../smooth_upgrade}/ddl/orders_load.sql | 0
.../smooth_upgrade}/ddl/orders_load_sequence.sql | 0
.../smooth_upgrade}/ddl/orders_part_delete.sql | 0
.../smooth_upgrade/ddl/orders_sequence.sql | 17 +
.../multi_cluster/smooth_upgrade/ddl/part.sql | 16 +
.../smooth_upgrade}/ddl/part_delete.sql | 0
.../smooth_upgrade}/ddl/part_load.sql | 0
.../smooth_upgrade}/ddl/part_load_sequence.sql | 0
.../smooth_upgrade}/ddl/part_part_delete.sql | 0
.../smooth_upgrade/ddl/part_sequence.sql | 18 +
.../multi_cluster/smooth_upgrade/ddl/partsupp.sql | 12 +
.../smooth_upgrade}/ddl/partsupp_delete.sql | 0
.../smooth_upgrade}/ddl/partsupp_load.sql | 0
.../smooth_upgrade}/ddl/partsupp_load_sequence.sql | 0
.../smooth_upgrade}/ddl/partsupp_part_delete.sql | 0
.../smooth_upgrade/ddl/partsupp_sequence.sql | 14 +
.../multi_cluster/smooth_upgrade/ddl/region.sql | 10 +
.../smooth_upgrade}/ddl/region_delete.sql | 0
.../smooth_upgrade}/ddl/region_load.sql | 0
.../smooth_upgrade}/ddl/region_load_sequence.sql | 0
.../smooth_upgrade}/ddl/region_part_delete.sql | 0
.../smooth_upgrade/ddl/region_sequence.sql | 12 +
.../multi_cluster/smooth_upgrade/ddl/supplier.sql | 13 +
.../smooth_upgrade}/ddl/supplier_delete.sql | 0
.../smooth_upgrade}/ddl/supplier_load.sql | 0
.../smooth_upgrade}/ddl/supplier_load_sequence.sql | 0
.../smooth_upgrade}/ddl/supplier_part_delete.sql | 0
.../smooth_upgrade/ddl/supplier_sequence.sql | 15 +
.../smooth_upgrade/smooth_upgrade.groovy | 165 +++
.../multi_cluster/stream_load/stream_load.groovy | 322 +++++
.../stream_load/stream_load_2pc.groovy | 396 ++++++
.../stream_load/stream_load_lb.groovy | 227 ++++
.../cloud_p0/multi_cluster/table_rebalance.groovy | 175 +++
.../test_apsaradb_internal_stage.groovy | 199 +++
.../multi_cluster/test_cloud_cluster.groovy | 202 +++
.../multi_cluster/test_drop_cluster.groovy | 96 ++
.../cloud_p0/multi_cluster/test_drop_userid.groovy | 121 ++
.../test_group_commit_multi_cluster.groovy | 129 ++
.../multi_cluster/test_overdue_instance.groovy | 243 ++++
.../cloud_p0/query_retry/test_retry_e-230.groovy | 1 -
.../cold_data_compaction.groovy | 6 +
.../spark_connector/spark_connector.groovy | 41 -
.../spark_connector/spark_connector_arrow.groovy | 20 +-
.../spark_connector_read_type.groovy | 21 +-
.../suites/correctness/test_compare_float.groovy | 2 +-
...est_distinct_streaming_agg_local_shuffle.groovy | 3 -
.../correctness_p0/test_grouping_sets_empty.groovy | 5 -
.../correctness_p0/test_mask_function.groovy | 16 +-
.../suites/correctness_p0/test_probe_clean.groovy | 21 -
.../suites/correctness_p0/test_time_round.groovy | 6 +
.../one_level_nestedtypes_with_s3data.groovy | 6 +-
.../string_len/test_string_len_complex.groovy | 229 ++++
.../create_view_nereids_fix_null.groovy | 2 +-
.../suites/demo_p0/test_external_ms_cluster.groovy | 83 ++
.../data-operate/scheduler/job-scheduler.md.groovy | 2 +-
.../suites/doc/external/jdbc/mysql.md.groovy | 2 +-
.../basic-elements/data-types/array-md.groovy | 8 +-
.../basic-elements/data-types/map-md.groovy | 8 +-
.../basic-elements/data-types/struct-md.groovy | 8 +-
.../sql-functions/doc_date_functions_test.groovy | 454 +++++++
.../outfile/test_outfile_parquet_schema.groovy | 93 ++
.../cache/test_file_cache_features.groovy | 222 +++
.../cache/test_file_cache_statistics.groovy | 199 ++-
.../export/test_export_external_table.groovy | 2 +-
.../hive/test_parquet_nested_types.groovy | 8 +-
.../jdbc/test_doris_jdbc_catalog.groovy | 2 +-
.../test_doris_jdbc_catalog_query_bitmap.groovy | 2 +-
.../external_table_p0/jdbc/test_jdbc_call.groovy | 4 +-
.../jdbc/test_jdbc_catalog_push_cast.groovy | 2 +-
.../jdbc/test_jdbc_query_mysql.groovy | 2 +-
.../jdbc/test_jdbc_query_tvf.groovy | 2 +-
.../jdbc/test_jdbc_row_count.groovy | 2 +-
.../jdbc/test_jni_complex_type.groovy | 2 +-
.../jdbc/test_mariadb_jdbc_catalog.groovy | 2 +-
.../jdbc/test_mysql_jdbc_catalog.groovy | 16 +-
.../jdbc/test_mysql_jdbc_catalog_nereids.groovy | 2 +-
.../test_mysql_jdbc_catalog_table_comment.groovy | 6 +-
.../jdbc/test_mysql_jdbc_statistics.groovy | 2 +-
.../test_switch_catalog_and_delete_internal.groovy | 2 +-
.../ctas/test_mysql_all_types_ctas.groovy | 2 +-
.../select/test_doris_all_types_select.groovy | 2 +-
.../select/test_mysql_all_types_select.groovy | 2 +-
.../select/test_mysql_varbinary_with_udf.groovy | 2 +-
.../type_test/tvf/test_mysql_all_types_tvf.groovy | 2 +-
.../lower_case/test_conflict_name.groovy | 2 +-
.../lower_case/test_lower_case_meta_include.groovy | 2 +-
.../test_lower_case_meta_show_and_select.groovy | 2 +-
...er_case_meta_with_lower_table_conf_grant.groovy | 2 +-
...ta_with_lower_table_conf_show_and_select.groovy | 2 +-
.../lower_case/test_lower_case_mtmv.groovy | 2 +-
.../test_meta_cache_select_without_refresh.groovy | 2 +-
.../lower_case/test_meta_names_mapping.groovy | 2 +-
.../lower_case/test_timing_refresh_catalog.groovy | 2 +-
.../lower_case/upgrade/load.groovy | 2 +-
.../test_nereids_refresh_catalog.groovy | 2 +-
.../nereids_commands/test_use_database_stmt.groovy | 2 +-
.../tvf/test_s3_tvf_number_range.groovy | 398 ++++++
.../suites/external_table_p0/upgrade/load.groovy | 4 +-
.../maxcompute/test_max_compute_schema.groovy | 239 ++++
.../mysql/test_external_catalog_mysql.groovy | 2 +-
.../mysql/test_external_resource_mysql.groovy | 2 +-
.../cloud/test_cloud_version_already_merged.groovy | 142 ++
.../memtable/test_memtable_too_many_rows.groovy | 49 +
...st_skip_index_compaction_fault_injection.groovy | 3 +
.../flink_connector_p0/flink_connector.groovy | 12 +-
.../flink_connector_syncdb.groovy | 19 +-
.../flink_connector_p0/flink_connector_type.groovy | 112 +-
.../test_cast_to_string_from_decimalv3.groovy | 53 +
.../to_string/test_cast_to_string_from_int.groovy | 51 +
.../test_ngram_bloomfilter_index_change.groovy | 3 +-
.../test_iot_auto_detect_fail.groovy | 4 +-
.../analyzer/test_custom_analyzer.groovy | 26 +-
.../analyzer/test_custom_analyzer1.groovy | 69 +-
.../analyzer/test_custom_analyzer2.groovy | 62 +
.../test_index_change_on_renamed_column.groovy | 2 +-
...est_cumulative_compaction_with_format_v2.groovy | 2 +-
.../test_mor_table_with_format_v2.groovy | 4 +-
.../test_mow_table_with_format_v2.groovy | 2 +-
.../test_storage_format_default.groovy | 91 ++
.../inverted_index_p0/test_inverted_is_null.groovy | 72 +
.../test_streaming_insert_job_alter.groovy | 155 +++
.../test_streaming_insert_job_crud.groovy | 494 +++++++
.../test_streaming_insert_job_offset.groovy | 226 ++++
.../test_streaming_insert_job_priv.groovy | 204 +++
.../test_streaming_job_schedule_task_error.groovy | 94 ++
.../suites/jsonb_p0/test_json_hash.groovy | 116 ++
.../load_p0/broker_load/test_array_load.groovy | 54 +-
.../test_broker_load_with_partition.groovy | 10 +-
.../broker_load/test_broker_load_with_where.groovy | 2 +-
.../test_broker_load_without_filepath.groovy | 2 +-
...where.groovy => test_s3_load_with_where.groovy} | 138 +-
.../test_ingestion_load_alter_column.groovy | 4 +-
.../insert/test_insert_move_memtable.groovy | 119 --
.../load_p0/merge_into/test_merge_into.groovy | 254 ++++
.../test_routine_load_adaptive_param.groovy | 91 ++
.../test_routine_load_delay_schedule.groovy | 83 ++
.../test_map_load_and_compaction.groovy | 4 +-
.../suites/manager/test_manager_interface_1.groovy | 2 +-
.../suites/manager/test_manager_interface_3.groovy | 4 +-
.../suites/mtmv_p0/test_build_mtmv.groovy | 15 -
.../mtmv_p0/test_create_mtmv_with_view.groovy | 112 ++
.../test_create_mtmv_with_view_alter.groovy | 94 ++
.../test_create_mtmv_with_view_alter_table.groovy | 103 ++
.../test_create_mtmv_with_view_commit.groovy | 69 +
.../mtmv_p0/test_create_mtmv_with_view_cte.groovy | 92 ++
.../mtmv_p0/test_create_mtmv_with_view_mtmv.groovy | 126 ++
.../mtmv_p0/test_create_mtmv_with_view_pct.groovy | 103 ++
.../test_create_mtmv_with_view_rollup.groovy | 81 ++
.../mtmv_p0/test_limit_partition_view_mtmv.groovy | 254 ++++
.../suites/mtmv_p0/test_mysql_mtmv.groovy | 2 +-
.../suites/mtmv_p0/test_view_hive_mtmv.groovy | 98 ++
.../mv_p0/test_create_mv/test_create_mv.groovy | 12 +
.../test_mv_with_keyword.groovy | 67 +
.../nereids_p0/cache/mtmv_with_sql_cache.groovy | 985 ++++++++++++++
.../nereids_p0/cache/mv_with_sql_cache.groovy | 129 ++
.../cache/parse_sql_from_sql_cache.groovy | 31 +
.../datatype/test_date_implicit_cast.groovy | 8 +-
.../partition_topn/check_orderkey.groovy | 61 +
.../session_variable/test_default_limit.groovy | 8 +-
.../show/test_nereids_show_resources.groovy | 2 +-
.../suites/nereids_p0/show/test_show_where.groovy | 2 +-
.../conditional_functions/test_nullif.groovy | 10 +-
.../test_date_function_v2.groovy | 68 +
.../json_function/test_query_json_object.groovy | 5 +-
.../nereids_p0/stats/partitionRowCount.groovy | 2 +-
.../avg_distinct_to_sum_div_count.groovy | 26 +-
.../expression/test_simplify_range.groovy | 2 +-
.../external/dml_query_has_external_table.groovy | 2 +-
.../partition_prune/one_key_list_part_test.groovy | 1 +
.../one_key_list_part_update_test.groovy | 1 +
.../partition_prune/one_key_range_part_test.groovy | 1 +
.../one_key_range_part_update_test.groovy | 3 +-
.../suites/nereids_syntax_p0/alias_conflict.groovy | 2 +-
.../suites/nereids_syntax_p0/explain.groovy | 2 +-
.../nereids_tpch_p0/tpch/push_topn_to_agg.groovy | 2 +-
.../auto_partition/test_auto_list_partition.groovy | 2 +-
.../test_auto_partition_behavior.groovy | 40 +-
...uto_partition_with_single_replica_insert.groovy | 4 +-
.../test_col_data_type_boundary.groovy | 20 +-
...t_dynamic_partition_mod_distribution_key.groovy | 1 +
.../suites/query_p0/cache/condition_cache.groovy | 290 ++++
.../suites/query_p0/explain/explain_alias.groovy | 56 +
.../query_p0/join/test_join_result_count.groovy | 3 -
.../join/test_partitioned_hash_join.groovy | 2 +-
.../schema_table/test_active_queries.groovy | 15 -
.../schema_table/test_backend_active_tasks.groovy | 15 -
.../session_variable/test_default_limit.groovy | 8 +-
.../query_p0/show/test_show_partitions.groovy | 2 +-
.../suites/query_p0/show/test_show_where.groovy | 2 +-
.../array_functions/test_array_map_function.groovy | 8 -
.../binary_functions/test_binary_function.groovy | 128 ++
.../bitmap_functions/test_bitmap_function.groovy | 26 -
.../test_cast_array_functions_by_literal.groovy | 36 +-
.../cast_function/test_cast_struct.groovy | 8 +-
.../conditional_functions/test_coalesce.groovy | 88 ++
.../conditional_functions/test_nullif.groovy | 10 +-
.../conditional_functions/test_query_in.groovy | 13 +
.../test_binary_for_digest.groovy | 2 +-
.../string_functions/test_string_all.groovy | 1095 +++++++++++++++
.../string_functions/test_string_function.groovy | 99 ++
.../struct_functions/test_struct_functions.groovy | 8 +-
.../sql_functions/table_function/explode.groovy | 47 +
.../system/test_query_sys_column_data_sizes.groovy | 130 ++
.../test_ranger_access_resource_catalog.groovy | 2 +-
.../test_ranger_access_resource_global.groovy | 2 +-
.../test_ranger_access_resource_resource.groovy | 2 +-
...test_ranger_access_role_resource_catalog.groovy | 2 +-
.../suites/rec_cte_p0/rec_cte/rec_cte.groovy | 271 ++++
.../rec_cte_from_ck_doc/rec_cte_from_ck_doc.groovy | 144 ++
.../rec_cte_from_duckdb_doc.groovy | 185 +++
.../rec_cte_from_mysql_doc.groovy | 140 ++
.../test_double_write_when_schema_change.groovy | 2 +-
.../test_alter_rollup_table.groovy | 9 +-
.../test_alter_table_modify_column.groovy | 2 +-
.../schema_change_p0/test_schema_change_agg.groovy | 4 +-
.../suites/search/test_search_mow_support.groovy | 78 ++
.../search/test_search_vs_match_consistency.groovy | 40 +
.../suites/show_p0/test_show_data_warehouse.groovy | 13 +-
.../ssb_sf0.1_p1/sql/pipelinex_flat_q1.1.sql | 24 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q1.2.sql | 24 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q1.3.sql | 25 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q2.1.sql | 24 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q2.2.sql | 27 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q2.3.sql | 26 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q3.1.sql | 29 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q3.2.sql | 29 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q3.3.sql | 29 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q3.4.sql | 29 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q4.1.sql | 27 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q4.2.sql | 33 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q4.3.sql | 29 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q1.1.sql | 24 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q1.2.sql | 24 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q1.3.sql | 25 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q2.1.sql | 26 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q2.2.sql | 27 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q2.3.sql | 26 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q3.1.sql | 28 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q3.2.sql | 28 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q3.3.sql | 30 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q3.4.sql | 30 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q4.1.sql | 30 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q4.2.sql | 31 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q4.3.sql | 29 -
.../test_temp_table_with_conn_timeout.groovy | 45 -
.../ddl/customer.sql | 0
.../ddl/customer_delete.sql | 0
.../ddl/lineitem.sql | 0
.../ddl/lineitem_delete.sql | 0
.../ddl/nation.sql | 0
.../ddl/nation_delete.sql | 0
.../ddl/orders.sql | 0
.../ddl/orders_delete.sql | 0
.../ddl/part.sql | 0
.../ddl/part_delete.sql | 0
.../ddl/partsupp.sql | 0
.../ddl/partsupp_delete.sql | 0
.../ddl/region.sql | 0
.../ddl/region_delete.sql | 0
.../ddl/revenue1.sql | 0
.../ddl/revenue1_delete.sql | 0
.../ddl/supplier.sql | 0
.../ddl/supplier_delete.sql | 0
.../load.groovy | 0
.../sql/q01.sql | 0
.../sql/q02.sql | 0
.../sql/q03.sql | 0
.../sql/q04.sql | 0
.../sql/q05.sql | 0
.../sql/q06.sql | 0
.../sql/q07.sql | 0
.../sql/q08.sql | 0
.../sql/q09.sql | 0
.../sql/q10.sql | 0
.../sql/q11.sql | 0
.../sql/q12.sql | 0
.../sql/q13.sql | 0
.../sql/q14.sql | 0
.../sql/q15.sql | 0
.../sql/q16.sql | 0
.../sql/q17.sql | 0
.../sql/q18.sql | 0
.../sql/q19.sql | 0
.../sql/q20.sql | 0
.../sql/q21.sql | 0
.../sql/q22.sql | 0
.../sql/test_left_anti_join_batch_size.sql | 0
.../test_key_bounds_truncation_basic.groovy | 12 +
...est_key_bounds_truncation_read_scenarios.groovy | 12 +
regression-test/suites/variant_p0/nested.groovy | 2 +-
.../predefine/test_multi_index_file.groovy | 2 +-
.../variant_p0/predefine/variant_with_mow.groovy | 4 +-
.../test_variant_search_subcolumn.groovy | 152 +++
run-regression-test.sh | 2 +-
samples/datalake/lakesoul/README.md | 2 +-
thirdparty/CHANGELOG.md | 7 +
thirdparty/build-thirdparty.sh | 120 +-
thirdparty/download-thirdparty.sh | 24 -
thirdparty/patches/googletest-release-1.11.0.patch | 71 -
thirdparty/patches/libevent-1532.patch | 188 ---
.../libevent-keepalive-accepted-socket.patch | 17 -
thirdparty/patches/libevent.patch | 193 ---
thirdparty/vars.sh | 39 +-
1756 files changed, 51938 insertions(+), 15090 deletions(-)
create mode 100644 be/benchmark/benchmark_block_bloom_filter.hpp
create mode 100644 be/benchmark/benchmark_string.hpp
create mode 100644 be/src/exec/schema_scanner/schema_column_data_sizes_scanner.cpp
create mode 100644 be/src/exec/schema_scanner/schema_column_data_sizes_scanner.h
copy be/src/olap/rowset/segment_v2/{binary_plain_page.h => binary_plain_page_v2.h} (56%)
create mode 100644 be/src/olap/rowset/segment_v2/condition_cache.cpp
create mode 100644 be/src/olap/rowset/segment_v2/condition_cache.h
copy be/src/olap/rowset/segment_v2/inverted_index/{tokenizer/tokenizer.h => char_filter/char_filter.h} (52%)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/char_filter/empty_char_filter_factory.h
copy be/src/olap/rowset/segment_v2/inverted_index/{tokenizer/tokenizer.h => token_filter/empty_token_filter_factory.h} (51%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/basic/basic_tokenizer.cpp (75%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/basic/basic_tokenizer.h (69%)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer_factory.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/empty/empty_tokenizer_factory.h
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/break_iterator_wrapper.cpp (97%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/break_iterator_wrapper.h (94%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/composite_break_iterator.cpp (97%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/composite_break_iterator.h (94%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/default_icu_tokenizer_config.cpp (98%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/default_icu_tokenizer_config.h (94%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/icu_common.h (93%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/icu_tokenizer.cpp (84%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/icu_tokenizer.h (82%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/icu_tokenizer_config.h (93%)
copy be/src/olap/rowset/segment_v2/inverted_index/{token_filter/token_filter.h => tokenizer/icu/icu_tokenizer_factory.h} (61%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/script_iterator.cpp (97%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/script_iterator.h (95%)
copy fe/fe-core/src/main/java/org/apache/doris/job/offset/Offset.java => be/src/olap/rowset/segment_v2/inverted_index/util/reader.h (76%)
create mode 100644 be/src/olap/rowset_version_mgr.cpp
create mode 100644 be/src/pipeline/revokable_task.h
create mode 100644 be/src/vec/functions/function_json_hash.cpp
create mode 100644 be/src/vec/functions/function_varbinary.h
create mode 100644 be/src/vec/utils/varbinaryop_subbinary.h
copy be/{src/common/version_internal.h => test/common/compare_test.cpp} (61%)
create mode 100644 be/test/olap/rowset/segment_v2/binary_plain_page_v2_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/analysis_factory_mgr_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/char_filter/char_filter_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/tokenizer/basic_tokenizer_factory_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/tokenizer/icu_tokenizer_factory_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/util/reader_test.cpp
create mode 100644 be/test/vec/function/function_varbinary_test.cpp
create mode 100644 cloud/src/recycler/snapshot_chain_compactor.cpp
create mode 100644 cloud/src/recycler/snapshot_chain_compactor.h
create mode 100644 cloud/src/recycler/snapshot_data_migrator.cpp
create mode 100644 cloud/src/recycler/snapshot_data_migrator.h
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/TableName.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketCalculator.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/McStructureHelper.java
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{AbstractHMSProperties.java => AbstractHiveProperties.java} (93%)
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{HMSAliyunDLFMetaStoreProperties.java => HiveAliyunDLFMetaStoreProperties.java} (94%)
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{HMSGlueMetaStoreProperties.java => HiveGlueMetaStoreProperties.java} (96%)
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{HMSProperties.java => HiveHMSProperties.java} (95%)
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{HMSPropertiesFactory.java => HivePropertiesFactory.java} (70%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AzurePropertyUtils.java
copy fe/fe-core/src/main/java/org/apache/doris/indexpolicy/{LowerCaseTokenFilterValidator.java => BasicTokenizerValidator.java} (59%)
copy fe/fe-core/src/main/java/org/apache/doris/indexpolicy/{KeywordTokenizerValidator.java => CharReplaceCharFilterValidator.java} (53%)
rename fe/fe-core/src/main/java/org/apache/doris/indexpolicy/{LowerCaseTokenFilterValidator.java => ICUTokenizerValidator.java} (83%)
rename fe/fe-core/src/main/java/org/apache/doris/indexpolicy/{KeywordTokenizerValidator.java => NoOperationValidator.java} (83%)
copy fe/fe-core/src/main/java/org/apache/doris/{nereids/rules/analysis/CollectOneLevelRelation.java => mtmv/MTMVAnalyzeQueryInfo.java} (50%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/GetFormatFunctionBinder.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/CaseWhenToCompoundPredicate.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/CondReplaceNullWithFalse.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConditionRewrite.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ReplaceNullWithFalseForCond.java
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{MaskLastN.java => ExportSet.java} (53%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Length.java => FromBase64Binary.java} (74%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Length.java => GetFormat.java} (63%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Length.java => JsonHash.java} (64%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{MaskFirstN.java => SubBinary.java} (56%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Length.java => ToBase64Binary.java} (75%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/{AlterJobStatusCommand.java => CreateIndexCharFilterCommand.java} (57%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/{DropJobCommand.java => DropIndexCharFilterCommand.java} (71%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/{PauseJobCommand.java => ShowIndexCharFilterCommand.java} (67%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/merge/MergeIntoCommand.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/merge/MergeMatchedClause.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/merge/MergeNotMatchedClause.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionExprUtilTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/AutoIncrementGeneratorTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketCalculatorTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/AzurePropertyUtilsTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVSchemaChangeTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/parser/UnboundFunctionWithUnboundStarTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/CaseWhenToCompoundPredicateTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/CondReplaceNullWithFalseTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/ReplaceNullWithFalseForCondTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/CTEInlineTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/merge/MergeIntoCommandTest.java
copy regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_flat_q3.4.out => ann_index_p0/product_quantization.out} (57%)
create mode 100644 regression-test/data/cast_p0/datelike_print.out
create mode 100644 regression-test/data/cloud_p0/multi_cluster/routine_load/test_routine_load.out
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_flat_q1.3.out => cloud_p0/multi_cluster/schema_change/test_schema_change_with_readd_cluster.out} (66%)
copy regression-test/data/{cloud/multi_cluster/vcluster => cloud_p0/multi_cluster/stream_load}/all_types.csv (100%)
copy regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_flat_q3.4.out => cloud_p0/multi_cluster/stream_load/stream_load.out} (58%)
create mode 100644 regression-test/data/cloud_p0/multi_cluster/stream_load/stream_load_2pc.out
create mode 100644 regression-test/data/cloud_p0/multi_cluster/stream_load/stream_load_lb.out
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_q1.2.out => cloud_p0/multi_cluster/test_apsaradb_internal_stage.out} (76%)
create mode 100644 regression-test/data/datatype_p0/nested_types/string_len/test_string_len_complex.out
create mode 100644 regression-test/data/export_p0/outfile/test_outfile_parquet_schema.out
create mode 100644 regression-test/data/external_table_p0/tvf/hdfs_data_1.txt
create mode 100644 regression-test/data/external_table_p0/tvf/hdfs_data_2.txt
create mode 100644 regression-test/data/external_table_p0/tvf/hdfs_data_3.txt
create mode 100644 regression-test/data/external_table_p0/tvf/test_s3_tvf_number_range.out
create mode 100644 regression-test/data/external_table_p2/maxcompute/test_max_compute_schema.out
create mode 100644 regression-test/data/fault_injection_p0/cloud/test_cloud_version_already_merged.out
create mode 100644 regression-test/data/function_p0/cast/to_string/test_cast_to_string_from_decimalv3.out
create mode 100644 regression-test/data/function_p0/cast/to_string/test_cast_to_string_from_int.out
create mode 100644 regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer2.out
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_q3.4.out => inverted_index_p0/storage_format/test_storage_format_default.out} (81%)
create mode 100644 regression-test/data/job_p0/streaming_job/test_streaming_insert_job_alter.out
create mode 100644 regression-test/data/job_p0/streaming_job/test_streaming_insert_job_crud.out
create mode 100644 regression-test/data/job_p0/streaming_job/test_streaming_insert_job_offset.out
create mode 100644 regression-test/data/jsonb_p0/test_json_hash.out
copy regression-test/data/load_p0/broker_load/{test_broker_load_with_where.out => test_s3_load_with_where.out} (91%)
delete mode 100644 regression-test/data/load_p0/insert/test_insert_move_memtable.out
create mode 100644 regression-test/data/load_p0/merge_into/test_merge_into.out
create mode 100644 regression-test/data/mtmv_p0/test_create_mtmv_with_view.out
create mode 100644 regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter.out
create mode 100644 regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter_table.out
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_flat_q1.1.out => mtmv_p0/test_create_mtmv_with_view_commit.out} (64%)
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_flat_q1.2.out => mtmv_p0/test_create_mtmv_with_view_cte.out} (63%)
create mode 100644 regression-test/data/mtmv_p0/test_create_mtmv_with_view_mtmv.out
create mode 100644 regression-test/data/mtmv_p0/test_create_mtmv_with_view_pct.out
create mode 100644 regression-test/data/mtmv_p0/test_create_mtmv_with_view_rollup.out
copy regression-test/data/mtmv_p0/{test_limit_partition_mtmv.out => test_limit_partition_view_mtmv.out} (100%)
create mode 100644 regression-test/data/mtmv_p0/test_view_hive_mtmv.out
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_q4.2.out => nereids_p0/partition_topn/check_orderkey.out} (81%)
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_flat_q3.4.out => nereids_rules_p0/avg_distinct_to_sum_div_count/avg_distinct_to_sum_div_count.out} (78%)
create mode 100644 regression-test/data/query_p0/cache/condition_cache.out
create mode 100644 regression-test/data/query_p0/sql_functions/string_functions/test_string_all.out
create mode 100644 regression-test/data/query_p0/system/test_query_sys_column_data_sizes.out
create mode 100644 regression-test/data/rec_cte_p0/rec_cte/rec_cte.out
create mode 100644 regression-test/data/rec_cte_p0/rec_cte_from_ck_doc/rec_cte_from_ck_doc.out
create mode 100644 regression-test/data/rec_cte_p0/rec_cte_from_duckdb_doc/rec_cte_from_duckdb_doc.out
create mode 100644 regression-test/data/rec_cte_p0/rec_cte_from_mysql_doc/rec_cte_from_mysql_doc.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q2.1.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q2.2.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q2.3.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q3.1.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q3.2.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q3.3.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q4.1.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q4.2.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q4.3.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q1.1.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q1.3.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q2.1.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q2.2.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q2.3.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q3.1.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q3.2.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q3.3.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q4.1.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q4.3.out
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q01.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q02.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q03.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q04.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q05.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q06.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q07.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q08.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q09.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q10.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q11.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q12.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q13.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q14.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q15.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q16.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q17.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q18.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q19.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q20.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q21.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q22.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/test_left_anti_join_batch_size.out (100%)
create mode 100644 regression-test/data/variant_p0/test_variant_search_subcolumn.out
create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/util/RoutineLoadTestUtils.groovy
create mode 100644 regression-test/suites/ann_index_p0/product_quantization.groovy
create mode 100644 regression-test/suites/cast_p0/datelike_print.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/async_copy_into/async_load.groovy
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/customer.sql (79%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/customer_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/lineitem.sql (91%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/lineitem_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/nation.sql (68%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/nation_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/orders.sql (81%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/orders_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/part.sql (81%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/part_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/partsupp.sql (82%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/partsupp_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/region.sql (64%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/region_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/revenue1.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/async_copy_into}/ddl/revenue1_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/supplier.sql (77%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/supplier_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/customer.sql (79%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/customer_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/lineitem.sql (91%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/lineitem_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/nation.sql (68%)
copy regression-test/suites/cloud_p0/multi_cluster/{virtual_compute_group => copy_into}/ddl/nation_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/orders.sql (81%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/orders_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/part.sql (81%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/part_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/partsupp.sql (82%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/partsupp_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/region.sql (64%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/region_delete.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/revenue1.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/revenue1_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/supplier.sql (77%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/supplier_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/copy_into/sync_laod.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/default_cluster.groovy
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/customer.sql (71%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/customer_delete.sql (100%)
copy regression-test/suites/cloud_p0/{cache/multi_cluster/read_write => multi_cluster/multi_cluster_s3_load}/ddl/customer_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/lineitem.sql (78%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/lineitem_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/multi_cluster_s3_load/ddl/lineitem_load.sql
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/nation.sql (68%)
copy regression-test/suites/cloud_p0/multi_cluster/{virtual_compute_group => multi_cluster_s3_load}/ddl/nation_delete.sql (100%)
copy regression-test/suites/cloud_p0/multi_cluster/{virtual_compute_group => multi_cluster_s3_load}/ddl/nation_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/orders.sql (73%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/orders_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/orders_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/part.sql (72%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/part_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/part_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/partsupp.sql (61%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/partsupp_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/partsupp_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/region.sql (64%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/region_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/region_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/supplier.sql (68%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/supplier_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/supplier_load.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/multi_cluster_s3_load/load.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/op/test_drop_and_add_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/rename_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/routine_load/test_routine_load.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_bitmap_index_with_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_bitmap_index_with_readd_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_materialized_view_with_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_materialized_view_with_readd_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_rollup_with_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_rollup_with_readd_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_schema_change_with_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_schema_change_with_readd_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/customer.sql
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/smooth_upgrade}/ddl/customer_delete.sql (100%)
copy regression-test/suites/cloud_p0/{cache/multi_cluster/read_write => multi_cluster/smooth_upgrade}/ddl/customer_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/customer_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/customer_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/customer_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/lineitem.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/lineitem_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/lineitem_load.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/lineitem_load_sequence.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/lineitem_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/lineitem_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/nation.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/nation_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/nation_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/nation_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/nation_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/nation_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/orders.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/orders_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/orders_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/orders_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/orders_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/orders_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/part.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/part_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/part_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/part_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/part_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/part_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/partsupp.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/partsupp_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/partsupp_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/partsupp_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/partsupp_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/partsupp_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/region.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/region_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/region_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/region_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/region_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/region_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/supplier.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/supplier_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/supplier_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/supplier_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/supplier_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/supplier_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/smooth_upgrade.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/stream_load/stream_load.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/stream_load/stream_load_2pc.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/stream_load/stream_load_lb.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/table_rebalance.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_apsaradb_internal_stage.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_cloud_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_drop_userid.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_group_commit_multi_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_overdue_instance.groovy
delete mode 100644 regression-test/suites/connector_p0/spark_connector/spark_connector.groovy
create mode 100644 regression-test/suites/datatype_p0/nested_types/string_len/test_string_len_complex.groovy
create mode 100644 regression-test/suites/demo_p0/test_external_ms_cluster.groovy
create mode 100644 regression-test/suites/export_p0/outfile/test_outfile_parquet_schema.groovy
create mode 100644 regression-test/suites/external_table_p0/cache/test_file_cache_features.groovy
create mode 100644 regression-test/suites/external_table_p0/tvf/test_s3_tvf_number_range.groovy
create mode 100644 regression-test/suites/external_table_p2/maxcompute/test_max_compute_schema.groovy
create mode 100644 regression-test/suites/fault_injection_p0/cloud/test_cloud_version_already_merged.groovy
create mode 100644 regression-test/suites/fault_injection_p0/memtable/test_memtable_too_many_rows.groovy
create mode 100644 regression-test/suites/function_p0/cast/to_string/test_cast_to_string_from_decimalv3.groovy
create mode 100644 regression-test/suites/function_p0/cast/to_string/test_cast_to_string_from_int.groovy
create mode 100644 regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer2.groovy
create mode 100644 regression-test/suites/inverted_index_p0/storage_format/test_storage_format_default.groovy
create mode 100644 regression-test/suites/inverted_index_p0/test_inverted_is_null.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_alter.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_crud.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_offset.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_priv.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_job_schedule_task_error.groovy
create mode 100644 regression-test/suites/jsonb_p0/test_json_hash.groovy
copy regression-test/suites/load_p0/broker_load/{test_broker_load_with_where.groovy => test_s3_load_with_where.groovy} (54%)
delete mode 100644 regression-test/suites/load_p0/insert/test_insert_move_memtable.groovy
create mode 100644 regression-test/suites/load_p0/merge_into/test_merge_into.groovy
create mode 100644 regression-test/suites/load_p0/routine_load/test_routine_load_adaptive_param.groovy
create mode 100644 regression-test/suites/load_p0/routine_load/test_routine_load_delay_schedule.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter_table.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view_commit.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view_cte.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view_mtmv.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view_pct.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view_rollup.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_limit_partition_view_mtmv.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_view_hive_mtmv.groovy
create mode 100644 regression-test/suites/mv_p0/test_mv_with_keyword/test_mv_with_keyword.groovy
create mode 100644 regression-test/suites/nereids_p0/cache/mtmv_with_sql_cache.groovy
create mode 100644 regression-test/suites/nereids_p0/cache/mv_with_sql_cache.groovy
create mode 100644 regression-test/suites/nereids_p0/partition_topn/check_orderkey.groovy
copy fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteContext.java => regression-test/suites/nereids_rules_p0/avg_distinct_to_sum_div_count/avg_distinct_to_sum_div_count.groovy (62%)
create mode 100755 regression-test/suites/query_p0/cache/condition_cache.groovy
create mode 100644 regression-test/suites/query_p0/explain/explain_alias.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/binary_functions/test_binary_function.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/string_functions/test_string_all.groovy
create mode 100644 regression-test/suites/query_p0/system/test_query_sys_column_data_sizes.groovy
create mode 100644 regression-test/suites/rec_cte_p0/rec_cte/rec_cte.groovy
create mode 100644 regression-test/suites/rec_cte_p0/rec_cte_from_ck_doc/rec_cte_from_ck_doc.groovy
create mode 100644 regression-test/suites/rec_cte_p0/rec_cte_from_duckdb_doc/rec_cte_from_duckdb_doc.groovy
create mode 100644 regression-test/suites/rec_cte_p0/rec_cte_from_mysql_doc/rec_cte_from_mysql_doc.groovy
create mode 100644 regression-test/suites/search/test_search_mow_support.groovy
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q1.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q1.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q1.3.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q2.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q2.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q2.3.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q3.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q3.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q3.3.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q3.4.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q4.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q4.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q4.3.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q1.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q1.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q1.3.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q2.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q2.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q2.3.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q3.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q3.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q3.3.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q3.4.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q4.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q4.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q4.3.sql
delete mode 100644 regression-test/suites/temp_table_p0/test_temp_table_with_conn_timeout.groovy
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/customer.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/customer_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/lineitem.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/lineitem_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/nation.sql (100%)
copy regression-test/suites/{cloud_p0/multi_cluster/virtual_compute_group => tpch_unique_sql_zstd_bucket1_p1}/ddl/nation_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/orders.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/orders_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/part.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/part_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/partsupp.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/partsupp_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/region.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/region_delete.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/revenue1.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/revenue1_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/supplier.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/supplier_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/load.groovy (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q01.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q02.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q03.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q04.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q05.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q06.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q07.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q08.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q09.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q10.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q11.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q12.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q13.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q14.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q15.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q16.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q17.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q18.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q19.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q20.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q21.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q22.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/test_left_anti_join_batch_size.sql (100%)
create mode 100644 regression-test/suites/variant_p0/test_variant_search_subcolumn.groovy
delete mode 100644 thirdparty/patches/googletest-release-1.11.0.patch
delete mode 100644 thirdparty/patches/libevent-1532.patch
delete mode 100644 thirdparty/patches/libevent-keepalive-accepted-socket.patch
delete mode 100644 thirdparty/patches/libevent.patch
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/03: add rec cte thrift struct
Posted by pa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch dev_rec2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8e2fda6d44a046765a3b325bfd49bf9a716de418
Author: BiteTheDDDDt <xl...@selectdb.com>
AuthorDate: Thu Sep 18 21:02:33 2025 +0800
add rec cte thrift struct
update
update thrift
update thrift
add result_expr_lists to TRecCTENode
add TRecCTEScanNode
---
gensrc/thrift/PlanNodes.thrift | 28 +++++++++++++++++++++++++++-
1 file changed, 27 insertions(+), 1 deletion(-)
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index e99f93b3ff1..4036ad7ea49 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -60,7 +60,9 @@ enum TPlanNodeType {
TEST_EXTERNAL_SCAN_NODE = 31,
PARTITION_SORT_NODE = 32,
GROUP_COMMIT_SCAN_NODE = 33,
- MATERIALIZATION_NODE = 34
+ MATERIALIZATION_NODE = 34,
+ REC_CTE_NODE = 35,
+ REC_CTE_SCAN_NODE = 36
}
struct TKeyRange {
@@ -692,6 +694,27 @@ struct TFileScanNode {
2: optional string table_name
}
+struct TRecCTETarget {
+ 1: optional Types.TNetworkAddress addr
+ 2: optional Types.TUniqueId fragment_instance_id
+ 3: optional i32 node_id
+}
+
+struct TRecCTEResetInfo {
+ 1: optional Types.TNetworkAddress addr
+ 2: optional i32 fragment_id
+}
+
+struct TRecCTENode {
+ 1: optional bool is_union_all
+ 2: optional list<TRecCTETarget> targets
+ 3: optional list<TRecCTEResetInfo> fragments_to_reset
+ 4: optional list<list<Exprs.TExpr>> result_expr_lists
+}
+
+struct TRecCTEScanNode {
+}
+
struct TEsScanNode {
1: required Types.TTupleId tuple_id
2: optional map<string,string> properties
@@ -1427,6 +1450,7 @@ struct TPlanNode {
36: optional list<TRuntimeFilterDesc> runtime_filters
37: optional TGroupCommitScanNode group_commit_scan_node
38: optional TMaterializationNode materialization_node
+ 39: optional TRecCTENode rec_cte_node
// Use in vec exec engine
40: optional Exprs.TExpr vconjunct
@@ -1449,6 +1473,8 @@ struct TPlanNode {
50: optional list<list<Exprs.TExpr>> distribute_expr_lists
51: optional bool is_serial_operator
+ 52: optional TRecCTEScanNode rec_cte_scan_node
+
// projections is final projections, which means projecting into results and materializing them into the output block.
101: optional list<Exprs.TExpr> projections
102: optional Types.TTupleId output_tuple_id
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/03: add rec cte thrift struct
Posted by pa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch dev_rec2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a0931099bfaa88139aee7a12d5072241128e3f51
Author: BiteTheDDDDt <xl...@selectdb.com>
AuthorDate: Thu Sep 18 21:02:33 2025 +0800
add rec cte thrift struct
update
update thrift
add base
update
update thrift
update _send_data_to_targets
add fragment reset interface
update
reset format
update send block
clean
fix
add result_expr_lists to TRecCTENode
update
format
add TRecCTEScanNode
update
format
update
update uniq
update
Revert "update"
This reverts commit f464ed4fac5089db27bc479d73646f542c26bddc.
update
update
update
fix
update
update
update
update
update
update
update
update
update
format
update
tmp delete case
Revert "tmp delete case"
This reverts commit 20329c58a04ebf9cdce366026b6403f563b1092d.
update
fix
update
update
update
fix
fix
update
update
clean
fix
update
---
be/src/pipeline/exec/exchange_source_operator.cpp | 5 +
be/src/pipeline/exec/exchange_source_operator.h | 19 +
be/src/pipeline/exec/operator.cpp | 10 +
be/src/pipeline/exec/operator.h | 2 +
.../pipeline/exec/rec_cte_anchor_sink_operator.cpp | 56 ++
.../pipeline/exec/rec_cte_anchor_sink_operator.h | 114 +++
be/src/pipeline/exec/rec_cte_scan_operator.h | 87 ++
be/src/pipeline/exec/rec_cte_sink_operator.cpp | 55 ++
be/src/pipeline/exec/rec_cte_sink_operator.h | 101 +++
be/src/pipeline/exec/rec_cte_source_operator.cpp | 86 ++
be/src/pipeline/exec/rec_cte_source_operator.h | 228 +++++
be/src/pipeline/exec/union_sink_operator.h | 42 +-
be/src/pipeline/pipeline.h | 2 +-
be/src/pipeline/pipeline_fragment_context.cpp | 236 +++--
be/src/pipeline/pipeline_fragment_context.h | 18 +-
be/src/pipeline/pipeline_task.cpp | 25 +-
be/src/pipeline/rec_cte_shared_state.h | 175 ++++
be/src/runtime/fragment_mgr.cpp | 60 ++
be/src/runtime/fragment_mgr.h | 11 +
be/src/runtime/query_context.cpp | 47 +
be/src/runtime/query_context.h | 28 +-
be/src/runtime/runtime_predicate.h | 1 +
be/src/runtime/runtime_state.cpp | 16 +-
be/src/runtime/runtime_state.h | 19 +-
be/src/runtime_filter/runtime_filter_consumer.h | 15 +-
be/src/runtime_filter/runtime_filter_merger.h | 2 +
be/src/runtime_filter/runtime_filter_mgr.cpp | 22 +-
be/src/runtime_filter/runtime_filter_mgr.h | 26 +-
be/src/service/internal_service.cpp | 51 ++
be/src/service/internal_service.h | 10 +
.../runtime_filter_consumer_test.cpp | 20 +-
be/test/runtime_filter/runtime_filter_mgr_test.cpp | 2 +-
.../java/org/apache/doris/qe/SessionVariable.java | 9 +-
gensrc/proto/internal_service.proto | 42 +
gensrc/thrift/PaloInternalService.thrift | 1 +
.../data/rec_cte_p0/rec_cte/rec_cte.out | 953 +++++++++++++++++++++
.../rec_cte_from_ck_doc/rec_cte_from_ck_doc.out | 29 +
.../rec_cte_from_duckdb_doc.out | 30 +
.../rec_cte_from_mysql_doc.out | 42 +
.../suites/rec_cte_p0/rec_cte/rec_cte.groovy | 271 ++++++
.../rec_cte_from_ck_doc/rec_cte_from_ck_doc.groovy | 144 ++++
.../rec_cte_from_duckdb_doc.groovy | 185 ++++
.../rec_cte_from_mysql_doc.groovy | 140 +++
43 files changed, 3316 insertions(+), 121 deletions(-)
diff --git a/be/src/pipeline/exec/exchange_source_operator.cpp b/be/src/pipeline/exec/exchange_source_operator.cpp
index b31b193aff2..8762b40fa40 100644
--- a/be/src/pipeline/exec/exchange_source_operator.cpp
+++ b/be/src/pipeline/exec/exchange_source_operator.cpp
@@ -224,4 +224,9 @@ Status ExchangeSourceOperatorX::close(RuntimeState* state) {
_is_closed = true;
return OperatorX<ExchangeLocalState>::close(state);
}
+
+Status ExchangeSourceOperatorX::reset(RuntimeState* state) {
+ auto& local_state = get_local_state(state);
+ return local_state.reset(state);
+}
} // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/exchange_source_operator.h b/be/src/pipeline/exec/exchange_source_operator.h
index 3008217e130..03f2a288cdf 100644
--- a/be/src/pipeline/exec/exchange_source_operator.h
+++ b/be/src/pipeline/exec/exchange_source_operator.h
@@ -48,6 +48,23 @@ public:
Status close(RuntimeState* state) override;
std::string debug_string(int indentation_level) const override;
+ Status reset(RuntimeState* state) {
+ if (stream_recvr) {
+ stream_recvr->close();
+ }
+ create_stream_recvr(state);
+
+ is_ready = false;
+ num_rows_skipped = 0;
+
+ const auto& queues = stream_recvr->sender_queues();
+ for (size_t i = 0; i < queues.size(); i++) {
+ deps[i]->block();
+ queues[i]->set_dependency(deps[i]);
+ }
+ return Status::OK();
+ }
+
std::vector<Dependency*> dependencies() const override {
std::vector<Dependency*> dep_vec;
std::for_each(deps.begin(), deps.end(),
@@ -83,6 +100,8 @@ public:
Status init(const TPlanNode& tnode, RuntimeState* state) override;
Status prepare(RuntimeState* state) override;
+ Status reset(RuntimeState* state);
+
Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos) override;
std::string debug_string(int indentation_level = 0) const override;
diff --git a/be/src/pipeline/exec/operator.cpp b/be/src/pipeline/exec/operator.cpp
index 0eea8c7e157..ceaa9148b23 100644
--- a/be/src/pipeline/exec/operator.cpp
+++ b/be/src/pipeline/exec/operator.cpp
@@ -62,6 +62,10 @@
#include "pipeline/exec/partitioned_aggregation_source_operator.h"
#include "pipeline/exec/partitioned_hash_join_probe_operator.h"
#include "pipeline/exec/partitioned_hash_join_sink_operator.h"
+#include "pipeline/exec/rec_cte_anchor_sink_operator.h"
+#include "pipeline/exec/rec_cte_scan_operator.h"
+#include "pipeline/exec/rec_cte_sink_operator.h"
+#include "pipeline/exec/rec_cte_source_operator.h"
#include "pipeline/exec/repeat_operator.h"
#include "pipeline/exec/result_file_sink_operator.h"
#include "pipeline/exec/result_sink_operator.h"
@@ -803,6 +807,8 @@ DECLARE_OPERATOR(PartitionedHashJoinSinkLocalState)
DECLARE_OPERATOR(GroupCommitBlockSinkLocalState)
DECLARE_OPERATOR(CacheSinkLocalState)
DECLARE_OPERATOR(DictSinkLocalState)
+DECLARE_OPERATOR(RecCTESinkLocalState)
+DECLARE_OPERATOR(RecCTEAnchorSinkLocalState)
#undef DECLARE_OPERATOR
@@ -836,6 +842,8 @@ DECLARE_OPERATOR(MetaScanLocalState)
DECLARE_OPERATOR(LocalExchangeSourceLocalState)
DECLARE_OPERATOR(PartitionedHashJoinProbeLocalState)
DECLARE_OPERATOR(CacheSourceLocalState)
+DECLARE_OPERATOR(RecCTESourceLocalState)
+DECLARE_OPERATOR(RecCTEScanLocalState)
#ifdef BE_TEST
DECLARE_OPERATOR(MockLocalState)
@@ -871,6 +879,7 @@ template class PipelineXSinkLocalState<SetSharedState>;
template class PipelineXSinkLocalState<LocalExchangeSharedState>;
template class PipelineXSinkLocalState<BasicSharedState>;
template class PipelineXSinkLocalState<DataQueueSharedState>;
+template class PipelineXSinkLocalState<RecCTESharedState>;
template class PipelineXLocalState<HashJoinSharedState>;
template class PipelineXLocalState<PartitionedHashJoinSharedState>;
@@ -888,6 +897,7 @@ template class PipelineXLocalState<PartitionSortNodeSharedState>;
template class PipelineXLocalState<SetSharedState>;
template class PipelineXLocalState<LocalExchangeSharedState>;
template class PipelineXLocalState<BasicSharedState>;
+template class PipelineXLocalState<RecCTESharedState>;
template class AsyncWriterSink<doris::vectorized::VFileResultWriter, ResultFileSinkOperatorX>;
template class AsyncWriterSink<doris::vectorized::VJdbcTableWriter, JdbcTableSinkOperatorX>;
diff --git a/be/src/pipeline/exec/operator.h b/be/src/pipeline/exec/operator.h
index d5630fd4aa6..7492ee75ae9 100644
--- a/be/src/pipeline/exec/operator.h
+++ b/be/src/pipeline/exec/operator.h
@@ -610,6 +610,8 @@ public:
// For agg/sort/join sink.
virtual Status init(const TPlanNode& tnode, RuntimeState* state);
+ virtual bool need_rerun(RuntimeState* state) const { return false; }
+
Status init(const TDataSink& tsink) override;
[[nodiscard]] virtual Status init(ExchangeType type, const int num_buckets,
const bool use_global_hash_shuffle,
diff --git a/be/src/pipeline/exec/rec_cte_anchor_sink_operator.cpp b/be/src/pipeline/exec/rec_cte_anchor_sink_operator.cpp
new file mode 100644
index 00000000000..f1552ed8e5e
--- /dev/null
+++ b/be/src/pipeline/exec/rec_cte_anchor_sink_operator.cpp
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "pipeline/exec/rec_cte_anchor_sink_operator.h"
+
+namespace doris::pipeline {
+#include "common/compile_check_begin.h"
+
+Status RecCTEAnchorSinkLocalState::open(RuntimeState* state) {
+ SCOPED_TIMER(exec_time_counter());
+ SCOPED_TIMER(_open_timer);
+ RETURN_IF_ERROR(Base::open(state));
+ auto& p = _parent->cast<Parent>();
+ _child_expr.resize(p._child_expr.size());
+ for (size_t i = 0; i < p._child_expr.size(); i++) {
+ RETURN_IF_ERROR(p._child_expr[i]->clone(state, _child_expr[i]));
+ }
+ return Status::OK();
+}
+
+Status RecCTEAnchorSinkOperatorX::init(const TPlanNode& tnode, RuntimeState* state) {
+ RETURN_IF_ERROR(Base::init(tnode, state));
+ DCHECK(tnode.__isset.rec_cte_node);
+ {
+ const auto& texprs = tnode.rec_cte_node.result_expr_lists[0];
+ vectorized::VExprContextSPtrs ctxs;
+ RETURN_IF_ERROR(vectorized::VExpr::create_expr_trees(texprs, ctxs));
+ _child_expr = ctxs;
+ }
+ _name = "REC_CTE_ANCHOR_SINK_OPERATOR";
+ return Status::OK();
+}
+
+Status RecCTEAnchorSinkOperatorX::prepare(RuntimeState* state) {
+ RETURN_IF_ERROR(Base::prepare(state));
+ RETURN_IF_ERROR(vectorized::VExpr::prepare(_child_expr, state, _child->row_desc()));
+ RETURN_IF_ERROR(vectorized::VExpr::check_expr_output_type(_child_expr, _child->row_desc()));
+ RETURN_IF_ERROR(vectorized::VExpr::open(_child_expr, state));
+ return Status::OK();
+}
+
+} // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/rec_cte_anchor_sink_operator.h b/be/src/pipeline/exec/rec_cte_anchor_sink_operator.h
new file mode 100644
index 00000000000..340949a2f79
--- /dev/null
+++ b/be/src/pipeline/exec/rec_cte_anchor_sink_operator.h
@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+
+#include "common/status.h"
+#include "operator.h"
+#include "pipeline/exec/union_sink_operator.h"
+#include "pipeline/rec_cte_shared_state.h"
+#include "vec/core/block.h"
+
+namespace doris {
+#include "common/compile_check_begin.h"
+class RuntimeState;
+
+namespace pipeline {
+class DataQueue;
+
+class RecCTEAnchorSinkOperatorX;
+class RecCTEAnchorSinkLocalState final : public PipelineXSinkLocalState<RecCTESharedState> {
+public:
+ ENABLE_FACTORY_CREATOR(RecCTEAnchorSinkLocalState);
+ RecCTEAnchorSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state)
+ : Base(parent, state) {}
+ Status open(RuntimeState* state) override;
+
+ bool is_blockable() const override { return true; }
+
+private:
+ friend class RecCTEAnchorSinkOperatorX;
+ using Base = PipelineXSinkLocalState<RecCTESharedState>;
+ using Parent = RecCTEAnchorSinkOperatorX;
+
+ vectorized::VExprContextSPtrs _child_expr;
+};
+
+class RecCTEAnchorSinkOperatorX MOCK_REMOVE(final)
+ : public DataSinkOperatorX<RecCTEAnchorSinkLocalState> {
+public:
+ using Base = DataSinkOperatorX<RecCTEAnchorSinkLocalState>;
+
+ friend class RecCTEAnchorSinkLocalState;
+ RecCTEAnchorSinkOperatorX(int sink_id, int dest_id, const TPlanNode& tnode,
+ const DescriptorTbl& descs)
+ : Base(sink_id, tnode.node_id, dest_id),
+ _row_descriptor(descs, tnode.row_tuples, tnode.nullable_tuples) {}
+
+ ~RecCTEAnchorSinkOperatorX() override = default;
+
+ Status init(const TPlanNode& tnode, RuntimeState* state) override;
+ Status prepare(RuntimeState* state) override;
+
+ bool is_serial_operator() const override { return true; }
+
+ DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
+ return {ExchangeType::NOOP};
+ }
+
+ Status sink(RuntimeState* state, vectorized::Block* input_block, bool eos) override {
+ auto& local_state = get_local_state(state);
+
+ if (_need_notify_rec_side_ready) {
+ RETURN_IF_ERROR(get_local_state(state)._shared_state->send_data_to_targets(state, 0));
+ _need_notify_rec_side_ready = false;
+ }
+
+ COUNTER_UPDATE(local_state.rows_input_counter(), (int64_t)input_block->rows());
+ if (input_block->rows() != 0) {
+ vectorized::Block block;
+ RETURN_IF_ERROR(materialize_block(local_state._child_expr, input_block, &block, true));
+ RETURN_IF_ERROR(local_state._shared_state->emplace_block(state, std::move(block)));
+ }
+
+ if (eos) {
+ local_state._shared_state->anchor_dep->set_ready();
+ }
+ return Status::OK();
+ }
+
+ std::shared_ptr<BasicSharedState> create_shared_state() const override {
+ std::shared_ptr<BasicSharedState> ss = std::make_shared<RecCTESharedState>();
+ ss->id = operator_id();
+ for (const auto& dest : dests_id()) {
+ ss->related_op_ids.insert(dest);
+ }
+ return ss;
+ }
+
+private:
+ const RowDescriptor _row_descriptor;
+ vectorized::VExprContextSPtrs _child_expr;
+
+ bool _need_notify_rec_side_ready = true;
+};
+
+} // namespace pipeline
+#include "common/compile_check_end.h"
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/pipeline/exec/rec_cte_scan_operator.h b/be/src/pipeline/exec/rec_cte_scan_operator.h
new file mode 100644
index 00000000000..661d97927d1
--- /dev/null
+++ b/be/src/pipeline/exec/rec_cte_scan_operator.h
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "common/status.h"
+#include "pipeline/exec/operator.h"
+
+namespace doris {
+#include "common/compile_check_begin.h"
+class RuntimeState;
+} // namespace doris
+
+namespace doris::pipeline {
+
+class RecCTEScanOperatorX;
+class RecCTEScanLocalState final : public PipelineXLocalState<> {
+public:
+ ENABLE_FACTORY_CREATOR(RecCTEScanLocalState);
+
+ RecCTEScanLocalState(RuntimeState* state, OperatorXBase* parent)
+ : PipelineXLocalState<>(state, parent) {
+ _scan_dependency = Dependency::create_shared(_parent->operator_id(), _parent->node_id(),
+ _parent->get_name() + "_DEPENDENCY");
+ state->get_query_ctx()->registe_cte_scan(state->fragment_instance_id(), parent->node_id(),
+ this);
+ }
+ ~RecCTEScanLocalState() override {
+ state()->get_query_ctx()->deregiste_cte_scan(state()->fragment_instance_id(),
+ parent()->node_id());
+ }
+
+ Status add_block(const PBlock& pblock) {
+ vectorized::Block block;
+ RETURN_IF_ERROR(block.deserialize(pblock));
+ _blocks.emplace_back(std::move(block));
+ return Status::OK();
+ }
+
+ void set_ready() { _scan_dependency->set_ready(); }
+
+ std::vector<Dependency*> dependencies() const override { return {_scan_dependency.get()}; }
+
+private:
+ friend class RecCTEScanOperatorX;
+ std::vector<vectorized::Block> _blocks;
+ DependencySPtr _scan_dependency = nullptr;
+};
+
+class RecCTEScanOperatorX final : public OperatorX<RecCTEScanLocalState> {
+public:
+ RecCTEScanOperatorX(ObjectPool* pool, const TPlanNode& tnode, int operator_id,
+ const DescriptorTbl& descs)
+ : OperatorX<RecCTEScanLocalState>(pool, tnode, operator_id, descs) {}
+
+ Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos) override {
+ auto& local_state = get_local_state(state);
+
+ if (local_state._blocks.empty()) {
+ *eos = true;
+ return Status::OK();
+ }
+ *block = std::move(local_state._blocks.back());
+ RETURN_IF_ERROR(local_state.filter_block(local_state.conjuncts(), block, block->columns()));
+ local_state._blocks.pop_back();
+ return Status::OK();
+ }
+
+ bool is_source() const override { return true; }
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris::pipeline
\ No newline at end of file
diff --git a/be/src/pipeline/exec/rec_cte_sink_operator.cpp b/be/src/pipeline/exec/rec_cte_sink_operator.cpp
new file mode 100644
index 00000000000..1508b478bca
--- /dev/null
+++ b/be/src/pipeline/exec/rec_cte_sink_operator.cpp
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "pipeline/exec/rec_cte_sink_operator.h"
+
+namespace doris::pipeline {
+#include "common/compile_check_begin.h"
+
+Status RecCTESinkLocalState::open(RuntimeState* state) {
+ SCOPED_TIMER(exec_time_counter());
+ SCOPED_TIMER(_open_timer);
+ RETURN_IF_ERROR(Base::open(state));
+ auto& p = _parent->cast<Parent>();
+ _child_expr.resize(p._child_expr.size());
+ for (size_t i = 0; i < p._child_expr.size(); i++) {
+ RETURN_IF_ERROR(p._child_expr[i]->clone(state, _child_expr[i]));
+ }
+ return Status::OK();
+}
+
+Status RecCTESinkOperatorX::init(const TPlanNode& tnode, RuntimeState* state) {
+ RETURN_IF_ERROR(Base::init(tnode, state));
+ DCHECK(tnode.__isset.rec_cte_node);
+ {
+ const auto& texprs = tnode.rec_cte_node.result_expr_lists[1];
+ vectorized::VExprContextSPtrs ctxs;
+ RETURN_IF_ERROR(vectorized::VExpr::create_expr_trees(texprs, ctxs));
+ _child_expr = ctxs;
+ }
+ return Status::OK();
+}
+
+Status RecCTESinkOperatorX::prepare(RuntimeState* state) {
+ RETURN_IF_ERROR(Base::prepare(state));
+ RETURN_IF_ERROR(vectorized::VExpr::prepare(_child_expr, state, _child->row_desc()));
+ RETURN_IF_ERROR(vectorized::VExpr::check_expr_output_type(_child_expr, _child->row_desc()));
+ RETURN_IF_ERROR(vectorized::VExpr::open(_child_expr, state));
+ return Status::OK();
+}
+
+} // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/rec_cte_sink_operator.h b/be/src/pipeline/exec/rec_cte_sink_operator.h
new file mode 100644
index 00000000000..2321ba16ea3
--- /dev/null
+++ b/be/src/pipeline/exec/rec_cte_sink_operator.h
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+
+#include "common/status.h"
+#include "operator.h"
+#include "pipeline/exec/union_sink_operator.h"
+#include "pipeline/rec_cte_shared_state.h"
+#include "vec/core/block.h"
+
+namespace doris {
+#include "common/compile_check_begin.h"
+class RuntimeState;
+
+namespace pipeline {
+class DataQueue;
+
+class RecCTESinkOperatorX;
+class RecCTESinkLocalState final : public PipelineXSinkLocalState<RecCTESharedState> {
+public:
+ ENABLE_FACTORY_CREATOR(RecCTESinkLocalState);
+ RecCTESinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state)
+ : Base(parent, state) {}
+ Status open(RuntimeState* state) override;
+
+private:
+ friend class RecCTESinkOperatorX;
+ using Base = PipelineXSinkLocalState<RecCTESharedState>;
+ using Parent = RecCTESinkOperatorX;
+
+ vectorized::VExprContextSPtrs _child_expr;
+};
+
+class RecCTESinkOperatorX MOCK_REMOVE(final) : public DataSinkOperatorX<RecCTESinkLocalState> {
+public:
+ using Base = DataSinkOperatorX<RecCTESinkLocalState>;
+
+ friend class RecCTESinkLocalState;
+ RecCTESinkOperatorX(int sink_id, int dest_id, const TPlanNode& tnode,
+ const DescriptorTbl& descs)
+ : Base(sink_id, tnode.node_id, dest_id),
+ _row_descriptor(descs, tnode.row_tuples, tnode.nullable_tuples) {}
+
+ ~RecCTESinkOperatorX() override = default;
+
+ Status init(const TPlanNode& tnode, RuntimeState* state) override;
+ Status prepare(RuntimeState* state) override;
+
+ bool need_rerun(RuntimeState* state) const override {
+ return get_local_state(state)._shared_state->ready_to_return == false;
+ }
+
+ std::shared_ptr<BasicSharedState> create_shared_state() const override { return nullptr; }
+
+ bool is_serial_operator() const override { return true; }
+
+ DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
+ return {ExchangeType::NOOP};
+ }
+
+ Status sink(RuntimeState* state, vectorized::Block* input_block, bool eos) override {
+ auto& local_state = get_local_state(state);
+
+ COUNTER_UPDATE(local_state.rows_input_counter(), (int64_t)input_block->rows());
+ if (input_block->rows() != 0) {
+ vectorized::Block block;
+ RETURN_IF_ERROR(materialize_block(local_state._child_expr, input_block, &block, true));
+ RETURN_IF_ERROR(local_state._shared_state->emplace_block(state, std::move(block)));
+ }
+
+ if (eos) {
+ local_state._shared_state->source_dep->set_ready();
+ }
+ return Status::OK();
+ }
+
+private:
+ const RowDescriptor _row_descriptor;
+ vectorized::VExprContextSPtrs _child_expr;
+};
+
+} // namespace pipeline
+#include "common/compile_check_end.h"
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/pipeline/exec/rec_cte_source_operator.cpp b/be/src/pipeline/exec/rec_cte_source_operator.cpp
new file mode 100644
index 00000000000..9a02bb947b8
--- /dev/null
+++ b/be/src/pipeline/exec/rec_cte_source_operator.cpp
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "pipeline/exec/rec_cte_source_operator.h"
+
+namespace doris::pipeline {
+#include "common/compile_check_begin.h"
+
+RecCTESourceLocalState::RecCTESourceLocalState(RuntimeState* state, OperatorXBase* parent)
+ : Base(state, parent) {}
+
+Status RecCTESourceLocalState::open(RuntimeState* state) {
+ SCOPED_TIMER(exec_time_counter());
+ SCOPED_TIMER(_open_timer);
+ RETURN_IF_ERROR(Base::open(state));
+ return Status::OK();
+}
+
+Status RecCTESourceLocalState::init(RuntimeState* state, LocalStateInfo& info) {
+ RETURN_IF_ERROR(Base::init(state, info));
+ _shared_state->targets = _parent->cast<RecCTESourceOperatorX>()._targets;
+ _shared_state->max_recursion_depth =
+ _parent->cast<RecCTESourceOperatorX>()._max_recursion_depth;
+ _shared_state->source_dep = _dependency;
+ _anchor_dependency = Dependency::create_shared(_parent->operator_id(), _parent->node_id(),
+ _parent->get_name() + "_ANCHOR_DEPENDENCY");
+ _shared_state->anchor_dep = _anchor_dependency.get();
+
+ auto& p = _parent->cast<Parent>();
+ _child_expr.resize(p._child_expr.size());
+ for (size_t i = 0; i < p._child_expr.size(); i++) {
+ RETURN_IF_ERROR(p._child_expr[i]->clone(state, _child_expr[i]));
+ }
+ if (!_parent->cast<RecCTESourceOperatorX>()._is_union_all) {
+ _shared_state->agg_data = std::make_unique<DistinctDataVariants>();
+ RETURN_IF_ERROR(init_hash_method<DistinctDataVariants>(_shared_state->agg_data.get(),
+ get_data_types(_child_expr), false));
+ }
+
+ _shared_state->hash_table_compute_timer =
+ ADD_TIMER(Base::custom_profile(), "HashTableComputeTime");
+ _shared_state->hash_table_emplace_timer =
+ ADD_TIMER(Base::custom_profile(), "HashTableEmplaceTime");
+ _shared_state->hash_table_input_counter =
+ ADD_COUNTER(Base::custom_profile(), "HashTableInputCount", TUnit::UNIT);
+ return Status::OK();
+}
+
+Status RecCTESourceOperatorX::init(const TPlanNode& tnode, RuntimeState* state) {
+ RETURN_IF_ERROR(Base::init(tnode, state));
+ DCHECK(tnode.__isset.rec_cte_node);
+
+ _max_recursion_depth = state->cte_max_recursion_depth();
+
+ {
+ const auto& texprs = tnode.rec_cte_node.result_expr_lists[1];
+ vectorized::VExprContextSPtrs ctxs;
+ RETURN_IF_ERROR(vectorized::VExpr::create_expr_trees(texprs, ctxs));
+ _child_expr = ctxs;
+ }
+ return Status::OK();
+}
+
+Status RecCTESourceOperatorX::prepare(RuntimeState* state) {
+ RETURN_IF_ERROR(Base::prepare(state));
+ RETURN_IF_ERROR(vectorized::VExpr::prepare(_child_expr, state, _child->row_desc()));
+ RETURN_IF_ERROR(vectorized::VExpr::check_expr_output_type(_child_expr, _child->row_desc()));
+ RETURN_IF_ERROR(vectorized::VExpr::open(_child_expr, state));
+ return Status::OK();
+}
+
+} // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/rec_cte_source_operator.h b/be/src/pipeline/exec/rec_cte_source_operator.h
new file mode 100644
index 00000000000..e36358161d2
--- /dev/null
+++ b/be/src/pipeline/exec/rec_cte_source_operator.h
@@ -0,0 +1,228 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <gen_cpp/internal_service.pb.h>
+
+#include "common/status.h"
+#include "operator.h"
+#include "pipeline/exec/union_sink_operator.h"
+#include "pipeline/rec_cte_shared_state.h"
+#include "util/brpc_client_cache.h"
+#include "util/uid_util.h"
+#include "vec/core/block.h"
+
+namespace doris {
+#include "common/compile_check_begin.h"
+class RuntimeState;
+
+namespace vectorized {
+class Block;
+} // namespace vectorized
+
+namespace pipeline {
+class DataQueue;
+
+class RecCTESourceOperatorX;
+class RecCTESourceLocalState final : public PipelineXLocalState<RecCTESharedState> {
+public:
+ ENABLE_FACTORY_CREATOR(RecCTESourceLocalState);
+ using Base = PipelineXLocalState<RecCTESharedState>;
+ using Parent = RecCTESourceOperatorX;
+ RecCTESourceLocalState(RuntimeState* state, OperatorXBase* parent);
+
+ Status open(RuntimeState* state) override;
+ Status init(RuntimeState* state, LocalStateInfo& info) override;
+
+ bool is_blockable() const override { return true; }
+
+ std::vector<Dependency*> dependencies() const override {
+ return std::vector<Dependency*> {_dependency, _anchor_dependency.get()};
+ }
+
+private:
+ friend class RecCTESourceOperatorX;
+ friend class OperatorX<RecCTESourceLocalState>;
+
+ vectorized::VExprContextSPtrs _child_expr;
+
+ std::shared_ptr<Dependency> _anchor_dependency = nullptr;
+};
+
+class RecCTESourceOperatorX : public OperatorX<RecCTESourceLocalState> {
+public:
+ using Base = OperatorX<RecCTESourceLocalState>;
+ RecCTESourceOperatorX(ObjectPool* pool, const TPlanNode& tnode, int operator_id,
+ const DescriptorTbl& descs)
+ : Base(pool, tnode, operator_id, descs),
+ _is_union_all(tnode.rec_cte_node.is_union_all),
+ _targets(tnode.rec_cte_node.targets),
+ _fragments_to_reset(tnode.rec_cte_node.fragments_to_reset),
+ _global_rf_ids(tnode.rec_cte_node.rec_side_runtime_filter_ids),
+ _is_used_by_other_rec_cte(tnode.rec_cte_node.is_used_by_other_rec_cte) {
+ DCHECK(tnode.__isset.rec_cte_node);
+ }
+ ~RecCTESourceOperatorX() override = default;
+
+ Status init(const TPlanNode& tnode, RuntimeState* state) override;
+ Status prepare(RuntimeState* state) override;
+
+ Status terminate(RuntimeState* state) override {
+ RETURN_IF_ERROR(_send_close(state));
+ return Base::terminate(state);
+ }
+
+ Status close(RuntimeState* state) override {
+ RETURN_IF_ERROR(_send_close(state));
+ return Base::close(state);
+ }
+
+ Status set_child(OperatorPtr child) override {
+ Base::_child = child;
+ return Status::OK();
+ }
+
+ bool is_serial_operator() const override { return true; }
+
+ DataDistribution required_data_distribution(RuntimeState* /*state*/) const override {
+ return {ExchangeType::NOOP};
+ }
+
+ Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos) override {
+ auto& local_state = get_local_state(state);
+ auto& ctx = local_state._shared_state;
+ ctx->update_ready_to_return();
+
+ if (!ctx->ready_to_return) {
+ if (ctx->current_round + 1 > _max_recursion_depth) {
+ return Status::Aborted("reach cte_max_recursion_depth {}", _max_recursion_depth);
+ }
+
+ ctx->source_dep->block();
+ // ctx->blocks.size() may be changed after _recursive_process
+ int current_blocks_size = int(ctx->blocks.size());
+ RETURN_IF_ERROR(_recursive_process(state, ctx->last_round_offset));
+ ctx->current_round++;
+ ctx->last_round_offset = current_blocks_size;
+ } else {
+ if (ctx->blocks.empty()) {
+ *eos = true;
+ } else {
+ block->swap(ctx->blocks.back());
+ RETURN_IF_ERROR(
+ local_state.filter_block(local_state.conjuncts(), block, block->columns()));
+ ctx->blocks.pop_back();
+ }
+ }
+ return Status::OK();
+ }
+
+ bool is_source() const override { return true; }
+
+private:
+ Status _send_close(RuntimeState* state) {
+ if (!_aready_send_close && !_is_used_by_other_rec_cte) {
+ RETURN_IF_ERROR(_send_rerun_fragments(state, PRerunFragmentParams::close));
+ _aready_send_close = true;
+
+ auto* round_counter = ADD_COUNTER(get_local_state(state).Base::custom_profile(),
+ "RecursiveRound", TUnit::UNIT);
+ round_counter->set(int64_t(get_local_state(state)._shared_state->current_round));
+ }
+ return Status::OK();
+ }
+
+ Status _recursive_process(RuntimeState* state, size_t last_round_offset) const {
+ RETURN_IF_ERROR(_send_rerun_fragments(state, PRerunFragmentParams::wait));
+ RETURN_IF_ERROR(_send_reset_global_rf(state));
+ RETURN_IF_ERROR(_send_rerun_fragments(state, PRerunFragmentParams::release));
+ RETURN_IF_ERROR(_send_rerun_fragments(state, PRerunFragmentParams::rebuild));
+ RETURN_IF_ERROR(_send_rerun_fragments(state, PRerunFragmentParams::submit));
+ RETURN_IF_ERROR(get_local_state(state)._shared_state->send_data_to_targets(
+ state, last_round_offset));
+ return Status::OK();
+ }
+
+ Status _send_reset_global_rf(RuntimeState* state) const {
+ TNetworkAddress addr;
+ RETURN_IF_ERROR(state->global_runtime_filter_mgr()->get_merge_addr(&addr));
+ auto stub =
+ state->get_query_ctx()->exec_env()->brpc_internal_client_cache()->get_client(addr);
+ PResetGlobalRfParams request;
+ request.mutable_query_id()->CopyFrom(UniqueId(state->query_id()).to_proto());
+ for (auto filter_id : _global_rf_ids) {
+ request.add_filter_ids(filter_id);
+ }
+
+ PResetGlobalRfResult result;
+ brpc::Controller controller;
+ controller.set_timeout_ms(
+ get_execution_rpc_timeout_ms(state->get_query_ctx()->execution_timeout()));
+ stub->reset_global_rf(&controller, &request, &result, brpc::DoNothing());
+ brpc::Join(controller.call_id());
+ return Status::create(result.status());
+ }
+
+ Status _send_rerun_fragments(RuntimeState* state, PRerunFragmentParams_Opcode stage) const {
+ for (auto fragment : _fragments_to_reset) {
+ if (state->fragment_id() == fragment.fragment_id) {
+ return Status::InternalError("Fragment {} contains a recursive CTE node",
+ fragment.fragment_id);
+ }
+ auto stub =
+ state->get_query_ctx()->exec_env()->brpc_internal_client_cache()->get_client(
+ fragment.addr);
+
+ PRerunFragmentParams request;
+ request.mutable_query_id()->CopyFrom(UniqueId(state->query_id()).to_proto());
+ request.set_fragment_id(fragment.fragment_id);
+ request.set_stage(stage);
+
+ PRerunFragmentResult result;
+ brpc::Controller controller;
+ controller.set_timeout_ms(
+ get_execution_rpc_timeout_ms(state->get_query_ctx()->execution_timeout()));
+ stub->rerun_fragment(&controller, &request, &result, brpc::DoNothing());
+ brpc::Join(controller.call_id());
+ if (controller.Failed()) {
+ return Status::InternalError(controller.ErrorText());
+ }
+
+ RETURN_IF_ERROR(Status::create(result.status()));
+ }
+ return Status::OK();
+ }
+
+ friend class RecCTESourceLocalState;
+
+ vectorized::VExprContextSPtrs _child_expr;
+
+ const bool _is_union_all = false;
+ std::vector<TRecCTETarget> _targets;
+ std::vector<TRecCTEResetInfo> _fragments_to_reset;
+ std::vector<int> _global_rf_ids;
+
+ int _max_recursion_depth = 0;
+
+ bool _aready_send_close = false;
+
+ bool _is_used_by_other_rec_cte = false;
+};
+
+} // namespace pipeline
+#include "common/compile_check_end.h"
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/pipeline/exec/union_sink_operator.h b/be/src/pipeline/exec/union_sink_operator.h
index 06b95cdf92d..764f8099427 100644
--- a/be/src/pipeline/exec/union_sink_operator.h
+++ b/be/src/pipeline/exec/union_sink_operator.h
@@ -29,6 +29,24 @@ namespace doris {
#include "common/compile_check_begin.h"
class RuntimeState;
+inline Status materialize_block(const vectorized::VExprContextSPtrs& exprs,
+ vectorized::Block* src_block, vectorized::Block* res_block,
+ bool need_clone) {
+ vectorized::ColumnsWithTypeAndName columns;
+ auto rows = src_block->rows();
+ for (const auto& expr : exprs) {
+ int result_column_id = -1;
+ RETURN_IF_ERROR(expr->execute(src_block, &result_column_id));
+ const auto& src_col_with_type = src_block->get_by_position(result_column_id);
+ vectorized::ColumnPtr cloned_col = need_clone
+ ? src_col_with_type.column->clone_resized(rows)
+ : src_col_with_type.column;
+ columns.emplace_back(cloned_col, src_col_with_type.type, src_col_with_type.name);
+ }
+ *res_block = {columns};
+ return Status::OK();
+}
+
namespace pipeline {
class DataQueue;
@@ -153,27 +171,17 @@ private:
vectorized::VectorizedUtils::build_mutable_mem_reuse_block(output_block,
row_descriptor());
vectorized::Block res;
- RETURN_IF_ERROR(materialize_block(state, input_block, child_id, &res));
+ auto& local_state = get_local_state(state);
+ {
+ SCOPED_TIMER(local_state._expr_timer);
+ RETURN_IF_ERROR(
+ materialize_block(local_state._child_expr, input_block, &res, false));
+ }
+ local_state._child_row_idx += res.rows();
RETURN_IF_ERROR(mblock.merge(res));
}
return Status::OK();
}
-
- Status materialize_block(RuntimeState* state, vectorized::Block* src_block, int child_idx,
- vectorized::Block* res_block) {
- auto& local_state = get_local_state(state);
- SCOPED_TIMER(local_state._expr_timer);
- const auto& child_exprs = local_state._child_expr;
- vectorized::ColumnsWithTypeAndName colunms;
- for (size_t i = 0; i < child_exprs.size(); ++i) {
- int result_column_id = -1;
- RETURN_IF_ERROR(child_exprs[i]->execute(src_block, &result_column_id));
- colunms.emplace_back(src_block->get_by_position(result_column_id));
- }
- local_state._child_row_idx += src_block->rows();
- *res_block = {colunms};
- return Status::OK();
- }
};
} // namespace pipeline
diff --git a/be/src/pipeline/pipeline.h b/be/src/pipeline/pipeline.h
index 1d0ddcf178e..2a20a5cd73d 100644
--- a/be/src/pipeline/pipeline.h
+++ b/be/src/pipeline/pipeline.h
@@ -133,7 +133,7 @@ public:
fmt::format_to(debug_string_buffer, "\n{}", _operators[i]->debug_string(i));
}
fmt::format_to(debug_string_buffer, "\n{}",
- _sink->debug_string(cast_set<int>(_operators.size())));
+ _sink ? _sink->debug_string(cast_set<int>(_operators.size())) : "null");
return fmt::to_string(debug_string_buffer);
}
diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp
index b5c2f7084dc..ba7937e9522 100644
--- a/be/src/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_fragment_context.cpp
@@ -82,6 +82,10 @@
#include "pipeline/exec/partitioned_aggregation_source_operator.h"
#include "pipeline/exec/partitioned_hash_join_probe_operator.h"
#include "pipeline/exec/partitioned_hash_join_sink_operator.h"
+#include "pipeline/exec/rec_cte_anchor_sink_operator.h"
+#include "pipeline/exec/rec_cte_scan_operator.h"
+#include "pipeline/exec/rec_cte_sink_operator.h"
+#include "pipeline/exec/rec_cte_source_operator.h"
#include "pipeline/exec/repeat_operator.h"
#include "pipeline/exec/result_file_sink_operator.h"
#include "pipeline/exec/result_sink_operator.h"
@@ -134,7 +138,9 @@ PipelineFragmentContext::PipelineFragmentContext(
_is_report_on_cancel(true),
_report_status_cb(std::move(report_status_cb)),
_params(request),
- _parallel_instances(_params.__isset.parallel_instances ? _params.parallel_instances : 0) {
+ _parallel_instances(_params.__isset.parallel_instances ? _params.parallel_instances : 0),
+ _need_notify_close(request.__isset.need_notify_close ? request.need_notify_close
+ : false) {
_fragment_watcher.start();
}
@@ -142,23 +148,8 @@ PipelineFragmentContext::~PipelineFragmentContext() {
LOG_INFO("PipelineFragmentContext::~PipelineFragmentContext")
.tag("query_id", print_id(_query_id))
.tag("fragment_id", _fragment_id);
- // The memory released by the query end is recorded in the query mem tracker.
- SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_query_ctx->query_mem_tracker());
- auto st = _query_ctx->exec_status();
- for (size_t i = 0; i < _tasks.size(); i++) {
- if (!_tasks[i].empty()) {
- _call_back(_tasks[i].front().first->runtime_state(), &st);
- }
- }
- _tasks.clear();
- _dag.clear();
- _pip_id_to_pipeline.clear();
- _pipelines.clear();
- _sink.reset();
- _root_op.reset();
+ _release_resource();
_runtime_state.reset();
- _runtime_filter_mgr_map.clear();
- _op_id_to_shared_state.clear();
_query_ctx.reset();
}
@@ -253,6 +244,54 @@ PipelinePtr PipelineFragmentContext::add_pipeline(PipelinePtr parent, int idx) {
return pipeline;
}
+Status PipelineFragmentContext::_build_and_prepare_full_pipeline(ThreadPool* thread_pool) {
+ {
+ SCOPED_TIMER(_build_pipelines_timer);
+ // 2. Build pipelines with operators in this fragment.
+ auto root_pipeline = add_pipeline();
+ RETURN_IF_ERROR(_build_pipelines(_runtime_state->obj_pool(), *_query_ctx->desc_tbl,
+ &_root_op, root_pipeline));
+
+ // 3. Create sink operator
+ if (!_params.fragment.__isset.output_sink) {
+ return Status::InternalError("No output sink in this fragment!");
+ }
+ RETURN_IF_ERROR(_create_data_sink(_runtime_state->obj_pool(), _params.fragment.output_sink,
+ _params.fragment.output_exprs, _params,
+ root_pipeline->output_row_desc(), _runtime_state.get(),
+ *_desc_tbl, root_pipeline->id()));
+ RETURN_IF_ERROR(_sink->init(_params.fragment.output_sink));
+ RETURN_IF_ERROR(root_pipeline->set_sink(_sink));
+
+ for (PipelinePtr& pipeline : _pipelines) {
+ DCHECK(pipeline->sink() != nullptr) << pipeline->operators().size();
+ RETURN_IF_ERROR(pipeline->sink()->set_child(pipeline->operators().back()));
+ }
+ }
+ // 4. Build local exchanger
+ if (_runtime_state->enable_local_shuffle()) {
+ SCOPED_TIMER(_plan_local_exchanger_timer);
+ RETURN_IF_ERROR(_plan_local_exchange(_params.num_buckets,
+ _params.bucket_seq_to_instance_idx,
+ _params.shuffle_idx_to_instance_idx));
+ }
+
+ // 5. Initialize global states in pipelines.
+ for (PipelinePtr& pipeline : _pipelines) {
+ SCOPED_TIMER(_prepare_all_pipelines_timer);
+ pipeline->children().clear();
+ RETURN_IF_ERROR(pipeline->prepare(_runtime_state.get()));
+ }
+
+ {
+ SCOPED_TIMER(_build_tasks_timer);
+ // 6. Build pipeline tasks and initialize local state.
+ RETURN_IF_ERROR(_build_pipeline_tasks(thread_pool));
+ }
+
+ return Status::OK();
+}
+
Status PipelineFragmentContext::prepare(ThreadPool* thread_pool) {
if (_prepared) {
return Status::InternalError("Already prepared");
@@ -277,7 +316,7 @@ Status PipelineFragmentContext::prepare(ThreadPool* thread_pool) {
auto* fragment_context = this;
- if (_params.query_options.__isset.is_report_success) {
+ if (!_need_notify_close && _params.query_options.__isset.is_report_success) {
fragment_context->set_is_report_success(_params.query_options.is_report_success);
}
@@ -322,49 +361,7 @@ Status PipelineFragmentContext::prepare(ThreadPool* thread_pool) {
}
}
- {
- SCOPED_TIMER(_build_pipelines_timer);
- // 2. Build pipelines with operators in this fragment.
- auto root_pipeline = add_pipeline();
- RETURN_IF_ERROR(_build_pipelines(_runtime_state->obj_pool(), *_query_ctx->desc_tbl,
- &_root_op, root_pipeline));
-
- // 3. Create sink operator
- if (!_params.fragment.__isset.output_sink) {
- return Status::InternalError("No output sink in this fragment!");
- }
- RETURN_IF_ERROR(_create_data_sink(_runtime_state->obj_pool(), _params.fragment.output_sink,
- _params.fragment.output_exprs, _params,
- root_pipeline->output_row_desc(), _runtime_state.get(),
- *_desc_tbl, root_pipeline->id()));
- RETURN_IF_ERROR(_sink->init(_params.fragment.output_sink));
- RETURN_IF_ERROR(root_pipeline->set_sink(_sink));
-
- for (PipelinePtr& pipeline : _pipelines) {
- DCHECK(pipeline->sink() != nullptr) << pipeline->operators().size();
- RETURN_IF_ERROR(pipeline->sink()->set_child(pipeline->operators().back()));
- }
- }
- // 4. Build local exchanger
- if (_runtime_state->enable_local_shuffle()) {
- SCOPED_TIMER(_plan_local_exchanger_timer);
- RETURN_IF_ERROR(_plan_local_exchange(_params.num_buckets,
- _params.bucket_seq_to_instance_idx,
- _params.shuffle_idx_to_instance_idx));
- }
-
- // 5. Initialize global states in pipelines.
- for (PipelinePtr& pipeline : _pipelines) {
- SCOPED_TIMER(_prepare_all_pipelines_timer);
- pipeline->children().clear();
- RETURN_IF_ERROR(pipeline->prepare(_runtime_state.get()));
- }
-
- {
- SCOPED_TIMER(_build_tasks_timer);
- // 6. Build pipeline tasks and initialize local state.
- RETURN_IF_ERROR(_build_pipeline_tasks(thread_pool));
- }
+ RETURN_IF_ERROR(_build_and_prepare_full_pipeline(thread_pool));
_init_next_report_time();
@@ -374,6 +371,7 @@ Status PipelineFragmentContext::prepare(ThreadPool* thread_pool) {
Status PipelineFragmentContext::_build_pipeline_tasks(ThreadPool* thread_pool) {
_total_tasks = 0;
+ _closed_tasks = 0;
const auto target_size = _params.local_params.size();
_tasks.resize(target_size);
_runtime_filter_mgr_map.resize(target_size);
@@ -422,6 +420,10 @@ Status PipelineFragmentContext::_build_pipeline_tasks(ThreadPool* thread_pool) {
task_runtime_state->set_task_execution_context(shared_from_this());
task_runtime_state->set_be_number(local_params.backend_num);
+ if (_need_notify_close) {
+ // rec cte require child rf to wait infinitely to make sure all rpc done
+ task_runtime_state->set_force_make_rf_wait_infinite();
+ }
if (_params.__isset.backend_id) {
task_runtime_state->set_backend_id(_params.backend_id);
@@ -1645,6 +1647,42 @@ Status PipelineFragmentContext::_create_operator(ObjectPool* pool, const TPlanNo
RETURN_IF_ERROR(cur_pipe->add_operator(op, _parallel_instances));
break;
}
+ case TPlanNodeType::REC_CTE_NODE: {
+ op = std::make_shared<RecCTESourceOperatorX>(pool, tnode, next_operator_id(), descs);
+ RETURN_IF_ERROR(cur_pipe->add_operator(op, _parallel_instances));
+
+ const auto downstream_pipeline_id = cur_pipe->id();
+ if (!_dag.contains(downstream_pipeline_id)) {
+ _dag.insert({downstream_pipeline_id, {}});
+ }
+
+ PipelinePtr anchor_side_pipe = add_pipeline(cur_pipe);
+ _dag[downstream_pipeline_id].push_back(anchor_side_pipe->id());
+
+ DataSinkOperatorPtr anchor_sink;
+ anchor_sink = std::make_shared<RecCTEAnchorSinkOperatorX>(next_sink_operator_id(),
+ op->operator_id(), tnode, descs);
+ RETURN_IF_ERROR(anchor_side_pipe->set_sink(anchor_sink));
+ RETURN_IF_ERROR(anchor_side_pipe->sink()->init(tnode, _runtime_state.get()));
+ _pipeline_parent_map.push(op->node_id(), anchor_side_pipe);
+
+ PipelinePtr rec_side_pipe = add_pipeline(cur_pipe);
+ _dag[downstream_pipeline_id].push_back(rec_side_pipe->id());
+
+ DataSinkOperatorPtr rec_sink;
+ rec_sink = std::make_shared<RecCTESinkOperatorX>(next_sink_operator_id(), op->operator_id(),
+ tnode, descs);
+ RETURN_IF_ERROR(rec_side_pipe->set_sink(rec_sink));
+ RETURN_IF_ERROR(rec_side_pipe->sink()->init(tnode, _runtime_state.get()));
+ _pipeline_parent_map.push(op->node_id(), rec_side_pipe);
+
+ break;
+ }
+ case TPlanNodeType::REC_CTE_SCAN_NODE: {
+ op = std::make_shared<RecCTEScanOperatorX>(pool, tnode, next_operator_id(), descs);
+ RETURN_IF_ERROR(cur_pipe->add_operator(op, _parallel_instances));
+ break;
+ }
default:
return Status::InternalError("Unsupported exec type in pipeline: {}",
print_plan_node_type(tnode.node_type));
@@ -1750,7 +1788,10 @@ void PipelineFragmentContext::_close_fragment_instance() {
if (_is_fragment_instance_closed) {
return;
}
- Defer defer_op {[&]() { _is_fragment_instance_closed = true; }};
+ Defer defer_op {[&]() {
+ _is_fragment_instance_closed = true;
+ _close_cv.notify_all();
+ }};
_fragment_level_profile->total_time_counter()->update(_fragment_watcher.elapsed_time());
static_cast<void>(send_report(true));
// Print profile content in info log is a tempoeray solution for stream load and external_connector.
@@ -1785,8 +1826,10 @@ void PipelineFragmentContext::_close_fragment_instance() {
collect_realtime_load_channel_profile());
}
- // all submitted tasks done
- _exec_env->fragment_mgr()->remove_pipeline_context({_query_id, _fragment_id});
+ if (!_need_notify_close) {
+ // all submitted tasks done
+ _exec_env->fragment_mgr()->remove_pipeline_context({_query_id, _fragment_id});
+ }
}
void PipelineFragmentContext::decrement_running_task(PipelineId pipeline_id) {
@@ -1923,8 +1966,10 @@ std::vector<PipelineTask*> PipelineFragmentContext::get_revocable_tasks() const
}
std::string PipelineFragmentContext::debug_string() {
+ std::lock_guard<std::mutex> l(_task_mutex);
fmt::memory_buffer debug_string_buffer;
- fmt::format_to(debug_string_buffer, "PipelineFragmentContext Info:\n");
+ fmt::format_to(debug_string_buffer, "PipelineFragmentContext Info:\nneed_notify_close: {}\n",
+ _need_notify_close);
for (size_t j = 0; j < _tasks.size(); j++) {
fmt::format_to(debug_string_buffer, "Tasks in instance {}:\n", j);
for (size_t i = 0; i < _tasks[j].size(); i++) {
@@ -1998,5 +2043,66 @@ PipelineFragmentContext::collect_realtime_load_channel_profile() const {
_runtime_state->profile_level());
return load_channel_profile;
}
+
+Status PipelineFragmentContext::wait_close(bool close) {
+ if (_exec_env->new_load_stream_mgr()->get(_query_id) != nullptr) {
+ return Status::InternalError("stream load do not support reset");
+ }
+ if (!_need_notify_close) {
+ return Status::InternalError("_need_notify_close is false, do not support reset");
+ }
+
+ {
+ std::unique_lock<std::mutex> lock(_task_mutex);
+ _close_cv.wait(lock, [this] { return _is_fragment_instance_closed.load(); });
+ }
+
+ if (close) {
+ _exec_env->fragment_mgr()->remove_pipeline_context({_query_id, _fragment_id});
+ }
+ return Status::OK();
+}
+
+Status PipelineFragmentContext::set_to_rerun() {
+ {
+ std::lock_guard<std::mutex> l(_task_mutex);
+ SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_query_ctx->query_mem_tracker());
+ for (size_t i = 0; i < _tasks.size(); i++) {
+ if (!_tasks[i].empty()) {
+ _tasks[i].front().first->runtime_state()->reset_to_rerun();
+ }
+ }
+ }
+ _release_resource();
+ _runtime_state->reset_to_rerun();
+ return Status::OK();
+}
+
+Status PipelineFragmentContext::rebuild(ThreadPool* thread_pool) {
+ _submitted = false;
+ _is_fragment_instance_closed = false;
+ return _build_and_prepare_full_pipeline(thread_pool);
+}
+
+void PipelineFragmentContext::_release_resource() {
+ std::lock_guard<std::mutex> l(_task_mutex);
+ // The memory released by the query end is recorded in the query mem tracker.
+ SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_query_ctx->query_mem_tracker());
+ auto st = _query_ctx->exec_status();
+ for (size_t i = 0; i < _tasks.size(); i++) {
+ if (!_tasks[i].empty()) {
+ _call_back(_tasks[i].front().first->runtime_state(), &st);
+ }
+ }
+ _tasks.clear();
+ _dag.clear();
+ _pip_id_to_pipeline.clear();
+ _pipelines.clear();
+ _sink.reset();
+ _root_op.reset();
+ _runtime_filter_mgr_map.clear();
+ _op_id_to_shared_state.clear();
+}
+
#include "common/compile_check_end.h"
} // namespace doris::pipeline
diff --git a/be/src/pipeline/pipeline_fragment_context.h b/be/src/pipeline/pipeline_fragment_context.h
index 81b3f57b01f..40bb80f72ec 100644
--- a/be/src/pipeline/pipeline_fragment_context.h
+++ b/be/src/pipeline/pipeline_fragment_context.h
@@ -115,6 +115,9 @@ public:
[[nodiscard]] std::vector<PipelineTask*> get_revocable_tasks() const;
void clear_finished_tasks() {
+ if (_need_notify_close) {
+ return;
+ }
for (size_t j = 0; j < _tasks.size(); j++) {
for (size_t i = 0; i < _tasks[j].size(); i++) {
_tasks[j][i].first->stop_if_finished();
@@ -125,7 +128,17 @@ public:
std::string get_load_error_url();
std::string get_first_error_msg();
+ Status wait_close(bool close);
+ Status rebuild(ThreadPool* thread_pool);
+ Status set_to_rerun();
+
+ bool need_notify_close() const { return _need_notify_close; }
+
private:
+ void _release_resource();
+
+ Status _build_and_prepare_full_pipeline(ThreadPool* thread_pool);
+
Status _build_pipelines(ObjectPool* pool, const DescriptorTbl& descs, OperatorPtr* root,
PipelinePtr cur_pipe);
Status _create_tree_helper(ObjectPool* pool, const std::vector<TPlanNode>& tnodes,
@@ -182,6 +195,7 @@ private:
Pipelines _pipelines;
PipelineId _next_pipeline_id = 0;
std::mutex _task_mutex;
+ std::condition_variable _close_cv;
int _closed_tasks = 0;
// After prepared, `_total_tasks` is equal to the size of `_tasks`.
// When submit fail, `_total_tasks` is equal to the number of tasks submitted.
@@ -203,7 +217,7 @@ private:
RuntimeProfile::Counter* _build_tasks_timer = nullptr;
std::function<void(RuntimeState*, Status*)> _call_back;
- bool _is_fragment_instance_closed = false;
+ std::atomic_bool _is_fragment_instance_closed = false;
// If this is set to false, and '_is_report_success' is false as well,
// This executor will not report status to FE on being cancelled.
@@ -323,6 +337,8 @@ private:
TPipelineFragmentParams _params;
int32_t _parallel_instances = 0;
+
+ bool _need_notify_close = false;
};
} // namespace pipeline
} // namespace doris
diff --git a/be/src/pipeline/pipeline_task.cpp b/be/src/pipeline/pipeline_task.cpp
index 930e885fc3b..edc56332058 100644
--- a/be/src/pipeline/pipeline_task.cpp
+++ b/be/src/pipeline/pipeline_task.cpp
@@ -30,6 +30,7 @@
#include "common/logging.h"
#include "common/status.h"
#include "pipeline/dependency.h"
+#include "pipeline/exec/exchange_source_operator.h"
#include "pipeline/exec/operator.h"
#include "pipeline/exec/scan_operator.h"
#include "pipeline/pipeline.h"
@@ -553,10 +554,6 @@ Status PipelineTask::execute(bool* done) {
}
}
- if (_eos) {
- RETURN_IF_ERROR(close(Status::OK(), false));
- }
-
DBUG_EXECUTE_IF("PipelineTask::execute.sink_eos_sleep", {
auto required_pipeline_id =
DebugPoints::instance()->get_debug_param_or_default<int32_t>(
@@ -595,6 +592,22 @@ Status PipelineTask::execute(bool* done) {
RETURN_IF_ERROR(block->check_type_and_column());
status = _sink->sink(_state, block, _eos);
+ if (_eos) {
+ if (_sink->need_rerun(_state)) {
+ if (auto* source = dynamic_cast<ExchangeSourceOperatorX*>(_root);
+ source != nullptr) {
+ RETURN_IF_ERROR(source->reset(_state));
+ _eos = false;
+ } else {
+ return Status::InternalError(
+ "Only ExchangeSourceOperatorX can be rerun, real is {}",
+ _root->get_name());
+ }
+ } else {
+ RETURN_IF_ERROR(close(Status::OK(), false));
+ }
+ }
+
if (status.is<ErrorCode::END_OF_FILE>()) {
set_wake_up_early();
return Status::OK();
@@ -857,7 +870,9 @@ Status PipelineTask::wake_up(Dependency* dep, std::unique_lock<std::mutex>& /* d
_blocked_dep = nullptr;
auto holder = std::dynamic_pointer_cast<PipelineTask>(shared_from_this());
RETURN_IF_ERROR(_state_transition(PipelineTask::State::RUNNABLE));
- RETURN_IF_ERROR(_state->get_query_ctx()->get_pipe_exec_scheduler()->submit(holder));
+ if (auto f = _fragment_context.lock(); f) {
+ RETURN_IF_ERROR(_state->get_query_ctx()->get_pipe_exec_scheduler()->submit(holder));
+ }
return Status::OK();
}
diff --git a/be/src/pipeline/rec_cte_shared_state.h b/be/src/pipeline/rec_cte_shared_state.h
new file mode 100644
index 00000000000..17fb00c82f5
--- /dev/null
+++ b/be/src/pipeline/rec_cte_shared_state.h
@@ -0,0 +1,175 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "dependency.h"
+#include "pipeline/common/distinct_agg_utils.h"
+#include "util/brpc_client_cache.h"
+
+namespace doris::pipeline {
+#include "common/compile_check_begin.h"
+
+struct RecCTESharedState : public BasicSharedState {
+ std::vector<TRecCTETarget> targets;
+ std::vector<vectorized::Block> blocks;
+ vectorized::IColumn::Selector distinct_row;
+ Dependency* source_dep = nullptr;
+ Dependency* anchor_dep = nullptr;
+ vectorized::Arena arena;
+ RuntimeProfile::Counter* hash_table_compute_timer = nullptr;
+ RuntimeProfile::Counter* hash_table_emplace_timer = nullptr;
+ RuntimeProfile::Counter* hash_table_input_counter = nullptr;
+
+ std::unique_ptr<DistinctDataVariants> agg_data = nullptr;
+
+ int current_round = 0;
+ int last_round_offset = 0;
+ int max_recursion_depth = 0;
+ bool ready_to_return = false;
+
+ void update_ready_to_return() {
+ if (last_round_offset == blocks.size()) {
+ ready_to_return = true;
+ }
+ }
+
+ Status emplace_block(RuntimeState* state, vectorized::Block&& block) {
+ if (agg_data) {
+ auto num_rows = uint32_t(block.rows());
+ vectorized::ColumnRawPtrs raw_columns;
+ std::vector<vectorized::ColumnPtr> columns = block.get_columns_and_convert();
+ for (auto& col : columns) {
+ raw_columns.push_back(col.get());
+ }
+
+ std::visit(vectorized::Overload {
+ [&](std::monostate& arg) -> void {
+ throw doris::Exception(ErrorCode::INTERNAL_ERROR,
+ "uninited hash table");
+ },
+ [&](auto& agg_method) -> void {
+ SCOPED_TIMER(hash_table_compute_timer);
+ using HashMethodType = std::decay_t<decltype(agg_method)>;
+ using AggState = typename HashMethodType::State;
+
+ AggState agg_state(raw_columns);
+ agg_method.init_serialized_keys(raw_columns, num_rows);
+ distinct_row.clear();
+
+ size_t row = 0;
+ auto creator = [&](const auto& ctor, auto& key, auto& origin) {
+ HashMethodType::try_presis_key(key, origin, arena);
+ ctor(key);
+ distinct_row.push_back(row);
+ };
+ auto creator_for_null_key = [&]() {
+ distinct_row.push_back(row);
+ };
+
+ SCOPED_TIMER(hash_table_emplace_timer);
+ for (; row < num_rows; ++row) {
+ agg_method.lazy_emplace(agg_state, row, creator,
+ creator_for_null_key);
+ }
+ COUNTER_UPDATE(hash_table_input_counter, num_rows);
+ }},
+ agg_data->method_variant);
+
+ if (distinct_row.size() == block.rows()) {
+ blocks.emplace_back(std::move(block));
+ } else if (!distinct_row.empty()) {
+ auto distinct_block = vectorized::MutableBlock(block.clone_empty());
+ RETURN_IF_ERROR(block.append_to_block_by_selector(&distinct_block, distinct_row));
+ blocks.emplace_back(distinct_block.to_block());
+ }
+ } else {
+ blocks.emplace_back(std::move(block));
+ }
+ return Status::OK();
+ }
+
+ PTransmitRecCTEBlockParams build_basic_param(RuntimeState* state,
+ const TRecCTETarget& target) const {
+ PTransmitRecCTEBlockParams request;
+ request.set_node_id(target.node_id);
+ request.mutable_query_id()->CopyFrom(UniqueId(state->query_id()).to_proto());
+ request.mutable_fragment_instance_id()->CopyFrom(
+ UniqueId(target.fragment_instance_id).to_proto());
+ return request;
+ }
+
+ Status send_data_to_targets(RuntimeState* state, size_t round_offset) const {
+ int send_multi_blocks_byte_size = state->query_options().exchange_multi_blocks_byte_size;
+ int block_number_per_target =
+ int(blocks.size() - round_offset + targets.size() - 1) / targets.size();
+ for (auto target : targets) {
+ auto stub =
+ state->get_query_ctx()->exec_env()->brpc_internal_client_cache()->get_client(
+ target.addr);
+ if (!stub) {
+ return Status::InternalError(fmt::format("Get rpc stub failed, host={}, port={}",
+ target.addr.hostname, target.addr.port));
+ }
+
+ // send blocks
+ int step = block_number_per_target;
+ while (round_offset < blocks.size() && step > 0) {
+ PTransmitRecCTEBlockParams request = build_basic_param(state, target);
+ auto current_bytes = 0;
+ while (round_offset < blocks.size() && step > 0 &&
+ current_bytes < send_multi_blocks_byte_size) {
+ auto* pblock = request.add_blocks();
+ size_t uncompressed_bytes = 0;
+ size_t compressed_bytes = 0;
+ RETURN_IF_ERROR(blocks[round_offset].serialize(
+ state->be_exec_version(), pblock, &uncompressed_bytes,
+ &compressed_bytes, state->fragement_transmission_compression_type()));
+ round_offset++;
+ step--;
+ current_bytes += compressed_bytes;
+ }
+ request.set_eos(false);
+
+ PTransmitRecCTEBlockResult result;
+ brpc::Controller controller;
+ controller.set_timeout_ms(
+ get_execution_rpc_timeout_ms(state->get_query_ctx()->execution_timeout()));
+
+ stub->transmit_rec_cte_block(&controller, &request, &result, brpc::DoNothing());
+ brpc::Join(controller.call_id());
+ RETURN_IF_ERROR(Status::create(result.status()));
+ }
+
+ // send eos
+ {
+ PTransmitRecCTEBlockParams request = build_basic_param(state, target);
+ request.set_eos(true);
+
+ PTransmitRecCTEBlockResult result;
+ brpc::Controller controller;
+ stub->transmit_rec_cte_block(&controller, &request, &result, brpc::DoNothing());
+ brpc::Join(controller.call_id());
+ RETURN_IF_ERROR(Status::create(result.status()));
+ }
+ }
+ return Status::OK();
+ }
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris::pipeline
diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp
index 47fd9e264bf..f531e1ef2b9 100644
--- a/be/src/runtime/fragment_mgr.cpp
+++ b/be/src/runtime/fragment_mgr.cpp
@@ -1403,6 +1403,66 @@ Status FragmentMgr::get_query_statistics(const TUniqueId& query_id, TQueryStatis
print_id(query_id), query_stats);
}
+Status FragmentMgr::transmit_rec_cte_block(
+ const TUniqueId& query_id, const TUniqueId& instance_id, int node_id,
+ const google::protobuf::RepeatedPtrField<doris::PBlock>& pblocks, bool eos) {
+ if (auto q_ctx = get_query_ctx(query_id)) {
+ SCOPED_ATTACH_TASK(q_ctx.get());
+ return q_ctx->send_block_to_cte_scan(instance_id, node_id, pblocks, eos);
+ } else {
+ return Status::EndOfFile(
+ "Transmit rec cte block failed: Query context (query-id: {}) not found, maybe "
+ "finished",
+ print_id(query_id));
+ }
+}
+
+Status FragmentMgr::rerun_fragment(const TUniqueId& query_id, int fragment,
+ PRerunFragmentParams_Opcode stage) {
+ if (auto q_ctx = get_query_ctx(query_id)) {
+ SCOPED_ATTACH_TASK(q_ctx.get());
+ auto fragment_ctx = _pipeline_map.find({query_id, fragment});
+ if (!fragment_ctx) {
+ return Status::NotFound("Fragment context (query-id: {}, fragment-id: {}) not found",
+ print_id(query_id), fragment);
+ }
+
+ if (stage == PRerunFragmentParams::wait) {
+ return fragment_ctx->wait_close(false);
+ }
+ if (stage == PRerunFragmentParams::release) {
+ return fragment_ctx->set_to_rerun();
+ }
+ if (stage == PRerunFragmentParams::rebuild) {
+ return fragment_ctx->rebuild(_thread_pool.get());
+ }
+ if (stage == PRerunFragmentParams::submit) {
+ return fragment_ctx->submit();
+ }
+ if (stage == PRerunFragmentParams::close) {
+ return fragment_ctx->wait_close(true);
+ }
+ } else {
+ return Status::NotFound(
+ "reset_fragment: Query context (query-id: {}) not found, maybe finished",
+ print_id(query_id));
+ }
+ return Status::OK();
+}
+
+Status FragmentMgr::reset_global_rf(const TUniqueId& query_id,
+ const google::protobuf::RepeatedField<int32_t>& filter_ids) {
+ if (auto q_ctx = get_query_ctx(query_id)) {
+ SCOPED_ATTACH_TASK(q_ctx.get());
+ return q_ctx->reset_global_rf(filter_ids);
+ } else {
+ return Status::NotFound(
+ "reset_fragment: Query context (query-id: {}) not found, maybe finished",
+ print_id(query_id));
+ }
+ return Status::OK();
+}
+
#include "common/compile_check_end.h"
} // namespace doris
diff --git a/be/src/runtime/fragment_mgr.h b/be/src/runtime/fragment_mgr.h
index b0c1a3ad592..58725a49a63 100644
--- a/be/src/runtime/fragment_mgr.h
+++ b/be/src/runtime/fragment_mgr.h
@@ -184,6 +184,17 @@ public:
std::shared_ptr<QueryContext> get_query_ctx(const TUniqueId& query_id);
+ Status transmit_rec_cte_block(const TUniqueId& query_id, const TUniqueId& instance_id,
+ int node_id,
+ const google::protobuf::RepeatedPtrField<PBlock>& pblocks,
+ bool eos);
+
+ Status rerun_fragment(const TUniqueId& query_id, int fragment,
+ PRerunFragmentParams_Opcode stage);
+
+ Status reset_global_rf(const TUniqueId& query_id,
+ const google::protobuf::RepeatedField<int32_t>& filter_ids);
+
private:
struct BrpcItem {
TNetworkAddress network_address;
diff --git a/be/src/runtime/query_context.cpp b/be/src/runtime/query_context.cpp
index 5332b886d58..b3097dde69f 100644
--- a/be/src/runtime/query_context.cpp
+++ b/be/src/runtime/query_context.cpp
@@ -34,6 +34,7 @@
#include "common/status.h"
#include "olap/olap_common.h"
#include "pipeline/dependency.h"
+#include "pipeline/exec/rec_cte_scan_operator.h"
#include "pipeline/pipeline_fragment_context.h"
#include "runtime/exec_env.h"
#include "runtime/fragment_mgr.h"
@@ -460,6 +461,10 @@ QueryContext::_collect_realtime_query_profile() {
continue;
}
+ if (fragment_ctx->need_notify_close()) {
+ continue;
+ }
+
auto profile = fragment_ctx->collect_realtime_profile();
if (profile.empty()) {
@@ -497,4 +502,46 @@ TReportExecStatusParams QueryContext::get_realtime_exec_status() {
return exec_status;
}
+Status QueryContext::send_block_to_cte_scan(
+ const TUniqueId& instance_id, int node_id,
+ const google::protobuf::RepeatedPtrField<doris::PBlock>& pblocks, bool eos) {
+ std::unique_lock<std::mutex> l(_cte_scan_lock);
+ auto it = _cte_scan.find(std::make_pair(instance_id, node_id));
+ if (it == _cte_scan.end()) {
+ return Status::InternalError("RecCTEScan not found for instance {}, node {}",
+ print_id(instance_id), node_id);
+ }
+ for (const auto& pblock : pblocks) {
+ RETURN_IF_ERROR(it->second->add_block(pblock));
+ }
+ if (eos) {
+ it->second->set_ready();
+ }
+ return Status::OK();
+}
+
+void QueryContext::registe_cte_scan(const TUniqueId& instance_id, int node_id,
+ pipeline::RecCTEScanLocalState* scan) {
+ std::unique_lock<std::mutex> l(_cte_scan_lock);
+ auto key = std::make_pair(instance_id, node_id);
+ DCHECK(!_cte_scan.contains(key)) << "Duplicate registe cte scan for instance "
+ << print_id(instance_id) << ", node " << node_id;
+ _cte_scan.emplace(key, scan);
+}
+
+void QueryContext::deregiste_cte_scan(const TUniqueId& instance_id, int node_id) {
+ std::lock_guard<std::mutex> l(_cte_scan_lock);
+ auto key = std::make_pair(instance_id, node_id);
+ DCHECK(_cte_scan.contains(key)) << "Duplicate deregiste cte scan for instance "
+ << print_id(instance_id) << ", node " << node_id;
+ _cte_scan.erase(key);
+}
+
+Status QueryContext::reset_global_rf(const google::protobuf::RepeatedField<int32_t>& filter_ids) {
+ if (_merge_controller_handler) {
+ return _merge_controller_handler->reset_global_rf(this, filter_ids);
+ }
+ return Status::OK();
+}
+
} // namespace doris
diff --git a/be/src/runtime/query_context.h b/be/src/runtime/query_context.h
index 32458cdb00e..874e889e368 100644
--- a/be/src/runtime/query_context.h
+++ b/be/src/runtime/query_context.h
@@ -32,6 +32,7 @@
#include "common/config.h"
#include "common/factory_creator.h"
#include "common/object_pool.h"
+#include "common/status.h"
#include "runtime/exec_env.h"
#include "runtime/memory/mem_tracker_limiter.h"
#include "runtime/runtime_predicate.h"
@@ -48,6 +49,7 @@ namespace pipeline {
class PipelineFragmentContext;
class PipelineTask;
class Dependency;
+class RecCTEScanLocalState;
} // namespace pipeline
struct ReportStatusRequest {
@@ -161,11 +163,6 @@ public:
return _query_options.runtime_filter_wait_time_ms;
}
- bool runtime_filter_wait_infinitely() const {
- return _query_options.__isset.runtime_filter_wait_infinitely &&
- _query_options.runtime_filter_wait_infinitely;
- }
-
int be_exec_version() const {
if (!_query_options.__isset.be_exec_version) {
return 0;
@@ -299,6 +296,23 @@ public:
void set_first_error_msg(std::string error_msg);
std::string get_first_error_msg();
+ Status send_block_to_cte_scan(const TUniqueId& instance_id, int node_id,
+ const google::protobuf::RepeatedPtrField<doris::PBlock>& pblocks,
+ bool eos);
+ void registe_cte_scan(const TUniqueId& instance_id, int node_id,
+ pipeline::RecCTEScanLocalState* scan);
+ void deregiste_cte_scan(const TUniqueId& instance_id, int node_id);
+
+ std::vector<int> get_fragment_ids() {
+ std::vector<int> fragment_ids;
+ for (const auto& it : _fragment_id_to_pipeline_ctx) {
+ fragment_ids.push_back(it.first);
+ }
+ return fragment_ids;
+ }
+
+ Status reset_global_rf(const google::protobuf::RepeatedField<int32_t>& filter_ids);
+
private:
friend class QueryTaskController;
@@ -377,6 +391,10 @@ private:
std::string _load_error_url;
std::string _first_error_msg;
+ // instance id + node id -> cte scan
+ std::map<std::pair<TUniqueId, int>, pipeline::RecCTEScanLocalState*> _cte_scan;
+ std::mutex _cte_scan_lock;
+
public:
// when fragment of pipeline is closed, it will register its profile to this map by using add_fragment_profile
void add_fragment_profile(
diff --git a/be/src/runtime/runtime_predicate.h b/be/src/runtime/runtime_predicate.h
index 51c79e1b426..b3e19d089fc 100644
--- a/be/src/runtime/runtime_predicate.h
+++ b/be/src/runtime/runtime_predicate.h
@@ -55,6 +55,7 @@ public:
void set_detected_source() {
std::unique_lock<std::shared_mutex> wlock(_rwlock);
+ _orderby_extrem = Field(PrimitiveType::TYPE_NULL);
_detected_source = true;
}
diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp
index 33db0f60e7c..faeda888cfb 100644
--- a/be/src/runtime/runtime_state.cpp
+++ b/be/src/runtime/runtime_state.cpp
@@ -503,7 +503,7 @@ Status RuntimeState::register_consumer_runtime_filter(
std::shared_ptr<RuntimeFilterConsumer>* consumer_filter) {
bool need_merge = desc.has_remote_targets || need_local_merge;
RuntimeFilterMgr* mgr = need_merge ? global_runtime_filter_mgr() : local_runtime_filter_mgr();
- return mgr->register_consumer_filter(_query_ctx, desc, node_id, consumer_filter);
+ return mgr->register_consumer_filter(this, desc, node_id, consumer_filter);
}
bool RuntimeState::is_nereids() const {
@@ -515,12 +515,22 @@ std::vector<std::shared_ptr<RuntimeProfile>> RuntimeState::pipeline_id_to_profil
return _pipeline_id_to_profile;
}
+void RuntimeState::reset_to_rerun() {
+ if (local_runtime_filter_mgr()) {
+ auto filter_ids = local_runtime_filter_mgr()->get_filter_ids();
+ filter_ids.merge(global_runtime_filter_mgr()->get_filter_ids());
+ local_runtime_filter_mgr()->remove_filters(filter_ids);
+ global_runtime_filter_mgr()->remove_filters(filter_ids);
+ }
+ std::unique_lock lc(_pipeline_profile_lock);
+ _pipeline_id_to_profile.clear();
+}
+
std::vector<std::shared_ptr<RuntimeProfile>> RuntimeState::build_pipeline_profile(
std::size_t pipeline_size) {
std::unique_lock lc(_pipeline_profile_lock);
if (!_pipeline_id_to_profile.empty()) {
- throw Exception(ErrorCode::INTERNAL_ERROR,
- "build_pipeline_profile can only be called once.");
+ return _pipeline_id_to_profile;
}
_pipeline_id_to_profile.resize(pipeline_size);
{
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index 3d89f4aa0d4..907f2c50a61 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -503,7 +503,7 @@ public:
_runtime_filter_mgr = runtime_filter_mgr;
}
- QueryContext* get_query_ctx() { return _query_ctx; }
+ QueryContext* get_query_ctx() const { return _query_ctx; }
[[nodiscard]] bool low_memory_mode() const;
@@ -520,6 +520,12 @@ public:
return _query_options.__isset.enable_profile && _query_options.enable_profile;
}
+ int cte_max_recursion_depth() const {
+ return _query_options.__isset.cte_max_recursion_depth
+ ? _query_options.cte_max_recursion_depth
+ : 0;
+ }
+
int rpc_verbose_profile_max_instance_count() const {
return _query_options.__isset.rpc_verbose_profile_max_instance_count
? _query_options.rpc_verbose_profile_max_instance_count
@@ -702,6 +708,17 @@ public:
_query_options.hnsw_bounded_queue);
}
+ void reset_to_rerun();
+
+ void set_force_make_rf_wait_infinite() {
+ _query_options.__set_runtime_filter_wait_infinitely(true);
+ }
+
+ bool runtime_filter_wait_infinitely() const {
+ return _query_options.__isset.runtime_filter_wait_infinitely &&
+ _query_options.runtime_filter_wait_infinitely;
+ }
+
private:
Status create_error_log_file();
diff --git a/be/src/runtime_filter/runtime_filter_consumer.h b/be/src/runtime_filter/runtime_filter_consumer.h
index e0e42e509d4..e9ef68f6d28 100644
--- a/be/src/runtime_filter/runtime_filter_consumer.h
+++ b/be/src/runtime_filter/runtime_filter_consumer.h
@@ -41,11 +41,11 @@ public:
APPLIED, // The consumer will switch to this state after the expression is acquired
};
- static Status create(const QueryContext* query_ctx, const TRuntimeFilterDesc* desc, int node_id,
+ static Status create(const RuntimeState* state, const TRuntimeFilterDesc* desc, int node_id,
std::shared_ptr<RuntimeFilterConsumer>* res) {
*res = std::shared_ptr<RuntimeFilterConsumer>(
- new RuntimeFilterConsumer(query_ctx, desc, node_id));
- RETURN_IF_ERROR((*res)->_init_with_desc(desc, &query_ctx->query_options()));
+ new RuntimeFilterConsumer(state, desc, node_id));
+ RETURN_IF_ERROR((*res)->_init_with_desc(desc, &state->query_options()));
return Status::OK();
}
@@ -86,17 +86,16 @@ public:
}
private:
- RuntimeFilterConsumer(const QueryContext* query_ctx, const TRuntimeFilterDesc* desc,
- int node_id)
+ RuntimeFilterConsumer(const RuntimeState* state, const TRuntimeFilterDesc* desc, int node_id)
: RuntimeFilter(desc),
_probe_expr(desc->planId_to_target_expr.find(node_id)->second),
_registration_time(MonotonicMillis()),
_rf_state(State::NOT_READY) {
// If bitmap filter is not applied, it will cause the query result to be incorrect
- bool wait_infinitely = query_ctx->runtime_filter_wait_infinitely() ||
+ bool wait_infinitely = state->runtime_filter_wait_infinitely() ||
_runtime_filter_type == RuntimeFilterType::BITMAP_FILTER;
- _rf_wait_time_ms = wait_infinitely ? query_ctx->execution_timeout() * 1000
- : query_ctx->runtime_filter_wait_time_ms();
+ _rf_wait_time_ms = wait_infinitely ? state->get_query_ctx()->execution_timeout() * 1000
+ : state->get_query_ctx()->runtime_filter_wait_time_ms();
DorisMetrics::instance()->runtime_filter_consumer_num->increment(1);
}
diff --git a/be/src/runtime_filter/runtime_filter_merger.h b/be/src/runtime_filter/runtime_filter_merger.h
index d373bb8be05..42e26b3d9c3 100644
--- a/be/src/runtime_filter/runtime_filter_merger.h
+++ b/be/src/runtime_filter/runtime_filter_merger.h
@@ -78,6 +78,8 @@ public:
_expected_producer_num = num;
}
+ int get_expected_producer_num() const { return _expected_producer_num; }
+
bool add_rf_size(uint64_t size) {
_received_rf_size_num++;
if (_expected_producer_num < _received_rf_size_num) {
diff --git a/be/src/runtime_filter/runtime_filter_mgr.cpp b/be/src/runtime_filter/runtime_filter_mgr.cpp
index ba9e9f5bc9d..dcddc29f06c 100644
--- a/be/src/runtime_filter/runtime_filter_mgr.cpp
+++ b/be/src/runtime_filter/runtime_filter_mgr.cpp
@@ -62,13 +62,13 @@ std::vector<std::shared_ptr<RuntimeFilterConsumer>> RuntimeFilterMgr::get_consum
}
Status RuntimeFilterMgr::register_consumer_filter(
- const QueryContext* query_ctx, const TRuntimeFilterDesc& desc, int node_id,
+ const RuntimeState* state, const TRuntimeFilterDesc& desc, int node_id,
std::shared_ptr<RuntimeFilterConsumer>* consumer) {
SCOPED_CONSUME_MEM_TRACKER(_tracker.get());
int32_t key = desc.filter_id;
std::lock_guard<std::mutex> l(_lock);
- RETURN_IF_ERROR(RuntimeFilterConsumer::create(query_ctx, &desc, node_id, consumer));
+ RETURN_IF_ERROR(RuntimeFilterConsumer::create(state, &desc, node_id, consumer));
_consumer_map[key].push_back(*consumer);
return Status::OK();
}
@@ -446,6 +446,24 @@ void RuntimeFilterMergeControllerEntity::release_undone_filters(QueryContext* qu
_filter_map.clear();
}
+Status RuntimeFilterMergeControllerEntity::reset_global_rf(
+ QueryContext* query_ctx, const google::protobuf::RepeatedField<int32_t>& filter_ids) {
+ for (const auto& filter_id : filter_ids) {
+ GlobalMergeContext* cnt_val;
+ {
+ std::unique_lock<std::shared_mutex> guard(_filter_map_mutex);
+ cnt_val = &_filter_map[filter_id]; // may inplace construct default object
+ }
+ int producer_size = cnt_val->merger->get_expected_producer_num();
+ RETURN_IF_ERROR(RuntimeFilterMerger::create(query_ctx, &cnt_val->runtime_filter_desc,
+ &cnt_val->merger));
+ cnt_val->merger->set_expected_producer_num(producer_size);
+ cnt_val->arrive_id.clear();
+ cnt_val->source_addrs.clear();
+ }
+ return Status::OK();
+}
+
std::string RuntimeFilterMergeControllerEntity::debug_string() {
std::string result = "RuntimeFilterMergeControllerEntity Info:\n";
std::shared_lock<std::shared_mutex> guard(_filter_map_mutex);
diff --git a/be/src/runtime_filter/runtime_filter_mgr.h b/be/src/runtime_filter/runtime_filter_mgr.h
index 6941e5e5631..a079f3b0ad9 100644
--- a/be/src/runtime_filter/runtime_filter_mgr.h
+++ b/be/src/runtime_filter/runtime_filter_mgr.h
@@ -83,7 +83,7 @@ public:
// get/set consumer
std::vector<std::shared_ptr<RuntimeFilterConsumer>> get_consume_filters(int filter_id);
- Status register_consumer_filter(const QueryContext* query_ctx, const TRuntimeFilterDesc& desc,
+ Status register_consumer_filter(const RuntimeState* state, const TRuntimeFilterDesc& desc,
int node_id,
std::shared_ptr<RuntimeFilterConsumer>* consumer_filter);
@@ -104,6 +104,27 @@ public:
std::string debug_string();
+ std::set<int32_t> get_filter_ids() {
+ std::set<int32_t> ids;
+ std::lock_guard<std::mutex> l(_lock);
+ for (const auto& id : _producer_id_set) {
+ ids.insert(id);
+ }
+ for (const auto& kv : _consumer_map) {
+ ids.insert(kv.first);
+ }
+ return ids;
+ }
+
+ void remove_filters(const std::set<int32_t>& filter_ids) {
+ std::lock_guard<std::mutex> l(_lock);
+ for (const auto& id : filter_ids) {
+ _consumer_map.erase(id);
+ _local_merge_map.erase(id);
+ _producer_id_set.erase(id);
+ }
+ }
+
private:
/**
* `_is_global = true` means this runtime filter manager menages query-level runtime filters.
@@ -152,6 +173,9 @@ public:
void release_undone_filters(QueryContext* query_ctx);
+ Status reset_global_rf(QueryContext* query_ctx,
+ const google::protobuf::RepeatedField<int32_t>& filter_ids);
+
private:
Status _init_with_desc(std::shared_ptr<QueryContext> query_ctx,
const TRuntimeFilterDesc* runtime_filter_desc,
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index daf45179f79..f9f7d2e4460 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -1597,6 +1597,57 @@ void PInternalService::fold_constant_expr(google::protobuf::RpcController* contr
}
}
+void PInternalService::transmit_rec_cte_block(google::protobuf::RpcController* controller,
+ const PTransmitRecCTEBlockParams* request,
+ PTransmitRecCTEBlockResult* response,
+ google::protobuf::Closure* done) {
+ bool ret = _light_work_pool.try_offer([this, request, response, done]() {
+ brpc::ClosureGuard closure_guard(done);
+ auto st = _exec_env->fragment_mgr()->transmit_rec_cte_block(
+ UniqueId(request->query_id()).to_thrift(),
+ UniqueId(request->fragment_instance_id()).to_thrift(), request->node_id(),
+ request->blocks(), request->eos());
+ st.to_protobuf(response->mutable_status());
+ });
+ if (!ret) {
+ offer_failed(response, done, _light_work_pool);
+ return;
+ }
+}
+
+void PInternalService::rerun_fragment(google::protobuf::RpcController* controller,
+ const PRerunFragmentParams* request,
+ PRerunFragmentResult* response,
+ google::protobuf::Closure* done) {
+ bool ret = _light_work_pool.try_offer([this, request, response, done]() {
+ brpc::ClosureGuard closure_guard(done);
+ auto st =
+ _exec_env->fragment_mgr()->rerun_fragment(UniqueId(request->query_id()).to_thrift(),
+ request->fragment_id(), request->stage());
+ st.to_protobuf(response->mutable_status());
+ });
+ if (!ret) {
+ offer_failed(response, done, _light_work_pool);
+ return;
+ }
+}
+
+void PInternalService::reset_global_rf(google::protobuf::RpcController* controller,
+ const PResetGlobalRfParams* request,
+ PResetGlobalRfResult* response,
+ google::protobuf::Closure* done) {
+ bool ret = _light_work_pool.try_offer([this, request, response, done]() {
+ brpc::ClosureGuard closure_guard(done);
+ auto st = _exec_env->fragment_mgr()->reset_global_rf(
+ UniqueId(request->query_id()).to_thrift(), request->filter_ids());
+ st.to_protobuf(response->mutable_status());
+ });
+ if (!ret) {
+ offer_failed(response, done, _light_work_pool);
+ return;
+ }
+}
+
void PInternalService::transmit_block(google::protobuf::RpcController* controller,
const PTransmitDataParams* request,
PTransmitDataResult* response,
diff --git a/be/src/service/internal_service.h b/be/src/service/internal_service.h
index d73501bfc80..781a7def178 100644
--- a/be/src/service/internal_service.h
+++ b/be/src/service/internal_service.h
@@ -140,6 +140,16 @@ public:
const ::doris::PPublishFilterRequestV2* request,
::doris::PPublishFilterResponse* response,
::google::protobuf::Closure* done) override;
+ void transmit_rec_cte_block(google::protobuf::RpcController* controller,
+ const PTransmitRecCTEBlockParams* request,
+ PTransmitRecCTEBlockResult* response,
+ google::protobuf::Closure* done) override;
+ void rerun_fragment(google::protobuf::RpcController* controller,
+ const PRerunFragmentParams* request, PRerunFragmentResult* response,
+ google::protobuf::Closure* done) override;
+ void reset_global_rf(google::protobuf::RpcController* controller,
+ const PResetGlobalRfParams* request, PResetGlobalRfResult* response,
+ google::protobuf::Closure* done) override;
void transmit_block(::google::protobuf::RpcController* controller,
const ::doris::PTransmitDataParams* request,
::doris::PTransmitDataResult* response,
diff --git a/be/test/runtime_filter/runtime_filter_consumer_test.cpp b/be/test/runtime_filter/runtime_filter_consumer_test.cpp
index 77ee21368c0..e0e352645c3 100644
--- a/be/test/runtime_filter/runtime_filter_consumer_test.cpp
+++ b/be/test/runtime_filter/runtime_filter_consumer_test.cpp
@@ -31,7 +31,7 @@ public:
void test_signal_aquire(TRuntimeFilterDesc desc) {
std::shared_ptr<RuntimeFilterConsumer> consumer;
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
- RuntimeFilterConsumer::create(_query_ctx.get(), &desc, 0, &consumer));
+ RuntimeFilterConsumer::create(_runtime_states[0].get(), &desc, 0, &consumer));
std::shared_ptr<RuntimeFilterProducer> producer;
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
@@ -59,7 +59,7 @@ TEST_F(RuntimeFilterConsumerTest, basic) {
std::shared_ptr<RuntimeFilterConsumer> consumer;
auto desc = TRuntimeFilterDescBuilder().add_planId_to_target_expr(0).build();
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
- RuntimeFilterConsumer::create(_query_ctx.get(), &desc, 0, &consumer));
+ RuntimeFilterConsumer::create(_runtime_states[1].get(), &desc, 0, &consumer));
std::shared_ptr<RuntimeFilterConsumer> registed_consumer;
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(_runtime_states[1]->register_consumer_runtime_filter(
@@ -116,7 +116,7 @@ TEST_F(RuntimeFilterConsumerTest, timeout_aquire) {
std::shared_ptr<RuntimeFilterConsumer> consumer;
auto desc = TRuntimeFilterDescBuilder().add_planId_to_target_expr(0).build();
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
- RuntimeFilterConsumer::create(_query_ctx.get(), &desc, 0, &consumer));
+ RuntimeFilterConsumer::create(_runtime_states[1].get(), &desc, 0, &consumer));
std::shared_ptr<RuntimeFilterProducer> producer;
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
@@ -141,7 +141,7 @@ TEST_F(RuntimeFilterConsumerTest, wait_infinity) {
const_cast<TQueryOptions&>(_query_ctx->_query_options)
.__set_runtime_filter_wait_infinitely(true);
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
- RuntimeFilterConsumer::create(_query_ctx.get(), &desc, 0, &consumer));
+ RuntimeFilterConsumer::create(_runtime_states[1].get(), &desc, 0, &consumer));
std::shared_ptr<RuntimeFilterConsumer> registed_consumer;
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(_runtime_states[1]->register_consumer_runtime_filter(
@@ -152,7 +152,7 @@ TEST_F(RuntimeFilterConsumerTest, aquire_disabled) {
std::shared_ptr<RuntimeFilterConsumer> consumer;
auto desc = TRuntimeFilterDescBuilder().add_planId_to_target_expr(0).build();
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
- RuntimeFilterConsumer::create(_query_ctx.get(), &desc, 0, &consumer));
+ RuntimeFilterConsumer::create(_runtime_states[1].get(), &desc, 0, &consumer));
std::shared_ptr<RuntimeFilterProducer> producer;
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
@@ -174,7 +174,7 @@ TEST_F(RuntimeFilterConsumerTest, bitmap_filter) {
std::shared_ptr<RuntimeFilterConsumer> consumer;
{
- auto st = RuntimeFilterConsumer::create(_query_ctx.get(), &desc, 0, &consumer);
+ auto st = RuntimeFilterConsumer::create(_runtime_states[1].get(), &desc, 0, &consumer);
ASSERT_FALSE(st.ok());
}
desc.__set_src_expr(
@@ -195,13 +195,13 @@ TEST_F(RuntimeFilterConsumerTest, bitmap_filter) {
.build());
{
- auto st = RuntimeFilterConsumer::create(_query_ctx.get(), &desc, 0, &consumer);
+ auto st = RuntimeFilterConsumer::create(_runtime_states[1].get(), &desc, 0, &consumer);
ASSERT_FALSE(st.ok());
}
{
desc.__set_has_local_targets(false);
desc.__set_has_remote_targets(true);
- auto st = RuntimeFilterConsumer::create(_query_ctx.get(), &desc, 0, &consumer);
+ auto st = RuntimeFilterConsumer::create(_runtime_states[1].get(), &desc, 0, &consumer);
ASSERT_FALSE(st.ok());
desc.__set_has_local_targets(true);
desc.__set_has_remote_targets(false);
@@ -209,7 +209,7 @@ TEST_F(RuntimeFilterConsumerTest, bitmap_filter) {
desc.__set_bitmap_target_expr(TRuntimeFilterDescBuilder::get_default_expr());
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
- RuntimeFilterConsumer::create(_query_ctx.get(), &desc, 0, &consumer));
+ RuntimeFilterConsumer::create(_runtime_states[1].get(), &desc, 0, &consumer));
}
TEST_F(RuntimeFilterConsumerTest, aquire_signal_at_same_time) {
@@ -217,7 +217,7 @@ TEST_F(RuntimeFilterConsumerTest, aquire_signal_at_same_time) {
std::shared_ptr<RuntimeFilterConsumer> consumer;
auto desc = TRuntimeFilterDescBuilder().add_planId_to_target_expr(0).build();
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
- RuntimeFilterConsumer::create(_query_ctx.get(), &desc, 0, &consumer));
+ RuntimeFilterConsumer::create(_runtime_states[1].get(), &desc, 0, &consumer));
std::shared_ptr<RuntimeFilterProducer> producer;
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
diff --git a/be/test/runtime_filter/runtime_filter_mgr_test.cpp b/be/test/runtime_filter/runtime_filter_mgr_test.cpp
index d8222e201d9..02b1349c1a6 100644
--- a/be/test/runtime_filter/runtime_filter_mgr_test.cpp
+++ b/be/test/runtime_filter/runtime_filter_mgr_test.cpp
@@ -65,7 +65,7 @@ TEST_F(RuntimeFilterMgrTest, TestRuntimeFilterMgr) {
EXPECT_TRUE(global_runtime_filter_mgr->get_consume_filters(filter_id).empty());
std::shared_ptr<RuntimeFilterConsumer> consumer_filter;
EXPECT_TRUE(global_runtime_filter_mgr
- ->register_consumer_filter(ctx.get(), desc, 0, &consumer_filter)
+ ->register_consumer_filter(&state, desc, 0, &consumer_filter)
.ok());
EXPECT_FALSE(global_runtime_filter_mgr->get_consume_filters(filter_id).empty());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 7582a7b6b6d..868263be5dc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -786,6 +786,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String READ_HIVE_JSON_IN_ONE_COLUMN = "read_hive_json_in_one_column";
+ public static final String CTE_MAX_RECURSION_DEPTH = "cte_max_recursion_depth";
+
/**
* Inserting overwrite for auto partition table allows creating partition for
* datas which cannot find partition to overwrite.
@@ -989,6 +991,11 @@ public class SessionVariable implements Serializable, Writable {
})
public int minScanSchedulerConcurrency = 0;
+ @VariableMgr.VarAttr(name = CTE_MAX_RECURSION_DEPTH, needForward = true, description = {
+ "CTE递归的最大深度，默认值100",
+ "The maximum depth of CTE recursion. Default is 100" })
+ public int cteMaxRecursionDepth = 100;
+
// By default, the number of Limit items after OrderBy is changed from 65535 items
// before v1.2.0 (not included), to return all items by default
@VariableMgr.VarAttr(name = DEFAULT_ORDER_BY_LIMIT, affectQueryResult = true)
@@ -4783,7 +4790,7 @@ public class SessionVariable implements Serializable, Writable {
tResult.setInvertedIndexSkipThreshold(invertedIndexSkipThreshold);
tResult.setInvertedIndexCompatibleRead(invertedIndexCompatibleRead);
-
+ tResult.setCteMaxRecursionDepth(cteMaxRecursionDepth);
tResult.setEnableParallelScan(enableParallelScan);
tResult.setParallelScanMaxScannersCount(parallelScanMaxScannersCount);
tResult.setParallelScanMinRowsPerScanner(parallelScanMinRowsPerScanner);
diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto
index 1ddfbcf2502..5b9c3579590 100644
--- a/gensrc/proto/internal_service.proto
+++ b/gensrc/proto/internal_service.proto
@@ -60,6 +60,45 @@ message PTransmitDataResult {
optional int64 receive_time = 2;
};
+message PTransmitRecCTEBlockParams {
+ repeated PBlock blocks = 1;
+ optional bool eos = 2;
+ optional PUniqueId query_id = 3;
+ optional PUniqueId fragment_instance_id = 4;
+ optional int32 node_id = 5;
+};
+
+message PTransmitRecCTEBlockResult {
+ optional PStatus status = 1;
+};
+
+
+message PRerunFragmentParams {
+ enum Opcode {
+ wait = 1; // wait fragment execute done
+ release = 2; // release current round's resource
+ rebuild = 3; // rebuild next round pipeline tasks
+ submit = 4; // submit tasks to execute
+ close = 5; // close fragment
+ }
+ optional PUniqueId query_id = 1;
+ optional int32 fragment_id = 2;
+ optional Opcode stage = 3;
+};
+
+message PRerunFragmentResult {
+ optional PStatus status = 1;
+};
+
+message PResetGlobalRfParams {
+ optional PUniqueId query_id = 1;
+ repeated int32 filter_ids = 2;
+};
+
+message PResetGlobalRfResult {
+ optional PStatus status = 1;
+};
+
message PTabletWithPartition {
required int64 partition_id = 1;
required int64 tablet_id = 2;
@@ -1107,6 +1146,9 @@ service PBackendService {
rpc sync_filter_size(PSyncFilterSizeRequest) returns (PSyncFilterSizeResponse);
rpc apply_filterv2(PPublishFilterRequestV2) returns (PPublishFilterResponse);
rpc fold_constant_expr(PConstantExprRequest) returns (PConstantExprResult);
+ rpc rerun_fragment(PRerunFragmentParams) returns (PRerunFragmentResult);
+ rpc reset_global_rf(PResetGlobalRfParams) returns (PResetGlobalRfResult);
+ rpc transmit_rec_cte_block(PTransmitRecCTEBlockParams) returns (PTransmitRecCTEBlockResult);
rpc transmit_block(PTransmitDataParams) returns (PTransmitDataResult);
rpc transmit_block_by_http(PEmptyRequest) returns (PTransmitDataResult);
rpc check_rpc_channel(PCheckRPCChannelRequest) returns (PCheckRPCChannelResponse);
diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
index cc018384f24..c4169768584 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -410,6 +410,7 @@ struct TQueryOptions {
175: optional bool enable_fuzzy_blockable_task = false;
176: optional list<i32> shuffled_agg_ids;
+ 177: optional i32 cte_max_recursion_depth;
// For cloud, to control if the content would be written into file cache
// In write path, to control if the content would be written into file cache.
diff --git a/regression-test/data/rec_cte_p0/rec_cte/rec_cte.out b/regression-test/data/rec_cte_p0/rec_cte/rec_cte.out
new file mode 100644
index 00000000000..ba843a71ee5
--- /dev/null
+++ b/regression-test/data/rec_cte_p0/rec_cte/rec_cte.out
@@ -0,0 +1,953 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+0.5403023058681398
+0.6542897904977791
+0.7013687736227565
+0.7221024250267077
+0.7314040424225098
+0.7356047404363474
+0.7375068905132428
+0.7383692041223232
+0.7387603198742113
+0.7389377567153445
+0.7390182624274122
+0.7390547907469174
+0.7390713652989449
+0.7390788859949921
+0.7390822985224024
+0.7390838469650002
+0.7390845495752126
+0.7390848683867142
+0.7390850130484203
+0.739085078689123
+0.7390851084737987
+0.7390851219886894
+0.7390851281211138
+0.7390851309037207
+0.7390851321663374
+0.7390851327392538
+0.7390851329992164
+0.7390851331171753
+0.7390851331706995
+0.7390851331949863
+0.7390851332060064
+0.7390851332110069
+0.7390851332132758
+0.7390851332143055
+0.7390851332147726
+0.7390851332149846
+0.7390851332150807
+0.7390851332151244
+0.7390851332151441
+0.7390851332151531
+0.7390851332151572
+0.7390851332151591
+0.7390851332151599
+0.7390851332151603
+0.7390851332151605
+0.7390851332151606
+0.7390851332151607
+0.7390851332151608
+0.7390851332151609
+0.7390851332151611
+0.7390851332151617
+0.7390851332151629
+0.7390851332151657
+0.7390851332151718
+0.7390851332151851
+0.7390851332152145
+0.7390851332152792
+0.739085133215422
+0.7390851332157367
+0.7390851332164302
+0.7390851332179587
+0.7390851332213271
+0.7390851332287504
+0.7390851332451103
+0.7390851332811648
+0.7390851333606233
+0.7390851335357372
+0.7390851339216605
+0.7390851347721744
+0.7390851366465718
+0.7390851407774467
+0.7390851498812394
+0.7390851699445544
+0.7390852141609171
+0.7390853116067619
+0.7390855263619245
+0.7390859996481299
+0.7390870426953322
+0.7390893414033927
+0.7390944073790913
+0.7391055719265363
+0.7391301765296711
+0.7391843997714936
+0.7393038923969059
+0.7395672022122561
+0.7401473355678757
+0.7414250866101092
+0.7442373549005569
+0.7504177617637605
+0.7639596829006542
+0.7934803587425656
+0.8575532158463934
+1
+
+-- !sql --
+55
+
+-- !sql --
+1 3
+1 5
+1 8
+2 4
+2 5
+2 10
+2 19
+3 1
+3 5
+3 8
+3 10
+3 24
+5 3
+5 4
+5 8
+5 15
+6 3
+6 4
+6 7
+7 4
+8 1
+9 4
+
+-- !sql --
+1 3
+1 5
+2 4
+2 5
+2 10
+3 1
+3 5
+3 8
+3 10
+5 3
+5 4
+5 8
+5 10
+6 3
+6 4
+6 5
+7 4
+8 1
+8 8
+9 4
+11 1
+12 3
+29 4
+
+-- !sql --
+1 3
+1 5
+2 4
+2 5
+2 10
+3 1
+3 5
+3 8
+3 10
+5 3
+5 4
+5 8
+6 3
+6 4
+7 4
+7 10
+8 1
+8 10
+9 4
+9 5
+9 10
+10 5
+10 8
+10 10
+11 5
+11 8
+11 10
+12 5
+12 8
+12 10
+13 1
+13 5
+13 8
+13 10
+14 1
+14 5
+14 8
+14 10
+15 1
+15 3
+15 5
+15 8
+15 10
+16 1
+16 3
+16 5
+16 8
+16 10
+17 1
+17 3
+17 5
+17 8
+17 10
+18 1
+18 3
+18 5
+18 8
+18 10
+19 1
+19 3
+19 5
+19 8
+19 10
+20 1
+20 3
+20 5
+20 8
+20 10
+21 1
+21 3
+21 5
+21 8
+21 10
+22 1
+22 3
+22 5
+22 8
+22 10
+23 1
+23 3
+23 5
+23 8
+23 10
+24 1
+24 3
+24 5
+24 8
+24 10
+25 1
+25 3
+25 5
+25 8
+25 10
+26 1
+26 3
+26 5
+26 8
+26 10
+27 1
+27 3
+27 5
+27 8
+27 10
+28 1
+28 3
+28 5
+28 8
+28 10
+29 1
+29 3
+29 5
+29 8
+29 10
+30 1
+30 3
+30 5
+30 8
+30 10
+31 1
+31 3
+31 5
+31 8
+31 10
+32 1
+32 3
+32 5
+32 8
+32 10
+33 1
+33 3
+33 5
+33 8
+33 10
+34 1
+34 3
+34 4
+34 5
+34 8
+34 10
+35 1
+35 3
+35 4
+35 5
+35 8
+35 10
+36 1
+36 3
+36 4
+36 5
+36 8
+36 10
+37 1
+37 3
+37 4
+37 5
+37 8
+37 10
+38 1
+38 3
+38 4
+38 5
+38 8
+38 10
+39 1
+39 3
+39 4
+39 5
+39 8
+39 10
+40 1
+40 3
+40 4
+40 5
+40 8
+40 10
+41 1
+41 3
+41 4
+41 5
+41 8
+41 10
+42 1
+42 3
+42 4
+42 5
+42 8
+42 10
+43 1
+43 3
+43 4
+43 5
+43 8
+43 10
+44 1
+44 3
+44 4
+44 5
+44 8
+44 10
+45 1
+45 3
+45 4
+45 5
+45 8
+45 10
+46 1
+46 3
+46 4
+46 5
+46 8
+46 10
+47 1
+47 3
+47 4
+47 5
+47 8
+47 10
+48 1
+48 3
+48 4
+48 5
+48 8
+48 10
+49 1
+49 3
+49 4
+49 5
+49 8
+49 10
+50 1
+50 3
+50 4
+50 5
+50 8
+50 10
+51 1
+51 3
+51 4
+51 5
+51 8
+51 10
+52 1
+52 3
+52 4
+52 5
+52 8
+52 10
+53 1
+53 3
+53 4
+53 5
+53 8
+53 10
+54 1
+54 3
+54 4
+54 5
+54 8
+54 10
+55 1
+55 3
+55 4
+55 5
+55 8
+55 10
+56 1
+56 3
+56 4
+56 5
+56 8
+56 10
+57 1
+57 3
+57 4
+57 5
+57 8
+57 10
+58 1
+58 3
+58 4
+58 5
+58 8
+58 10
+59 1
+59 3
+59 4
+59 5
+59 8
+59 10
+60 1
+60 3
+60 4
+60 5
+60 8
+60 10
+61 1
+61 3
+61 4
+61 5
+61 8
+61 10
+62 1
+62 3
+62 4
+62 5
+62 8
+62 10
+63 1
+63 3
+63 4
+63 5
+63 8
+63 10
+64 1
+64 3
+64 4
+64 5
+64 8
+64 10
+65 1
+65 3
+65 4
+65 5
+65 8
+65 10
+66 1
+66 3
+66 4
+66 5
+66 8
+66 10
+67 1
+67 3
+67 4
+67 5
+67 8
+67 10
+68 1
+68 3
+68 4
+68 5
+68 8
+68 10
+69 1
+69 3
+69 4
+69 5
+69 8
+69 10
+70 1
+70 3
+70 4
+70 5
+70 8
+70 10
+71 1
+71 3
+71 4
+71 5
+71 8
+71 10
+72 1
+72 3
+72 4
+72 5
+72 8
+72 10
+73 1
+73 3
+73 4
+73 5
+73 8
+73 10
+74 1
+74 3
+74 4
+74 5
+74 8
+74 10
+75 1
+75 3
+75 4
+75 5
+75 8
+75 10
+76 1
+76 3
+76 4
+76 5
+76 8
+76 10
+77 1
+77 3
+77 4
+77 5
+77 8
+77 10
+78 1
+78 3
+78 4
+78 5
+78 8
+78 10
+79 1
+79 3
+79 4
+79 5
+79 8
+79 10
+80 1
+80 3
+80 4
+80 5
+80 8
+80 10
+81 1
+81 3
+81 4
+81 5
+81 8
+81 10
+82 1
+82 3
+82 4
+82 5
+82 8
+82 10
+83 1
+83 3
+83 4
+83 5
+83 8
+83 10
+84 1
+84 3
+84 4
+84 5
+84 8
+84 10
+85 1
+85 3
+85 4
+85 5
+85 8
+85 10
+86 1
+86 3
+86 4
+86 5
+86 8
+86 10
+87 1
+87 3
+87 4
+87 5
+87 8
+87 10
+88 1
+88 3
+88 4
+88 5
+88 8
+88 10
+89 1
+89 3
+89 4
+89 5
+89 8
+89 10
+90 1
+90 3
+90 4
+90 5
+90 8
+90 10
+91 1
+91 3
+91 4
+91 5
+91 8
+91 10
+92 1
+92 3
+92 4
+92 5
+92 8
+92 10
+93 1
+93 3
+93 4
+93 5
+93 8
+93 10
+94 1
+94 3
+94 4
+94 5
+94 8
+94 10
+95 1
+95 3
+95 4
+95 5
+95 8
+95 10
+96 1
+96 3
+96 4
+96 5
+96 8
+96 10
+97 1
+97 3
+97 4
+97 5
+97 8
+97 10
+98 1
+98 3
+98 4
+98 5
+98 8
+98 10
+99 1
+99 3
+99 4
+99 5
+99 8
+99 10
+100 1
+100 3
+100 4
+100 5
+100 8
+100 10
+
+-- !sql --
+1 3
+1 5
+1 8
+2 \N
+2 4
+2 5
+2 9
+2 10
+2 15
+2 19
+2 28
+2 34
+2 43
+2 71
+2 77
+2 105
+2 176
+2 182
+2 253
+2 429
+2 435
+2 611
+2 1040
+2 1046
+2 1475
+2 2515
+2 2521
+2 3561
+2 6076
+2 6082
+2 8597
+2 14673
+2 14679
+2 20755
+2 35428
+2 35434
+2 50107
+2 85535
+2 85541
+2 120969
+2 206504
+2 206510
+2 292045
+2 498549
+2 498555
+2 705059
+2 1203608
+2 1203614
+2 1702163
+2 2905771
+2 2905777
+2 4109385
+2 7015156
+2 7015162
+2 9920933
+2 16936089
+2 16936095
+2 23951251
+2 40887340
+2 40887346
+2 57823435
+2 98710775
+2 98710781
+2 139598121
+2 238308896
+2 238308902
+2 337019677
+2 575328573
+2 575328579
+2 813637475
+2 1388966048
+2 1388966054
+2 1964294627
+3 \N
+3 1
+3 5
+3 6
+3 8
+3 10
+3 14
+3 18
+3 20
+3 23
+3 41
+3 43
+3 55
+3 63
+3 96
+3 118
+3 139
+3 181
+3 235
+3 320
+3 353
+3 501
+3 588
+3 854
+3 908
+3 1355
+3 1496
+3 2263
+3 2350
+3 3618
+3 3846
+3 5968
+3 6109
+3 9586
+3 9955
+3 15695
+3 15923
+3 25281
+3 25878
+3 41204
+3 41573
+3 66485
+3 67451
+3 108058
+3 108655
+3 174543
+3 176106
+3 283198
+3 284164
+3 457741
+3 460270
+3 741905
+3 743468
+3 1199646
+3 1203738
+3 1943114
+3 1945643
+3 3142760
+3 3149381
+3 5088403
+3 5092495
+3 8231163
+3 8241876
+3 13323658
+3 13330279
+3 21554821
+3 21572155
+3 34885100
+3 34895813
+3 56439921
+3 56467968
+3 91335734
+3 91353068
+3 147775655
+3 147821036
+3 239128723
+3 239156770
+3 386904378
+3 386977806
+3 626061148
+3 626106529
+3 1012965526
+3 1013084335
+3 1639072055
+3 1639145483
+5 \N
+5 3
+5 4
+5 7
+5 8
+5 12
+5 15
+5 22
+5 27
+5 34
+5 56
+5 61
+5 83
+5 139
+5 144
+5 200
+5 339
+5 344
+5 483
+5 822
+5 827
+5 1166
+5 1988
+5 1993
+5 2815
+5 4803
+5 4808
+5 6796
+5 11599
+5 11604
+5 16407
+5 28006
+5 28011
+5 39610
+5 67616
+5 67621
+5 95627
+5 163243
+5 163248
+5 230864
+5 394107
+5 394112
+5 557355
+5 951462
+5 951467
+5 1345574
+5 2297036
+5 2297041
+5 3248503
+5 5545539
+5 5545544
+5 7842580
+5 13388119
+5 13388124
+5 18933663
+5 32321782
+5 32321787
+5 45709906
+5 78031688
+5 78031693
+5 110353475
+5 188385163
+5 188385168
+5 266416856
+5 454802019
+5 454802024
+5 643187187
+5 1097989206
+5 1097989211
+5 1552791230
+6 3
+6 4
+6 7
+7 4
+8 1
+9 4
+
+-- !sql --
+1 2
+
+-- !sql --
+1 2
+3 4
+
+-- !sql --
+1 2
+3 4
+11 22
+33 44
+
+-- !sql --
+1 2
+3 4
+11 22
+
+-- !sql --
+1 22
+3 22
+11 22
+
+-- !sql --
+1 2
+2 3
+
+-- !sql --
+1 2
+3 4
+11 22
+
diff --git a/regression-test/data/rec_cte_p0/rec_cte_from_ck_doc/rec_cte_from_ck_doc.out b/regression-test/data/rec_cte_p0/rec_cte_from_ck_doc/rec_cte_from_ck_doc.out
new file mode 100644
index 00000000000..3a055b01acc
--- /dev/null
+++ b/regression-test/data/rec_cte_p0/rec_cte_from_ck_doc/rec_cte_from_ck_doc.out
@@ -0,0 +1,29 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q1 --
+5050
+
+-- !q2 --
+0 \N ROOT
+1 0 Child_1
+2 0 Child_2
+3 1 Child_1_1
+
+-- !q3 --
+0 \N ROOT [0]
+1 0 Child_1 [0, 1]
+3 1 Child_1_1 [0, 1, 3]
+2 0 Child_2 [0, 2]
+
+-- !q4 --
+0 \N ROOT [0] 0
+1 0 Child_1 [0, 1] 1
+2 0 Child_2 [0, 2] 1
+3 1 Child_1_1 [0, 1, 3] 2
+
+-- !q5 --
+1 2 1 -> 2
+1 3 1 -> 3
+1 4 1 -> 4
+2 3 2 -> 3
+4 5 4 -> 5
+
diff --git a/regression-test/data/rec_cte_p0/rec_cte_from_duckdb_doc/rec_cte_from_duckdb_doc.out b/regression-test/data/rec_cte_p0/rec_cte_from_duckdb_doc/rec_cte_from_duckdb_doc.out
new file mode 100644
index 00000000000..958b6848838
--- /dev/null
+++ b/regression-test/data/rec_cte_p0/rec_cte_from_duckdb_doc/rec_cte_from_duckdb_doc.out
@@ -0,0 +1,30 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q1 --
+0 0 1
+1 1 1
+2 1 2
+3 2 3
+4 3 5
+5 5 8
+6 8 13
+7 13 21
+8 21 34
+9 34 55
+
+-- !q2 --
+["Oasis", "Rock", "Music", "Art"]
+
+-- !q3 --
+1 3 [1, 3]
+1 5 [1, 5]
+1 5 [1, 3, 5]
+1 8 [1, 3, 8]
+1 10 [1, 3, 10]
+1 3 [1, 5, 3]
+1 4 [1, 5, 4]
+1 8 [1, 5, 8]
+1 4 [1, 3, 5, 4]
+1 8 [1, 3, 5, 8]
+1 8 [1, 5, 3, 8]
+1 10 [1, 5, 3, 10]
+
diff --git a/regression-test/data/rec_cte_p0/rec_cte_from_mysql_doc/rec_cte_from_mysql_doc.out b/regression-test/data/rec_cte_p0/rec_cte_from_mysql_doc/rec_cte_from_mysql_doc.out
new file mode 100644
index 00000000000..3bc4cf27785
--- /dev/null
+++ b/regression-test/data/rec_cte_p0/rec_cte_from_mysql_doc/rec_cte_from_mysql_doc.out
@@ -0,0 +1,42 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q1 --
+1 abc
+2 abcabc
+3 abcabcabcabc
+
+-- !q2 --
+1 1 -1
+2 -2 2
+3 4 -4
+4 -8 8
+5 16 -16
+
+-- !q4 --
+2017-01-03
+2017-01-04
+2017-01-05
+2017-01-06
+2017-01-07
+2017-01-08
+2017-01-09
+2017-01-10
+
+-- !q5 --
+2017-01-03 300
+2017-01-04 0
+2017-01-05 0
+2017-01-06 50
+2017-01-07 0
+2017-01-08 180
+2017-01-09 0
+2017-01-10 5
+
+-- !q6 --
+333 Yasmina 333
+198 John 333,198
+29 Pedro 333,198,29
+4610 Sarah 333,198,29,4610
+72 Pierre 333,198,29,72
+692 Tarek 333,692
+123 Adil 333,692,123
+
diff --git a/regression-test/suites/rec_cte_p0/rec_cte/rec_cte.groovy b/regression-test/suites/rec_cte_p0/rec_cte/rec_cte.groovy
new file mode 100644
index 00000000000..d456f6ba998
--- /dev/null
+++ b/regression-test/suites/rec_cte_p0/rec_cte/rec_cte.groovy
@@ -0,0 +1,271 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("rec_cte") {
+ qt_sql """
+ WITH RECURSIVE test_table AS (
+ SELECT
+ cast(1.0 as double) AS number
+ UNION
+ SELECT
+ cos(number)
+ FROM
+ test_table
+ )
+ SELECT
+ number
+ FROM
+ test_table order by number;
+ """
+
+ qt_sql """
+ WITH RECURSIVE test_table AS (
+ SELECT cast(10 as int) AS number
+ UNION ALL
+ SELECT cast(number - 1 as int) FROM test_table WHERE number > 0
+ )
+ SELECT sum(number) FROM test_table;
+ """
+
+
+ sql "DROP TABLE IF EXISTS edge;"
+ sql """
+ CREATE TABLE edge
+ (
+ node1id int,
+ node2id int
+ ) DUPLICATE KEY (node1id)
+ DISTRIBUTED BY HASH(node1id) BUCKETS 1 PROPERTIES ('replication_num' = '1');
+ """
+ sql """
+ INSERT INTO edge VALUES
+ (1, 3), (1, 5), (2, 4), (2, 5), (2, 10), (3, 1),
+ (3, 5), (3, 8), (3, 10), (5, 3), (5, 4), (5, 8),
+ (6, 3), (6, 4), (7, 4), (8, 1), (9, 4);
+ """
+
+ qt_sql """
+ WITH RECURSIVE t1(k1, k2) AS (
+ SELECT
+ node1id AS k1,
+ node2id AS k2
+ FROM edge
+ UNION
+ SELECT
+ k1,
+ cast(sum(k2) as int)
+ FROM t1 GROUP BY k1
+ )
+ SELECT * FROM t1 ORDER BY 1,2;
+ """
+
+ qt_sql """
+ WITH RECURSIVE t1(k1, k2) AS (
+ SELECT
+ node1id AS k1,
+ node2id AS k2
+ FROM edge
+ UNION
+ SELECT
+ cast(sum(k1) as int),
+ k2
+ FROM t1 GROUP BY k2
+ )
+ SELECT * FROM t1 ORDER BY 1,2;
+ """
+
+ test {
+ sql """
+ WITH RECURSIVE t1(k1, k2) AS (
+ SELECT
+ node1id AS k1,
+ node2id AS k2
+ FROM edge
+ UNION
+ SELECT
+ cast(sum(k1 + 1) as int),
+ k2
+ FROM t1 GROUP BY k2
+ )
+ SELECT * FROM t1 ORDER BY 1,2;
+ """
+ exception "ABORTED"
+ }
+
+ qt_sql """
+ WITH RECURSIVE t1(k1, k2) AS (
+ SELECT
+ node1id AS k1,
+ node2id AS k2
+ FROM edge
+ UNION
+ SELECT
+ cast(sum(k1 + 1) as int),
+ k2
+ FROM t1 WHERE k1 < 100 GROUP BY k2
+ )
+ SELECT * FROM t1 ORDER BY 1,2;
+ """
+
+ qt_sql """
+ WITH RECURSIVE t1(k1, k2) AS (
+ SELECT
+ node1id AS k1,
+ node2id AS k2
+ FROM edge
+ UNION
+ SELECT
+ k1,
+ cast(sum(k2) OVER (PARTITION BY k1 ORDER BY k1 ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) as int)
+ FROM t1
+ )
+ SELECT * FROM t1 ORDER BY 1,2;
+ """
+
+ test {
+ sql """
+ WITH RECURSIVE t1(k1, k2) AS (
+ SELECT
+ 1,2
+ UNION ALL
+ SELECT
+ 1,2
+ FROM t1 GROUP BY k1
+ )
+ SELECT * FROM t1 ORDER BY 1,2;
+ """
+ exception "ABORTED"
+ }
+
+ qt_sql """
+ WITH RECURSIVE t1(k1, k2) AS (
+ SELECT
+ 1,2
+ UNION
+ SELECT
+ 1,2
+ FROM t1 GROUP BY k1
+ )
+ SELECT * FROM t1 ORDER BY 1,2;
+ """
+
+ qt_sql """
+ WITH RECURSIVE t1(k1, k2) AS (
+ SELECT
+ 1,2
+ UNION
+ SELECT
+ 3,4
+ FROM t1 GROUP BY k1
+ )
+ SELECT * FROM t1 ORDER BY 1,2;
+ """
+
+ qt_sql """
+ WITH RECURSIVE t1(k1, k2) AS (
+ SELECT
+ 1,2
+ UNION
+ SELECT
+ 3,4
+ FROM t1 GROUP BY k1
+ ),
+ t2(k1, k2) AS (
+ SELECT
+ 11,22
+ UNION
+ SELECT
+ 33,44
+ FROM t2 GROUP BY k1
+ )
+ SELECT * FROM t1 UNION select * from t2 ORDER BY 1,2;
+ """
+
+ qt_sql """
+ WITH RECURSIVE t1(k1, k2) AS (
+ SELECT
+ 1,2
+ UNION
+ SELECT
+ 3,4
+ FROM t1 GROUP BY k1
+ ),
+ t2(k1, k2) AS (
+ SELECT
+ 11,22
+ UNION
+ SELECT t2.k1, t2.k2 FROM t1,t2
+ )
+ SELECT * FROM t1 UNION select * from t2 ORDER BY 1,2;
+ """
+
+ qt_sql """
+ WITH RECURSIVE t1(k1, k2) AS (
+ SELECT
+ 1,2
+ UNION
+ SELECT
+ 3,4
+ FROM t1 GROUP BY k1
+ ),
+ t2(k1, k2) AS (
+ SELECT
+ 11,22
+ UNION
+ SELECT t1.k1, t2.k2 FROM t1,t2
+ )
+ select * from t2 ORDER BY 1,2;
+ """
+
+ qt_sql """
+ WITH RECURSIVE t1(k1, k2) AS (
+ SELECT
+ 1,2
+ UNION
+ SELECT
+ 3,4
+ FROM t1 GROUP BY k1
+ ),
+ t2(k1, k2) AS (
+ SELECT
+ 2,3
+ UNION
+ SELECT least(t1.k1,t2.k1), least(t1.k2,t2.k2) FROM t1,t2
+ )
+ select * from t2 ORDER BY 1,2;
+ """
+
+ qt_sql """
+ WITH RECURSIVE t1(k1, k2) AS (
+ SELECT
+ 1,2
+ UNION
+ SELECT
+ 3,4
+ FROM t1 GROUP BY k1
+ ),
+ t2(k1, k2) AS (
+ SELECT
+ 11,22
+ UNION
+ SELECT t1.k1, t1.k2 FROM t1
+ )
+ SELECT * FROM t1 UNION select * from t2 ORDER BY 1,2;
+ """
+}
diff --git a/regression-test/suites/rec_cte_p0/rec_cte_from_ck_doc/rec_cte_from_ck_doc.groovy b/regression-test/suites/rec_cte_p0/rec_cte_from_ck_doc/rec_cte_from_ck_doc.groovy
new file mode 100644
index 00000000000..a01ab03db34
--- /dev/null
+++ b/regression-test/suites/rec_cte_p0/rec_cte_from_ck_doc/rec_cte_from_ck_doc.groovy
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+// https://clickhouse.com/docs/sql-reference/statements/select/with
+suite ("rec_cte_from_ck_doc") {
+ qt_q1 """
+ WITH RECURSIVE test_table AS (
+ SELECT cast(1 as int) AS number
+ UNION ALL
+ SELECT cast(number + 1 as int) FROM test_table WHERE number < 100
+ )
+ SELECT sum(number) FROM test_table;
+ """
+
+ sql "DROP TABLE IF EXISTS tree;"
+ sql """
+ CREATE TABLE tree
+ (
+ id int,
+ parent_id int,
+ data varchar(100)
+ ) DUPLICATE KEY (id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES ('replication_num' = '1');
+ """
+ sql """INSERT INTO tree VALUES (0, NULL, 'ROOT'), (1, 0, 'Child_1'), (2, 0, 'Child_2'), (3, 1, 'Child_1_1');"""
+
+ qt_q2 """
+ WITH RECURSIVE search_tree AS (
+ SELECT id, parent_id, data
+ FROM tree t
+ WHERE t.id = 0
+ UNION ALL
+ SELECT t.id, t.parent_id, t.data
+ FROM tree t, search_tree st
+ WHERE t.parent_id = st.id
+ )
+ SELECT * FROM search_tree order BY id;
+ """
+
+ qt_q3 """
+ WITH RECURSIVE search_tree AS (
+ SELECT id, parent_id, data, array(t.id) AS path
+ FROM tree t
+ WHERE t.id = 0
+ UNION ALL
+ SELECT t.id, t.parent_id, t.data, array_concat(path, array(t.id))
+ FROM tree t, search_tree st
+ WHERE t.parent_id = st.id
+ )
+ SELECT * FROM search_tree ORDER BY path;
+ """
+
+ qt_q4 """
+ WITH RECURSIVE search_tree AS (
+ SELECT id, parent_id, data, array(t.id) AS path, cast(0 as int) AS depth
+ FROM tree t
+ WHERE t.id = 0
+ UNION ALL
+ SELECT t.id, t.parent_id, t.data, array_concat(path, array(t.id)), cast(depth + 1 as int)
+ FROM tree t, search_tree st
+ WHERE t.parent_id = st.id
+ )
+ SELECT * FROM search_tree ORDER BY depth, id;
+ """
+
+ sql "DROP TABLE IF EXISTS graph;"
+ sql """
+ CREATE TABLE graph
+ (
+ c_from int,
+ c_to int,
+ label varchar(100)
+ ) DUPLICATE KEY (c_from) DISTRIBUTED BY HASH(c_from) BUCKETS 1 PROPERTIES ('replication_num' = '1');
+ """
+ sql """INSERT INTO graph VALUES (1, 2, '1 -> 2'), (1, 3, '1 -> 3'), (2, 3, '2 -> 3'), (1, 4, '1 -> 4'), (4, 5, '4 -> 5');"""
+
+ qt_q5 """
+ WITH RECURSIVE search_graph AS (
+ SELECT c_from, c_to, label FROM graph g
+ UNION ALL
+ SELECT g.c_from, g.c_to, g.label
+ FROM graph g, search_graph sg
+ WHERE g.c_from = sg.c_to
+ )
+ SELECT DISTINCT * FROM search_graph ORDER BY c_from, c_to;
+ """
+
+ sql "INSERT INTO graph VALUES (5, 1, '5 -> 1');"
+ test {
+ sql """
+ WITH RECURSIVE search_graph AS (
+ SELECT c_from, c_to, label FROM graph g
+ UNION ALL
+ SELECT g.c_from, g.c_to, g.label
+ FROM graph g, search_graph sg
+ WHERE g.c_from = sg.c_to
+ )
+ SELECT DISTINCT * FROM search_graph ORDER BY c_from, c_to;
+ """
+ exception "ABORTED"
+ }
+
+ // test global rf
+ sql "set enable_runtime_filter_prune = false;"
+ test {
+ sql """
+ WITH RECURSIVE search_graph AS (
+ SELECT c_from, c_to, label FROM graph g
+ UNION ALL
+ SELECT g.c_from, g.c_to, g.label
+ FROM graph g join [shuffle] search_graph sg
+ on g.c_from = sg.c_to
+ )
+ SELECT DISTINCT * FROM search_graph ORDER BY c_from, c_to;
+ """
+ exception "ABORTED"
+ }
+
+ // do not support use limit to stop recursion now
+ //qt_q6 """
+ //WITH RECURSIVE test_table AS (
+ // SELECT cast(1 as int) AS number
+ //UNION ALL
+ // SELECT cast(number + 1 as int) FROM test_table
+ //)
+ //SELECT sum(number) FROM test_table LIMIT 100;
+ //"""
+}
diff --git a/regression-test/suites/rec_cte_p0/rec_cte_from_duckdb_doc/rec_cte_from_duckdb_doc.groovy b/regression-test/suites/rec_cte_p0/rec_cte_from_duckdb_doc/rec_cte_from_duckdb_doc.groovy
new file mode 100644
index 00000000000..62d70558163
--- /dev/null
+++ b/regression-test/suites/rec_cte_p0/rec_cte_from_duckdb_doc/rec_cte_from_duckdb_doc.groovy
@@ -0,0 +1,185 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+// https://duckdb.org/docs/stable/sql/query_syntax/with#recursive-ctes
+suite ("rec_cte_from_duckdb_doc") {
+ qt_q1 """
+ WITH RECURSIVE FibonacciNumbers (
+ RecursionDepth,
+ FibonacciNumber,
+ NextNumber
+ ) AS (
+ -- Base case
+ SELECT
+ cast(0 as int) AS RecursionDepth,
+ cast(0 as int) AS FibonacciNumber,
+ cast(1 as int) AS NextNumber
+ UNION
+ ALL -- Recursive step
+ SELECT
+ cast((fib.RecursionDepth + 1) as int) AS RecursionDepth,
+ fib.NextNumber AS FibonacciNumber,
+ cast((fib.FibonacciNumber + fib.NextNumber) as int) AS NextNumber
+ FROM
+ FibonacciNumbers fib
+ WHERE
+ cast((fib.RecursionDepth + 1) as int) < 10
+ )
+ SELECT
+ *
+ FROM
+ FibonacciNumbers fn ORDER BY fn.RecursionDepth;
+ """
+
+ sql "DROP TABLE IF EXISTS tag;"
+ sql """
+ CREATE TABLE tag
+ (
+ id int,
+ name varchar(100),
+ subclassof int
+ ) DUPLICATE KEY (id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES ('replication_num' = '1');
+ """
+ sql """INSERT INTO tag VALUES
+ (1, 'U2', 5),
+ (2, 'Blur', 5),
+ (3, 'Oasis', 5),
+ (4, '2Pac', 6),
+ (5, 'Rock', 7),
+ (6, 'Rap', 7),
+ (7, 'Music', 9),
+ (8, 'Movies', 9),
+ (9, 'Art', NULL);"""
+
+ qt_q2 """
+ WITH RECURSIVE tag_hierarchy(id, source, path) AS (
+ SELECT id, name, array(name) AS path
+ FROM tag
+ WHERE subclassof IS NULL
+ UNION ALL
+ SELECT tag.id, tag.name, array_concat(array(tag.name), tag_hierarchy.path)
+ FROM tag, tag_hierarchy
+ WHERE tag.subclassof = tag_hierarchy.id
+ )
+ SELECT path
+ FROM tag_hierarchy
+ WHERE source = 'Oasis';
+ """
+
+ sql "DROP TABLE IF EXISTS edge;"
+ sql """
+ CREATE TABLE edge
+ (
+ node1id int,
+ node2id int
+ ) DUPLICATE KEY (node1id)
+ DISTRIBUTED BY HASH(node1id) BUCKETS 1 PROPERTIES ('replication_num' = '1');
+ """
+ sql """
+ INSERT INTO edge VALUES
+ (1, 3), (1, 5), (2, 4), (2, 5), (2, 10), (3, 1),
+ (3, 5), (3, 8), (3, 10), (5, 3), (5, 4), (5, 8),
+ (6, 3), (6, 4), (7, 4), (8, 1), (9, 4);
+ """
+
+ qt_q3 """
+ WITH RECURSIVE paths(startNode, endNode, path) AS (
+ SELECT -- Define the path as the first edge of the traversal
+ node1id AS startNode,
+ node2id AS endNode,
+ array_concat(array(node1id), array(node2id)) AS path
+ FROM edge
+ WHERE node1id = 1
+ UNION ALL
+ SELECT -- Concatenate new edge to the path
+ paths.startNode AS startNode,
+ node2id AS endNode,
+ array_concat(path, array(node2id)) AS path
+ FROM paths
+ JOIN edge ON paths.endNode = node1id
+ -- Prevent adding a repeated node to the path.
+ -- This ensures that no cycles occur.
+ WHERE array_contains(paths.path, node2id) = false
+ )
+ SELECT startNode, endNode, path
+ FROM paths
+ ORDER BY array_size(path), path;
+ """
+
+ // do not support subquery containing recursive cte
+ //qt_q4 """
+ //WITH RECURSIVE paths(startNode, endNode, path) AS (
+ // SELECT -- Define the path as the first edge of the traversal
+ // node1id AS startNode,
+ // node2id AS endNode,
+ // array_concat(array(node1id), array(node2id)) AS path
+ // FROM edge
+ // WHERE startNode = 1
+ // UNION ALL
+ // SELECT -- Concatenate new edge to the path
+ // paths.startNode AS startNode,
+ // node2id AS endNode,
+ // array_concat(path, array(node2id)) AS path
+ // FROM paths
+ // JOIN edge ON paths.endNode = node1id
+ // -- Prevent adding a node that was visited previously by any path.
+ // -- This ensures that (1) no cycles occur and (2) only nodes that
+ // -- were not visited by previous (shorter) paths are added to a path.
+ // WHERE NOT EXISTS (
+ // SELECT 1 FROM paths previous_paths
+ // WHERE array_contains(previous_paths.path, node2id)
+ // )
+ // )
+ //SELECT startNode, endNode, path
+ //FROM paths
+ //ORDER BY array_size(path), path;
+ //"""
+
+ //qt_q5 """
+ //WITH RECURSIVE paths(startNode, endNode, path, endReached) AS (
+ //SELECT -- Define the path as the first edge of the traversal
+ // node1id AS startNode,
+ // node2id AS endNode,
+ // array_concat(array(node1id), array(node2id)) AS path,
+ // (node2id = 8) AS endReached
+ // FROM edge
+ // WHERE startNode = 1
+ //UNION ALL
+ //SELECT -- Concatenate new edge to the path
+ // paths.startNode AS startNode,
+ // node2id AS endNode,
+ // array_concat(path, array(node2id)) AS path,
+ // max(CASE WHEN node2id = 8 THEN 1 ELSE 0 END)
+ // OVER (ROWS BETWEEN UNBOUNDED PRECEDING
+ // AND UNBOUNDED FOLLOWING) AS endReached
+ // FROM paths
+ // JOIN edge ON paths.endNode = node1id
+ // WHERE NOT EXISTS (
+ // FROM paths previous_paths
+ // WHERE array_contains(previous_paths.path, node2id)
+ // )
+ // AND paths.endReached = 0
+ //)
+ //SELECT startNode, endNode, path
+ //FROM paths
+ //WHERE endNode = 8
+ //ORDER BY array_size(path), path;
+ //"""
+}
diff --git a/regression-test/suites/rec_cte_p0/rec_cte_from_mysql_doc/rec_cte_from_mysql_doc.groovy b/regression-test/suites/rec_cte_p0/rec_cte_from_mysql_doc/rec_cte_from_mysql_doc.groovy
new file mode 100644
index 00000000000..cee07dc6612
--- /dev/null
+++ b/regression-test/suites/rec_cte_p0/rec_cte_from_mysql_doc/rec_cte_from_mysql_doc.groovy
@@ -0,0 +1,140 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+// https://dev.mysql.com/doc/refman/8.4/en/with.html#common-table-expressions-recursive
+suite ("rec_cte_from_mysql_doc") {
+ qt_q1 """
+ WITH RECURSIVE cte AS
+ (
+ SELECT cast(1 as int) AS n, cast('abc' as varchar(65533)) AS str
+ UNION ALL
+ SELECT cast(n + 1 as int), cast(CONCAT(str, str) as varchar(65533)) FROM cte WHERE n < 3
+ )
+ SELECT * FROM cte order by n;
+ """
+
+ qt_q2 """
+ WITH RECURSIVE cte AS
+ (
+ SELECT cast(1 as int) AS n, cast(1 as int) AS p, cast(-1 as int) AS q
+ UNION ALL
+ SELECT cast(n + 1 as int), cast(q * 2 as int), cast(p * 2 as int) FROM cte WHERE n < 5
+ )
+ SELECT * FROM cte order by n;
+ """
+
+ test {
+ sql """
+ WITH RECURSIVE cte (n) AS
+ (
+ SELECT cast(1 as int)
+ UNION ALL
+ SELECT cast(n + 1 as int) FROM cte
+ )
+ SELECT n FROM cte order by n;
+ """
+ exception "ABORTED"
+ }
+
+ // do not support use limit to stop recursion now
+ //qt_q3 """
+ //WITH RECURSIVE cte (n) AS
+ //(
+ //SELECT cast(1 as int)
+ //UNION ALL
+ //SELECT cast(n + 1 as int) FROM cte LIMIT 10000
+ //)
+ //SELECT n FROM cte order by n;
+ //"""
+
+ sql "DROP TABLE IF EXISTS sales;"
+ sql """
+ CREATE TABLE sales
+ (
+ c_date date,
+ c_price double
+ ) DUPLICATE KEY (c_date)
+ DISTRIBUTED BY HASH(c_date) BUCKETS 1 PROPERTIES ('replication_num' = '1');
+ """
+ sql """insert into sales values
+ ('2017-01-03', 100.0),
+ ('2017-01-03', 200.0),
+ ('2017-01-06', 50.0),
+ ('2017-01-08', 10.0),
+ ('2017-01-08', 20.0),
+ ('2017-01-08', 150.0),
+ ('2017-01-10', 5.0);"""
+
+ qt_q4 """
+ WITH RECURSIVE dates (c_date) AS
+ (
+ SELECT MIN(c_date) FROM sales
+ UNION ALL
+ SELECT c_date + INTERVAL 1 DAY FROM dates
+ WHERE c_date + INTERVAL 1 DAY <= (SELECT MAX(c_date) FROM sales)
+ )
+ SELECT * FROM dates order by 1;
+ """
+
+ qt_q5 """
+ WITH RECURSIVE dates (c_date) AS
+ (
+ SELECT MIN(c_date) FROM sales
+ UNION ALL
+ SELECT c_date + INTERVAL 1 DAY FROM dates
+ WHERE c_date + INTERVAL 1 DAY <= (SELECT MAX(c_date) FROM sales)
+ )
+ SELECT dates.c_date, COALESCE(SUM(c_price), 0) AS sum_price
+ FROM dates LEFT JOIN sales ON dates.c_date = sales.c_date
+ GROUP BY dates.c_date
+ ORDER BY dates.c_date;
+ """
+
+ sql "DROP TABLE IF EXISTS employees;"
+ sql """
+ CREATE TABLE employees (
+ id INT NOT NULL,
+ name VARCHAR(100) NOT NULL,
+ manager_id INT NULL
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES ('replication_num' = '1');
+ """
+ sql """INSERT INTO employees VALUES
+ (333, "Yasmina", NULL),
+ (198, "John", 333),
+ (692, "Tarek", 333),
+ (29, "Pedro", 198),
+ (4610, "Sarah", 29),
+ (72, "Pierre", 29),
+ (123, "Adil", 692);
+ """
+
+ qt_q6 """
+ WITH RECURSIVE employee_paths (id, name, path) AS
+ (
+ SELECT id, name, CAST(id AS varchar(65533))
+ FROM employees
+ WHERE manager_id IS NULL
+ UNION ALL
+ SELECT e.id, e.name, cast(CONCAT(ep.path, ',', e.id) as varchar(65533))
+ FROM employee_paths AS ep JOIN employees AS e
+ ON ep.id = e.manager_id
+ )
+ SELECT * FROM employee_paths ORDER BY path;
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/03: recursive cte fe part
Posted by pa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch dev_rec2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1e349c4dd4cc3f4fb6312cfc49a7af60fc061ec5
Author: lichi <li...@selectdb.com>
AuthorDate: Thu Oct 9 15:14:12 2025 +0800
recursive cte fe part
fix some bug in fe
fix fe plan
update fe code
update fe
update fe part
update fe
add ut
---
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 3 +-
.../org/apache/doris/analysis/DescriptorTable.java | 5 +
.../doris/catalog/RecursiveCteTempTable.java | 28 ++
.../java/org/apache/doris/catalog/TableIf.java | 6 +-
.../org/apache/doris/nereids/CascadesContext.java | 41 ++-
.../glue/translator/PhysicalPlanTranslator.java | 107 +++++++-
.../doris/nereids/parser/LogicalPlanBuilder.java | 10 +-
.../apache/doris/nereids/pattern/MemoPatterns.java | 24 ++
.../processor/post/RuntimeFilterPruner.java | 9 +
.../processor/pre/PullUpSubqueryAliasToCTE.java | 8 +-
.../properties/ChildOutputPropertyDeriver.java | 20 ++
.../nereids/properties/RequestPropertyDeriver.java | 11 +
.../org/apache/doris/nereids/rules/RuleSet.java | 6 +
.../org/apache/doris/nereids/rules/RuleType.java | 3 +
.../doris/nereids/rules/analysis/AnalyzeCTE.java | 163 ++++++++++--
.../doris/nereids/rules/analysis/BindRelation.java | 32 ++-
.../nereids/rules/analysis/CollectRelation.java | 11 +-
.../nereids/rules/analysis/SubExprAnalyzer.java | 4 +-
...eChildToPhysicalRecursiveCteRecursiveChild.java | 37 +++
...RecursiveCteScanToPhysicalRecursiveCteScan.java | 42 +++
.../LogicalRecursiveCteToPhysicalRecursiveCte.java | 39 +++
.../nereids/rules/rewrite/AdjustNullable.java | 48 ++++
.../doris/nereids/rules/rewrite/CTEInline.java | 44 +++-
.../doris/nereids/rules/rewrite/ColumnPruning.java | 55 ++++
.../doris/nereids/stats/StatsCalculator.java | 109 ++++++++
.../trees/copier/LogicalPlanDeepCopier.java | 26 ++
.../apache/doris/nereids/trees/plans/PlanType.java | 6 +
.../nereids/trees/plans/algebra/RecursiveCte.java | 36 +++
.../plans/commands/UpdateMvByPartitionCommand.java | 3 +-
.../worker/job/UnassignedJobBuilder.java | 13 +
.../worker/job/UnassignedRecursiveCteScanJob.java | 65 +++++
.../nereids/trees/plans/logical/LogicalCTE.java | 27 +-
.../trees/plans/logical/LogicalRecursiveCte.java | 284 ++++++++++++++++++++
.../logical/LogicalRecursiveCteRecursiveChild.java | 117 +++++++++
.../plans/logical/LogicalRecursiveCteScan.java | 87 ++++++
.../trees/plans/logical/LogicalSubQueryAlias.java | 22 ++
.../trees/plans/physical/PhysicalRecursiveCte.java | 292 +++++++++++++++++++++
.../PhysicalRecursiveCteRecursiveChild.java | 121 +++++++++
.../plans/physical/PhysicalRecursiveCteScan.java | 85 ++++++
.../nereids/trees/plans/visitor/PlanVisitor.java | 22 ++
.../trees/plans/visitor/RelationVisitor.java | 10 +
.../org/apache/doris/planner/RecursiveCteNode.java | 87 ++++++
.../apache/doris/planner/RecursiveCteScanNode.java | 122 +++++++++
.../doris/qe/runtime/ThriftPlansBuilder.java | 151 ++++++++++-
.../apache/doris/statistics/StatisticalType.java | 2 +
.../nereids/rules/analysis/AnalyzeCTETest.java | 219 ++++++++++++++++
.../doris/nereids/rules/rewrite/CTEInlineTest.java | 84 ++++++
.../nereids/rules/rewrite/ColumnPruningTest.java | 30 +++
gensrc/thrift/PaloInternalService.thrift | 1 +
gensrc/thrift/PlanNodes.thrift | 2 +
.../recursive_cte/test_recursive_cte.groovy | 102 +++++++
52 files changed, 2819 insertions(+), 63 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index 397b961353c..1cdf170b9e2 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -440,6 +440,7 @@ REAL: 'REAL';
REBALANCE: 'REBALANCE';
RECENT: 'RECENT';
RECOVER: 'RECOVER';
+RECURSIVE: 'RECURSIVE';
RECYCLE: 'RECYCLE';
REFRESH: 'REFRESH';
REFERENCES: 'REFERENCES';
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 8ee10fe28ab..9e3bbf6f3ec 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -1214,7 +1214,7 @@ querySpecification
;
cte
- : WITH aliasQuery (COMMA aliasQuery)*
+ : WITH RECURSIVE? aliasQuery (COMMA aliasQuery)*
;
aliasQuery
@@ -2143,6 +2143,7 @@ nonReserved
| RANDOM
| RECENT
| RECOVER
+ | RECURSIVE
| RECYCLE
| REFRESH
| REPEATABLE
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java
index fb6cc7df0a8..197293ac69a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java
@@ -20,6 +20,7 @@
package org.apache.doris.analysis;
+import org.apache.doris.catalog.RecursiveCteTempTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.IdGenerator;
@@ -212,6 +213,10 @@ public class DescriptorTable {
}
for (TableIf tbl : referencedTbls.values()) {
+ if (tbl instanceof RecursiveCteTempTable) {
+ // skip recursive cte temp table
+ continue;
+ }
result.addToTableDescriptors(tbl.toThrift());
}
thriftDescTable = result;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RecursiveCteTempTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RecursiveCteTempTable.java
new file mode 100644
index 00000000000..ee924466fbb
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RecursiveCteTempTable.java
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.common.SystemIdGenerator;
+
+import java.util.List;
+
+public class RecursiveCteTempTable extends Table {
+ public RecursiveCteTempTable(String tableName, List<Column> fullSchema) {
+ super(SystemIdGenerator.getNextId(), tableName, TableType.RECURSIVE_CTE_TEMP_TABLE, fullSchema);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
index d15fabd6b75..5aa7f9ead76 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
@@ -398,7 +398,8 @@ public interface TableIf {
@Deprecated ICEBERG, @Deprecated HUDI, JDBC,
TABLE_VALUED_FUNCTION, HMS_EXTERNAL_TABLE, ES_EXTERNAL_TABLE, MATERIALIZED_VIEW, JDBC_EXTERNAL_TABLE,
ICEBERG_EXTERNAL_TABLE, TEST_EXTERNAL_TABLE, PAIMON_EXTERNAL_TABLE, MAX_COMPUTE_EXTERNAL_TABLE,
- HUDI_EXTERNAL_TABLE, TRINO_CONNECTOR_EXTERNAL_TABLE, LAKESOUl_EXTERNAL_TABLE, DICTIONARY;
+ HUDI_EXTERNAL_TABLE, TRINO_CONNECTOR_EXTERNAL_TABLE, LAKESOUl_EXTERNAL_TABLE, DICTIONARY,
+ RECURSIVE_CTE_TEMP_TABLE;
public String toEngineName() {
switch (this) {
@@ -437,6 +438,8 @@ public interface TableIf {
return "iceberg";
case DICTIONARY:
return "dictionary";
+ case RECURSIVE_CTE_TEMP_TABLE:
+ return "RecursiveCteTempTable";
default:
return null;
}
@@ -475,6 +478,7 @@ public interface TableIf {
case PAIMON_EXTERNAL_TABLE:
case MATERIALIZED_VIEW:
case TRINO_CONNECTOR_EXTERNAL_TABLE:
+ case RECURSIVE_CTE_TEMP_TABLE:
return "BASE TABLE";
default:
return null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
index 6980765049f..1517d6f2a7a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
@@ -133,6 +133,8 @@ public class CascadesContext implements ScheduleContext {
private final boolean isEnableExprTrace;
private int groupExpressionCount = 0;
+ private Optional<String> currentRecursiveCteName;
+ private List<Slot> recursiveCteOutputs;
/**
* Constructor of OptimizerContext.
@@ -142,7 +144,8 @@ public class CascadesContext implements ScheduleContext {
*/
private CascadesContext(Optional<CascadesContext> parent, Optional<CTEId> currentTree,
StatementContext statementContext, Plan plan, Memo memo,
- CTEContext cteContext, PhysicalProperties requireProperties, boolean isLeadingDisableJoinReorder) {
+ CTEContext cteContext, PhysicalProperties requireProperties, boolean isLeadingDisableJoinReorder,
+ Optional<String> currentRecursiveCteName, List<Slot> recursiveCteOutputs) {
this.parent = Objects.requireNonNull(parent, "parent should not null");
this.currentTree = Objects.requireNonNull(currentTree, "currentTree should not null");
this.statementContext = Objects.requireNonNull(statementContext, "statementContext should not null");
@@ -167,6 +170,8 @@ public class CascadesContext implements ScheduleContext {
this.isEnableExprTrace = false;
}
this.isLeadingDisableJoinReorder = isLeadingDisableJoinReorder;
+ this.currentRecursiveCteName = currentRecursiveCteName;
+ this.recursiveCteOutputs = recursiveCteOutputs;
}
/** init a temporary context to rewrite expression */
@@ -181,7 +186,7 @@ public class CascadesContext implements ScheduleContext {
}
return newContext(Optional.empty(), Optional.empty(),
statementContext, DUMMY_PLAN,
- new CTEContext(), PhysicalProperties.ANY, false);
+ new CTEContext(), PhysicalProperties.ANY, false, Optional.empty(), ImmutableList.of());
}
/**
@@ -190,24 +195,25 @@ public class CascadesContext implements ScheduleContext {
public static CascadesContext initContext(StatementContext statementContext,
Plan initPlan, PhysicalProperties requireProperties) {
return newContext(Optional.empty(), Optional.empty(), statementContext,
- initPlan, new CTEContext(), requireProperties, false);
+ initPlan, new CTEContext(), requireProperties, false, Optional.empty(), ImmutableList.of());
}
/**
* use for analyze cte. we must pass CteContext from outer since we need to get right scope of cte
*/
public static CascadesContext newContextWithCteContext(CascadesContext cascadesContext,
- Plan initPlan, CTEContext cteContext) {
+ Plan initPlan, CTEContext cteContext, Optional<String> currentRecursiveCteName,
+ List<Slot> recursiveCteOutputs) {
return newContext(Optional.of(cascadesContext), Optional.empty(),
cascadesContext.getStatementContext(), initPlan, cteContext, PhysicalProperties.ANY,
- cascadesContext.isLeadingDisableJoinReorder
- );
+ cascadesContext.isLeadingDisableJoinReorder, currentRecursiveCteName, recursiveCteOutputs);
}
public static CascadesContext newCurrentTreeContext(CascadesContext context) {
return CascadesContext.newContext(context.getParent(), context.getCurrentTree(), context.getStatementContext(),
context.getRewritePlan(), context.getCteContext(),
- context.getCurrentJobContext().getRequiredProperties(), context.isLeadingDisableJoinReorder);
+ context.getCurrentJobContext().getRequiredProperties(), context.isLeadingDisableJoinReorder,
+ Optional.empty(), ImmutableList.of());
}
/**
@@ -216,14 +222,17 @@ public class CascadesContext implements ScheduleContext {
public static CascadesContext newSubtreeContext(Optional<CTEId> subtree, CascadesContext context,
Plan plan, PhysicalProperties requireProperties) {
return CascadesContext.newContext(Optional.of(context), subtree, context.getStatementContext(),
- plan, context.getCteContext(), requireProperties, context.isLeadingDisableJoinReorder);
+ plan, context.getCteContext(), requireProperties, context.isLeadingDisableJoinReorder, Optional.empty(),
+ ImmutableList.of());
}
private static CascadesContext newContext(Optional<CascadesContext> parent, Optional<CTEId> subtree,
StatementContext statementContext, Plan initPlan, CTEContext cteContext,
- PhysicalProperties requireProperties, boolean isLeadingDisableJoinReorder) {
+ PhysicalProperties requireProperties, boolean isLeadingDisableJoinReorder,
+ Optional<String> currentRecursiveCteName, List<Slot> recursiveCteOutputs) {
return new CascadesContext(parent, subtree, statementContext, initPlan, null,
- cteContext, requireProperties, isLeadingDisableJoinReorder);
+ cteContext, requireProperties, isLeadingDisableJoinReorder, currentRecursiveCteName,
+ recursiveCteOutputs);
}
public CascadesContext getRoot() {
@@ -250,6 +259,18 @@ public class CascadesContext implements ScheduleContext {
return isTimeout;
}
+ public Optional<String> getCurrentRecursiveCteName() {
+ return currentRecursiveCteName;
+ }
+
+ public List<Slot> getRecursiveCteOutputs() {
+ return recursiveCteOutputs;
+ }
+
+ public boolean isAnalyzingRecursiveCteAnchorChild() {
+ return currentRecursiveCteName.isPresent() && recursiveCteOutputs.isEmpty();
+ }
+
/**
* Init memo with plan
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 44186fe3711..ee8fec61401 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -156,6 +156,9 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteRecursiveChild;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink;
@@ -205,6 +208,8 @@ import org.apache.doris.planner.OlapTableSink;
import org.apache.doris.planner.PartitionSortNode;
import org.apache.doris.planner.PlanFragment;
import org.apache.doris.planner.PlanNode;
+import org.apache.doris.planner.RecursiveCteNode;
+import org.apache.doris.planner.RecursiveCteScanNode;
import org.apache.doris.planner.RepeatNode;
import org.apache.doris.planner.ResultFileSink;
import org.apache.doris.planner.ResultSink;
@@ -1070,6 +1075,28 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return planFragment;
}
+ @Override
+ public PlanFragment visitPhysicalRecursiveCteScan(PhysicalRecursiveCteScan recursiveCteScan,
+ PlanTranslatorContext context) {
+ TableIf table = recursiveCteScan.getTable();
+ List<Slot> slots = ImmutableList.copyOf(recursiveCteScan.getOutput());
+ TupleDescriptor tupleDescriptor = generateTupleDesc(slots, null, context);
+
+ RecursiveCteScanNode scanNode = new RecursiveCteScanNode(table != null ? table.getName() : "",
+ context.nextPlanNodeId(), tupleDescriptor);
+ scanNode.setNereidsId(recursiveCteScan.getId());
+ context.getNereidsIdToPlanNodeIdMap().put(recursiveCteScan.getId(), scanNode.getId());
+ Utils.execWithUncheckedException(scanNode::initScanRangeLocations);
+
+ translateRuntimeFilter(recursiveCteScan, scanNode, context);
+
+ context.addScanNode(scanNode, recursiveCteScan);
+ PlanFragment planFragment = createPlanFragment(scanNode, DataPartition.RANDOM, recursiveCteScan);
+ context.addPlanFragment(planFragment);
+ updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), recursiveCteScan);
+ return planFragment;
+ }
+
private List<Expr> translateToExprs(List<Expression> expressions, PlanTranslatorContext context) {
List<Expr> exprs = Lists.newArrayListWithCapacity(expressions.size());
for (Expression expression : expressions) {
@@ -2229,8 +2256,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
if (inputPlanNode instanceof OlapScanNode) {
((OlapScanNode) inputPlanNode).updateRequiredSlots(context, requiredByProjectSlotIdSet);
}
- updateScanSlotsMaterialization((ScanNode) inputPlanNode, requiredSlotIdSet,
- requiredByProjectSlotIdSet, context);
+ if (!(inputPlanNode instanceof RecursiveCteScanNode)) {
+ updateScanSlotsMaterialization((ScanNode) inputPlanNode, requiredSlotIdSet,
+ requiredByProjectSlotIdSet, context);
+ }
} else {
if (project.child() instanceof PhysicalDeferMaterializeTopN) {
inputFragment.setOutputExprs(allProjectionExprs);
@@ -2243,6 +2272,80 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return inputFragment;
}
+ @Override
+ public PlanFragment visitPhysicalRecursiveCte(PhysicalRecursiveCte recursiveCte, PlanTranslatorContext context) {
+ List<PlanFragment> childrenFragments = new ArrayList<>();
+ for (Plan plan : recursiveCte.children()) {
+ childrenFragments.add(plan.accept(this, context));
+ }
+
+ TupleDescriptor setTuple = generateTupleDesc(recursiveCte.getOutput(), null, context);
+ List<SlotDescriptor> outputSlotDescs = new ArrayList<>(setTuple.getSlots());
+
+ RecursiveCteNode recursiveCteNode = new RecursiveCteNode(context.nextPlanNodeId(), setTuple.getId(),
+ recursiveCte.getCteName(), recursiveCte.isUnionAll());
+ List<List<Expr>> distributeExprLists = getDistributeExprs(recursiveCte.children().toArray(new Plan[0]));
+ recursiveCteNode.setChildrenDistributeExprLists(distributeExprLists);
+ recursiveCteNode.setNereidsId(recursiveCte.getId());
+ List<List<Expression>> resultExpressionLists = Lists.newArrayList();
+ context.getNereidsIdToPlanNodeIdMap().put(recursiveCte.getId(), recursiveCteNode.getId());
+ for (List<SlotReference> regularChildrenOutput : recursiveCte.getRegularChildrenOutputs()) {
+ resultExpressionLists.add(new ArrayList<>(regularChildrenOutput));
+ }
+
+ for (PlanFragment childFragment : childrenFragments) {
+ recursiveCteNode.addChild(childFragment.getPlanRoot());
+ }
+
+ List<List<Expr>> materializedResultExprLists = Lists.newArrayList();
+ for (int i = 0; i < resultExpressionLists.size(); ++i) {
+ List<Expression> resultExpressionList = resultExpressionLists.get(i);
+ List<Expr> exprList = Lists.newArrayList();
+ Preconditions.checkState(resultExpressionList.size() == outputSlotDescs.size());
+ for (int j = 0; j < resultExpressionList.size(); ++j) {
+ if (outputSlotDescs.get(j).isMaterialized()) {
+ exprList.add(ExpressionTranslator.translate(resultExpressionList.get(j), context));
+ // TODO: reconsider this, we may change nullable info in previous nereids rules not here.
+ outputSlotDescs.get(j)
+ .setIsNullable(outputSlotDescs.get(j).getIsNullable() || exprList.get(j).isNullable());
+ }
+ }
+ materializedResultExprLists.add(exprList);
+ }
+ recursiveCteNode.setMaterializedResultExprLists(materializedResultExprLists);
+ Preconditions.checkState(recursiveCteNode.getMaterializedResultExprLists().size()
+ == recursiveCteNode.getChildren().size());
+
+ PlanFragment recursiveCteFragment;
+ if (childrenFragments.isEmpty()) {
+ recursiveCteFragment = createPlanFragment(recursiveCteNode,
+ DataPartition.UNPARTITIONED, recursiveCte);
+ context.addPlanFragment(recursiveCteFragment);
+ } else {
+ int childrenSize = childrenFragments.size();
+ recursiveCteFragment = childrenFragments.get(childrenSize - 1);
+ for (int i = childrenSize - 2; i >= 0; i--) {
+ context.mergePlanFragment(childrenFragments.get(i), recursiveCteFragment);
+ for (PlanFragment child : childrenFragments.get(i).getChildren()) {
+ recursiveCteFragment.addChild(child);
+ }
+ }
+ setPlanRoot(recursiveCteFragment, recursiveCteNode, recursiveCte);
+ }
+
+ recursiveCteFragment.updateDataPartition(DataPartition.UNPARTITIONED);
+ recursiveCteFragment.setOutputPartition(DataPartition.UNPARTITIONED);
+
+ return recursiveCteFragment;
+ }
+
+ @Override
+ public PlanFragment visitPhysicalRecursiveCteRecursiveChild(
+ PhysicalRecursiveCteRecursiveChild<? extends Plan> recursiveChild,
+ PlanTranslatorContext context) {
+ return recursiveChild.child().accept(this, context);
+ }
+
/**
* Returns a new fragment with a UnionNode as its root. The data partition of the
* returned fragment and how the data of the child fragments is consumed depends on the
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 3557a77e3ff..c2981f5928c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -1117,6 +1117,8 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
private final Map<Integer, ParserRuleContext> selectHintMap;
+ private boolean isInRecursiveCteContext = false;
+
public LogicalPlanBuilder(Map<Integer, ParserRuleContext> selectHintMap) {
this.selectHintMap = selectHintMap;
}
@@ -2183,7 +2185,11 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
if (ctx == null) {
return plan;
}
- return new LogicalCTE<>((List) visit(ctx.aliasQuery(), LogicalSubQueryAlias.class), plan);
+ isInRecursiveCteContext = ctx.RECURSIVE() != null;
+ LogicalCTE<Plan> logicalCTE = new LogicalCTE<>(isInRecursiveCteContext,
+ (List) visit(ctx.aliasQuery(), LogicalSubQueryAlias.class), plan);
+ isInRecursiveCteContext = false;
+ return logicalCTE;
}
/**
@@ -2377,7 +2383,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
public LogicalPlan visitSetOperation(SetOperationContext ctx) {
return ParserUtils.withOrigin(ctx, () -> {
- if (ctx.UNION() != null) {
+ if (ctx.UNION() != null && !isInRecursiveCteContext) {
Qualifier qualifier = getQualifier(ctx);
List<QueryTermContext> contexts = Lists.newArrayList(ctx.right);
QueryTermContext current = ctx.left;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java
index fa9d191f5e2..ec53deafaad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java
@@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnary;
@@ -205,6 +206,29 @@ public interface MemoPatterns extends Patterns {
defaultPromise());
}
+ /**
+ * create a LogicalRecursiveCte pattern.
+ */
+ default PatternDescriptor<LogicalRecursiveCte>
+ logicalRecursiveCte(
+ PatternDescriptor... children) {
+ return new PatternDescriptor(
+ new TypePattern(LogicalRecursiveCte.class,
+ Arrays.stream(children)
+ .map(PatternDescriptor::getPattern)
+ .toArray(Pattern[]::new)),
+ defaultPromise());
+ }
+
+ /**
+ * create a logicalRecursiveCte group.
+ */
+ default PatternDescriptor<LogicalRecursiveCte> logicalRecursiveCte() {
+ return new PatternDescriptor(
+ new TypePattern(LogicalRecursiveCte.class, multiGroup().pattern),
+ defaultPromise());
+ }
+
/**
* create a logicalExcept pattern.
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
index 51a9003b78f..d7e4e5b67cb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
@@ -33,6 +33,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN;
@@ -76,6 +77,14 @@ public class RuntimeFilterPruner extends PlanPostProcessor {
return plan;
}
+ @Override
+ public PhysicalRecursiveCte visitPhysicalRecursiveCte(PhysicalRecursiveCte recursiveCte, CascadesContext context) {
+ for (Plan child : recursiveCte.children()) {
+ child.accept(this, context);
+ }
+ return recursiveCte;
+ }
+
@Override
public PhysicalSetOperation visitPhysicalSetOperation(PhysicalSetOperation setOperation, CascadesContext context) {
for (Plan child : setOperation.children()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java
index 31a205d5ed5..1dfe63c9e67 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java
@@ -47,10 +47,11 @@ public class PullUpSubqueryAliasToCTE extends PlanPreprocessor {
subQueryAliases.addAll(logicalCTE.getAliasQueries());
subQueryAliases.addAll(aliasQueries);
return topPlan.withChildren(
- new LogicalCTE<>(subQueryAliases, (LogicalPlan) ((UnboundResultSink) topPlan).child()));
+ new LogicalCTE<>(logicalCTE.isRecursiveCte(), subQueryAliases,
+ (LogicalPlan) ((UnboundResultSink) topPlan).child()));
}
return topPlan.withChildren(
- new LogicalCTE<>(aliasQueries, (LogicalPlan) ((UnboundResultSink) topPlan).child()));
+ new LogicalCTE<>(false, aliasQueries, (LogicalPlan) ((UnboundResultSink) topPlan).child()));
}
return topPlan;
}
@@ -86,7 +87,8 @@ public class PullUpSubqueryAliasToCTE extends PlanPreprocessor {
subQueryAliasesOfCte.addAll(logicalCTE.getAliasQueries());
subQueryAliasesOfCte.addAll(aliasQueries);
aliasQueries = new ArrayList<>();
- return new LogicalCTE<>(subQueryAliasesOfCte, (LogicalPlan) newLogicalCTE.child());
+ return new LogicalCTE<>(newLogicalCTE.isRecursiveCte(), subQueryAliasesOfCte,
+ (LogicalPlan) newLogicalCTE.child());
}
return cte;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
index d1c8bccb421..56433c82d33 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
@@ -52,6 +52,9 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteRecursiveChild;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSink;
@@ -147,6 +150,11 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
return PhysicalProperties.STORAGE_ANY;
}
+ @Override
+ public PhysicalProperties visitPhysicalRecursiveCteScan(PhysicalRecursiveCteScan cteScan, PlanContext context) {
+ return PhysicalProperties.ANY;
+ }
+
/**
* TODO return ANY after refactor coordinator
* return STORAGE_ANY not ANY, in order to generate distribute on jdbc scan.
@@ -463,6 +471,18 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
return PhysicalProperties.createHash(request, firstType);
}
+ @Override
+ public PhysicalProperties visitPhysicalRecursiveCte(PhysicalRecursiveCte recursiveCte, PlanContext context) {
+ return PhysicalProperties.GATHER;
+ }
+
+ @Override
+ public PhysicalProperties visitPhysicalRecursiveCteRecursiveChild(
+ PhysicalRecursiveCteRecursiveChild<? extends Plan> recursiveChild,
+ PlanContext context) {
+ return PhysicalProperties.MUST_SHUFFLE;
+ }
+
@Override
public PhysicalProperties visitPhysicalUnion(PhysicalUnion union, PlanContext context) {
if (union.getConstantExprsList().isEmpty()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
index 2322b39a62b..c26d1abade1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
@@ -54,6 +54,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion;
@@ -315,6 +316,16 @@ public class RequestPropertyDeriver extends PlanVisitor<Void, PlanContext> {
return null;
}
+ @Override
+ public Void visitPhysicalRecursiveCte(PhysicalRecursiveCte recursiveCte, PlanContext context) {
+ List<PhysicalProperties> requestGather = Lists.newArrayListWithCapacity(context.arity());
+ for (int i = context.arity(); i > 0; --i) {
+ requestGather.add(PhysicalProperties.GATHER);
+ }
+ addRequestPropertyToChildren(requestGather);
+ return null;
+ }
+
@Override
public Void visitAbstractPhysicalSort(AbstractPhysicalSort<? extends Plan> sort, PlanContext context) {
if (!sort.getSortPhase().isLocal()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 45cb22e6b52..62a5d7702c3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -80,6 +80,9 @@ import org.apache.doris.nereids.rules.implementation.LogicalOlapTableSinkToPhysi
import org.apache.doris.nereids.rules.implementation.LogicalOneRowRelationToPhysicalOneRowRelation;
import org.apache.doris.nereids.rules.implementation.LogicalPartitionTopNToPhysicalPartitionTopN;
import org.apache.doris.nereids.rules.implementation.LogicalProjectToPhysicalProject;
+import org.apache.doris.nereids.rules.implementation.LogicalRecursiveCteRecursiveChildToPhysicalRecursiveCteRecursiveChild;
+import org.apache.doris.nereids.rules.implementation.LogicalRecursiveCteScanToPhysicalRecursiveCteScan;
+import org.apache.doris.nereids.rules.implementation.LogicalRecursiveCteToPhysicalRecursiveCte;
import org.apache.doris.nereids.rules.implementation.LogicalRepeatToPhysicalRepeat;
import org.apache.doris.nereids.rules.implementation.LogicalResultSinkToPhysicalResultSink;
import org.apache.doris.nereids.rules.implementation.LogicalSchemaScanToPhysicalSchemaScan;
@@ -194,6 +197,7 @@ public class RuleSet {
.add(new LogicalJdbcScanToPhysicalJdbcScan())
.add(new LogicalOdbcScanToPhysicalOdbcScan())
.add(new LogicalEsScanToPhysicalEsScan())
+ .add(new LogicalRecursiveCteScanToPhysicalRecursiveCteScan())
.add(new LogicalProjectToPhysicalProject())
.add(new LogicalLimitToPhysicalLimit())
.add(new LogicalWindowToPhysicalWindow())
@@ -209,6 +213,8 @@ public class RuleSet {
.add(SplitAggWithoutDistinct.INSTANCE)
.add(SplitAggMultiPhase.INSTANCE)
.add(SplitAggMultiPhaseWithoutGbyKey.INSTANCE)
+ .add(new LogicalRecursiveCteToPhysicalRecursiveCte())
+ .add(new LogicalRecursiveCteRecursiveChildToPhysicalRecursiveCteRecursiveChild())
.add(new LogicalUnionToPhysicalUnion())
.add(new LogicalExceptToPhysicalExcept())
.add(new LogicalIntersectToPhysicalIntersect())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index a8bc2052348..75550513068 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -496,6 +496,7 @@ public enum RuleType {
LOGICAL_JDBC_SCAN_TO_PHYSICAL_JDBC_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_ODBC_SCAN_TO_PHYSICAL_ODBC_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_ES_SCAN_TO_PHYSICAL_ES_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
+ LOGICAL_RECURSIVE_CTE_SCAN_TO_PHYSICAL_RECUSIVE_CTE_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_BLACKHOLE_SINK_TO_PHYSICAL_BLACKHOLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_OLAP_TABLE_SINK_TO_PHYSICAL_OLAP_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_HIVE_TABLE_SINK_TO_PHYSICAL_HIVE_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
@@ -517,6 +518,8 @@ public enum RuleType {
COUNT_ON_INDEX_WITHOUT_PROJECT(RuleTypeClass.IMPLEMENTATION),
TWO_PHASE_AGGREGATE_WITH_DISTINCT(RuleTypeClass.IMPLEMENTATION),
LOGICAL_UNION_TO_PHYSICAL_UNION(RuleTypeClass.IMPLEMENTATION),
+ LOGICAL_RECURSIVE_CTE_TO_PHYSICAL_RECURSIVE_CTE(RuleTypeClass.IMPLEMENTATION),
+ LOGICAL_RECURSIVE_CTE_RECURSIVE_CHILD_TO_PHYSICAL_RECURSIVE_CTE_RECURSIVE_CHILD(RuleTypeClass.IMPLEMENTATION),
LOGICAL_EXCEPT_TO_PHYSICAL_EXCEPT(RuleTypeClass.IMPLEMENTATION),
LOGICAL_INTERSECT_TO_PHYSICAL_INTERSECT(RuleTypeClass.IMPLEMENTATION),
LOGICAL_GENERATE_TO_PHYSICAL_GENERATE(RuleTypeClass.IMPLEMENTATION),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
index 4287e8ca7de..b734292d321 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java
@@ -23,22 +23,36 @@ import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.CTEId;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.SetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteRecursiveChild;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+import org.apache.doris.nereids.trees.plans.logical.ProjectProcessor;
+import org.apache.doris.nereids.types.DataType;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -70,7 +84,7 @@ public class AnalyzeCTE extends OneAnalysisRuleFactory {
// step 1. analyzed all cte plan
Pair<CTEContext, List<LogicalCTEProducer<Plan>>> result = analyzeCte(logicalCTE, ctx.cascadesContext);
CascadesContext outerCascadesCtx = CascadesContext.newContextWithCteContext(
- ctx.cascadesContext, logicalCTE.child(), result.first);
+ ctx.cascadesContext, logicalCTE.child(), result.first, Optional.empty(), ImmutableList.of());
outerCascadesCtx.withPlanProcess(ctx.cascadesContext.showPlanProcess(), () -> {
outerCascadesCtx.newAnalyzer().analyze();
});
@@ -95,25 +109,142 @@ public class AnalyzeCTE extends OneAnalysisRuleFactory {
List<LogicalCTEProducer<Plan>> cteProducerPlans = new ArrayList<>();
for (LogicalSubQueryAlias<Plan> aliasQuery : aliasQueries) {
// we should use a chain to ensure visible of cte
- LogicalPlan parsedCtePlan = (LogicalPlan) aliasQuery.child();
- CascadesContext innerCascadesCtx = CascadesContext.newContextWithCteContext(
- cascadesContext, parsedCtePlan, outerCteCtx);
- innerCascadesCtx.withPlanProcess(cascadesContext.showPlanProcess(), () -> {
- innerCascadesCtx.newAnalyzer().analyze();
- });
- cascadesContext.addPlanProcesses(innerCascadesCtx.getPlanProcesses());
- LogicalPlan analyzedCtePlan = (LogicalPlan) innerCascadesCtx.getRewritePlan();
- checkColumnAlias(aliasQuery, analyzedCtePlan.getOutput());
- CTEId cteId = StatementScopeIdGenerator.newCTEId();
- LogicalSubQueryAlias<Plan> logicalSubQueryAlias =
- aliasQuery.withChildren(ImmutableList.of(analyzedCtePlan));
- outerCteCtx = new CTEContext(cteId, logicalSubQueryAlias, outerCteCtx);
- outerCteCtx.setAnalyzedPlan(logicalSubQueryAlias);
- cteProducerPlans.add(new LogicalCTEProducer<>(cteId, logicalSubQueryAlias));
+ if (aliasQuery.isRecursiveCte() && logicalCTE.isRecursiveCte()) {
+ Pair<CTEContext, LogicalCTEProducer<Plan>> result = analyzeRecursiveCte(aliasQuery, outerCteCtx,
+ cascadesContext);
+ outerCteCtx = result.first;
+ cteProducerPlans.add(result.second);
+ } else {
+ LogicalPlan parsedCtePlan = (LogicalPlan) aliasQuery.child();
+ CascadesContext innerCascadesCtx = CascadesContext.newContextWithCteContext(
+ cascadesContext, parsedCtePlan, outerCteCtx, Optional.empty(), ImmutableList.of());
+ innerCascadesCtx.withPlanProcess(cascadesContext.showPlanProcess(), () -> {
+ innerCascadesCtx.newAnalyzer().analyze();
+ });
+ cascadesContext.addPlanProcesses(innerCascadesCtx.getPlanProcesses());
+ LogicalPlan analyzedCtePlan = (LogicalPlan) innerCascadesCtx.getRewritePlan();
+ checkColumnAlias(aliasQuery, analyzedCtePlan.getOutput());
+ CTEId cteId = StatementScopeIdGenerator.newCTEId();
+ LogicalSubQueryAlias<Plan> logicalSubQueryAlias = aliasQuery
+ .withChildren(ImmutableList.of(analyzedCtePlan));
+ outerCteCtx = new CTEContext(cteId, logicalSubQueryAlias, outerCteCtx);
+ outerCteCtx.setAnalyzedPlan(logicalSubQueryAlias);
+ cteProducerPlans.add(new LogicalCTEProducer<>(cteId, logicalSubQueryAlias));
+ }
}
return Pair.of(outerCteCtx, cteProducerPlans);
}
+ private Pair<CTEContext, LogicalCTEProducer<Plan>> analyzeRecursiveCte(LogicalSubQueryAlias<Plan> aliasQuery,
+ CTEContext outerCteCtx, CascadesContext cascadesContext) {
+ Preconditions.checkArgument(aliasQuery.isRecursiveCte(), "alias query must be recursive cte");
+ LogicalPlan parsedCtePlan = (LogicalPlan) aliasQuery.child();
+ if (!(parsedCtePlan instanceof LogicalUnion) || parsedCtePlan.children().size() != 2) {
+ throw new AnalysisException(String.format("recursive cte must be union, don't support %s",
+ parsedCtePlan.getClass().getSimpleName()));
+ }
+ // analyze anchor child, its output list will be recursive cte temp table's schema
+ LogicalPlan anchorChild = (LogicalPlan) parsedCtePlan.child(0);
+ CascadesContext innerAnchorCascadesCtx = CascadesContext.newContextWithCteContext(
+ cascadesContext, anchorChild, outerCteCtx, Optional.of(aliasQuery.getAlias()), ImmutableList.of());
+ innerAnchorCascadesCtx.withPlanProcess(cascadesContext.showPlanProcess(), () -> {
+ innerAnchorCascadesCtx.newAnalyzer().analyze();
+ });
+ cascadesContext.addPlanProcesses(innerAnchorCascadesCtx.getPlanProcesses());
+ LogicalPlan analyzedAnchorChild = (LogicalPlan) innerAnchorCascadesCtx.getRewritePlan();
+ if (!analyzedAnchorChild.collect(LogicalRecursiveCteScan.class::isInstance).isEmpty()) {
+ throw new AnalysisException(
+ String.format("recursive reference to query %s must not appear within its non-recursive term",
+ aliasQuery.getAlias()));
+ }
+ checkColumnAlias(aliasQuery, analyzedAnchorChild.getOutput());
+ // make all output nullable
+ analyzedAnchorChild = forceOutputNullable(analyzedAnchorChild,
+ aliasQuery.getColumnAliases().orElse(ImmutableList.of()));
+ // analyze recursive child
+ LogicalPlan recursiveChild = (LogicalPlan) parsedCtePlan.child(1);
+ CascadesContext innerRecursiveCascadesCtx = CascadesContext.newContextWithCteContext(
+ cascadesContext, recursiveChild, outerCteCtx, Optional.of(aliasQuery.getAlias()),
+ analyzedAnchorChild.getOutput());
+ innerRecursiveCascadesCtx.withPlanProcess(cascadesContext.showPlanProcess(), () -> {
+ innerRecursiveCascadesCtx.newAnalyzer().analyze();
+ });
+ cascadesContext.addPlanProcesses(innerRecursiveCascadesCtx.getPlanProcesses());
+ LogicalPlan analyzedRecursiveChild = (LogicalPlan) innerRecursiveCascadesCtx.getRewritePlan();
+ List<LogicalRecursiveCteScan> recursiveCteScanList = analyzedRecursiveChild
+ .collectToList(LogicalRecursiveCteScan.class::isInstance);
+ if (recursiveCteScanList.size() > 1) {
+ throw new AnalysisException(String.format("recursive reference to query %s must not appear more than once",
+ aliasQuery.getAlias()));
+ }
+ List<Slot> anchorChildOutputs = analyzedAnchorChild.getOutput();
+ List<DataType> anchorChildOutputTypes = new ArrayList<>(anchorChildOutputs.size());
+ for (Slot slot : anchorChildOutputs) {
+ anchorChildOutputTypes.add(slot.getDataType());
+ }
+ List<Slot> recursiveChildOutputs = analyzedRecursiveChild.getOutput();
+ for (int i = 0; i < recursiveChildOutputs.size(); ++i) {
+ if (!recursiveChildOutputs.get(i).getDataType().equals(anchorChildOutputTypes.get(i))) {
+ throw new AnalysisException(String.format("%s recursive child's %d column's datatype in select list %s "
+ + "is different from anchor child's output datatype %s, please add cast manually "
+ + "to get expect datatype", aliasQuery.getAlias(), i + 1,
+ recursiveChildOutputs.get(i).getDataType(), anchorChildOutputTypes.get(i)));
+ }
+ }
+ analyzedRecursiveChild = new LogicalRecursiveCteRecursiveChild<>(aliasQuery.getAlias(),
+ forceOutputNullable(analyzedRecursiveChild, ImmutableList.of()));
+
+ // create LogicalRecursiveCte
+ LogicalUnion logicalUnion = (LogicalUnion) parsedCtePlan;
+ LogicalRecursiveCte analyzedCtePlan = new LogicalRecursiveCte(aliasQuery.getAlias(),
+ logicalUnion.getQualifier() == SetOperation.Qualifier.ALL,
+ ImmutableList.of(analyzedAnchorChild, analyzedRecursiveChild));
+ List<List<NamedExpression>> childrenProjections = analyzedCtePlan.collectChildrenProjections();
+ int childrenProjectionSize = childrenProjections.size();
+ ImmutableList.Builder<List<SlotReference>> childrenOutputs = ImmutableList
+ .builderWithExpectedSize(childrenProjectionSize);
+ ImmutableList.Builder<Plan> newChildren = ImmutableList.builderWithExpectedSize(childrenProjectionSize);
+ for (int i = 0; i < childrenProjectionSize; i++) {
+ Plan newChild;
+ Plan child = analyzedCtePlan.child(i);
+ if (childrenProjections.get(i).stream().allMatch(SlotReference.class::isInstance)) {
+ newChild = child;
+ } else {
+ List<NamedExpression> parentProject = childrenProjections.get(i);
+ newChild = ProjectProcessor.tryProcessProject(parentProject, child)
+ .orElseGet(() -> new LogicalProject<>(parentProject, child));
+ }
+ newChildren.add(newChild);
+ childrenOutputs.add((List<SlotReference>) (List) newChild.getOutput());
+ }
+ analyzedCtePlan = analyzedCtePlan.withChildrenAndTheirOutputs(newChildren.build(), childrenOutputs.build());
+ List<NamedExpression> newOutputs = analyzedCtePlan.buildNewOutputs();
+ analyzedCtePlan = analyzedCtePlan.withNewOutputs(newOutputs);
+
+ CTEId cteId = StatementScopeIdGenerator.newCTEId();
+ LogicalSubQueryAlias<Plan> logicalSubQueryAlias = aliasQuery.withChildren(ImmutableList.of(analyzedCtePlan));
+ outerCteCtx = new CTEContext(cteId, logicalSubQueryAlias, outerCteCtx);
+ outerCteCtx.setAnalyzedPlan(logicalSubQueryAlias);
+ LogicalCTEProducer<Plan> cteProducer = new LogicalCTEProducer<>(cteId, logicalSubQueryAlias);
+ return Pair.of(outerCteCtx, cteProducer);
+ }
+
+ private LogicalPlan forceOutputNullable(LogicalPlan logicalPlan, List<String> aliasNames) {
+ List<Slot> oldOutputs = logicalPlan.getOutput();
+ int size = oldOutputs.size();
+ List<NamedExpression> newOutputs = new ArrayList<>(oldOutputs.size());
+ if (!aliasNames.isEmpty()) {
+ for (int i = 0; i < size; ++i) {
+ newOutputs.add(new Alias(new Nullable(oldOutputs.get(i)), aliasNames.get(i)));
+ }
+ } else {
+ for (Slot slot : oldOutputs) {
+ newOutputs.add(new Alias(new Nullable(slot), slot.getName()));
+ }
+ }
+ return new LogicalProject<>(newOutputs, logicalPlan);
+ }
+
/**
* check columnAliases' size and name
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index 84652b9eed0..b6ef89a26e8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -27,6 +27,7 @@ import org.apache.doris.catalog.FunctionRegistry;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.RecursiveCteTempTable;
import org.apache.doris.catalog.SchemaTable;
import org.apache.doris.catalog.SchemaTable.SchemaColumn;
import org.apache.doris.catalog.TableIf;
@@ -89,6 +90,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
@@ -168,16 +170,32 @@ public class BindRelation extends OneAnalysisRuleFactory {
return consumer;
}
}
+ LogicalPlan scan;
List<String> tableQualifier = RelationUtil.getQualifierName(
cascadesContext.getConnectContext(), unboundRelation.getNameParts());
- TableIf table = cascadesContext.getStatementContext().getAndCacheTable(tableQualifier, TableFrom.QUERY,
- Optional.of(unboundRelation));
+ if (tableName.equalsIgnoreCase(cascadesContext.getCurrentRecursiveCteName().orElse(""))) {
+ if (cascadesContext.isAnalyzingRecursiveCteAnchorChild()) {
+ throw new AnalysisException(
+ String.format("recursive reference to query %s must not appear within its non-recursive term",
+ cascadesContext.getCurrentRecursiveCteName().get()));
+ }
+ ImmutableList.Builder<Column> schema = new ImmutableList.Builder<>();
+ for (Slot slot : cascadesContext.getRecursiveCteOutputs()) {
+ schema.add(new Column(slot.getName(), slot.getDataType().toCatalogDataType(), slot.nullable()));
+ }
+ RecursiveCteTempTable cteTempTable = new RecursiveCteTempTable(tableName, schema.build());
+ scan = new LogicalRecursiveCteScan(cascadesContext.getStatementContext().getNextRelationId(),
+ cteTempTable, tableQualifier);
+ } else {
+ TableIf table = cascadesContext.getStatementContext().getAndCacheTable(tableQualifier, TableFrom.QUERY,
+ Optional.of(unboundRelation));
- LogicalPlan scan = getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
- if (cascadesContext.isLeadingJoin()) {
- LeadingHint leading = (LeadingHint) cascadesContext.getHintMap().get("Leading");
- leading.putRelationIdAndTableName(Pair.of(unboundRelation.getRelationId(), tableName));
- leading.getRelationIdToScanMap().put(unboundRelation.getRelationId(), scan);
+ scan = getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
+ if (cascadesContext.isLeadingJoin()) {
+ LeadingHint leading = (LeadingHint) cascadesContext.getHintMap().get("Leading");
+ leading.putRelationIdAndTableName(Pair.of(unboundRelation.getRelationId(), tableName));
+ leading.getRelationIdToScanMap().put(unboundRelation.getRelationId(), scan);
+ }
}
return scan;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
index 914220f177e..e11a456e205 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
@@ -106,8 +106,10 @@ public class CollectRelation implements AnalysisRuleFactory {
for (LogicalSubQueryAlias<Plan> aliasQuery : aliasQueries) {
// we should use a chain to ensure visible of cte
LogicalPlan parsedCtePlan = (LogicalPlan) aliasQuery.child();
+ // 看起来需要在CascadesContext中添加当前CTE的name，以便判断自引用
CascadesContext innerCascadesCtx = CascadesContext.newContextWithCteContext(
- cascadesContext, parsedCtePlan, outerCteCtx);
+ cascadesContext, parsedCtePlan, outerCteCtx, aliasQuery.isRecursiveCte()
+ ? Optional.of(aliasQuery.getAlias()) : Optional.empty(), ImmutableList.of());
innerCascadesCtx.newTableCollector(true).collect();
LogicalPlan analyzedCtePlan = (LogicalPlan) innerCascadesCtx.getRewritePlan();
// cteId is not used in CollectTable stage
@@ -126,7 +128,8 @@ public class CollectRelation implements AnalysisRuleFactory {
if (e instanceof SubqueryExpr) {
SubqueryExpr subqueryExpr = (SubqueryExpr) e;
CascadesContext subqueryContext = CascadesContext.newContextWithCteContext(
- ctx.cascadesContext, subqueryExpr.getQueryPlan(), ctx.cteContext);
+ ctx.cascadesContext, subqueryExpr.getQueryPlan(), ctx.cteContext, Optional.empty(),
+ ImmutableList.of());
subqueryContext.keepOrShowPlanProcess(ctx.cascadesContext.showPlanProcess(),
() -> subqueryContext.newTableCollector(true).collect());
ctx.cascadesContext.addPlanProcesses(subqueryContext.getPlanProcesses());
@@ -178,6 +181,10 @@ public class CollectRelation implements AnalysisRuleFactory {
List<String> nameParts, TableFrom tableFrom, Optional<UnboundRelation> unboundRelation) {
if (nameParts.size() == 1) {
String tableName = nameParts.get(0);
+ if (cascadesContext.getCurrentRecursiveCteName().isPresent()
+ && tableName.equalsIgnoreCase(cascadesContext.getCurrentRecursiveCteName().get())) {
+ return;
+ }
// check if it is a CTE's name
CTEContext cteContext = cascadesContext.getCteContext().findCTEContext(tableName).orElse(null);
if (cteContext != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java
index 69d79934992..05a35dc6ed8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java
@@ -52,6 +52,7 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
/**
@@ -227,7 +228,8 @@ class SubExprAnalyzer<T> extends DefaultExpressionRewriter<T> {
throw new IllegalStateException("Missing CascadesContext");
}
CascadesContext subqueryContext = CascadesContext.newContextWithCteContext(
- cascadesContext, expr.getQueryPlan(), cascadesContext.getCteContext());
+ cascadesContext, expr.getQueryPlan(), cascadesContext.getCteContext(), Optional.empty(),
+ ImmutableList.of());
// don't use `getScope()` because we only need `getScope().getOuterScope()` and `getScope().getSlots()`
// otherwise unexpected errors may occur
Scope subqueryScope = new Scope(getScope().getOuterScope(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteRecursiveChildToPhysicalRecursiveCteRecursiveChild.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteRecursiveChildToPhysicalRecursiveCteRecursiveChild.java
new file mode 100644
index 00000000000..689e550fa3a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteRecursiveChildToPhysicalRecursiveCteRecursiveChild.java
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.implementation;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteRecursiveChild;
+
+/**
+ * Implementation rule that convert logical recursive cte's recursive child to physical recursive child.
+ */
+public class LogicalRecursiveCteRecursiveChildToPhysicalRecursiveCteRecursiveChild
+ extends OneImplementationRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalRecursiveCteRecursiveChild().then(recursiveCte -> new PhysicalRecursiveCteRecursiveChild(
+ recursiveCte.getCteName(),
+ recursiveCte.getLogicalProperties(),
+ recursiveCte.child()))
+ .toRule(RuleType.LOGICAL_RECURSIVE_CTE_RECURSIVE_CHILD_TO_PHYSICAL_RECURSIVE_CTE_RECURSIVE_CHILD);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteScanToPhysicalRecursiveCteScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteScanToPhysicalRecursiveCteScan.java
new file mode 100644
index 00000000000..8714c280bb9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteScanToPhysicalRecursiveCteScan.java
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.implementation;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteScan;
+
+import java.util.Optional;
+
+/**
+ * Implementation rule that convert logical Recursive Cte Scan to physical Recursive Cte Scan.
+ */
+public class LogicalRecursiveCteScanToPhysicalRecursiveCteScan extends OneImplementationRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalRecursiveCteScan().then(recursiveCteScan ->
+ new PhysicalRecursiveCteScan(
+ recursiveCteScan.getRelationId(),
+ recursiveCteScan.getTable(),
+ recursiveCteScan.getQualifier(),
+ Optional.empty(),
+ recursiveCteScan.getLogicalProperties(),
+ recursiveCteScan.getOperativeSlots())
+ ).toRule(RuleType.LOGICAL_RECURSIVE_CTE_SCAN_TO_PHYSICAL_RECUSIVE_CTE_SCAN_RULE);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteToPhysicalRecursiveCte.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteToPhysicalRecursiveCte.java
new file mode 100644
index 00000000000..12ac89df19c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteToPhysicalRecursiveCte.java
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.implementation;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
+
+/**
+ * Implementation rule that convert logical Recursive Cte to Physical Recursive Cte.
+ */
+public class LogicalRecursiveCteToPhysicalRecursiveCte extends OneImplementationRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalRecursiveCte().then(recursiveCte ->
+ new PhysicalRecursiveCte(recursiveCte.getCteName(),
+ recursiveCte.isUnionAll(),
+ recursiveCte.getOutputs(),
+ recursiveCte.getRegularChildrenOutputs(),
+ recursiveCte.getLogicalProperties(),
+ recursiveCte.children())
+ ).toRule(RuleType.LOGICAL_RECURSIVE_CTE_TO_PHYSICAL_RECURSIVE_CTE);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
index 8592d3f22a0..204a018fbc7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
@@ -41,6 +41,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
@@ -289,6 +290,53 @@ public class AdjustNullable extends DefaultPlanRewriter<Map<ExprId, Slot>> imple
return repeat.withGroupSetsAndOutput(repeat.getGroupingSets(), newOutputs).recomputeLogicalProperties();
}
+ @Override
+ public Plan visitLogicalRecursiveCte(LogicalRecursiveCte recursiveCte, Map<ExprId, Slot> replaceMap) {
+ recursiveCte = (LogicalRecursiveCte) super.visit(recursiveCte, replaceMap);
+ ImmutableList.Builder<List<SlotReference>> newChildrenOutputs = ImmutableList.builder();
+ List<Boolean> inputNullable = null;
+ if (!recursiveCte.children().isEmpty()) {
+ inputNullable = Lists.newArrayListWithCapacity(recursiveCte.getOutputs().size());
+ for (int i = 0; i < recursiveCte.getOutputs().size(); i++) {
+ inputNullable.add(false);
+ }
+ for (int i = 0; i < recursiveCte.arity(); i++) {
+ List<Slot> childOutput = recursiveCte.child(i).getOutput();
+ List<SlotReference> setChildOutput = recursiveCte.getRegularChildOutput(i);
+ ImmutableList.Builder<SlotReference> newChildOutputs = ImmutableList.builder();
+ for (int j = 0; j < setChildOutput.size(); j++) {
+ for (Slot slot : childOutput) {
+ if (slot.getExprId().equals(setChildOutput.get(j).getExprId())) {
+ inputNullable.set(j, slot.nullable() || inputNullable.get(j));
+ newChildOutputs.add((SlotReference) slot);
+ break;
+ }
+ }
+ }
+ newChildrenOutputs.add(newChildOutputs.build());
+ }
+ }
+ if (inputNullable == null) {
+ // this is a fail-safe
+ // means there is no children and having no getConstantExprsList
+ // no way to update the nullable flag, so just do nothing
+ return recursiveCte;
+ }
+ List<NamedExpression> outputs = recursiveCte.getOutputs();
+ List<NamedExpression> newOutputs = Lists.newArrayListWithCapacity(outputs.size());
+ for (int i = 0; i < inputNullable.size(); i++) {
+ NamedExpression ne = outputs.get(i);
+ Slot slot = ne instanceof Alias ? (Slot) ((Alias) ne).child() : (Slot) ne;
+ slot = slot.withNullable(inputNullable.get(i));
+ NamedExpression newOutput = ne instanceof Alias ? (NamedExpression) ne.withChildren(slot) : slot;
+ newOutputs.add(newOutput);
+ replaceMap.put(newOutput.getExprId(), newOutput.toSlot());
+ }
+ return recursiveCte.withNewOutputs(newOutputs)
+ .withChildrenAndTheirOutputs(recursiveCte.children(), newChildrenOutputs.build())
+ .recomputeLogicalProperties();
+ }
+
@Override
public Plan visitLogicalSetOperation(LogicalSetOperation setOperation, Map<ExprId, Slot> replaceMap) {
setOperation = (LogicalSetOperation) super.visit(setOperation, replaceMap);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java
index 22ec72c99c5..9bbcb2e1e8d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java
@@ -31,15 +31,19 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteRecursiveChild;
import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
/**
* pull up LogicalCteAnchor to the top of plan to avoid CteAnchor break other rewrite rules pattern
@@ -48,9 +52,13 @@ import java.util.List;
* and put all of them to the top of plan depends on dependency tree of them.
*/
public class CTEInline extends DefaultPlanRewriter<LogicalCTEProducer<?>> implements CustomRewriter {
+ // all cte used by recursive cte's recursive child should be inline
+ private Set<LogicalCTEConsumer> mustInlineCteConsumers = new HashSet<>();
@Override
public Plan rewriteRoot(Plan plan, JobContext jobContext) {
+ collectMustInlineCteConsumers(plan, false, mustInlineCteConsumers);
+
Plan root = plan.accept(this, null);
// collect cte id to consumer
root.foreach(p -> {
@@ -78,17 +86,24 @@ public class CTEInline extends DefaultPlanRewriter<LogicalCTEProducer<?>> implem
}
return false;
});
- ConnectContext connectContext = ConnectContext.get();
- if (connectContext.getSessionVariable().enableCTEMaterialize
- && consumers.size() > connectContext.getSessionVariable().inlineCTEReferencedThreshold) {
- // not inline
- Plan right = cteAnchor.right().accept(this, null);
- return cteAnchor.withChildren(cteAnchor.left(), right);
- } else {
+ if (!Sets.intersection(mustInlineCteConsumers, Sets.newHashSet(consumers)).isEmpty()) {
// should inline
Plan root = cteAnchor.right().accept(this, (LogicalCTEProducer<?>) cteAnchor.left());
// process child
return root.accept(this, null);
+ } else {
+ ConnectContext connectContext = ConnectContext.get();
+ if (connectContext.getSessionVariable().enableCTEMaterialize
+ && consumers.size() > connectContext.getSessionVariable().inlineCTEReferencedThreshold) {
+ // not inline
+ Plan right = cteAnchor.right().accept(this, null);
+ return cteAnchor.withChildren(cteAnchor.left(), right);
+ } else {
+ // should inline
+ Plan root = cteAnchor.right().accept(this, (LogicalCTEProducer<?>) cteAnchor.left());
+ // process child
+ return root.accept(this, null);
+ }
}
}
}
@@ -113,4 +128,19 @@ public class CTEInline extends DefaultPlanRewriter<LogicalCTEProducer<?>> implem
}
return cteConsumer;
}
+
+ private void collectMustInlineCteConsumers(Plan planNode, boolean needCollect,
+ Set<LogicalCTEConsumer> cteConsumers) {
+ if (planNode instanceof LogicalCTEConsumer) {
+ if (needCollect) {
+ cteConsumers.add((LogicalCTEConsumer) planNode);
+ }
+ } else if (planNode instanceof LogicalRecursiveCteRecursiveChild) {
+ collectMustInlineCteConsumers(planNode.child(0), true, cteConsumers);
+ } else {
+ for (Plan child : planNode.children()) {
+ collectMustInlineCteConsumers(child, needCollect, cteConsumers);
+ }
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
index 90e94d578ba..dfbf13cedb7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
@@ -44,6 +44,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
@@ -213,6 +214,12 @@ public class ColumnPruning extends DefaultPlanRewriter<PruneContext> implements
return pruneChildren(plan, new RoaringBitmap());
}
+ @Override
+ public Plan visitLogicalRecursiveCte(LogicalRecursiveCte recursiveCte, PruneContext context) {
+ // keep LogicalRecursiveCte's output unchanged
+ return skipPruneThisAndFirstLevelChildren(recursiveCte);
+ }
+
// union can not prune children by the common logic, we must override visit method to write special code.
@Override
public Plan visitLogicalUnion(LogicalUnion union, PruneContext context) {
@@ -403,6 +410,54 @@ public class ColumnPruning extends DefaultPlanRewriter<PruneContext> implements
}
}
+ private LogicalRecursiveCte pruneRecursiveCteOutput(LogicalRecursiveCte recursiveCte, PruneContext context) {
+ List<NamedExpression> originOutput = recursiveCte.getOutputs();
+ if (originOutput.isEmpty()) {
+ return recursiveCte;
+ }
+ List<NamedExpression> prunedOutputs = Lists.newArrayList();
+ List<List<SlotReference>> regularChildrenOutputs = recursiveCte.getRegularChildrenOutputs();
+ List<Plan> children = recursiveCte.children();
+ List<Integer> extractColumnIndex = Lists.newArrayList();
+ for (int i = 0; i < originOutput.size(); i++) {
+ NamedExpression output = originOutput.get(i);
+ if (context.requiredSlotsIds.contains(output.getExprId().asInt())) {
+ prunedOutputs.add(output);
+ extractColumnIndex.add(i);
+ }
+ }
+
+ if (prunedOutputs.isEmpty()) {
+ // process prune all columns
+ NamedExpression originSlot = originOutput.get(0);
+ prunedOutputs = ImmutableList.of(new SlotReference(originSlot.getExprId(), originSlot.getName(),
+ TinyIntType.INSTANCE, false, originSlot.getQualifier()));
+ regularChildrenOutputs = Lists.newArrayListWithCapacity(regularChildrenOutputs.size());
+ children = Lists.newArrayListWithCapacity(children.size());
+ for (int i = 0; i < recursiveCte.children().size(); i++) {
+ Plan child = recursiveCte.child(i);
+ List<NamedExpression> newProjectOutput = ImmutableList.of(new Alias(new TinyIntLiteral((byte) 1)));
+ LogicalProject<?> project;
+ if (child instanceof LogicalProject) {
+ LogicalProject<Plan> childProject = (LogicalProject<Plan>) child;
+ List<NamedExpression> mergeProjections = PlanUtils.mergeProjections(
+ childProject.getProjects(), newProjectOutput);
+ project = new LogicalProject<>(mergeProjections, childProject.child());
+ } else {
+ project = new LogicalProject<>(newProjectOutput, child);
+ }
+ regularChildrenOutputs.add((List) project.getOutput());
+ children.add(project);
+ }
+ }
+
+ if (prunedOutputs.equals(originOutput) && !context.requiredSlotsIds.isEmpty()) {
+ return recursiveCte;
+ } else {
+ return recursiveCte.withNewOutputsAndChildren(prunedOutputs, children, regularChildrenOutputs);
+ }
+ }
+
private LogicalUnion pruneUnionOutput(LogicalUnion union, PruneContext context) {
List<NamedExpression> originOutput = union.getOutputs();
if (originOutput.isEmpty()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index f0ca1f1e6ba..930674b873b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -60,6 +60,7 @@ import org.apache.doris.nereids.trees.plans.algebra.Limit;
import org.apache.doris.nereids.trees.plans.algebra.OlapScan;
import org.apache.doris.nereids.trees.plans.algebra.PartitionTopN;
import org.apache.doris.nereids.trees.plans.algebra.Project;
+import org.apache.doris.nereids.trees.plans.algebra.RecursiveCte;
import org.apache.doris.nereids.trees.plans.algebra.Relation;
import org.apache.doris.nereids.trees.plans.algebra.Repeat;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation;
@@ -90,6 +91,9 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteRecursiveChild;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
@@ -123,6 +127,9 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteRecursiveChild;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSchemaScan;
@@ -869,6 +876,12 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
return computeCatalogRelation(esScan);
}
+ @Override
+ public Statistics visitLogicalRecursiveCteScan(LogicalRecursiveCteScan recursiveCteScan, Void context) {
+ recursiveCteScan.getExpressions();
+ return computeCatalogRelation(recursiveCteScan);
+ }
+
@Override
public Statistics visitLogicalProject(LogicalProject<? extends Plan> project, Void context) {
return computeProject(project, groupExpression.childStatistics(0));
@@ -912,6 +925,20 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
return computeAssertNumRows(assertNumRows.getAssertNumRowsElement(), groupExpression.childStatistics(0));
}
+ @Override
+ public Statistics visitLogicalRecursiveCte(
+ LogicalRecursiveCte recursiveCte, Void context) {
+ return computeRecursiveCte(recursiveCte,
+ groupExpression.children()
+ .stream().map(Group::getStatistics).collect(Collectors.toList()));
+ }
+
+ @Override
+ public Statistics visitLogicalRecursiveCteRecursiveChild(LogicalRecursiveCteRecursiveChild recursiveChild,
+ Void context) {
+ return groupExpression.childStatistics(0);
+ }
+
@Override
public Statistics visitLogicalUnion(
LogicalUnion union, Void context) {
@@ -997,6 +1024,11 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
return computeCatalogRelation(schemaScan);
}
+ @Override
+ public Statistics visitPhysicalRecursiveCteScan(PhysicalRecursiveCteScan recursiveCteScan, Void context) {
+ return computeCatalogRelation(recursiveCteScan);
+ }
+
@Override
public Statistics visitPhysicalFileScan(PhysicalFileScan fileScan, Void context) {
return computeCatalogRelation(fileScan);
@@ -1083,6 +1115,18 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
return computeAssertNumRows(assertNumRows.getAssertNumRowsElement(), groupExpression.childStatistics(0));
}
+ @Override
+ public Statistics visitPhysicalRecursiveCte(PhysicalRecursiveCte recursiveCte, Void context) {
+ return computeRecursiveCte(recursiveCte, groupExpression.children()
+ .stream().map(Group::getStatistics).collect(Collectors.toList()));
+ }
+
+ @Override
+ public Statistics visitPhysicalRecursiveCteRecursiveChild(PhysicalRecursiveCteRecursiveChild recursiveChild,
+ Void context) {
+ return groupExpression.childStatistics(0);
+ }
+
@Override
public Statistics visitPhysicalUnion(PhysicalUnion union, Void context) {
return computeUnion(union, groupExpression.children()
@@ -1464,6 +1508,71 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
return new Statistics(rowCount, 1, columnStatsMap);
}
+ /**
+ * computeRecursiveCte
+ */
+ public Statistics computeRecursiveCte(RecursiveCte recursiveCte, List<Statistics> childStats) {
+ // TODO: refactor this for one row relation
+ List<SlotReference> head;
+ Statistics headStats;
+ List<List<SlotReference>> childOutputs = Lists.newArrayList(recursiveCte.getRegularChildrenOutputs());
+
+ head = childOutputs.get(0);
+ headStats = new StatisticsBuilder(childStats.get(0)).build();
+
+ StatisticsBuilder statisticsBuilder = new StatisticsBuilder();
+ List<NamedExpression> unionOutput = recursiveCte.getOutputs();
+ double unionRowCount = childStats.stream().mapToDouble(Statistics::getRowCount).sum();
+ statisticsBuilder.setRowCount(unionRowCount);
+
+ for (int i = 0; i < head.size(); i++) {
+ Slot headSlot = head.get(i);
+ ColumnStatisticBuilder colStatsBuilder = new ColumnStatisticBuilder(
+ headStats.findColumnStatistics(headSlot));
+ for (int j = 1; j < childOutputs.size(); j++) {
+ Slot slot = childOutputs.get(j).get(i);
+ ColumnStatistic rightStatistic = childStats.get(j).findColumnStatistics(slot);
+ double rightRowCount = childStats.get(j).getRowCount();
+ colStatsBuilder = unionColumn(colStatsBuilder,
+ headStats.getRowCount(), rightStatistic, rightRowCount, headSlot.getDataType());
+ }
+
+ //update hot values
+ Map<Literal, Float> unionHotValues = new HashMap<>();
+ for (int j = 0; j < childOutputs.size(); j++) {
+ Slot slot = childOutputs.get(j).get(i);
+ ColumnStatistic slotStats = childStats.get(j).findColumnStatistics(slot);
+ if (slotStats.getHotValues() != null) {
+ for (Map.Entry<Literal, Float> entry : slotStats.getHotValues().entrySet()) {
+ Float value = unionHotValues.get(entry.getKey());
+ if (value == null) {
+ unionHotValues.put(entry.getKey(),
+ (float) (entry.getValue() * childStats.get(j).getRowCount()));
+ } else {
+ unionHotValues.put(entry.getKey(),
+ (float) (value + entry.getValue() * childStats.get(j).getRowCount()));
+ }
+ }
+ }
+ }
+
+ Map<Literal, Float> resultHotValues = new LinkedHashMap<>();
+ for (Literal hot : unionHotValues.keySet()) {
+ float ratio = (float) (unionHotValues.get(hot) / unionRowCount);
+ if (ratio * colStatsBuilder.getNdv() >= SessionVariable.getSkewValueThreshold()
+ || ratio >= SessionVariable.getHotValueThreshold()) {
+ resultHotValues.put(hot, ratio);
+ }
+ }
+ if (!resultHotValues.isEmpty()) {
+ colStatsBuilder.setHotValues(resultHotValues);
+ }
+ statisticsBuilder.putColumnStatistics(unionOutput.get(i), colStatsBuilder.build());
+ }
+
+ return statisticsBuilder.setWidthInJoinCluster(1).build();
+ }
+
/**
* computeUnion
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
index e85c6eb8dae..b13644dc10d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
@@ -53,6 +53,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteRecursiveChild;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
@@ -354,6 +356,30 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
constantExprsList, union.hasPushedFilter(), children);
}
+ @Override
+ public Plan visitLogicalRecursiveCte(LogicalRecursiveCte recursiveCte, DeepCopierContext context) {
+ List<Plan> children = recursiveCte.children().stream()
+ .map(c -> c.accept(this, context))
+ .collect(ImmutableList.toImmutableList());
+ List<NamedExpression> outputs = recursiveCte.getOutputs().stream()
+ .map(o -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(o, context))
+ .collect(ImmutableList.toImmutableList());
+ List<List<SlotReference>> childrenOutputs = recursiveCte.getRegularChildrenOutputs().stream()
+ .map(childOutputs -> childOutputs.stream()
+ .map(o -> (SlotReference) ExpressionDeepCopier.INSTANCE.deepCopy(o, context))
+ .collect(ImmutableList.toImmutableList()))
+ .collect(ImmutableList.toImmutableList());
+ return new LogicalRecursiveCte(recursiveCte.getCteName(), recursiveCte.isUnionAll(), outputs,
+ childrenOutputs, children);
+ }
+
+ @Override
+ public Plan visitLogicalRecursiveCteRecursiveChild(LogicalRecursiveCteRecursiveChild<? extends Plan> recursiveChild,
+ DeepCopierContext context) {
+ Plan child = recursiveChild.child().accept(this, context);
+ return new LogicalRecursiveCteRecursiveChild<>(recursiveChild.getCteName(), child);
+ }
+
@Override
public Plan visitLogicalExcept(LogicalExcept except, DeepCopierContext context) {
List<Plan> children = except.children().stream()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index ab364b08ff8..8288cb2ac64 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -83,6 +83,9 @@ public enum PlanType {
LOGICAL_PARTITION_TOP_N,
LOGICAL_PROJECT,
LOGICAL_QUALIFY,
+ LOGICAL_RECURSIVE_CTE,
+ LOGICAL_RECURSIVE_CTE_RECURSIVE_CHILD,
+ LOGICAL_RECURSIVE_CTE_SCAN,
LOGICAL_REPEAT,
LOGICAL_SELECT_HINT,
LOGICAL_SUBQUERY_ALIAS,
@@ -107,6 +110,7 @@ public enum PlanType {
PHYSICAL_OLAP_SCAN,
PHYSICAL_SCHEMA_SCAN,
PHYSICAL_TVF_RELATION,
+ PHYSICAL_RECURSIVE_CTE_SCAN,
// physical sinks
PHYSICAL_FILE_SINK,
@@ -123,6 +127,8 @@ public enum PlanType {
PHYSICAL_ASSERT_NUM_ROWS,
PHYSICAL_CTE_PRODUCER,
PHYSICAL_CTE_ANCHOR,
+ PHYSICAL_RECURSIVE_CTE,
+ PHYSICAL_RECURSIVE_CTE_RECURSIVE_CHILD,
PHYSICAL_DISTRIBUTE,
PHYSICAL_EXCEPT,
PHYSICAL_FILTER,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/RecursiveCte.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/RecursiveCte.java
new file mode 100644
index 00000000000..d7fd8f985a1
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/RecursiveCte.java
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.algebra;
+
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+
+import java.util.List;
+
+/**
+ * Common interface for logical/physical recursive cte.
+ */
+public interface RecursiveCte {
+ boolean isUnionAll();
+
+ List<SlotReference> getRegularChildOutput(int i);
+
+ List<NamedExpression> getOutputs();
+
+ List<List<SlotReference>> getRegularChildrenOutputs();
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
index 8411bf14085..9e4bfedb9a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
@@ -269,7 +269,8 @@ public class UpdateMvByPartitionCommand extends InsertOverwriteTableCommand {
);
rewrittenSubQueryAlias.add(subQueryAlias.withChildren(subQueryAliasChildren));
}
- return super.visitLogicalCTE(new LogicalCTE<>(rewrittenSubQueryAlias, cte.child()), predicates);
+ return super.visitLogicalCTE(new LogicalCTE<>(cte.isRecursiveCte(),
+ rewrittenSubQueryAlias, cte.child()), predicates);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java
index bc20d3efa17..1229bc59455 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java
@@ -29,6 +29,7 @@ import org.apache.doris.planner.OlapScanNode;
import org.apache.doris.planner.PlanFragment;
import org.apache.doris.planner.PlanFragmentId;
import org.apache.doris.planner.PlanNodeId;
+import org.apache.doris.planner.RecursiveCteScanNode;
import org.apache.doris.planner.ScanNode;
import org.apache.doris.planner.SchemaScanNode;
import org.apache.doris.thrift.TExplainLevel;
@@ -126,6 +127,10 @@ public class UnassignedJobBuilder {
unassignedJob = buildScanMetadataJob(
statementContext, planFragment, (SchemaScanNode) scanNode, scanWorkerSelector
);
+ } else if (scanNode instanceof RecursiveCteScanNode) {
+ unassignedJob = buildScanRecursiveCteJob(
+ statementContext, planFragment, (RecursiveCteScanNode) scanNode, inputJobs, scanWorkerSelector
+ );
} else {
// only scan external tables or cloud tables or table valued functions
// e,g. select * from numbers('number'='100')
@@ -196,6 +201,14 @@ public class UnassignedJobBuilder {
return new UnassignedScanMetadataJob(statementContext, fragment, schemaScanNode, scanWorkerSelector);
}
+ private UnassignedJob buildScanRecursiveCteJob(
+ StatementContext statementContext, PlanFragment fragment,
+ RecursiveCteScanNode recursiveCteScanNode,
+ ListMultimap<ExchangeNode, UnassignedJob> inputJobs, ScanWorkerSelector scanWorkerSelector) {
+ return new UnassignedRecursiveCteScanJob(statementContext, fragment, recursiveCteScanNode,
+ inputJobs, scanWorkerSelector);
+ }
+
private UnassignedJob buildScanRemoteTableJob(
StatementContext statementContext, PlanFragment planFragment, List<ScanNode> scanNodes,
ListMultimap<ExchangeNode, UnassignedJob> inputJobs,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedRecursiveCteScanJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedRecursiveCteScanJob.java
new file mode 100644
index 00000000000..cfd3ebe7b2b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedRecursiveCteScanJob.java
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.trees.plans.distribute.DistributeContext;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager;
+import org.apache.doris.nereids.trees.plans.distribute.worker.ScanWorkerSelector;
+import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.ScanNode;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ListMultimap;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * UnassignedRecursiveCteScanJob
+ */
+public class UnassignedRecursiveCteScanJob extends AbstractUnassignedScanJob {
+ private final ScanWorkerSelector scanWorkerSelector;
+
+ public UnassignedRecursiveCteScanJob(
+ StatementContext statementContext, PlanFragment fragment, ScanNode scanNode,
+ ListMultimap<ExchangeNode, UnassignedJob> exchangeToChildJob, ScanWorkerSelector scanWorkerSelector) {
+ super(statementContext, fragment, ImmutableList.of(scanNode), exchangeToChildJob);
+ this.scanWorkerSelector = Objects.requireNonNull(scanWorkerSelector, "scanWorkerSelector is not null");
+ }
+
+ @Override
+ protected Map<DistributedPlanWorker, UninstancedScanSource> multipleMachinesParallelization(
+ DistributeContext distributeContext, ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+ return scanWorkerSelector.selectReplicaAndWorkerWithoutBucket(
+ scanNodes.get(0), statementContext.getConnectContext()
+ );
+ }
+
+ @Override
+ protected List<AssignedJob> fillUpAssignedJobs(List<AssignedJob> assignedJobs,
+ DistributedPlanWorkerManager workerManager, ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+ Preconditions.checkArgument(!assignedJobs.isEmpty(),
+ "assignedJobs is empty for UnassignedRecursiveCteScanJob");
+ return assignedJobs;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java
index 4f810c3b6de..5ceb0e55bca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java
@@ -41,21 +41,28 @@ import java.util.stream.Collectors;
public class LogicalCTE<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE> implements PropagateFuncDeps {
private final List<LogicalSubQueryAlias<Plan>> aliasQueries;
+ private final boolean isRecursiveCte;
- public LogicalCTE(List<LogicalSubQueryAlias<Plan>> aliasQueries, CHILD_TYPE child) {
- this(aliasQueries, Optional.empty(), Optional.empty(), child);
+ public LogicalCTE(boolean isRecursiveCte, List<LogicalSubQueryAlias<Plan>> aliasQueries, CHILD_TYPE child) {
+ this(isRecursiveCte, aliasQueries, Optional.empty(), Optional.empty(), child);
}
- public LogicalCTE(List<LogicalSubQueryAlias<Plan>> aliasQueries, Optional<GroupExpression> groupExpression,
- Optional<LogicalProperties> logicalProperties, CHILD_TYPE child) {
+ public LogicalCTE(boolean isRecursiveCte, List<LogicalSubQueryAlias<Plan>> aliasQueries,
+ Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
+ CHILD_TYPE child) {
super(PlanType.LOGICAL_CTE, groupExpression, logicalProperties, child);
this.aliasQueries = ImmutableList.copyOf(Objects.requireNonNull(aliasQueries, "aliasQueries can not be null"));
+ this.isRecursiveCte = isRecursiveCte;
}
public List<LogicalSubQueryAlias<Plan>> getAliasQueries() {
return aliasQueries;
}
+ public boolean isRecursiveCte() {
+ return isRecursiveCte;
+ }
+
@Override
public List<? extends Plan> extraPlans() {
return aliasQueries;
@@ -74,6 +81,7 @@ public class LogicalCTE<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE
@Override
public String toString() {
return Utils.toSqlString("LogicalCTE",
+ "isRecursiveCte", isRecursiveCte,
"aliasQueries", aliasQueries
);
}
@@ -105,18 +113,18 @@ public class LogicalCTE<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE
return false;
}
LogicalCTE that = (LogicalCTE) o;
- return aliasQueries.equals(that.aliasQueries);
+ return aliasQueries.equals(that.aliasQueries) && isRecursiveCte == that.isRecursiveCte;
}
@Override
public int hashCode() {
- return Objects.hash(aliasQueries);
+ return Objects.hash(aliasQueries, isRecursiveCte);
}
@Override
public Plan withChildren(List<Plan> children) {
Preconditions.checkArgument(aliasQueries.size() > 0);
- return new LogicalCTE<>(aliasQueries, children.get(0));
+ return new LogicalCTE<>(isRecursiveCte, aliasQueries, children.get(0));
}
@Override
@@ -131,13 +139,14 @@ public class LogicalCTE<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE
@Override
public LogicalCTE<CHILD_TYPE> withGroupExpression(Optional<GroupExpression> groupExpression) {
- return new LogicalCTE<>(aliasQueries, groupExpression, Optional.of(getLogicalProperties()), child());
+ return new LogicalCTE<>(isRecursiveCte, aliasQueries, groupExpression,
+ Optional.of(getLogicalProperties()), child());
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
Preconditions.checkArgument(aliasQueries.size() > 0);
- return new LogicalCTE<>(aliasQueries, groupExpression, logicalProperties, children.get(0));
+ return new LogicalCTE<>(isRecursiveCte, aliasQueries, groupExpression, logicalProperties, children.get(0));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java
new file mode 100644
index 00000000000..bef00db5f72
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java
@@ -0,0 +1,284 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Cast;
+import org.apache.doris.nereids.trees.expressions.ExprId;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.algebra.RecursiveCte;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.util.TypeCoercionUtils;
+import org.apache.doris.nereids.util.Utils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * LogicalRecursiveCte is basically like LogicalUnion
+ */
+public class LogicalRecursiveCte extends AbstractLogicalPlan implements RecursiveCte, OutputPrunable {
+ private final String cteName;
+ private final List<NamedExpression> outputs;
+ private final List<List<SlotReference>> regularChildrenOutputs;
+ private final boolean isUnionAll;
+
+ /** LogicalRecursiveCte */
+ public LogicalRecursiveCte(String cteName, boolean isUnionAll, List<Plan> children) {
+ this(cteName, isUnionAll, ImmutableList.of(), ImmutableList.of(), children);
+ }
+
+ /** LogicalRecursiveCte */
+ public LogicalRecursiveCte(String cteName, boolean isUnionAll, List<NamedExpression> outputs,
+ List<List<SlotReference>> childrenOutputs, List<Plan> children) {
+ this(cteName, isUnionAll, outputs, childrenOutputs, Optional.empty(),
+ Optional.empty(),
+ children);
+ }
+
+ /** LogicalRecursiveCte */
+ public LogicalRecursiveCte(String cteName, boolean isUnionAll, List<NamedExpression> outputs,
+ List<List<SlotReference>> childrenOutputs,
+ Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
+ List<Plan> children) {
+ super(PlanType.LOGICAL_RECURSIVE_CTE, groupExpression, logicalProperties, children);
+ this.cteName = cteName;
+ this.isUnionAll = isUnionAll;
+ this.outputs = ImmutableList.copyOf(outputs);
+ this.regularChildrenOutputs = ImmutableList.copyOf(childrenOutputs);
+ }
+
+ @Override
+ public boolean isUnionAll() {
+ return isUnionAll;
+ }
+
+ public String getCteName() {
+ return cteName;
+ }
+
+ @Override
+ public List<SlotReference> getRegularChildOutput(int i) {
+ return regularChildrenOutputs.get(i);
+ }
+
+ @Override
+ public List<List<SlotReference>> getRegularChildrenOutputs() {
+ return regularChildrenOutputs;
+ }
+
+ public List<List<NamedExpression>> collectChildrenProjections() {
+ return castCommonDataTypeOutputs();
+ }
+
+ private List<List<NamedExpression>> castCommonDataTypeOutputs() {
+ int childOutputSize = child(0).getOutput().size();
+ ImmutableList.Builder<NamedExpression> newLeftOutputs = ImmutableList.builderWithExpectedSize(
+ childOutputSize);
+ ImmutableList.Builder<NamedExpression> newRightOutputs = ImmutableList.builderWithExpectedSize(
+ childOutputSize
+ );
+ // Ensure that the output types of the left and right children are consistent and expand upward.
+ for (int i = 0; i < childOutputSize; ++i) {
+ Slot left = child(0).getOutput().get(i);
+ Slot right = child(1).getOutput().get(i);
+ DataType compatibleType;
+ try {
+ compatibleType = LogicalSetOperation.getAssignmentCompatibleType(left.getDataType(),
+ right.getDataType());
+ } catch (Exception e) {
+ throw new AnalysisException(
+ "Can not find compatible type for " + left + " and " + right + ", " + e.getMessage());
+ }
+ Expression newLeft = TypeCoercionUtils.castIfNotSameTypeStrict(left, compatibleType);
+ Expression newRight = TypeCoercionUtils.castIfNotSameTypeStrict(right, compatibleType);
+ if (newLeft instanceof Cast) {
+ newLeft = new Alias(newLeft, left.getName());
+ }
+ if (newRight instanceof Cast) {
+ newRight = new Alias(newRight, right.getName());
+ }
+ newLeftOutputs.add((NamedExpression) newLeft);
+ newRightOutputs.add((NamedExpression) newRight);
+ }
+
+ return ImmutableList.of(newLeftOutputs.build(), newRightOutputs.build());
+ }
+
+ /**
+ * Generate new output for Recursive Cte.
+ */
+ public List<NamedExpression> buildNewOutputs() {
+ List<Slot> slots = resetNullableForLeftOutputs();
+ ImmutableList.Builder<NamedExpression> newOutputs = ImmutableList.builderWithExpectedSize(slots.size());
+
+ for (int i = 0; i < slots.size(); i++) {
+ Slot slot = slots.get(i);
+ ExprId exprId = i < outputs.size() ? outputs.get(i).getExprId() : StatementScopeIdGenerator.newExprId();
+ newOutputs.add(
+ new SlotReference(exprId, slot.toSql(), slot.getDataType(), slot.nullable(), ImmutableList.of())
+ );
+ }
+ return newOutputs.build();
+ }
+
+ // If the right child is nullable, need to ensure that the left child is also nullable
+ private List<Slot> resetNullableForLeftOutputs() {
+ int rightChildOutputSize = child(1).getOutput().size();
+ ImmutableList.Builder<Slot> resetNullableForLeftOutputs
+ = ImmutableList.builderWithExpectedSize(rightChildOutputSize);
+ for (int i = 0; i < rightChildOutputSize; ++i) {
+ if (child(1).getOutput().get(i).nullable() && !child(0).getOutput().get(i).nullable()) {
+ resetNullableForLeftOutputs.add(child(0).getOutput().get(i).withNullable(true));
+ } else {
+ resetNullableForLeftOutputs.add(child(0).getOutput().get(i));
+ }
+ }
+ return resetNullableForLeftOutputs.build();
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlStringSkipNull("LogicalRecursiveCte",
+ "cteName", cteName,
+ "isUnionAll", isUnionAll,
+ "outputs", outputs,
+ "regularChildrenOutputs", regularChildrenOutputs,
+ "stats", statistics);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ LogicalRecursiveCte that = (LogicalRecursiveCte) o;
+ return cteName.equals(that.cteName) && isUnionAll == that.isUnionAll && Objects.equals(outputs, that.outputs)
+ && Objects.equals(regularChildrenOutputs, that.regularChildrenOutputs);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(cteName, isUnionAll, outputs, regularChildrenOutputs);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitLogicalRecursiveCte(this, context);
+ }
+
+ @Override
+ public List<? extends Expression> getExpressions() {
+ return regularChildrenOutputs.stream().flatMap(List::stream).collect(ImmutableList.toImmutableList());
+ }
+
+ @Override
+ public List<Slot> computeOutput() {
+ return outputs.stream()
+ .map(NamedExpression::toSlot)
+ .collect(ImmutableList.toImmutableList());
+ }
+
+ @Override
+ public LogicalRecursiveCte withChildren(List<Plan> children) {
+ return new LogicalRecursiveCte(cteName, isUnionAll, outputs, regularChildrenOutputs, children);
+ }
+
+ public LogicalRecursiveCte withChildrenAndTheirOutputs(List<Plan> children,
+ List<List<SlotReference>> childrenOutputs) {
+ Preconditions.checkArgument(children.size() == childrenOutputs.size(),
+ "children size %s is not equals with children outputs size %s",
+ children.size(), childrenOutputs.size());
+ return new LogicalRecursiveCte(cteName, isUnionAll, outputs, childrenOutputs, children);
+ }
+
+ @Override
+ public LogicalRecursiveCte withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new LogicalRecursiveCte(cteName, isUnionAll, outputs, regularChildrenOutputs,
+ groupExpression, Optional.of(getLogicalProperties()), children);
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ return new LogicalRecursiveCte(cteName, isUnionAll, outputs, regularChildrenOutputs,
+ groupExpression, logicalProperties, children);
+ }
+
+ public LogicalRecursiveCte withNewOutputs(List<NamedExpression> newOutputs) {
+ return new LogicalRecursiveCte(cteName, isUnionAll, newOutputs, regularChildrenOutputs,
+ Optional.empty(), Optional.empty(), children);
+ }
+
+ public LogicalRecursiveCte withNewOutputsAndChildren(List<NamedExpression> newOutputs,
+ List<Plan> children,
+ List<List<SlotReference>> childrenOutputs) {
+ return new LogicalRecursiveCte(cteName, isUnionAll, newOutputs, childrenOutputs,
+ Optional.empty(), Optional.empty(), children);
+ }
+
+ @Override
+ public List<NamedExpression> getOutputs() {
+ return outputs;
+ }
+
+ @Override
+ public LogicalRecursiveCte pruneOutputs(List<NamedExpression> prunedOutputs) {
+ return withNewOutputs(prunedOutputs);
+ }
+
+ @Override
+ public void computeUnique(DataTrait.Builder builder) {
+ }
+
+ @Override
+ public void computeUniform(DataTrait.Builder builder) {
+ }
+
+ @Override
+ public boolean hasUnboundExpression() {
+ return outputs.isEmpty();
+ }
+
+ @Override
+ public void computeEqualSet(DataTrait.Builder builder) {
+ }
+
+ @Override
+ public void computeFd(DataTrait.Builder builder) {
+ // don't generate
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteRecursiveChild.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteRecursiveChild.java
new file mode 100644
index 00000000000..d2766a86f95
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteRecursiveChild.java
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * LogicalRecursiveCteRecursiveChild is sentinel plan for must_shuffle
+ */
+public class LogicalRecursiveCteRecursiveChild<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE> {
+ private final String cteName;
+
+ public LogicalRecursiveCteRecursiveChild(String cteName, CHILD_TYPE child) {
+ this(cteName, Optional.empty(), Optional.empty(), child);
+ }
+
+ public LogicalRecursiveCteRecursiveChild(String cteName, Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, CHILD_TYPE child) {
+ this(cteName, groupExpression, logicalProperties, ImmutableList.of(child));
+ }
+
+ public LogicalRecursiveCteRecursiveChild(String cteName, Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> child) {
+ super(PlanType.LOGICAL_RECURSIVE_CTE_RECURSIVE_CHILD, groupExpression, logicalProperties, child);
+ this.cteName = cteName;
+ }
+
+ public String getCteName() {
+ return cteName;
+ }
+
+ @Override
+ public Plan withChildren(List<Plan> children) {
+ return new LogicalRecursiveCteRecursiveChild<>(cteName, Optional.empty(), Optional.empty(), children);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitLogicalRecursiveCteRecursiveChild(this, context);
+ }
+
+ @Override
+ public List<? extends Expression> getExpressions() {
+ return ImmutableList.of();
+ }
+
+ @Override
+ public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new LogicalRecursiveCteRecursiveChild<>(cteName, groupExpression,
+ Optional.of(getLogicalProperties()), children);
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ return new LogicalRecursiveCteRecursiveChild<>(cteName, groupExpression, logicalProperties, children);
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlStringSkipNull("LogicalRecursiveCteRecursiveChild",
+ "cteName", cteName);
+ }
+
+ @Override
+ public void computeUnique(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public void computeUniform(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public void computeEqualSet(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public void computeFd(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public List<Slot> computeOutput() {
+ return child().getOutput();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteScan.java
new file mode 100644
index 00000000000..31b335260b9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteScan.java
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * LogicalRecursiveCteScan.
+ */
+public class LogicalRecursiveCteScan extends LogicalCatalogRelation {
+ public LogicalRecursiveCteScan(RelationId relationId, TableIf table, List<String> qualifier) {
+ this(relationId, table, qualifier, Optional.empty(), Optional.empty());
+ }
+
+ private LogicalRecursiveCteScan(RelationId relationId, TableIf table, List<String> qualifier,
+ Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties) {
+ super(relationId, PlanType.LOGICAL_RECURSIVE_CTE_SCAN, table, qualifier, groupExpression, logicalProperties);
+ }
+
+ private LogicalRecursiveCteScan(RelationId relationId, TableIf table, List<String> qualifier,
+ Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
+ String tableAlias) {
+ super(relationId, PlanType.LOGICAL_RECURSIVE_CTE_SCAN, table, qualifier, groupExpression,
+ logicalProperties, tableAlias);
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlString("LogicalRecursiveCteScan",
+ "cteName", table.getName());
+ }
+
+ @Override
+ public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new LogicalRecursiveCteScan(relationId, table, qualifier,
+ groupExpression, Optional.ofNullable(getLogicalProperties()), tableAlias);
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ return new LogicalRecursiveCteScan(relationId, table, qualifier, groupExpression, logicalProperties,
+ tableAlias);
+ }
+
+ @Override
+ public LogicalCatalogRelation withRelationId(RelationId relationId) {
+ return new LogicalRecursiveCteScan(relationId, table, qualifier,
+ groupExpression, Optional.ofNullable(getLogicalProperties()), tableAlias);
+ }
+
+ @Override
+ public LogicalCatalogRelation withTableAlias(String tableAlias) {
+ return new LogicalRecursiveCteScan(relationId, table, qualifier,
+ groupExpression, Optional.ofNullable(getLogicalProperties()), tableAlias);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitLogicalRecursiveCteScan(this, context);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
index 2f49dc1ce85..8b18d3ce49d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.logical;
+import org.apache.doris.nereids.analyzer.UnboundRelation;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.DataTrait;
import org.apache.doris.nereids.properties.LogicalProperties;
@@ -27,6 +28,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.LazyCompute;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
@@ -42,6 +44,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
/**
@@ -55,6 +58,7 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
protected RelationId relationId;
private final List<String> qualifier;
private final Optional<List<String>> columnAliases;
+ private final Supplier<Boolean> isRecursiveCte;
public LogicalSubQueryAlias(String tableAlias, CHILD_TYPE child) {
this(ImmutableList.of(tableAlias), Optional.empty(), Optional.empty(), Optional.empty(), child);
@@ -78,6 +82,7 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
super(PlanType.LOGICAL_SUBQUERY_ALIAS, groupExpression, logicalProperties, child);
this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier is null"));
this.columnAliases = columnAliases;
+ this.isRecursiveCte = computeIsRecursiveCte();
}
@Override
@@ -121,6 +126,23 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
return currentOutput.build();
}
+ private Supplier<Boolean> computeIsRecursiveCte() {
+ return LazyCompute.of(() -> {
+ List<UnboundRelation> relationList = collectToList(UnboundRelation.class::isInstance);
+ for (UnboundRelation relation : relationList) {
+ List<String> nameParts = relation.getNameParts();
+ if (nameParts.size() == 1 && nameParts.get(0).equalsIgnoreCase(getAlias())) {
+ return true;
+ }
+ }
+ return false;
+ });
+ }
+
+ public boolean isRecursiveCte() {
+ return isRecursiveCte.get();
+ }
+
public String getAlias() {
return qualifier.get(qualifier.size() - 1);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCte.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCte.java
new file mode 100644
index 00000000000..44aab38fc30
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCte.java
@@ -0,0 +1,292 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.physical;
+
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.algebra.RecursiveCte;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.statistics.Statistics;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * PhysicalRecursiveCte is basically like PhysicalUnion
+ */
+public class PhysicalRecursiveCte extends AbstractPhysicalPlan implements RecursiveCte {
+ private final String cteName;
+ private final List<NamedExpression> outputs;
+ private final List<List<SlotReference>> regularChildrenOutputs;
+ private final boolean isUnionAll;
+
+ /** PhysicalRecursiveCte */
+ public PhysicalRecursiveCte(String cteName, boolean isUnionAll,
+ List<NamedExpression> outputs,
+ List<List<SlotReference>> childrenOutputs,
+ LogicalProperties logicalProperties,
+ List<Plan> children) {
+ this(cteName, isUnionAll, outputs, childrenOutputs, Optional.empty(), logicalProperties, children);
+ }
+
+ /** PhysicalRecursiveCte */
+ public PhysicalRecursiveCte(String cteName, boolean isUnionAll,
+ List<NamedExpression> outputs,
+ List<List<SlotReference>> childrenOutputs,
+ Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties,
+ List<Plan> children) {
+ this(cteName, isUnionAll, outputs, childrenOutputs, groupExpression, logicalProperties,
+ PhysicalProperties.ANY, null, children);
+ }
+
+ /** PhysicalRecursiveCte */
+ public PhysicalRecursiveCte(String cteName, boolean isUnionAll, List<NamedExpression> outputs,
+ List<List<SlotReference>> childrenOutputs,
+ Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties,
+ PhysicalProperties physicalProperties, Statistics statistics, List<Plan> children) {
+ super(PlanType.PHYSICAL_RECURSIVE_CTE, groupExpression, logicalProperties, physicalProperties,
+ statistics, children.toArray(new Plan[0]));
+ this.cteName = cteName;
+ this.isUnionAll = isUnionAll;
+ this.outputs = ImmutableList.copyOf(outputs);
+ this.regularChildrenOutputs = ImmutableList.copyOf(childrenOutputs);
+ }
+
+ @Override
+ public boolean isUnionAll() {
+ return isUnionAll;
+ }
+
+ public String getCteName() {
+ return cteName;
+ }
+
+ @Override
+ public List<SlotReference> getRegularChildOutput(int i) {
+ return regularChildrenOutputs.get(i);
+ }
+
+ @Override
+ public List<NamedExpression> getOutputs() {
+ return outputs;
+ }
+
+ @Override
+ public List<Slot> computeOutput() {
+ return outputs.stream()
+ .map(NamedExpression::toSlot)
+ .collect(ImmutableList.toImmutableList());
+ }
+
+ @Override
+ public List<List<SlotReference>> getRegularChildrenOutputs() {
+ return regularChildrenOutputs;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PhysicalRecursiveCte that = (PhysicalRecursiveCte) o;
+ return cteName.equals(that.cteName) && isUnionAll == that.isUnionAll && Objects.equals(outputs, that.outputs)
+ && Objects.equals(regularChildrenOutputs, that.regularChildrenOutputs);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(cteName, isUnionAll, outputs, regularChildrenOutputs);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitPhysicalRecursiveCte(this, context);
+ }
+
+ @Override
+ public List<? extends Expression> getExpressions() {
+ return regularChildrenOutputs.stream().flatMap(List::stream).collect(ImmutableList.toImmutableList());
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlString("PhysicalRecursiveCte" + "[" + id.asInt() + "]" + getGroupIdWithPrefix(),
+ "stats", statistics,
+ "cteName", cteName,
+ "isUnionAll", isUnionAll,
+ "outputs", outputs,
+ "regularChildrenOutputs", regularChildrenOutputs);
+ }
+
+ @Override
+ public String shapeInfo() {
+ ConnectContext context = ConnectContext.get();
+ if (context != null
+ && context.getSessionVariable().getDetailShapePlanNodesSet().contains(getClass().getSimpleName())) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(getClass().getSimpleName());
+ builder.append(")");
+ return builder.toString();
+ } else {
+ return super.shapeInfo();
+ }
+ }
+
+ @Override
+ public PhysicalRecursiveCte withChildren(List<Plan> children) {
+ return new PhysicalRecursiveCte(cteName, isUnionAll, outputs, regularChildrenOutputs, groupExpression,
+ getLogicalProperties(), children);
+ }
+
+ @Override
+ public PhysicalRecursiveCte withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new PhysicalRecursiveCte(cteName, isUnionAll, outputs, regularChildrenOutputs,
+ groupExpression, getLogicalProperties(), children);
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ return new PhysicalRecursiveCte(cteName, isUnionAll, outputs, regularChildrenOutputs,
+ groupExpression, logicalProperties.get(), children);
+ }
+
+ @Override
+ public PhysicalRecursiveCte withPhysicalPropertiesAndStats(
+ PhysicalProperties physicalProperties, Statistics statistics) {
+ return new PhysicalRecursiveCte(cteName, isUnionAll, outputs, regularChildrenOutputs,
+ groupExpression, getLogicalProperties(), physicalProperties, statistics, children);
+ }
+
+ @Override
+ public PhysicalRecursiveCte resetLogicalProperties() {
+ return new PhysicalRecursiveCte(cteName, isUnionAll, outputs, regularChildrenOutputs,
+ Optional.empty(), null, physicalProperties, statistics, children);
+ }
+
+ @Override
+ public void computeUnique(DataTrait.Builder builder) {
+ if (!isUnionAll) {
+ builder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
+ }
+ }
+
+ @Override
+ public void computeUniform(DataTrait.Builder builder) {
+ // don't propagate uniform slots
+ }
+
+ private List<BitSet> mapSlotToIndex(Plan plan, List<Set<Slot>> equalSlotsList) {
+ Map<Slot, Integer> slotToIndex = new HashMap<>();
+ for (int i = 0; i < plan.getOutput().size(); i++) {
+ slotToIndex.put(plan.getOutput().get(i), i);
+ }
+ List<BitSet> equalSlotIndicesList = new ArrayList<>();
+ for (Set<Slot> equalSlots : equalSlotsList) {
+ BitSet equalSlotIndices = new BitSet();
+ for (Slot slot : equalSlots) {
+ if (slotToIndex.containsKey(slot)) {
+ equalSlotIndices.set(slotToIndex.get(slot));
+ }
+ }
+ if (equalSlotIndices.cardinality() > 1) {
+ equalSlotIndicesList.add(equalSlotIndices);
+ }
+ }
+ return equalSlotIndicesList;
+ }
+
+ @Override
+ public void computeEqualSet(DataTrait.Builder builder) {
+ if (children.isEmpty()) {
+ return;
+ }
+
+ // Get the list of equal slot sets and their corresponding index mappings for the first child
+ List<Set<Slot>> childEqualSlotsList = child(0).getLogicalProperties()
+ .getTrait().calAllEqualSet();
+ List<BitSet> childEqualSlotsIndicesList = mapSlotToIndex(child(0), childEqualSlotsList);
+ List<BitSet> unionEqualSlotIndicesList = new ArrayList<>(childEqualSlotsIndicesList);
+
+ // Traverse all children and find the equal sets that exist in all children
+ for (int i = 1; i < children.size(); i++) {
+ Plan child = children.get(i);
+
+ // Get the equal slot sets for the current child
+ childEqualSlotsList = child.getLogicalProperties().getTrait().calAllEqualSet();
+
+ // Map slots to indices for the current child
+ childEqualSlotsIndicesList = mapSlotToIndex(child, childEqualSlotsList);
+
+ // Only keep the equal pairs that exist in all children of the union
+ // This is done by calculating the intersection of all children's equal slot indices
+ for (BitSet unionEqualSlotIndices : unionEqualSlotIndicesList) {
+ BitSet intersect = new BitSet();
+ for (BitSet childEqualSlotIndices : childEqualSlotsIndicesList) {
+ if (unionEqualSlotIndices.intersects(childEqualSlotIndices)) {
+ intersect = childEqualSlotIndices;
+ break;
+ }
+ }
+ unionEqualSlotIndices.and(intersect);
+ }
+ }
+
+ // Build the functional dependencies for the output slots
+ List<Slot> outputList = getOutput();
+ for (BitSet equalSlotIndices : unionEqualSlotIndicesList) {
+ if (equalSlotIndices.cardinality() <= 1) {
+ continue;
+ }
+ int first = equalSlotIndices.nextSetBit(0);
+ int next = equalSlotIndices.nextSetBit(first + 1);
+ while (next > 0) {
+ builder.addEqualPair(outputList.get(first), outputList.get(next));
+ next = equalSlotIndices.nextSetBit(next + 1);
+ }
+ }
+ }
+
+ @Override
+ public void computeFd(DataTrait.Builder builder) {
+ // don't generate
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteRecursiveChild.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteRecursiveChild.java
new file mode 100644
index 00000000000..9aef71e7ee9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteRecursiveChild.java
@@ -0,0 +1,121 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.physical;
+
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.statistics.Statistics;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * PhysicalRecursiveCteRecursiveChild is sentinel plan for must_shuffle
+ */
+public class PhysicalRecursiveCteRecursiveChild<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD_TYPE> {
+ private final String cteName;
+
+ public PhysicalRecursiveCteRecursiveChild(String cteName, LogicalProperties logicalProperties, CHILD_TYPE child) {
+ this(cteName, Optional.empty(), logicalProperties, child);
+ }
+
+ public PhysicalRecursiveCteRecursiveChild(String cteName, Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties, CHILD_TYPE child) {
+ this(cteName, groupExpression, logicalProperties, PhysicalProperties.ANY, null, child);
+ }
+
+ public PhysicalRecursiveCteRecursiveChild(String cteName, Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties, @Nullable PhysicalProperties physicalProperties, Statistics statistics,
+ CHILD_TYPE child) {
+ super(PlanType.PHYSICAL_RECURSIVE_CTE_RECURSIVE_CHILD, groupExpression, logicalProperties, physicalProperties,
+ statistics, child);
+ this.cteName = cteName;
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlStringSkipNull("PhysicalRecursiveCteRecursiveChild",
+ "cteName", cteName);
+ }
+
+ @Override
+ public Plan withChildren(List<Plan> children) {
+ Preconditions.checkArgument(children.size() == 1);
+ return new PhysicalRecursiveCteRecursiveChild<>(cteName, groupExpression, getLogicalProperties(),
+ children.get(0));
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitPhysicalRecursiveCteRecursiveChild(this, context);
+ }
+
+ @Override
+ public List<? extends Expression> getExpressions() {
+ return ImmutableList.of();
+ }
+
+ @Override
+ public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new PhysicalRecursiveCteRecursiveChild<>(cteName, groupExpression, getLogicalProperties(), child());
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ Preconditions.checkArgument(children.size() == 1);
+ return new PhysicalRecursiveCteRecursiveChild<>(cteName, groupExpression, logicalProperties.get(), child());
+ }
+
+ @Override
+ public void computeUnique(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public void computeUniform(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public void computeEqualSet(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public void computeFd(DataTrait.Builder builder) {
+
+ }
+
+ @Override
+ public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) {
+ return new PhysicalRecursiveCteRecursiveChild<>(cteName, groupExpression, getLogicalProperties(),
+ physicalProperties, statistics, child());
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteScan.java
new file mode 100644
index 00000000000..3450ae0de18
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteScan.java
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.physical;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.statistics.Statistics;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * PhysicalRecursiveCteScan.
+ */
+public class PhysicalRecursiveCteScan extends PhysicalCatalogRelation {
+ public PhysicalRecursiveCteScan(RelationId relationId, TableIf table, List<String> qualifier,
+ Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties,
+ Collection<Slot> operativeSlots) {
+ this(relationId, table, qualifier, groupExpression, logicalProperties, PhysicalProperties.ANY, null,
+ operativeSlots);
+ }
+
+ public PhysicalRecursiveCteScan(RelationId relationId, TableIf table, List<String> qualifier,
+ Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties,
+ PhysicalProperties physicalProperties, Statistics statistics, Collection<Slot> operativeSlots) {
+ super(relationId, PlanType.PHYSICAL_RECURSIVE_CTE_SCAN, table, qualifier, groupExpression, logicalProperties,
+ physicalProperties, statistics, operativeSlots);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitPhysicalRecursiveCteScan(this, context);
+ }
+
+ @Override
+ public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new PhysicalRecursiveCteScan(relationId, table, qualifier, groupExpression, getLogicalProperties(),
+ physicalProperties, statistics, operativeSlots);
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ return new PhysicalRecursiveCteScan(relationId, table, qualifier, groupExpression, getLogicalProperties(),
+ physicalProperties, statistics, operativeSlots);
+ }
+
+ @Override
+ public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) {
+ return new PhysicalRecursiveCteScan(relationId, table, qualifier, groupExpression, getLogicalProperties(),
+ physicalProperties, statistics, operativeSlots);
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlString("PhysicalRecursiveCteScan[" + table.getName() + "]" + getGroupIdWithPrefix(),
+ "stats", statistics,
+ "qualified", Utils.qualifiedName(qualifier, table.getName()),
+ "operativeCols", getOperativeSlots());
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java
index 291f9ed5d80..5b0067bbd7a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java
@@ -44,6 +44,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPreAggOnHint;
import org.apache.doris.nereids.trees.plans.logical.LogicalPreFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalQualify;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteRecursiveChild;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint;
@@ -78,6 +80,8 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCte;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteRecursiveChild;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
@@ -211,6 +215,15 @@ public abstract class PlanVisitor<R, C> implements CommandVisitor<R, C>, Relatio
return visit(join, context);
}
+ public R visitLogicalRecursiveCte(LogicalRecursiveCte recursiveCte, C context) {
+ return visit(recursiveCte, context);
+ }
+
+ public R visitLogicalRecursiveCteRecursiveChild(LogicalRecursiveCteRecursiveChild<? extends Plan> recursiveChild,
+ C context) {
+ return visit(recursiveChild, context);
+ }
+
public R visitLogicalLimit(LogicalLimit<? extends Plan> limit, C context) {
return visit(limit, context);
}
@@ -376,6 +389,15 @@ public abstract class PlanVisitor<R, C> implements CommandVisitor<R, C>, Relatio
return visitPhysicalSetOperation(union, context);
}
+ public R visitPhysicalRecursiveCte(PhysicalRecursiveCte recursiveCte, C context) {
+ return visit(recursiveCte, context);
+ }
+
+ public R visitPhysicalRecursiveCteRecursiveChild(PhysicalRecursiveCteRecursiveChild<? extends Plan> recursiveChild,
+ C context) {
+ return visit(recursiveChild, context);
+ }
+
public R visitAbstractPhysicalSort(AbstractPhysicalSort<? extends Plan> sort, C context) {
return visit(sort, context);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
index fef94ff52f9..b325849b1b1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
@@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
@@ -45,6 +46,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveCteScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSchemaScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation;
@@ -139,6 +141,10 @@ public interface RelationVisitor<R, C> {
return visitLogicalCatalogRelation(testScan, context);
}
+ default R visitLogicalRecursiveCteScan(LogicalRecursiveCteScan recursiveCteScan, C context) {
+ return visitLogicalCatalogRelation(recursiveCteScan, context);
+ }
+
// *******************************
// physical relations
// *******************************
@@ -176,6 +182,10 @@ public interface RelationVisitor<R, C> {
return visitPhysicalCatalogRelation(deferMaterializeOlapScan, context);
}
+ default R visitPhysicalRecursiveCteScan(PhysicalRecursiveCteScan recursiveCteScan, C context) {
+ return visitPhysicalCatalogRelation(recursiveCteScan, context);
+ }
+
default R visitPhysicalOneRowRelation(PhysicalOneRowRelation oneRowRelation, C context) {
return visitPhysicalRelation(oneRowRelation, context);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java
new file mode 100644
index 00000000000..c8259cc2821
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.planner;
+
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.TupleId;
+import org.apache.doris.statistics.StatisticalType;
+import org.apache.doris.thrift.TExplainLevel;
+import org.apache.doris.thrift.TPlanNode;
+import org.apache.doris.thrift.TPlanNodeType;
+import org.apache.doris.thrift.TRecCTENode;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+public class RecursiveCteNode extends PlanNode {
+ private String cteName;
+ private boolean isUnionAll;
+ private List<List<Expr>> materializedResultExprLists = Lists.newArrayList();
+ private TRecCTENode tRecCTENode;
+
+ public RecursiveCteNode(PlanNodeId id, TupleId tupleId, String cteName, boolean isUnionAll) {
+ super(id, tupleId.asList(), "RECURSIVE_CTE", StatisticalType.RECURSIVE_CTE_NODE);
+ this.cteName = cteName;
+ this.isUnionAll = isUnionAll;
+ }
+
+ public boolean isUnionAll() {
+ return isUnionAll;
+ }
+
+ public void setMaterializedResultExprLists(List<List<Expr>> exprs) {
+ this.materializedResultExprLists = exprs;
+ }
+
+ public List<List<Expr>> getMaterializedResultExprLists() {
+ return materializedResultExprLists;
+ }
+
+ public void settRecCTENode(TRecCTENode tRecCTENode) {
+ this.tRecCTENode = tRecCTENode;
+ }
+
+ @Override
+ protected void toThrift(TPlanNode msg) {
+ msg.node_type = TPlanNodeType.REC_CTE_NODE;
+ msg.rec_cte_node = tRecCTENode;
+ }
+
+ @Override
+ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
+ StringBuilder output = new StringBuilder();
+ output.append(prefix).append("Recursive Cte: ").append(cteName).append("\n");
+ output.append(prefix).append("isUnionAll: ").append(isUnionAll).append("\n");
+ if (!conjuncts.isEmpty()) {
+ Expr expr = convertConjunctsToAndCompoundPredicate(conjuncts);
+ output.append(prefix).append("PREDICATES: ").append(expr.toSql()).append("\n");
+ }
+ return output.toString();
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("name", cteName)
+ .add("id", getId().asInt())
+ .add("tid", tupleIds.get(0).asInt())
+ .add("isUnionAll", isUnionAll).toString();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
new file mode 100644
index 00000000000..9603e3cfa63
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.planner;
+
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.UserException;
+import org.apache.doris.statistics.StatisticalType;
+import org.apache.doris.system.Backend;
+import org.apache.doris.thrift.TExplainLevel;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TPlanNode;
+import org.apache.doris.thrift.TPlanNodeType;
+import org.apache.doris.thrift.TScanRange;
+import org.apache.doris.thrift.TScanRangeLocation;
+import org.apache.doris.thrift.TScanRangeLocations;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.Lists;
+
+import java.util.Collections;
+import java.util.List;
+
+// Full scan of recursive cte temp table
+public class RecursiveCteScanNode extends ScanNode {
+ private final String recursiveCteName;
+
+ public RecursiveCteScanNode(String recursiveCteName, PlanNodeId id, TupleDescriptor desc) {
+ super(id, desc, "RECURSIVE_CTE_SCAN", StatisticalType.CTE_SCAN_NODE);
+ this.recursiveCteName = recursiveCteName;
+ }
+
+ public void initScanRangeLocations() throws UserException {
+ createScanRangeLocations();
+ }
+
+ @Override
+ protected void createScanRangeLocations() throws UserException {
+ scanRangeLocations = Lists.newArrayList();
+ // randomly select 1 backend
+ List<Backend> backendList = Lists.newArrayList();
+ for (Backend be : Env.getCurrentSystemInfo().getBackendsByCurrentCluster().values()) {
+ if (be.isAlive()) {
+ backendList.add(be);
+ }
+ }
+ if (backendList.isEmpty()) {
+ throw new UserException("No Alive backends");
+ }
+ Collections.shuffle(backendList);
+ Backend selectedBackend = backendList.get(0);
+
+ // create scan range locations
+ TScanRangeLocation location = new TScanRangeLocation();
+ location.setBackendId(selectedBackend.getId());
+ location.setServer(new TNetworkAddress(selectedBackend.getHost(), selectedBackend.getBePort()));
+ TScanRangeLocations locations = new TScanRangeLocations();
+ locations.addToLocations(location);
+ locations.setScanRange(new TScanRange());
+ scanRangeLocations.add(locations);
+ }
+
+ @Override
+ public List<TScanRangeLocations> getScanRangeLocations(long maxScanRangeLength) {
+ return scanRangeLocations;
+ }
+
+ @Override
+ public int getNumInstances() {
+ return 1;
+ }
+
+ @Override
+ public int getScanRangeNum() {
+ return 1;
+ }
+
+ @Override
+ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
+ StringBuilder output = new StringBuilder();
+ output.append(prefix).append("Recursive Cte: ").append(recursiveCteName).append("\n");
+ if (!conjuncts.isEmpty()) {
+ Expr expr = convertConjunctsToAndCompoundPredicate(conjuncts);
+ output.append(prefix).append("PREDICATES: ").append(expr.toSql()).append("\n");
+ }
+ return output.toString();
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("recursiveCteName", recursiveCteName)
+ .add("id", getId().asInt())
+ .add("tid", desc.getId().asInt()).toString();
+ }
+
+ @Override
+ protected void toThrift(TPlanNode msg) {
+ msg.node_type = TPlanNodeType.REC_CTE_SCAN_NODE;
+ }
+
+ @Override
+ public boolean isSerialOperator() {
+ return true;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
index 394e0cd5b1c..2733385154c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
@@ -17,6 +17,7 @@
package org.apache.doris.qe.runtime;
+import org.apache.doris.analysis.Expr;
import org.apache.doris.catalog.AIResource;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Resource;
@@ -41,6 +42,11 @@ import org.apache.doris.planner.MultiCastDataSink;
import org.apache.doris.planner.OlapScanNode;
import org.apache.doris.planner.OlapTableSink;
import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.PlanFragmentId;
+import org.apache.doris.planner.PlanNode;
+import org.apache.doris.planner.RecursiveCteNode;
+import org.apache.doris.planner.RecursiveCteScanNode;
+import org.apache.doris.planner.RuntimeFilter;
import org.apache.doris.planner.ScanNode;
import org.apache.doris.planner.SortNode;
import org.apache.doris.qe.ConnectContext;
@@ -48,6 +54,7 @@ import org.apache.doris.qe.CoordinatorContext;
import org.apache.doris.thrift.PaloInternalServiceVersion;
import org.apache.doris.thrift.TAIResource;
import org.apache.doris.thrift.TDataSinkType;
+import org.apache.doris.thrift.TExpr;
import org.apache.doris.thrift.TFileScanRangeParams;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TPipelineFragmentParams;
@@ -56,6 +63,9 @@ import org.apache.doris.thrift.TPipelineInstanceParams;
import org.apache.doris.thrift.TPlanFragment;
import org.apache.doris.thrift.TPlanFragmentDestination;
import org.apache.doris.thrift.TQueryOptions;
+import org.apache.doris.thrift.TRecCTENode;
+import org.apache.doris.thrift.TRecCTEResetInfo;
+import org.apache.doris.thrift.TRecCTETarget;
import org.apache.doris.thrift.TRuntimeFilterInfo;
import org.apache.doris.thrift.TRuntimeFilterParams;
import org.apache.doris.thrift.TScanRangeParams;
@@ -75,10 +85,13 @@ import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
import java.util.function.BiConsumer;
import java.util.function.Supplier;
@@ -89,6 +102,8 @@ public class ThriftPlansBuilder {
CoordinatorContext coordinatorContext) {
List<PipelineDistributedPlan> distributedPlans = coordinatorContext.distributedPlans;
+ Set<Integer> fragmentToNotifyClose = setParamsForRecursiveCteNode(distributedPlans,
+ coordinatorContext.runtimeFilters);
// we should set runtime predicate first, then we can use heap sort and to thrift
setRuntimePredicateIfNeed(coordinatorContext.scanNodes);
@@ -114,7 +129,7 @@ public class ThriftPlansBuilder {
TPipelineFragmentParams currentFragmentParam = fragmentToThriftIfAbsent(
currentFragmentPlan, instanceJob, workerToCurrentFragment,
instancesPerWorker, exchangeSenderNum, sharedFileScanRangeParams,
- workerProcessInstanceNum, coordinatorContext);
+ workerProcessInstanceNum, fragmentToNotifyClose, coordinatorContext);
TPipelineInstanceParams instanceParam = instanceToThrift(
currentFragmentParam, instanceJob, currentInstanceIndex++);
@@ -325,6 +340,7 @@ public class ThriftPlansBuilder {
Map<Integer, Integer> exchangeSenderNum,
Map<Integer, TFileScanRangeParams> fileScanRangeParamsMap,
Multiset<DistributedPlanWorker> workerProcessInstanceNum,
+ Set<Integer> fragmentToNotifyClose,
CoordinatorContext coordinatorContext) {
DistributedPlanWorker worker = assignedJob.getAssignedWorker();
return workerToFragmentParams.computeIfAbsent(worker, w -> {
@@ -338,6 +354,9 @@ public class ThriftPlansBuilder {
params.setDescTbl(coordinatorContext.descriptorTable);
params.setQueryId(coordinatorContext.queryId);
params.setFragmentId(fragment.getFragmentId().asInt());
+ if (fragmentToNotifyClose.contains(params.getFragmentId())) {
+ params.setNeedNotifyClose(true);
+ }
// Each tParam will set the total number of Fragments that need to be executed on the same BE,
// and the BE will determine whether all Fragments have been executed based on this information.
@@ -576,6 +595,136 @@ public class ThriftPlansBuilder {
}
}
+ private static Set<Integer> setParamsForRecursiveCteNode(List<PipelineDistributedPlan> distributedPlans,
+ List<RuntimeFilter> runtimeFilters) {
+ Set<Integer> fragmentToNotifyClose = new HashSet<>();
+ Map<PlanFragmentId, TRecCTETarget> fragmentIdToRecCteTargetMap = new TreeMap<>();
+ Map<PlanFragmentId, Set<TNetworkAddress>> fragmentIdToNetworkAddressMap = new TreeMap<>();
+ // distributedPlans is ordered in bottom up way, so does the fragments
+ for (PipelineDistributedPlan plan : distributedPlans) {
+ List<AssignedJob> fragmentAssignedJobs = plan.getInstanceJobs();
+ Set<TNetworkAddress> networkAddresses = new TreeSet<>();
+ for (AssignedJob assignedJob : fragmentAssignedJobs) {
+ DistributedPlanWorker distributedPlanWorker = assignedJob.getAssignedWorker();
+ networkAddresses.add(new TNetworkAddress(distributedPlanWorker.host(),
+ distributedPlanWorker.brpcPort()));
+ }
+ PlanFragment planFragment = plan.getFragmentJob().getFragment();
+ fragmentIdToNetworkAddressMap.put(planFragment.getFragmentId(), networkAddresses);
+
+ List<RecursiveCteScanNode> recursiveCteScanNodes = planFragment.getPlanRoot()
+ .collectInCurrentFragment(RecursiveCteScanNode.class::isInstance);
+ if (!recursiveCteScanNodes.isEmpty()) {
+ if (recursiveCteScanNodes.size() != 1) {
+ throw new IllegalStateException(
+ String.format("one fragment can only have 1 recursive cte scan node, but there is %d",
+ recursiveCteScanNodes.size()));
+ }
+ if (fragmentAssignedJobs.isEmpty()) {
+ throw new IllegalStateException(
+ "fragmentAssignedJobs is empty for recursive cte scan node");
+ }
+ TRecCTETarget tRecCTETarget = new TRecCTETarget();
+ DistributedPlanWorker distributedPlanWorker = fragmentAssignedJobs.get(0).getAssignedWorker();
+ tRecCTETarget.setAddr(new TNetworkAddress(distributedPlanWorker.host(),
+ distributedPlanWorker.brpcPort()));
+ tRecCTETarget.setFragmentInstanceId(fragmentAssignedJobs.get(0).instanceId());
+ tRecCTETarget.setNodeId(recursiveCteScanNodes.get(0).getId().asInt());
+ // find all RecursiveCteScanNode and its fragment id
+ fragmentIdToRecCteTargetMap.put(planFragment.getFragmentId(), tRecCTETarget);
+ }
+
+ List<RecursiveCteNode> recursiveCteNodes = planFragment.getPlanRoot()
+ .collectInCurrentFragment(RecursiveCteNode.class::isInstance);
+ if (!recursiveCteNodes.isEmpty()) {
+ if (recursiveCteNodes.size() != 1) {
+ throw new IllegalStateException(
+ String.format("one fragment can only have 1 recursive cte node, but there is %d",
+ recursiveCteNodes.size()));
+ }
+
+ List<TRecCTETarget> targets = new ArrayList<>();
+ List<TRecCTEResetInfo> fragmentsToReset = new ArrayList<>();
+ // PhysicalPlanTranslator will swap recursiveCteNode's child fragment,
+ // so we get recursive one by 1st child and collect all child fragment of recursive side
+ List<PlanFragment> childFragments = new ArrayList<>();
+ planFragment.getChild(0).collectAll(PlanFragment.class::isInstance, childFragments);
+ for (PlanFragment child : childFragments) {
+ PlanFragmentId childFragmentId = child.getFragmentId();
+ // the fragment need to be notified to close
+ fragmentToNotifyClose.add(childFragmentId.asInt());
+ TRecCTETarget tRecCTETarget = fragmentIdToRecCteTargetMap.getOrDefault(childFragmentId, null);
+ if (tRecCTETarget != null) {
+ // one RecursiveCteNode can only have one corresponding RecursiveCteScanNode
+ targets.add(tRecCTETarget);
+ // because we traverse the fragments in bottom-up way
+ // we can safely remove accessed RecursiveCteScanNode
+ // so the parent RecursiveCteNode won't see its grandson RecursiveCteScanNode
+ // but can only see its child RecursiveCteScanNode
+ fragmentIdToRecCteTargetMap.remove(childFragmentId);
+ }
+ Set<TNetworkAddress> tNetworkAddresses = fragmentIdToNetworkAddressMap.get(childFragmentId);
+ if (tNetworkAddresses == null) {
+ throw new IllegalStateException(
+ String.format("can't find TNetworkAddress for fragment %d", childFragmentId));
+ }
+ for (TNetworkAddress address : tNetworkAddresses) {
+ TRecCTEResetInfo tRecCTEResetInfo = new TRecCTEResetInfo();
+ tRecCTEResetInfo.setFragmentId(childFragmentId.asInt());
+ tRecCTEResetInfo.setAddr(address);
+ fragmentsToReset.add(tRecCTEResetInfo);
+ }
+ }
+
+ RecursiveCteNode recursiveCteNode = recursiveCteNodes.get(0);
+ List<List<Expr>> materializedResultExprLists = recursiveCteNode.getMaterializedResultExprLists();
+ List<List<TExpr>> texprLists = new ArrayList<>(materializedResultExprLists.size());
+ for (List<Expr> exprList : materializedResultExprLists) {
+ texprLists.add(Expr.treesToThrift(exprList));
+ }
+ // the recursive side's rf need to be reset
+ List<Integer> runtimeFiltersToReset = new ArrayList<>(runtimeFilters.size());
+ for (RuntimeFilter rf : runtimeFilters) {
+ if (rf.hasRemoteTargets()
+ && recursiveCteNode.getChild(1).contains(node -> node == rf.getBuilderNode())) {
+ runtimeFiltersToReset.add(rf.getFilterId().asInt());
+ }
+ }
+ // find recursiveCte used by other recursive cte
+ Set<RecursiveCteNode> recursiveCteNodesInRecursiveSide = new HashSet<>();
+ PlanNode rootPlan = distributedPlans.get(distributedPlans.size() - 1)
+ .getFragmentJob().getFragment().getPlanRoot();
+ collectAllRecursiveCteNodesInRecursiveSide(rootPlan, false, recursiveCteNodesInRecursiveSide);
+ boolean isUsedByOtherRecCte = recursiveCteNodesInRecursiveSide.contains(recursiveCteNode);
+
+ TRecCTENode tRecCTENode = new TRecCTENode();
+ tRecCTENode.setIsUnionAll(recursiveCteNode.isUnionAll());
+ tRecCTENode.setTargets(targets);
+ tRecCTENode.setFragmentsToReset(fragmentsToReset);
+ tRecCTENode.setResultExprLists(texprLists);
+ tRecCTENode.setRecSideRuntimeFilterIds(runtimeFiltersToReset);
+ tRecCTENode.setIsUsedByOtherRecCte(isUsedByOtherRecCte);
+ recursiveCteNode.settRecCTENode(tRecCTENode);
+ }
+ }
+ return fragmentToNotifyClose;
+ }
+
+ private static void collectAllRecursiveCteNodesInRecursiveSide(PlanNode planNode, boolean needCollect,
+ Set<RecursiveCteNode> recursiveCteNodes) {
+ if (planNode instanceof RecursiveCteNode) {
+ if (needCollect) {
+ recursiveCteNodes.add((RecursiveCteNode) planNode);
+ }
+ collectAllRecursiveCteNodesInRecursiveSide(planNode.getChild(0), needCollect, recursiveCteNodes);
+ collectAllRecursiveCteNodesInRecursiveSide(planNode.getChild(1), true, recursiveCteNodes);
+ } else {
+ for (PlanNode child : planNode.getChildren()) {
+ collectAllRecursiveCteNodesInRecursiveSide(child, needCollect, recursiveCteNodes);
+ }
+ }
+ }
+
private static class PerNodeScanParams {
Map<Integer, List<TScanRangeParams>> perNodeScanRanges;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java
index 42a930c2471..a2633b51449 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java
@@ -41,6 +41,8 @@ public enum StatisticalType {
ODBC_SCAN_NODE,
OLAP_SCAN_NODE,
PARTITION_TOPN_NODE,
+ RECURSIVE_CTE_NODE,
+ RECURSIVE_CTE_SCAN_NODE,
REPEAT_NODE,
SELECT_NODE,
SET_OPERATION_NODE,
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java
index a91c0dd4712..1921ae9025d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java
@@ -33,10 +33,14 @@ import org.apache.doris.nereids.rules.rewrite.InApplyToJoin;
import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderApply;
import org.apache.doris.nereids.rules.rewrite.UnCorrelatedApplyFilter;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
+import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.qe.OriginStatement;
import org.apache.doris.utframe.TestWithFeService;
import com.google.common.collect.ImmutableList;
@@ -264,6 +268,92 @@ public class AnalyzeCTETest extends TestWithFeService implements MemoPatternMatc
);
}
+ @Test
+ public void testRecCteOutputNullable() {
+ String sql = new StringBuilder()
+ .append("WITH RECURSIVE test_table AS (\n")
+ .append(" SELECT 1 UNION ALL\n")
+ .append(" SELECT 2 FROM test_table\n")
+ .append(")\n")
+ .append("SELECT * FROM test_table;")
+ .toString();
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .matches(
+ logicalRecursiveCte(
+ logicalProject(
+ logicalOneRowRelation(
+ )
+ ).when(project -> project.getProjects().get(0).child(0) instanceof Nullable),
+ logicalRecursiveCteRecursiveChild(
+ logicalProject(
+ logicalProject(
+ logicalCTEConsumer()
+ )
+ ).when(project -> project.getProjects().get(0).child(0) instanceof Nullable)
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testRecCteWithoutRecKeyword() {
+ String sql = new StringBuilder()
+ .append("WITH RECURSIVE t1 AS (\n")
+ .append(" SELECT 1\n")
+ .append("UNION ALL\n")
+ .append(" SELECT 2 FROM t1\n")
+ .append("),\n").append("t2 AS (\n")
+ .append(" SELECT 3\n")
+ .append("UNION ALL\n")
+ .append(" SELECT 4 FROM t1, t2\n")
+ .append(")\n")
+ .append("SELECT * FROM t2;")
+ .toString();
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .matches(
+ logicalRecursiveCte(
+ logicalProject(
+ logicalOneRowRelation(
+ )
+ ),
+ logicalRecursiveCteRecursiveChild(
+ logicalProject(
+ logicalProject(
+ logicalJoin()
+ )
+ )
+ )
+ ).when(cte -> cte.getCteName().equals("t2"))
+ );
+ }
+
+ @Test
+ public void testRecCteMultipleUnion() {
+ String sql = new StringBuilder().append("with recursive t1 as (\n").append(" select\n")
+ .append(" 1 as c1,\n").append(" 1 as c2\n").append("),\n").append("t2 as (\n")
+ .append(" select\n").append(" 2 as c1,\n").append(" 2 as c2\n").append("),\n")
+ .append("xx as (\n").append(" select\n").append(" c1,\n").append(" c2\n")
+ .append(" from\n").append(" t1\n").append(" union\n").append(" select\n")
+ .append(" c1,\n").append(" c2\n").append(" from\n").append(" t2\n")
+ .append(" union\n").append(" select\n").append(" c1,\n").append(" c2\n")
+ .append(" from\n").append(" xx\n").append(")\n").append("select\n").append(" *\n")
+ .append("from\n").append(" xx;").toString();
+ LogicalPlan unboundPlan = new NereidsParser().parseSingle(sql);
+ StatementContext statementContext = new StatementContext(connectContext,
+ new OriginStatement(sql, 0));
+ NereidsPlanner planner = new NereidsPlanner(statementContext);
+ planner.planWithLock(unboundPlan, PhysicalProperties.ANY,
+ ExplainCommand.ExplainLevel.ANALYZED_PLAN);
+ MemoTestUtils.initMemoAndValidState(planner.getCascadesContext());
+ PlanChecker.from(planner.getCascadesContext()).matches(
+ logicalRecursiveCte(
+ logicalProject(
+ logicalUnion()),
+ logicalRecursiveCteRecursiveChild()).when(cte -> cte.getCteName().equals("xx")));
+ }
+
/* ********************************************************************************************
* Test CTE Exceptions
@@ -333,4 +423,133 @@ public class AnalyzeCTETest extends TestWithFeService implements MemoPatternMatc
() -> PlanChecker.from(connectContext).analyze(sql), "Not throw expected exception.");
Assertions.assertTrue(exception.getMessage().contains("Table [cte2] does not exist in database"));
}
+
+ @Test
+ public void testRecCteWithoutRecKeywordException() {
+ String sql = new StringBuilder()
+ .append("WITH t1 AS (\n")
+ .append(" SELECT 1 UNION ALL\n")
+ .append(" SELECT 2 FROM t1\n")
+ .append(")\n")
+ .append("SELECT * FROM t1;")
+ .toString();
+ LogicalPlan unboundPlan = new NereidsParser().parseSingle(sql);
+ StatementContext statementContext = new StatementContext(connectContext,
+ new OriginStatement(sql, 0));
+ NereidsPlanner planner = new NereidsPlanner(statementContext);
+ AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
+ () -> planner.planWithLock(unboundPlan, PhysicalProperties.ANY,
+ ExplainCommand.ExplainLevel.ANALYZED_PLAN), "Not throw expected exception.");
+ Assertions.assertTrue(exception.getMessage().contains("Table [t1] does not exist in database"));
+ }
+
+ @Test
+ public void testRecCteDatatypeException() {
+ String sql = new StringBuilder().append("WITH RECURSIVE t1 AS (\n").append(" SELECT 1 AS number\n")
+ .append("UNION ALL\n").append(" SELECT number + 1 FROM t1 WHERE number < 100\n").append(")\n")
+ .append("SELECT number FROM t1;").toString();
+ LogicalPlan unboundPlan = new NereidsParser().parseSingle(sql);
+ StatementContext statementContext = new StatementContext(connectContext,
+ new OriginStatement(sql, 0));
+ NereidsPlanner planner = new NereidsPlanner(statementContext);
+ AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
+ () -> planner.planWithLock(unboundPlan, PhysicalProperties.ANY,
+ ExplainCommand.ExplainLevel.ANALYZED_PLAN),
+ "Not throw expected exception.");
+ Assertions.assertTrue(exception.getMessage().contains("please add cast manually to get expect datatype"));
+ }
+
+ @Test
+ public void testRecCteMultipleUnionException() {
+ String sql = new StringBuilder().append("with recursive t1 as (\n").append(" select\n")
+ .append(" 1 as c1,\n").append(" 1 as c2\n").append("),\n").append("t2 as (\n")
+ .append(" select\n").append(" 2 as c1,\n").append(" 2 as c2\n").append("),\n")
+ .append("xx as (\n").append(" select\n").append(" c1,\n").append(" c2\n")
+ .append(" from\n").append(" t1\n").append(" union\n").append(" select\n")
+ .append(" c1,\n").append(" c2\n").append(" from\n").append(" xx\n")
+ .append(" union\n").append(" select\n").append(" c1,\n").append(" c2\n")
+ .append(" from\n").append(" t2\n").append(")\n").append("select\n").append(" *\n")
+ .append("from\n").append(" xx").toString();
+ LogicalPlan unboundPlan = new NereidsParser().parseSingle(sql);
+ StatementContext statementContext = new StatementContext(connectContext,
+ new OriginStatement(sql, 0));
+ NereidsPlanner planner = new NereidsPlanner(statementContext);
+ AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
+ () -> planner.planWithLock(unboundPlan, PhysicalProperties.ANY,
+ ExplainCommand.ExplainLevel.ANALYZED_PLAN),
+ "Not throw expected exception.");
+ Assertions.assertTrue(exception.getMessage()
+ .contains("recursive reference to query xx must not appear within its non-recursive term"));
+ }
+
+ @Test
+ public void testRecCteNoUnionException() {
+ String sql = new StringBuilder().append("with recursive t1 as (\n").append(" select 1 \n")
+ .append(" intersect\n").append(" select 2 from t1\n").append(")\n").append("select\n")
+ .append(" *\n").append("from\n").append(" t1").toString();
+ LogicalPlan unboundPlan = new NereidsParser().parseSingle(sql);
+ StatementContext statementContext = new StatementContext(connectContext,
+ new OriginStatement(sql, 0));
+ NereidsPlanner planner = new NereidsPlanner(statementContext);
+ AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
+ () -> planner.planWithLock(unboundPlan, PhysicalProperties.ANY,
+ ExplainCommand.ExplainLevel.ANALYZED_PLAN),
+ "Not throw expected exception.");
+ Assertions.assertTrue(exception.getMessage().contains("recursive cte must be union"));
+ }
+
+ @Test
+ public void testRecCteAnchorException() {
+ String sql = new StringBuilder().append("with recursive t1 as (\n").append(" select 1 from t1\n")
+ .append(" union\n").append(" select 2 from t1\n").append(")\n").append("select\n")
+ .append(" *\n").append("from\n").append(" t1;").toString();
+ LogicalPlan unboundPlan = new NereidsParser().parseSingle(sql);
+ StatementContext statementContext = new StatementContext(connectContext,
+ new OriginStatement(sql, 0));
+ NereidsPlanner planner = new NereidsPlanner(statementContext);
+ AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
+ () -> planner.planWithLock(unboundPlan, PhysicalProperties.ANY,
+ ExplainCommand.ExplainLevel.ANALYZED_PLAN),
+ "Not throw expected exception.");
+ Assertions.assertTrue(exception.getMessage()
+ .contains("recursive reference to query t1 must not appear within its non-recursive term"));
+ }
+
+ @Test
+ public void testRecCteMoreThanOnceException() {
+ String sql = new StringBuilder().append("with recursive t1 as (\n").append(" select 1\n")
+ .append(" union\n").append(" select 2 from t1 x, t1 y\n").append(")\n").append("select\n")
+ .append(" *\n").append("from\n").append(" t1").toString();
+ LogicalPlan unboundPlan = new NereidsParser().parseSingle(sql);
+ StatementContext statementContext = new StatementContext(connectContext,
+ new OriginStatement(sql, 0));
+ NereidsPlanner planner = new NereidsPlanner(statementContext);
+ AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
+ () -> planner.planWithLock(unboundPlan, PhysicalProperties.ANY,
+ ExplainCommand.ExplainLevel.ANALYZED_PLAN),
+ "Not throw expected exception.");
+ Assertions.assertTrue(
+ exception.getMessage().contains("recursive reference to query t1 must not appear more than once"));
+ }
+
+ @Test
+ public void testRecCteInSubqueryException() {
+ String sql = new StringBuilder().append("with recursive t1 as (\n").append(" select\n")
+ .append(" 1 as c1,\n").append(" 1 as c2\n").append("),\n").append("xx as (\n")
+ .append(" select\n").append(" 2 as c1,\n").append(" 2 as c2\n").append(" from\n")
+ .append(" t1\n").append(" union\n").append(" select\n").append(" 3 as c1,\n")
+ .append(" 3 as c2\n").append(" from\n")
+ .append(" t1 where t1.c1 in (select c1 from xx)\n").append(")\n").append("select\n")
+ .append(" *\n").append("from\n").append(" xx").toString();
+ LogicalPlan unboundPlan = new NereidsParser().parseSingle(sql);
+ StatementContext statementContext = new StatementContext(connectContext,
+ new OriginStatement(sql, 0));
+ NereidsPlanner planner = new NereidsPlanner(statementContext);
+ AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
+ () -> planner.planWithLock(unboundPlan, PhysicalProperties.ANY,
+ ExplainCommand.ExplainLevel.ANALYZED_PLAN),
+ "Not throw expected exception.");
+ Assertions.assertTrue(
+ exception.getMessage().contains("Table [xx] does not exist in database"));
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/CTEInlineTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/CTEInlineTest.java
new file mode 100644
index 00000000000..0a0efc8b5db
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/CTEInlineTest.java
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.util.MemoPatternMatchSupported;
+import org.apache.doris.nereids.util.MemoTestUtils;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.qe.OriginStatement;
+import org.apache.doris.utframe.TestWithFeService;
+
+import org.junit.jupiter.api.Test;
+
+public class CTEInlineTest extends TestWithFeService implements MemoPatternMatchSupported {
+ @Override
+ protected void runBeforeAll() throws Exception {
+ createDatabase("test");
+ connectContext.setDatabase("test");
+ }
+
+ @Test
+ public void recCteInline() {
+ String sql = new StringBuilder().append("with recursive t1 as (\n").append(" select\n")
+ .append(" 1 as c1,\n").append(" 1 as c2\n").append("),\n").append("t2 as (\n")
+ .append(" select\n").append(" 2 as c1,\n").append(" 2 as c2\n").append("),\n")
+ .append("t3 as (\n").append(" select\n").append(" 3 as c1,\n").append(" 3 as c2\n")
+ .append("),\n").append("xx as (\n").append(" select\n").append(" c1,\n")
+ .append(" c2\n").append(" from\n").append(" t1\n").append(" union\n")
+ .append(" select\n").append(" t2.c1,\n").append(" t2.c2\n").append(" from\n")
+ .append(" t2,\n").append(" xx\n").append(" where\n").append(" t2.c1 = xx.c1\n")
+ .append("),\n").append("yy as (\n").append(" select\n").append(" c1,\n")
+ .append(" c2\n").append(" from\n").append(" t3\n").append(" union\n")
+ .append(" select\n").append(" t3.c1,\n").append(" t3.c2\n").append(" from\n")
+ .append(" t3,\n").append(" yy,\n").append(" xx\n").append(" where\n")
+ .append(" t3.c1 = yy.c1\n").append(" and t3.c2 = xx.c1\n").append(")\n")
+ .append("select\n").append(" *\n").append("from\n").append(" yy y1,\n").append(" yy y2;")
+ .toString();
+ LogicalPlan unboundPlan = new NereidsParser().parseSingle(sql);
+ StatementContext statementContext = new StatementContext(connectContext,
+ new OriginStatement(sql, 0));
+ NereidsPlanner planner = new NereidsPlanner(statementContext);
+ planner.planWithLock(unboundPlan, PhysicalProperties.ANY,
+ ExplainCommand.ExplainLevel.REWRITTEN_PLAN);
+ MemoTestUtils.initMemoAndValidState(planner.getCascadesContext());
+ PlanChecker.from(planner.getCascadesContext()).matches(
+ logicalRecursiveCte(
+ any(
+ ),
+ logicalRecursiveCteRecursiveChild(
+ logicalProject(
+ logicalJoin(
+ any(),
+ logicalProject(
+ logicalFilter(
+ logicalRecursiveCte().when(cte -> cte.getCteName().equals("xx"))
+ )
+ )
+ )
+ )
+ )
+ ).when(cte -> cte.getCteName().equals("yy"))
+ );
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ColumnPruningTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ColumnPruningTest.java
index 12e9a1ad381..353f0c13863 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ColumnPruningTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ColumnPruningTest.java
@@ -17,15 +17,23 @@
package org.apache.doris.nereids.rules.rewrite;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.types.DoubleType;
import org.apache.doris.nereids.types.TinyIntType;
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
+import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.qe.OriginStatement;
import org.apache.doris.utframe.TestWithFeService;
import com.google.common.collect.ImmutableList;
@@ -328,6 +336,28 @@ public class ColumnPruningTest extends TestWithFeService implements MemoPatternM
);
}
+ @Test
+ public void pruneRecCte() {
+ String sql = new StringBuilder().append("WITH RECURSIVE t1(col1, col2, col3) AS (\n")
+ .append(" SELECT 1, 1, 1\n").append(" UNION ALL\n").append(" SELECT 2, 2, 2\n")
+ .append(" FROM student, t1\n").append(" WHERE t1.col1 = student.id\n").append(" )\n")
+ .append("SELECT col1\n").append("FROM t1\n").append("WHERE col2 = 2;").toString();
+ LogicalPlan unboundPlan = new NereidsParser().parseSingle(sql);
+ StatementContext statementContext = new StatementContext(connectContext,
+ new OriginStatement(sql, 0));
+ NereidsPlanner planner = new NereidsPlanner(statementContext);
+ planner.planWithLock(unboundPlan, PhysicalProperties.ANY,
+ ExplainCommand.ExplainLevel.REWRITTEN_PLAN);
+ MemoTestUtils.initMemoAndValidState(planner.getCascadesContext());
+ PlanChecker.from(planner.getCascadesContext()).matches(
+ logicalProject(
+ logicalFilter(
+ logicalRecursiveCte().when(cte -> cte.getOutput().size() == 3)
+ )
+ ).when(project -> project.getOutputs().size() == 1)
+ );
+ }
+
private List<String> getOutputQualifiedNames(LogicalProject<? extends Plan> p) {
return getOutputQualifiedNames(p.getOutputs());
}
diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
index 415e1b40d95..cc018384f24 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -657,6 +657,7 @@ struct TPipelineFragmentParams {
// Used by 2.1
44: optional list<i32> topn_filter_source_node_ids
45: optional map<string, TAIResource> ai_resources
+ 46: optional bool need_notify_close
// For cloud
1000: optional bool is_mow_table;
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 4036ad7ea49..a13f511f9eb 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -710,6 +710,8 @@ struct TRecCTENode {
2: optional list<TRecCTETarget> targets
3: optional list<TRecCTEResetInfo> fragments_to_reset
4: optional list<list<Exprs.TExpr>> result_expr_lists
+ 5: optional list<i32> rec_side_runtime_filter_ids
+ 6: optional bool is_used_by_other_rec_cte
}
struct TRecCTEScanNode {
diff --git a/regression-test/suites/nereids_p0/recursive_cte/test_recursive_cte.groovy b/regression-test/suites/nereids_p0/recursive_cte/test_recursive_cte.groovy
new file mode 100644
index 00000000000..6055f9798cd
--- /dev/null
+++ b/regression-test/suites/nereids_p0/recursive_cte/test_recursive_cte.groovy
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+suite("test_recursive_cte") {
+ sql """drop table if exists t1"""
+ sql """drop table if exists t2"""
+ sql """drop table if exists t3"""
+ sql """drop table if exists t4"""
+
+ sql """
+ create table if not exists t1 (
+ c2 int ,
+ c1 int ,
+ c3 int ,
+ c4 int ,
+ pk int
+ )
+ distributed by hash(pk) buckets 10
+ properties("replication_num" = "1");
+ """
+
+ sql """
+ create table if not exists t2 (
+ c1 int ,
+ c2 int ,
+ c3 int ,
+ c4 int ,
+ pk int
+ )
+ distributed by hash(pk) buckets 10
+ properties("replication_num" = "1");
+ """
+
+ sql """
+ create table if not exists t3 (
+ c2 int ,
+ c1 int ,
+ c3 int ,
+ c4 int ,
+ pk int
+ )
+ distributed by hash(pk) buckets 10
+ properties("replication_num" = "1");
+ """
+
+ sql """
+ create table if not exists t4 (
+ c1 int ,
+ c2 int ,
+ c3 int ,
+ c4 int ,
+ pk int
+ )
+ distributed by hash(pk) buckets 10
+ properties("replication_num" = "1");
+ """
+
+ sql """
+ insert into t1(pk,c1,c2,c3,c4) values (0,7,2,3328056,7),(1,3,5,3,3045349),(2,2130015,0,7,-7116176),(3,4411710,1203314,1,2336164),(4,4,-8001461,0,8),(5,9,3,6,2),(6,-8088092,null,-7256698,-2025142),(7,8,2,5,1),(8,4,4953685,3,null),(9,-6662413,-3845449,4,2),(10,5315281,0,5,null),(11,9,3,7,7),(12,4341905,null,null,8),(13,3,6,5,1),(14,5,9,6541164,3),(15,1,-582319,1,9),(16,5533636,4,39841,0),(17,1,1,null,7),(18,742881,-1420303,6,1),(19,281430,6753011,3,2),(20,7,1,4,-31350),(21,-5663089 [...]
+ """
+
+ sql """
+ insert into t2(pk,c1,c2,c3,c4) values (0,5,4,189864,-7663457),(1,7,null,6,1),(2,null,8,-3362640,9),(3,3,2,5,-2197130),(4,2,3,7160615,1),(5,null,-57834,420441,3),(6,0,null,2,2),(7,1,-3681539,3,4),(8,548866,3,0,5),(9,8,-2824887,0,3246956),(10,5,3,7,2),(11,8,8,6,8),(12,0,2,7,9),(13,8,6,null,null),(14,-4103729,4,5,8),(15,-3659292,2,7,5),(16,8,7,1,null),(17,2526018,4,8069607,5),(18,6,6,5,2802235),(19,9,0,6379201,null),(20,3,null,4,3),(21,0,8,-5506402,2),(22,6,4,3,1),(23,4,5225086,3,1) [...]
+ """
+
+ sql """
+ insert into t3(pk,c1,c2,c3,c4) values (0,3,2,6,-3164679),(1,-6216443,3437690,-288827,6),(2,4,-5352286,-1005469,4118240),(3,9,6795167,5,1616205),(4,8,-4659990,-4816829,6),(5,0,9,4,8),(6,-4454766,2,2510766,3),(7,7860071,-3434966,8,3),(8,null,0,2,1),(9,8031908,2,-6673194,-5981416),(10,5,6716310,8,2529959),(11,null,-3622116,1,-7891010),(12,null,3527222,7993802,null),(13,null,1,2,1),(14,2,8,7,7),(15,0,9,5,null),(16,7452083,null,-4620796,0),(17,9,9,null,6),(18,3,1,-1578776,5),(19,9,253 [...]
+ """
+
+ sql """
+ insert into t4(pk,c1,c2,c3,c4) values (0,-4263513,null,null,6),(1,1,3,4,null),(2,2460936,6,5,6299003),(3,null,7,7107446,-2366754),(4,6247611,4785035,3,-8014875),(5,0,2,5249218,3),(6,null,253825,4,3),(7,null,2,9,-350785),(8,6,null,null,4),(9,1,3,1,3422691),(10,0,-6596165,1808018,3),(11,2,752342,null,1),(12,-5220927,2676278,9,7),(13,6025864,2,1,4),(14,7,4,4,9),(15,5,9,9,849881),(16,-4253076,null,-4404479,-6365351),(17,null,6,4240023,3),(18,7,1276495,7,6),(19,null,-4459040,178194,-6 [...]
+ """
+
+ sql """
+ EXPLAIN ALL PLAN WITH XX AS (
+ SELECT C1
+ FROM t1
+ ),
+ RECURSIVE YY AS (
+ SELECT C1 + 1.0 as c2
+ FROM XX
+ UNION
+ SELECT c1
+ FROM XX t2 join YY
+ WHERE t2.c1 = YY.c2
+ )
+ SELECT * FROM YY;
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org