You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by hu...@apache.org on 2025/10/28 07:21:48 UTC
(doris) branch nested_column_prune updated (22c43e0b18f -> 462492fe28b)
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a change to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
discard 22c43e0b18f fix access path case-insensitive if the type is struct or root column
omit 1fd25254466 fix prune nested column when use union with cast
omit f9079ca71c8 fix some bugs of prune nested columns
omit 79b18201e23 Support new thrift protocol(data/meta)
omit 7bfc0dac459 refactor TDataAccessPath
omit 2795841d875 use enablePruneNestedColumns to control PushDownNestedColumn
omit 7e7ad8ad500 [fix] Adapt to the pruned data type
omit a2d41ad43ca support iceberg's id access path
omit e54ee541bb3 push down project through partition top n
omit 7297f64d937 push down project through window
omit 25009bd5158 fix missing column name
omit 70cb7f63446 push down project through join/union
omit 91d6df510e6 add test cases
omit f2dd53d2515 fix missing access path when select * from tbl
omit c56e05f0e77 [opt](olap) Support column pruning for struct, array, and map types (#57195)
omit e2f2c3e091e support prune nested column in fe
omit 2c9392e15ae refactor interface and support fe part
omit 908ce063f13 define the interface of nested column prune
add 3e8e6d296f8 [fix](agg) Fix agg's input distribution (#56801)
add 85e0a723813 [fix](stats) do not forward drop cached stats command (#56453)
add 1dc731fe678 [chore](recycler) Log task type when delete data (#56797)
add d30170a7437 [feature](file-cache) Implement cache warm up select functionality. (#54822)
add aecdd030e29 [improve](job) hidden secret key in streaming job (#56742)
add 42745950b70 [fix](fold constant) fix result's scale of some datetime related function (#56671)
add 8fc232a28d5 [fix](case) fix test_group_commit_stream_load (#56774)
add cd8d8413e93 [chore](cloud) Run all test binaries even if some report failure (#56839)
add 491d628569e [minor](pipeline) Add lock restriction (#56822)
add 42408f18786 [fix](error) Avoid deadlock if task status is unexpected (#56816)
add 3cf779fde47 [Chore](nereids) remove TableRef (#56661)
add 596cb9fac1a [feat](cloud): add file cache consistency check (#41280) (#54295)
add 557efa356d2 [fix](test) fix unstable regression test (#56814)
add 00580dd417c [Fix](warmup) one-time and periodic warm up job should be triggered regardless whether the rowset has been warmed up before (#56737)
add d20036ea29b [improvement](json load) Compatible with "$" and "$." that appear during json load (#56703)
add 158a27f8d4b [opt](test) Use better way to check `enableStoragevault` for regression test (#56847)
add 5a7ff703a2a [fix](UT) Fix cloud string codec test when len is 0 (#56859)
add a73b6e81042 [feature](function) support sort_json_object_keys/normalize_json_numbers_to_double function (#56576)
add ac8833ef696 [improvement](load & cache) Set disable_file_cache to true for load data into disposable LRU queue (#56637)
add f0f12729363 [test](streaming job) add streaming job restart fe test (#56808)
add e917c59eb4c [fix](decimal256) fix unstable overflow error when casting decimal256 to float (#56848)
add ff97e9ab6d9 [fix](regression) fix typo error for suites/tpch_sf0.1_p1/sql/cse.groovy (#56857)
add ff904e45dd3 [test](streaming job) add streaming job auto resume test (#56881)
add d0af6bad651 [fix](insert) do not register group commit and insert into value job to load manager (#56852)
add 1d7967ae5be [minor](task) Remove unnecessary status result (#56863)
add 06bb48f90d7 [opt](regression) ./run-regression-test.sh support -f parameter to run case in specified file (#56799)
add 8785a474926 [fix](DecimalV2)keep the original precision and scale in get_data_type_with_default_argument (#56821)
add e09c08d11c5 [Fix](Chore) Fix cumu compaction fault injection case (#56867)
add 6fca73394f9 [fix](tde) make get_exec_env interface static (#56820)
add 4db09207ae2 [feat](s3client) Use `CustomAwsCredentialsProviderChain` for aws sdk (#56065)
add e3181d5cc88 [fix](nereids)ReorderJoin rule should not absorb mark join to multi join (#56837)
add 3e91ad98535 [chore](jsonb) remove unused code from jsonb_document (#56496)
add 063f3534c2e [fix](docker) modify readiness probe for ipv6 on k8s (#56917)
add 7e886ee237a [opt](fe) Add alias element_at as struct_element (#56898)
add c36027bec4e [test](sqlcache)Add sql cache with mtmv test case (#56753)
add ed674c7fe8f [test](nereids)fix prune partitions test case (#56892)
add 74109427066 [improvement](cloud) only add filecache profile for olap table in cloud deploy mode (#52118)
add 2594f8c7927 [Improve](fix) add streamjob case with create alter select delete and resume (#56849)
add fc38d1f0c65 [feature](regression-plugin) support version compare plugin with cloud (#56850)
add 16fc34e0cb2 [fix](mysql buffer) fix buffer overflow of output complex type columns (#56922)
add 7f41367e2f0 [opt](function) Make coalesce support more types (#56430)
add 807a87fe970 [fix](search) fix search result for non-tokenizer match (#56914)
add a91c067fabb [Enhancement](nereids)remove the class TableName (#56862)
add dca62cd5862 [fix](field) implement move constructor (#56803)
add ed2e880b44c [chore](code)Remove unnecessary accurate code and reduce template parameters (#56909)
add 27687930852 [feat](nereids) add table alias for catalog relation (#56802)
add d26e4953148 [Feature](func) Support function EXPORT_SET (#56798)
add 032ebaed6bd [fix](hdfs)Fix be coredump in HDFS reader during profile collection. (#56806)
add 588c37bc678 [fix](nereids)fix some show command's redirect policy (#55185)
add 483cd1c6448 [fix](nereids)forbid create sync mv based on annother sync mv (#56912)
add a5cf96230f4 [refactor](query) Clear query context once it is finished (#56911)
add fbb64e1f5ca [test](cache) add regression test cases for filecache statistics and cache eviction patterns (#54062)
add 8c1e26037fe [fix](insert) do not generate next id when fe is not master (#56959)
add 4971c1dc0b6 [feature] (nereids) generate sql digest for nereids prased stmt (#56256)
add ed6fb629e4c [fix](cloud) Fix unhandled error when get_partition_boundaries (#56935)
add 071a7e3fd30 [chore](build) Add feature list to version info (#56604)
add ac66ed0d232 [chore](checker) Add `get_all_tablet_ids` for MetaReader skeleton (#56888)
add 6ca89e31b17 [fix](schema-change) Fix distribution columns with varchar type could not increase length (#56743)
add 1d91724d5a6 [enhance](cloud)enable tablets meta url show rs_meta for cloud (#55167)
add b8688964673 [fix](iceberg) support reading iceberg partition table without partition value stored in file (#56918)
add fffbeabab16 [fix](nereids) partition topn optimization requires all window expressions are in the same order (#56622)
add 886fef76e73 [Enhancement](Snapshot) Add some snapshot interface (#56953)
add 31079ef49ca [Enhancement](nereids) remove class PartitionNames (#56947)
add c4ff082d9c3 [fix](search) fix mow support for search function (#56927)
add d1b0a19c083 [opt](inverted index) add custom analyzer support with char_filter, basic and icu tokenizer (#56243)
add 23ada5ec5c9 [fix](editlog) do not write editlog if not master (#56896)
add 996a54482d1 [opt](if) vec exec for more type in function if (#56905)
add 106e5b3f294 [refactor](insert) unified INSERT_JOB to INSERT for EtlJobType (#56780)
add 9bdc9695a3c [feature](agg) Add a knob to shuffled streaming agg (#56956)
add 7b9c3fea6cf [feat](snapshot) fix Unknown snapshot status when select snapshots (#56726)
add db0593bc591 [fix](function) Crash caused by explode function (#56991)
add e138c3e25b0 [Fix](Repository)Handle non-StorageException errors in Storage parameter validation (#56929)
add 753f3467f6e [fix](string) fix string len check of nested types (#56963)
add 3038a44b8fc [fix](inverted index) fix is null predicate for inverted index evaluate (#56964)
add f0190dbb459 [fix](ubsan) fix ub error of uninitialized variable (#56854)
add ce665f71aa3 [fix](iceberg) fix logic of getting namespaces (#56874)
add 65b2a5f0aa0 [Opt](sys table) Add a system table column_data_sizes (#56815)
add 8d7da73b0e4 [Improvement](execute) optimize for ColumnNullable's serialize_vec/deserialize_vec (#56885)
add 0a2b8d8a9af [enhance](memtable) support adaptive memtable write buffer size (#56948)
add 9a16dcd3fec [opt](case) move some time consuming case to p1 (#56986)
add 3199414db51 [chore](cloud) Add decode_*_key (#57008)
add fd47c36ceb8 [test](cloud) Add more multi cluster case for cloud (#56840)
add d57b02adb42 [chore](tde) Pick rotate open source code (#56838)
add 58203c8e278 [feat](case when) rewrite case when to compound predicate (#56756)
add c436e9224ad [chore](cloud) expose split_tablet_stats_into_load_and_compact_parts (#57009)
add 286b47f04a2 [feat](cloud) Add snapshot data migrator & chain compactor skeleton (#56939)
add ce7b17aea79 [opt](memory) remove useless replicaToTabletMap in TabletInvertedIndex (#57011)
add aad5e485bdc [refine](float) Add NormalizeFloat to handle special values of floating-point numbers. (#56937)
add 01f83399600 [test](auth)Fix backup backup of auth test case (#56988)
add 46d27a21f43 [fix](pipeline) Crashing caused by repeated spill operations (#56755)
add 456af82122a [opt](nereids)move runtime filter info from instance level to BE level (#56978)
add 9451276b524 [feature](search) add variant subcolumn suppport for search function (#56718)
add 7f908857eaa [chore](UT) Make cloud mgr test stable (#57033)
add 02cd275d7f3 [Improve](variant) optimize cast performance when variant only contains null values (#57003)
add 16c62be4598 [fix](nereids) fix rule AvgDistinctToSumDivCount need add cast (#56887)
add 302aec1316a [Enhancement](auto partition) Auto partition could use auto bucket (#56921)
add ea1b7fabe66 [refine](DECIMALV2)Only DECIMALV2 has the member variables original_precision and original_scale. (#56913)
add bdca890342a [Improve](job) add priv check for streaming job (#56965)
add 0578af22b20 [improvement](cache) (log) add logs for mode switching and optimize log output frequency (#54085)
add 1f4cd9a22ab [chore](MS)Rename HMSProperties to HiveProperties to avoid confusion with Hive/HMS configs (#57064)
add 04e5604246c [improve](case) Fix spark flink case in jdk17 (#56992)
add b31c45444ae [fix](schema_change) fix incorrect unique id in rollup after schema change (#57073)
add 7165d4ee6f6 [Fix](metrics) Fix FE's incorrect prometheus metrics format (#57059)
add 490ade94225 [Fix](startup) Skip wrong file when loading tzdata (#57061)
add 78d8cb8ff98 [fix](exception) avoid not set required field exception (#57069)
add b6d920081ca [fix](fe)show config command use readline instead of ready (#54791)
add 508b5712db8 [Feature](func) Support GET_FORAMT function (#56691)
add 7ce60da00f8 [fix](cloud) Fix /get_snapshot_property result (#57067)
add 50123ad8a83 [opt](function) Enhance the handling of a single struct-type argument in the EXPLODE function (#57050)
add 71480ff12ab [fix](cloud) Return error if migrate to versioned keys/compact snapsh… (#57066)
add 35f7fab36b4 [env](case) remove disable pipeline engine case (#57046)
add 19cf4c4a4e1 [feature](function) support json_hash function (#56962)
add af9fb9d82e4 [refine](type)use new type dispatch (#56990)
add 6b41b1ba55c [fix](cache) fix incorrectly incrementing s3_read_counter (#55805)
add b3ff40587b7 [Chore](runtime-filter) catch Exception from runtime filter rpc (#57085)
add b276af3bb68 [Fix](autoinc) Fix wrong auto inc value after FE do checkpoint and restart (#57104)
add e2cb3d6b877 [enhancement](filecache) add wait time for other downloader in profile (#57090)
add 82d5a3aac83 [fix](cloud) Fix /set_snapshot_property property names (#57099)
add bb121e30810 [chore](external catalog) deprecated lakesoul external catalog (#56843)
add 175e6ebcc38 [chore](inverted index) remove duplicate code in comparison evaluate index (#57115)
add 26aacf428d2 [opt](ann index) omp_threads_limit working better & minor change for observability (#56796)
add 1b6476e3840 [enhance](job) delay load task schedule when transaction fail (#57092)
add c1e72907efd [chore](search) remove useless params (#57117)
add 3e3efb572f6 [Fix](job) fix streaming job block when fe restart (#57103)
add e46317fdbb6 [function](mask) mask_last/first second parameter can be non-const (#57062)
add 2794fcc9c90 [fix](case) add query to force sync rowset meta (#56949)
add 5c610cd4b96 [opt](inverted index) Add built_in_analyzer parser alias and modify custom analyzer logic (#57055)
add 26645e9da84 [fix] remove useless const_cast and explain const_cast for vec (2) (#56464)
add ab1a05a4d07 [mem](jemalloc) Improve memory reuse efficiency and reduce page faults to change jemalloc conf (#57076)
add ad7fc73265f [fix](docker) readiness-modify for http code (#57113)
add f34b8c1a1b7 [enhance](job) support adaptive batch param for routine load job (#56930)
add 9e6926909ed [Enhancement](grammar) Support omit auto keyword for auto range partition (#57060)
add 017f9e49c72 [fix](catalog)Disable Hadoop FileSystem cache for multi-configurationobject storage catalogs (#57063)
add 04e333b442b [fix](parquet) incorrect skip files when returning eof (#57071)
add 72bcfd6d2fe [opt](ann index) Support product quantization (#56954)
add fa36b1c6cc2 [feature](mtmv)create mtmv can use view (#56423)
add e08709fba10 [chore](cloud) support notifing itself to refresh instance (#57107)
add f06196e9a24 [chore](regression) fix java home inconsistency (#57095)
add f27070c2d13 [fix](load) fix load plan when using WHERE clause with CompoundPredicate (#57128)
add 0dfdfcd1a01 [fix](filecache) fix flaky case populate_empty_cache_with_normal (#57091)
add 5ef4666eb13 [Bug](cloud restore) Fix npe in show restore command after fe restart (#57051)
add d24e675cb1a [Fix](column) Fix `ColumnFixedLengthObject::{insert_many_continuous_binary_data, insert_many_strings}` (#57164)
add e51d64d7ffd [chore](compile) fix macos compile be failed (#57150)
add 5f0a8c68ded [fix](inverted index) remove shadow prefix when matching index columns in schema change (#57006)
add b36f517e1a6 [Feature](varbinary) add length, from_base64_binary, to_base64_bianry, sub_binary for VarBinary type (#56648)
add 87a53626d48 [Exec](Cache) Support condition cache in Apache doris (#55534)
add 0f82adf9856 [chore](cloud) Make key info declaration more clear (#57034)
add 7be788844c8 [Enhancement] add column_uid and schema_version field for table schema open api (#56919)
add b99f8e972ed [Fix](ai) Fix the wrong check in AI_FILTER (#56960)
add 69d48801dc4 [fix](docker) update node to v14 in docker compilation image (#56029)
add 322546caf2e [fix](nereids)fix sync mv parse error if it contains sql KEYWORD (#57052)
add 7945b29db60 [fix](nullable) Fix nullable property for array (#57179)
add 33db05e5c1e [fix](nereids)set RuntimeFilterInfo only on BE which is merge node (#57108)
add 94d7f2eabca [refactor](segment iterator) rename _non_predicate_column_ids to _common_expr_column_ids (#56931)
add d29b3fe32eb [improve](job) support change offset for streaming job (#57138)
add a301e313569 [fix](job) fix streaming job repeated load after FE master change in cloud mode (#57205)
add f7876530964 [fix](olap) Need to handle floating-point Infinity values in the predicate (#57100)
add 3f87ec38d43 [fix](segcompaction) Fix segcompaction coredump when adding key column (#57162)
add 903d052924e [test](jdbc) Upgraded the mysql jdbc driver jar under test to 8.4.0 (#57142)
add 9177047d17c [feat](storage)Support Azure Blob Storage (#56861)
add 5641a274d4b [test](nereids)fix sql cache of mtmv test case (#57177)
add b7612ab0d9c [chore](cold) add a score threshold for cold data compaction (#55333)
add dc7e1d82c0e [feat](nereids) Add fold constant for nullif function (#56932)
add 8226a2a3bac [Opt](encoding) Add new encoding `BinaryPlainPageV2` (#56846)
add cb1e3bd9f8c [feat](nereids) opt condition expression in filter / join / if /case when (#57025)
add 69f82ad67db [Feature](variable) Support session variable lc_time_names (#56915)
add bd8a44f2db0 [fix](case) fix case bug (#56871)
add d9daa81c18e [chore](regression) add conf for run ccr cases in regression (#57222)
add 8f22466b6f8 [opt](inverted index) default index format uses v3 (#57140)
add 3eca43c56a3 [Chore](compile) fix compile fail coz base64_encode_part_num (#57225)
add b9bac89b66f [improve](sort) opt heap sort without runtime predicate (#57149)
add 6719ea0f144 [chore](checker) Add skeleton for mvcc meta checker (#57174)
add 5a9cb5aa255 [optimize](function) modify to_base64, from_base64, unhex to avoid a extra copy (#57182)
add 1553bc61dc8 [fix](outfile) fix analysis error when specifying parquet schema (#57232)
add dce846d5576 [Fix](function) Fix wrong nullable of floor/ceil functions (#57238)
add 4a4c2b223de [chore](cloud) Log the large value info (#57237)
add c4193bdb676 [feat](regression) Add dockers to manage multi docker clusters (#57236)
add 72385a5f364 [fix](cloud) Change CloudGlobalTransactionMgr some log to debug level (#57168)
add 6ee5dae4950 [fix](cloud) Fix cloud balance warm up lost idx file (#57114)
add 641b3861ee4 [Fix](job) add thread num config for streaming task exec (#57230)
add 8e919ba0aa1 [fix](olap) Crash caused by out-of-bounds PODArray access (#57239)
add 324fd59bd09 [opt](nereids) print slotReference in explain verbose (#57167)
add 0b20f0313ee [improvement](tvf load)add the data_{1..200}.csv wildcard in tvf load (#56705)
add d739136485c [fix](paimon)Support user-defined S3 config prefixes and unify to HDFS S3A protocol (#57116)
add 95c55bbc217 [fix](case) make case stable (#57040)
add e1884e4d323 [fix](cloud) fix correct pipeline task number calculation in cloud mode (#57129)
add 08a5143b8c1 [opt](arm) Improve BlockBloomFilter performance with NEON intrinsics (#57183)
add 75d45605d59 [opt](rowset) Remote fetch rowsets to avoid -230 error when capturing rowsets (#52995)
add 3c8cc786ba1 [External](parquet) pass non predicates column's offset index to RowGroupReader (#55795)
add 008b51bb157 [Bug](memtracker) fix wrong memtracker scope on PipelineTask destructor (#57276)
add 821aae2e769 [feat](deps) add hadoop 3.4.2 (#57181)
add fad1907e74b [chore](cloud) Add find_derived_instance_ids function (#57274)
add 890c778bacb [feature](maxcompute)support maxcompte catalog read project-schema-table. (#57012)
add 4f00ca1cbad [feature](analyzer) support * as json_object's parameter (#57256)
add c21b125e2ce [feat](load) support merge into (#57044)
add f1faa1a3fdd [opt](var) change merge_io_read_slice_size to merge_io_read_slice_size_bytes (#57272)
add ff322803ac0 [fix](case)drop test_temp_table_with_conn_timeout (#56996)
add eea9d23fdb2 [fix](ut)fix ut AlterMTMVTest (#56997)
add ebc8c679f38 [fix](ann index)Remove redundant omp_set_num_threads (#57224)
add 765c98f771a [fix](array) fix arr funcs with ret which BE should eginore precision because of deduce that the FE planner cannot infer (#57018)
add 84c1966b6d0 [chore](typo)Fix typos in thrift (#57267)
add 26392bfe032 [improvement](storage) Release unused memory earlier, rather than leaving it to the memory pool for guaranteed release (#57215)
add ef01efc780b [chore](regression) Reject getJdbcUrl request in dockers() context (#57251)
add 8df7321cc79 [fix](bin) Remove leading - of the cluster_snapshot arg (#57289)
add 002579e86b0 [fix](binlog) Avoid to issue getVersion RPC during replay journals (#57290)
add 62b00a68fb2 [fix](case) broker load cases compactibility (#56870)
add ca8c12d8f1a [fix](nereids)relase physical plan in Profile.releaseMemory() (#57257)
add b6c001756d9 [chore](cloud) Disable auto snapshot by default (#57285)
add f976ef7921a [fix](cloud)Delete debug log because it reveals too much sensitive information (#57277)
add 58c9a18dd6e [Fix](brpc) Fix wrong usage of brpc call_id (#56825)
add e08e53efb23 [refine](type)use new type dispatch for agg function (#57112)
add ef415d2990e [feature](search) add another two params for search (#57312)
add 5130ee07eec [cases](map) Update test_map_load_and_compaction.groovy for not stable out rowsets in cumative compaction (#56562)
add 77b8f787f53 [fix](regression-test) Add some test for str functions (#57315)
add d9f3044ecfa [fix] remove parse_id (#57240)
add 955d04d13a6 [fix] remove useless const_cast and explain const_cast(3) (#56441)
add e7e2b5bdc31 [fix](cast)Allow cast array, struct and map type to stringlike type. (#57326)
add 3da6abc4ba9 [cases](print) Add print date/datetime/time cases (#57332)
add 0513457e2e0 [p0](to_string) add p0 tests for casting int and decimal to string (#57335)
add 226576b1de6 [Feature](func) Support function mmh64_v2 (#57180)
add 8a6f0f91c3f [opt](variant) deserialize sparse binary data to variant subcolumn (#56977)
add 5b7acb0c583 [opt](cloud) Exposes cloud balance metrics (#57200)
add f66e51738c9 [Feature](func) Support function PERIOD_ADD and PERIOD_DIFF (#56945)
add 71486a2278a [refactor](parquet) Use column predicates to do filtering (#56904)
add a42e7631ea4 [feature](inverted index) query_v2 add regexp, wildcard, phrase query (#57007)
add bd45c50e652 [chore](regression) Support init docker cluster with cluster_snapshot parameter (#57345)
add 870a134e464 [Fix](job) fix drop db error and duplicate fragment_id when retry (#57307)
add fa2d23f9f06 [fix](cast)Support FE to string cast (#57338)
add d513ae49ab8 [Opt](storage) Interger type use plain encoding by default (#57220)
add 1823be98624 [fix](Rollup) When creating a table with rollup and setting a sequence column, we should correctly configure the uniqueId of the sequence column (#56143)
add fba8f6a5b1b [fix](nereids) fix copy into load orc/parquet format fail (#57362)
add aa50f7fb638 [metrics](storage) Add metrics (just bvar) for memtable flushes triggered by different reasons. (#57229)
add 44272386114 [pipeline](conf) make blocking scheduler configurable (#57354)
add c23b9bb15a9 [fix](cloud) Skip tablet report when CloudTabletRebalancer is not inited (#56989)
add 52d3b59ef05 [fix](compile) Fix BE UT (#57391)
add 7babfce839f [test](nerieds)fix sql cache with mtmv test case (#57366)
add 1baeaef9b41 [fix](external catalog) Distinguish between functions used to refresh cache and change catalog properties (#56639)
add cbd56024517 [feat](expression rewrite) simplify expression equals true/false (#56899)
add e29025ea39c [Chore](exchange) add some check about eos with exhcnage source (#57357)
new 462492fe28b support prune nested columns
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (22c43e0b18f)
\
N -- N -- N refs/heads/nested_column_prune (462492fe28b)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/build-thirdparty.yml | 6 +
be/benchmark/benchmark_bit_pack.hpp | 8 +-
be/benchmark/benchmark_block_bloom_filter.hpp | 97 ++
be/benchmark/benchmark_main.cpp | 2 +
be/benchmark/benchmark_string.hpp | 394 ++++++
be/src/cloud/cloud_full_compaction.cpp | 5 +-
be/src/cloud/cloud_internal_service.cpp | 88 +-
be/src/cloud/cloud_schema_change_job.cpp | 18 +-
be/src/cloud/cloud_snapshot_mgr.cpp | 2 +-
be/src/cloud/cloud_tablet.cpp | 236 ++--
be/src/cloud/cloud_tablet.h | 62 +-
be/src/cloud/cloud_tablet_mgr.cpp | 17 +-
be/src/cloud/cloud_tablet_mgr.h | 2 +-
be/src/cloud/cloud_warm_up_manager.cpp | 29 +-
be/src/cloud/cloud_warm_up_manager.h | 7 -
be/src/common/compare.h | 12 +
be/src/common/config.cpp | 43 +-
be/src/common/config.h | 19 +-
be/src/common/version_internal.cpp | 3 +
be/src/common/version_internal.h | 2 +
be/src/exec/schema_scanner.cpp | 3 +
.../schema_cluster_snapshots_scanner.cpp | 19 +-
.../schema_column_data_sizes_scanner.cpp | 363 +++++
.../schema_column_data_sizes_scanner.h | 71 +
be/src/exprs/block_bloom_filter.hpp | 109 +-
be/src/exprs/block_bloom_filter_avx_impl.cc | 8 +-
be/src/exprs/block_bloom_filter_impl.cc | 58 +-
be/src/http/action/check_encryption_action.cpp | 80 +-
be/src/http/action/file_cache_action.cpp | 28 +-
be/src/http/action/meta_action.cpp | 2 +-
be/src/io/cache/block_file_cache.cpp | 160 ++-
be/src/io/cache/block_file_cache.h | 5 +
be/src/io/cache/block_file_cache_downloader.cpp | 16 +-
be/src/io/cache/block_file_cache_profile.cpp | 3 +
be/src/io/cache/block_file_cache_profile.h | 1 +
be/src/io/cache/cached_remote_file_reader.cpp | 8 +-
be/src/io/cache/file_cache_common.cpp | 41 +-
be/src/io/cache/file_cache_common.h | 48 +-
be/src/io/cache/file_cache_storage.h | 7 +
be/src/io/cache/fs_file_cache_storage.cpp | 69 +-
be/src/io/cache/fs_file_cache_storage.h | 3 +
be/src/io/file_factory.h | 2 +
be/src/io/fs/azure_obj_storage_client.cpp | 6 +-
be/src/io/fs/hdfs_file_reader.cpp | 10 +
be/src/io/io_common.h | 1 +
be/src/olap/base_tablet.cpp | 57 +-
be/src/olap/base_tablet.h | 90 +-
be/src/olap/block_column_predicate.h | 29 +
be/src/olap/column_predicate.h | 10 +
be/src/olap/comparison_predicate.h | 47 +
be/src/olap/full_compaction.cpp | 5 +-
be/src/olap/in_list_predicate.h | 36 +
be/src/olap/inverted_index_parser.cpp | 16 +-
be/src/olap/inverted_index_parser.h | 2 +
be/src/olap/iterators.h | 24 +-
be/src/olap/memtable.cpp | 38 +-
be/src/olap/memtable.h | 6 +
be/src/olap/memtable_memory_limiter.cpp | 11 +-
be/src/olap/memtable_writer.cpp | 26 +-
be/src/olap/memtable_writer.h | 2 +
be/src/olap/merger.cpp | 8 +-
be/src/olap/null_predicate.h | 11 +
be/src/olap/olap_common.h | 4 +
be/src/olap/olap_server.cpp | 2 +-
be/src/olap/parallel_scanner_builder.cpp | 100 +-
be/src/olap/parallel_scanner_builder.h | 13 +-
be/src/olap/predicate_creator.h | 37 +-
be/src/olap/primary_key_index.cpp | 2 +-
be/src/olap/push_handler.cpp | 9 +-
be/src/olap/rowset/beta_rowset_reader.cpp | 16 +
be/src/olap/rowset/rowset_reader_context.h | 4 +-
be/src/olap/rowset/rowset_writer_context.h | 4 +-
.../olap/rowset/segment_v2/ann_index/ann_index.h | 14 +-
.../segment_v2/ann_index/ann_index_writer.cpp | 4 +-
.../rowset/segment_v2/ann_index/ann_index_writer.h | 2 +
.../segment_v2/ann_index/faiss_ann_index.cpp | 139 +-
.../rowset/segment_v2/ann_index/faiss_ann_index.h | 62 +-
be/src/olap/rowset/segment_v2/binary_dict_page.cpp | 79 +-
be/src/olap/rowset/segment_v2/binary_dict_page.h | 18 +-
be/src/olap/rowset/segment_v2/binary_plain_page.h | 12 +-
...{binary_plain_page.h => binary_plain_page_v2.h} | 261 ++--
.../olap/rowset/segment_v2/binary_prefix_page.cpp | 1 +
be/src/olap/rowset/segment_v2/binary_prefix_page.h | 4 +
.../olap/rowset/segment_v2/bitmap_index_writer.cpp | 4 +-
be/src/olap/rowset/segment_v2/bitshuffle_page.h | 5 +
be/src/olap/rowset/segment_v2/bloom_filter.h | 2 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 19 +-
be/src/olap/rowset/segment_v2/column_writer.cpp | 35 +-
be/src/olap/rowset/segment_v2/column_writer.h | 119 +-
be/src/olap/rowset/segment_v2/condition_cache.cpp | 52 +
be/src/olap/rowset/segment_v2/condition_cache.h | 135 ++
be/src/olap/rowset/segment_v2/encoding_info.cpp | 223 ++-
be/src/olap/rowset/segment_v2/encoding_info.h | 69 +-
.../rowset/segment_v2/frame_of_reference_page.h | 5 +
.../rowset/segment_v2/indexed_column_reader.cpp | 2 +-
.../rowset/segment_v2/indexed_column_writer.cpp | 2 +-
.../inverted_index/analysis_factory_mgr.cpp | 61 +-
.../inverted_index/analysis_factory_mgr.h | 8 +-
.../inverted_index/analyzer/analyzer.cpp | 23 +-
.../segment_v2/inverted_index/analyzer/analyzer.h | 5 +-
.../inverted_index/analyzer/basic/basic_analyzer.h | 47 +-
.../inverted_index/analyzer/custom_analyzer.cpp | 51 +-
.../inverted_index/analyzer/custom_analyzer.h | 28 +-
.../analyzer/custom_analyzer_config.cpp | 10 +
.../analyzer/custom_analyzer_config.h | 4 +
.../inverted_index/analyzer/icu/icu_analyzer.h | 49 +-
.../tokenizer.h => char_filter/char_filter.h} | 41 +-
.../char_filter/char_filter_factory.h | 24 +-
.../char_filter/char_replace_char_filter.cpp | 26 +-
.../char_filter/char_replace_char_filter.h | 16 +-
.../char_filter/char_replace_char_filter_factory.h | 75 ++
.../char_filter/empty_char_filter_factory.h | 61 +
.../bit_set_query.h} | 14 +-
.../bit_set_scorer.h} | 16 +-
.../bit_set_weight.h} | 13 +-
.../query_v2/boolean_query/boolean_weight.h | 4 +-
.../const_score_scorer.h} | 30 +-
.../segment_v2/inverted_index/query_v2/doc_set.h | 74 +
.../inverted_index/query_v2/intersection.cpp | 175 +++
.../inverted_index/query_v2/intersection.h | 59 +
.../query_v2/phrase_query/phrase_query.h | 54 +
.../query_v2/phrase_query/phrase_scorer.cpp | 188 +++
.../query_v2/phrase_query/phrase_scorer.h | 115 ++
.../query_v2/phrase_query/phrase_weight.h | 81 ++
.../query_v2/phrase_query/postings_with_offset.h | 54 +
.../bitmap_query.h => regexp_query/regexp_query.h} | 31 +-
.../query_v2/regexp_query/regexp_weight.cpp | 223 +++
.../regexp_weight.h} | 40 +-
.../inverted_index/query_v2/segment_postings.h | 40 +-
.../query_v2/term_query/term_scorer.h | 10 +-
.../query_v2/term_query/term_weight.h | 21 +-
.../segment_v2/inverted_index/query_v2/weight.h | 22 +-
.../wildcard_query.h} | 31 +-
.../query_v2/wildcard_query/wildcard_weight.h | 62 +
.../rowset/segment_v2/inverted_index/setting.h | 61 +-
.../inverted_index/similarity/bm25_similarity.cpp | 23 +
.../inverted_index/similarity/bm25_similarity.h | 2 +
.../inverted_index/similarity/similarity.h | 2 +
.../empty_token_filter_factory.h} | 43 +-
.../inverted_index/token_filter/token_filter.h | 2 -
.../token_filter/word_delimiter_filter_factory.h | 2 +
.../segment_v2/inverted_index/token_stream.h | 37 +-
.../basic/basic_tokenizer.cpp | 43 +-
.../basic/basic_tokenizer.h | 26 +-
.../tokenizer/basic/basic_tokenizer_factory.h | 50 +
.../tokenizer/empty/empty_tokenizer_factory.h | 72 +
.../icu/break_iterator_wrapper.cpp | 4 +-
.../icu/break_iterator_wrapper.h | 4 +-
.../icu/composite_break_iterator.cpp | 4 +-
.../icu/composite_break_iterator.h | 4 +-
.../icu/default_icu_tokenizer_config.cpp | 4 +-
.../icu/default_icu_tokenizer_config.h | 4 +-
.../{analyzer => tokenizer}/icu/icu_common.h | 4 +-
.../{analyzer => tokenizer}/icu/icu_tokenizer.cpp | 18 +-
.../{analyzer => tokenizer}/icu/icu_tokenizer.h | 13 +-
.../icu/icu_tokenizer_config.h | 4 +-
.../icu/icu_tokenizer_factory.h} | 21 +-
.../icu/script_iterator.cpp | 4 +-
.../{analyzer => tokenizer}/icu/script_iterator.h | 4 +-
.../tokenizer/standard/standard_tokenizer_impl.h | 7 +-
.../inverted_index/tokenizer/tokenizer.h | 8 +-
.../segment_v2/inverted_index/util/reader.h} | 14 +-
.../segment_v2/inverted_index_fs_directory.cpp | 22 +-
.../segment_v2/inverted_index_fs_directory.h | 8 +-
.../rowset/segment_v2/inverted_index_writer.cpp | 21 +-
.../olap/rowset/segment_v2/inverted_index_writer.h | 8 +-
be/src/olap/rowset/segment_v2/ngram_bloom_filter.h | 4 +-
be/src/olap/rowset/segment_v2/page_builder.h | 13 +
be/src/olap/rowset/segment_v2/page_decoder.h | 4 +
be/src/olap/rowset/segment_v2/page_io.cpp | 1 +
be/src/olap/rowset/segment_v2/plain_page.h | 73 +-
be/src/olap/rowset/segment_v2/rle_page.h | 5 +
be/src/olap/rowset/segment_v2/row_ranges.h | 14 +
be/src/olap/rowset/segment_v2/segment.cpp | 9 +
be/src/olap/rowset/segment_v2/segment.h | 2 +
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 127 +-
be/src/olap/rowset/segment_v2/segment_iterator.h | 11 +-
be/src/olap/rowset/segment_v2/segment_writer.cpp | 18 +-
be/src/olap/rowset/segment_v2/segment_writer.h | 2 +-
.../variant/hierarchical_data_iterator.cpp | 13 +-
.../variant/variant_column_writer_impl.cpp | 2 +-
.../rowset/segment_v2/vertical_segment_writer.cpp | 18 +-
.../rowset/segment_v2/vertical_segment_writer.h | 2 +-
be/src/olap/rowset/segment_v2/zone_map_index.cpp | 2 +-
be/src/olap/rowset_builder.cpp | 1 +
be/src/olap/rowset_version_mgr.cpp | 449 +++++++
be/src/olap/schema_change.cpp | 17 +-
be/src/olap/snapshot_manager.cpp | 21 +-
be/src/olap/tablet.cpp | 99 +-
be/src/olap/tablet.h | 13 +-
be/src/olap/tablet_meta.cpp | 57 +-
be/src/olap/tablet_meta.h | 14 +-
be/src/olap/tablet_reader.cpp | 16 +-
be/src/olap/tablet_reader.h | 18 +-
be/src/olap/tablet_schema.cpp | 115 +-
be/src/olap/tablet_schema.h | 1 +
be/src/olap/task/engine_checksum_task.cpp | 12 +-
be/src/olap/task/engine_storage_migration_task.cpp | 9 +-
be/src/pipeline/dependency.cpp | 4 +-
be/src/pipeline/exec/aggregation_sink_operator.cpp | 19 +-
be/src/pipeline/exec/aggregation_sink_operator.h | 7 +-
be/src/pipeline/exec/analytic_sink_operator.h | 2 +-
be/src/pipeline/exec/assert_num_rows_operator.h | 2 +-
be/src/pipeline/exec/blackhole_sink_operator.cpp | 112 ++
be/src/pipeline/exec/blackhole_sink_operator.h | 90 ++
.../exec/distinct_streaming_aggregation_operator.h | 4 +-
be/src/pipeline/exec/exchange_source_operator.cpp | 2 +-
be/src/pipeline/exec/exchange_source_operator.h | 2 +-
be/src/pipeline/exec/hashjoin_build_sink.h | 2 +-
be/src/pipeline/exec/hashjoin_probe_operator.h | 2 +-
.../exec/nested_loop_join_build_operator.h | 2 +-
.../exec/nested_loop_join_probe_operator.h | 2 +-
be/src/pipeline/exec/olap_scan_operator.cpp | 23 +-
be/src/pipeline/exec/olap_scan_operator.h | 6 +-
be/src/pipeline/exec/operator.cpp | 8 +-
be/src/pipeline/exec/operator.h | 5 +-
.../pipeline/exec/partition_sort_sink_operator.h | 2 +-
.../exec/partitioned_aggregation_sink_operator.h | 4 +-
.../exec/partitioned_hash_join_probe_operator.h | 2 +-
.../exec/partitioned_hash_join_sink_operator.h | 2 +-
be/src/pipeline/exec/result_sink_operator.cpp | 18 +-
be/src/pipeline/exec/result_sink_operator.h | 4 +-
be/src/pipeline/exec/scan_operator.cpp | 20 +-
be/src/pipeline/exec/scan_operator.h | 15 +-
be/src/pipeline/exec/set_probe_sink_operator.h | 2 +-
be/src/pipeline/exec/set_sink_operator.h | 2 +-
be/src/pipeline/exec/sort_sink_operator.cpp | 2 +-
be/src/pipeline/exec/sort_sink_operator.h | 2 +-
be/src/pipeline/exec/spill_sort_sink_operator.h | 4 +-
be/src/pipeline/exec/spill_utils.h | 4 +-
.../exec/streaming_aggregation_operator.cpp | 20 +-
.../pipeline/exec/streaming_aggregation_operator.h | 16 +-
be/src/pipeline/exec/table_function_operator.h | 2 +-
be/src/pipeline/exec/union_sink_operator.h | 4 +-
be/src/pipeline/exec/union_source_operator.h | 2 +-
be/src/pipeline/pipeline.h | 4 +-
be/src/pipeline/pipeline_fragment_context.cpp | 39 +-
be/src/pipeline/pipeline_task.cpp | 103 +-
be/src/pipeline/pipeline_task.h | 44 +-
be/src/pipeline/revokable_task.h | 76 ++
be/src/pipeline/task_scheduler.cpp | 11 +-
be/src/pipeline/task_scheduler.h | 7 +-
be/src/runtime/exec_env.h | 8 +
be/src/runtime/exec_env_init.cpp | 16 +-
be/src/runtime/fragment_mgr.cpp | 7 +
be/src/runtime/index_policy/index_policy_mgr.cpp | 84 +-
be/src/runtime/index_policy/index_policy_mgr.h | 7 +
be/src/runtime/memory/cache_policy.h | 5 +
be/src/runtime/query_context.cpp | 3 +-
be/src/runtime/query_context.h | 6 +
be/src/runtime/runtime_query_statistics_mgr.cpp | 20 +-
be/src/runtime/runtime_state.cpp | 5 +-
be/src/runtime/runtime_state.h | 2 +
be/src/runtime/workload_group/workload_group.cpp | 7 +
be/src/runtime/workload_group/workload_group.h | 1 +
be/src/runtime/workload_management/io_context.h | 10 +
.../workload_management/resource_context.cpp | 1 +
be/src/service/arrow_flight/flight_sql_service.cpp | 8 +-
be/src/service/internal_service.cpp | 96 +-
be/src/service/internal_service.h | 5 +
be/src/tools/meta_tool.cpp | 2 +-
be/src/util/brpc_closure.h | 6 +-
be/src/util/debug_util.cpp | 4 +
be/src/util/doris_metrics.cpp | 11 +
be/src/util/doris_metrics.h | 5 +
be/src/util/hash_util.hpp | 11 +
be/src/util/jsonb_document.cpp | 20 +
be/src/util/jsonb_document.h | 23 +-
be/src/util/murmur_hash3.cpp | 34 +-
be/src/util/murmur_hash3.h | 4 +
be/src/util/mysql_row_buffer.cpp | 11 +-
be/src/util/mysql_row_buffer.h | 7 +-
be/src/util/s3_util.cpp | 51 +-
be/src/util/s3_util.h | 4 +
be/src/util/thrift_rpc_helper.h | 2 +-
be/src/util/timezone_utils.cpp | 21 +-
be/src/util/uid_util.cpp | 23 -
be/src/util/uid_util.h | 5 -
.../aggregate_function_array_agg.cpp | 74 +-
.../aggregate_function_collect.cpp | 79 +-
.../aggregate_functions/aggregate_function_map.cpp | 59 +-
be/src/vec/columns/column.cpp | 41 +
be/src/vec/columns/column.h | 15 +-
be/src/vec/columns/column_array.cpp | 4 +-
be/src/vec/columns/column_array.h | 4 +-
be/src/vec/columns/column_const.h | 9 +-
be/src/vec/columns/column_decimal.cpp | 48 +-
be/src/vec/columns/column_decimal.h | 8 +-
be/src/vec/columns/column_fixed_length_object.h | 4 +-
be/src/vec/columns/column_map.cpp | 6 +-
be/src/vec/columns/column_map.h | 4 +-
be/src/vec/columns/column_nullable.cpp | 36 +-
be/src/vec/columns/column_nullable.h | 4 +-
be/src/vec/columns/column_string.cpp | 47 +-
be/src/vec/columns/column_string.h | 9 +-
be/src/vec/columns/column_struct.cpp | 4 +-
be/src/vec/columns/column_struct.h | 4 +-
be/src/vec/columns/column_varbinary.h | 8 +
be/src/vec/columns/column_variant.cpp | 274 +---
be/src/vec/columns/column_variant.h | 23 +-
be/src/vec/columns/column_vector.cpp | 61 +-
be/src/vec/columns/column_vector.h | 8 +-
be/src/vec/common/hash_table/hash_map_context.h | 4 +-
be/src/vec/common/pod_array.h | 5 +
be/src/vec/common/sort/heap_sorter.cpp | 64 +-
be/src/vec/common/sort/heap_sorter.h | 13 +-
be/src/vec/common/sort/sorter.cpp | 21 +-
be/src/vec/common/sort/sorter.h | 2 +-
be/src/vec/common/string_view.h | 2 +
be/src/vec/core/accurate_comparison.h | 303 +----
be/src/vec/core/call_on_type_index.h | 250 ++--
be/src/vec/core/decimal_comparison.h | 7 +-
be/src/vec/core/field.cpp | 332 ++++-
be/src/vec/core/field.h | 8 +-
be/src/vec/core/sort_cursor.h | 5 +
be/src/vec/data_types/data_type_decimal.cpp | 16 +-
be/src/vec/data_types/data_type_decimal.h | 195 +--
be/src/vec/data_types/data_type_factory.cpp | 4 +-
.../vec/data_types/serde/data_type_array_serde.cpp | 40 +
.../vec/data_types/serde/data_type_array_serde.h | 5 +
.../data_types/serde/data_type_decimal_serde.cpp | 62 +
.../vec/data_types/serde/data_type_decimal_serde.h | 5 +
.../vec/data_types/serde/data_type_jsonb_serde.cpp | 20 +
.../vec/data_types/serde/data_type_jsonb_serde.h | 5 +
.../data_types/serde/data_type_number_serde.cpp | 111 ++
.../vec/data_types/serde/data_type_number_serde.h | 6 +
be/src/vec/data_types/serde/data_type_serde.cpp | 128 ++
be/src/vec/data_types/serde/data_type_serde.h | 37 +
.../vec/data_types/serde/data_type_string_serde.h | 19 +
be/src/vec/exec/format/column_type_convert.cpp | 3 +-
be/src/vec/exec/format/csv/csv_reader.cpp | 9 +-
be/src/vec/exec/format/generic_reader.cpp | 252 ++++
be/src/vec/exec/format/generic_reader.h | 26 +
be/src/vec/exec/format/json/new_json_reader.cpp | 16 +-
be/src/vec/exec/format/orc/vorc_reader.cpp | 16 +-
be/src/vec/exec/format/orc/vorc_reader.h | 10 +-
.../format/parquet/byte_stream_split_decoder.cpp | 2 +
.../format/parquet/fix_length_dict_decoder.hpp | 2 +
.../exec/format/parquet/fix_length_plain_decoder.h | 2 +
.../exec/format/parquet/parquet_column_convert.cpp | 3 +-
.../exec/format/parquet/parquet_column_convert.h | 8 +-
be/src/vec/exec/format/parquet/parquet_pred_cmp.h | 64 +-
be/src/vec/exec/format/parquet/schema_desc.cpp | 2 +-
be/src/vec/exec/format/parquet/schema_desc.h | 4 +-
.../parquet/vparquet_column_chunk_reader.cpp | 2 +-
.../format/parquet/vparquet_column_chunk_reader.h | 2 +-
.../exec/format/parquet/vparquet_column_reader.cpp | 35 +-
.../exec/format/parquet/vparquet_column_reader.h | 24 +-
.../exec/format/parquet/vparquet_file_metadata.h | 1 +
.../exec/format/parquet/vparquet_group_reader.cpp | 7 +-
.../exec/format/parquet/vparquet_group_reader.h | 4 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 299 ++---
be/src/vec/exec/format/parquet/vparquet_reader.h | 25 +-
be/src/vec/exec/format/table/iceberg_reader.cpp | 14 +-
be/src/vec/exec/format/table/iceberg_reader.h | 3 +-
.../vec/exec/format/table/lakesoul_jni_reader.cpp | 3 +
be/src/vec/exec/format/table/lakesoul_jni_reader.h | 3 +
be/src/vec/exec/jni_connector.cpp | 28 +-
be/src/vec/exec/jni_connector.h | 8 +-
be/src/vec/exec/scan/file_scanner.cpp | 28 +-
be/src/vec/exec/scan/file_scanner.h | 1 +
be/src/vec/exec/scan/olap_scanner.cpp | 54 +-
be/src/vec/exec/scan/olap_scanner.h | 2 +-
be/src/vec/exprs/table_function/vexplode_v2.cpp | 75 +-
be/src/vec/exprs/vbitmap_predicate.h | 3 +
be/src/vec/exprs/vbloom_predicate.cpp | 7 +
be/src/vec/exprs/vbloom_predicate.h | 2 +
be/src/vec/exprs/vdirect_in_predicate.h | 2 +
be/src/vec/exprs/vexpr.cpp | 19 +
be/src/vec/exprs/vexpr.h | 2 +
be/src/vec/exprs/vexpr_context.cpp | 5 +-
be/src/vec/exprs/vexpr_context.h | 2 +
be/src/vec/exprs/vin_predicate.h | 2 +
be/src/vec/exprs/vlambda_function_call_expr.h | 2 +
be/src/vec/exprs/vlambda_function_expr.h | 2 +
be/src/vec/exprs/vliteral.cpp | 5 +
be/src/vec/exprs/vliteral.h | 2 +
be/src/vec/exprs/vruntimefilter_wrapper.h | 2 +
be/src/vec/exprs/vsearch.cpp | 87 +-
be/src/vec/exprs/vsearch.h | 2 +
be/src/vec/exprs/vslot_ref.cpp | 9 +
be/src/vec/exprs/vslot_ref.h | 3 +
be/src/vec/functions/ai/ai_functions.h | 6 +-
be/src/vec/functions/array/function_array_binary.h | 2 +-
.../array/function_array_contains_all.cpp | 133 +-
.../vec/functions/array/function_array_distinct.h | 76 +-
.../vec/functions/array/function_array_element.h | 104 +-
.../array/function_array_enumerate_uniq.cpp | 78 +-
be/src/vec/functions/array/function_array_index.h | 76 +-
be/src/vec/functions/array/function_array_remove.h | 76 +-
.../vec/functions/array/function_arrays_overlap.h | 122 +-
.../functions/cast/cast_to_basic_number_common.h | 9 +-
be/src/vec/functions/cast/cast_to_variant.h | 3 +-
be/src/vec/functions/date_time_transforms.h | 15 +-
be/src/vec/functions/function.h | 1 +
be/src/vec/functions/function_coalesce.cpp | 94 +-
.../function_date_or_datetime_computation.cpp | 5 +
.../function_date_or_datetime_computation.h | 201 +++
.../function_date_or_datetime_to_string.cpp | 78 +-
.../vec/functions/function_datetime_floor_ceil.cpp | 2 +-
be/src/vec/functions/function_fake.cpp | 6 +-
be/src/vec/functions/function_hash.cpp | 41 +-
be/src/vec/functions/function_hash.h | 10 +-
be/src/vec/functions/function_json_hash.cpp | 104 ++
be/src/vec/functions/function_jsonb_transform.cpp | 182 +++
be/src/vec/functions/function_map.cpp | 128 +-
.../vec/functions/function_needs_to_handle_null.h | 154 +++
.../vec/functions/function_other_types_to_date.cpp | 2 +-
be/src/vec/functions/function_search.cpp | 85 +-
be/src/vec/functions/function_search.h | 24 +-
be/src/vec/functions/function_string.cpp | 147 +-
be/src/vec/functions/function_string.h | 282 ++--
be/src/vec/functions/function_tokenize.cpp | 4 +-
be/src/vec/functions/function_totype.h | 81 +-
be/src/vec/functions/function_varbinary.cpp | 136 +-
be/src/vec/functions/function_varbinary.h | 53 +
be/src/vec/functions/function_variant_element.cpp | 5 +-
be/src/vec/functions/functions_comparison.h | 105 +-
be/src/vec/functions/if.cpp | 68 +-
be/src/vec/functions/if.h | 39 +-
be/src/vec/functions/is_null.h | 23 +-
be/src/vec/functions/least_greast.cpp | 241 +---
be/src/vec/functions/match.cpp | 6 +-
be/src/vec/functions/simple_function_factory.h | 4 +
be/src/vec/olap/vcollect_iterator.cpp | 2 +-
be/src/vec/runtime/vdata_stream_recvr.cpp | 10 +-
be/src/vec/runtime/vdatetime_value.cpp | 34 +
be/src/vec/runtime/vdatetime_value.h | 4 +
be/src/vec/runtime/vparquet_transformer.cpp | 21 -
be/src/vec/runtime/vparquet_transformer.h | 17 +-
be/src/vec/sink/vtablet_block_convertor.cpp | 156 ++-
be/src/vec/utils/varbinaryop_subbinary.h | 119 ++
be/test/CMakeLists.txt | 1 -
be/test/cloud/cloud_meta_mgr_test.cpp | 135 +-
.../cloud/cloud_tablet_query_prefer_cache_test.cpp | 14 +-
.../cloud_tablet_query_with_tolerance_test.cpp | 12 +-
be/test/cloud/cloud_tablet_test.cpp | 513 +++++--
.../common/compare_test.cpp} | 43 +-
be/test/http/file_cache_action_test.cpp | 163 +++
be/test/io/cache/block_file_cache_test.cpp | 149 ++-
be/test/io/s3_client_factory_test.cpp | 106 ++
be/test/olap/block_column_predicate_test.cpp | 1006 ++++++++++++++
be/test/olap/delete_handler_test.cpp | 9 +-
be/test/olap/index_builder_test.cpp | 6 +-
.../rowset/segment_v2/binary_dict_page_test.cpp | 868 ++++++++++--
.../segment_v2/binary_plain_page_v2_test.cpp | 576 ++++++++
.../rowset/segment_v2/block_bloom_filter_test.cpp | 2 +-
.../olap/rowset/segment_v2/encoding_info_test.cpp | 78 +-
.../inverted_index/analysis_factory_mgr_test.cpp | 249 ++++
.../inverted_index/analyzer/icu_analyzer_test.cpp | 59 +-
.../analyzer/simple_analyzer_test.cpp | 8 +-
.../ananlyzer/custom_analyzer_test.cpp | 328 +++--
.../char_filter/char_filter_test.cpp | 76 ++
.../char_replace_char_filter_factory_test.cpp | 193 +++
.../inverted_index/query/query_helper_test.cpp | 4 +
.../inverted_index/query_v2/boolean_query_test.cpp | 9 +-
.../inverted_index/query_v2/intersection_test.cpp | 425 ++++++
.../inverted_index/query_v2/phrase_query_test.cpp | 558 ++++++++
.../inverted_index/query_v2/regexp_query_test.cpp | 392 ++++++
.../query_v2/wildcard_query_test.cpp | 527 ++++++++
.../segment_v2/inverted_index/setting_test.cpp | 95 +-
.../ascii_folding_filter_factory_test.cpp | 6 +-
.../lower_case_filter_factory_test.cpp | 6 +-
.../token_filter/word_delimiter_filter_test.cpp | 6 +-
.../tokenizer/basic_tokenizer_factory_test.cpp | 235 ++++
.../char_group_tokenizer_factory_test.cpp | 6 +-
.../tokenizer/edge_ngram_tokenizer_test.cpp | 6 +-
.../tokenizer/icu_tokenizer_factory_test.cpp | 212 +++
.../tokenizer/keyword_analyzer_test.cpp | 6 +-
.../tokenizer/ngram_tokenizer_test.cpp | 6 +-
.../tokenizer/standard_tokenizer_factory_test.cpp | 6 +-
.../segment_v2/inverted_index/util/reader_test.cpp | 92 ++
be/test/olap/segcompaction_mow_test.cpp | 4 +-
be/test/olap/segcompaction_test.cpp | 1 +
be/test/olap/tablet_meta_test.cpp | 10 +-
.../operator/hashjoin_probe_operator_test.cpp | 6 +-
.../operator/spill_sort_sink_operator_test.cpp | 5 +-
be/test/pipeline/pipeline_task_test.cpp | 33 -
be/test/pipeline/pipeline_test.cpp | 46 +-
be/test/util/mysql_row_buffer_test.cpp | 29 +
.../deserialize_from_sparse_column_test.bin | Bin 0 -> 410 bytes
.../columns/column_fixed_length_object_test.cpp | 22 +-
.../column_nullable_seriazlization_test.cpp | 72 +-
be/test/vec/columns/column_variant_test.cpp | 125 +-
be/test/vec/columns/common_column_test.h | 24 +-
be/test/vec/core/accurate_comparison_test.cpp | 63 +-
be/test/vec/core/field_test.cpp | 88 ++
.../vec/data_types/serde/data_type_serde_test.cpp | 376 ++++++
.../vec/exec/format/parquet/parquet_expr_test.cpp | 70 +-
be/test/vec/function/cast/cast_to_float_double.cpp | 57 +-
be/test/vec/function/function_hash_test.cpp | 86 +-
be/test/vec/function/function_is_null_test.cpp | 147 ++
be/test/vec/function/function_num_if_test.cpp | 6 +-
be/test/vec/function/function_string_test.cpp | 87 +-
be/test/vec/function/function_test_util.cpp | 4 +-
be/test/vec/function/function_test_util.h | 32 +-
be/test/vec/function/function_varbinary_test.cpp | 426 ++++++
be/test/vec/function/table_function_test.cpp | 46 +-
bin/start_fe.sh | 2 +-
build.sh | 17 +-
cloud/script/run_all_tests.sh | 9 +-
cloud/src/common/config.h | 9 +
cloud/src/common/stopwatch.h | 4 +
cloud/src/main.cpp | 6 +-
cloud/src/meta-service/meta_service_helper.h | 2 +-
cloud/src/meta-service/meta_service_http.cpp | 34 +-
cloud/src/meta-service/meta_service_job.cpp | 23 -
cloud/src/meta-service/meta_service_resource.cpp | 37 +-
.../src/meta-service/meta_service_tablet_stats.cpp | 23 +
cloud/src/meta-service/meta_service_tablet_stats.h | 3 +
cloud/src/meta-store/keys.cpp | 332 ++++-
cloud/src/meta-store/keys.h | 163 ++-
cloud/src/meta-store/meta_reader.cpp | 197 +--
cloud/src/meta-store/meta_reader.h | 27 +-
cloud/src/meta-store/txn_kv.cpp | 51 +-
cloud/src/recycler/azure_obj_client.cpp | 7 +-
cloud/src/recycler/checker.cpp | 2 +-
cloud/src/recycler/checker.h | 4 +-
cloud/src/recycler/checker_snapshot.cpp | 2 +-
cloud/src/recycler/meta_checker.cpp | 526 ++++----
cloud/src/recycler/meta_checker.h | 81 +-
cloud/src/recycler/recycler.cpp | 52 +-
cloud/src/recycler/recycler.h | 4 +
cloud/src/recycler/recycler_service.cpp | 5 +-
cloud/src/recycler/s3_accessor.cpp | 44 +-
cloud/src/recycler/s3_accessor.h | 6 +
cloud/src/recycler/snapshot_chain_compactor.cpp | 323 +++++
cloud/src/recycler/snapshot_chain_compactor.h | 111 ++
cloud/src/recycler/snapshot_data_migrator.cpp | 362 +++++
cloud/src/recycler/snapshot_data_migrator.h | 132 ++
cloud/src/snapshot/snapshot_manager.cpp | 14 +
cloud/src/snapshot/snapshot_manager.h | 13 +
cloud/test/codec_test.cpp | 48 +-
cloud/test/keys_test.cpp | 350 ++++-
cloud/test/meta_reader_test.cpp | 171 +++
cloud/test/meta_server_test.cpp | 6 +-
cloud/test/meta_service_test.cpp | 2 +-
.../cpp/custom_aws_credentials_provider_chain.cpp | 103 ++
.../cpp/custom_aws_credentials_provider_chain.h | 15 +-
conf/be.conf | 2 +-
docker/compilation/Dockerfile | 10 +-
.../runtime/be/resource/be_disaggregated_probe.sh | 37 +-
docker/runtime/doris-compose/Readme.md | 70 +
docker/runtime/doris-compose/cluster.py | 109 +-
docker/runtime/doris-compose/command.py | 127 +-
docker/runtime/doris-compose/resource/common.sh | 78 ++
docker/runtime/doris-compose/resource/init_be.sh | 6 +-
.../runtime/doris-compose/resource/init_cloud.sh | 35 +-
docker/runtime/doris-compose/resource/init_fe.sh | 74 +-
docker/runtime/doris-compose/utils.py | 6 +-
.../runtime/fe/resource/fe_disaggregated_probe.sh | 4 +-
fe/be-java-extensions/lakesoul-scanner/pom.xml | 1 +
.../apache/doris/lakesoul/LakeSoulJniScanner.java | 4 +
.../org/apache/doris/lakesoul/LakeSoulUtils.java | 4 +
.../apache/doris/lakesoul/arrow/ArrowUtils.java | 4 +
.../lakesoul/arrow/LakeSoulArrowJniScanner.java | 4 +
fe/be-java-extensions/pom.xml | 3 +-
.../main/java/org/apache/doris/common/Config.java | 21 +-
fe/fe-core/pom.xml | 9 +-
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 2 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 28 +
.../org/apache/doris/nereids/search/SearchLexer.g4 | 1 +
.../apache/doris/nereids/search/SearchParser.g4 | 7 +-
.../src/main/java/org/apache/doris/DorisFE.java | 5 +
.../main/java/org/apache/doris/alter/Alter.java | 9 +-
.../apache/doris/alter/SchemaChangeHandler.java | 1 +
.../org/apache/doris/analysis/AddColumnClause.java | 8 +-
.../apache/doris/analysis/AlterTableClause.java | 7 +-
.../doris/analysis/AnnIndexPropertiesChecker.java | 54 +-
.../org/apache/doris/analysis/BaseTableRef.java | 69 -
.../java/org/apache/doris/analysis/BrokerDesc.java | 2 +-
.../apache/doris/analysis/BuildIndexClause.java | 10 +-
.../apache/doris/analysis/ChannelDescription.java | 4 +-
.../org/apache/doris/analysis/CopyFromParam.java | 15 +-
.../apache/doris/analysis/CreateIndexClause.java | 17 +-
.../org/apache/doris/analysis/DataDescription.java | 44 +-
.../org/apache/doris/analysis/DropIndexClause.java | 13 +-
.../java/org/apache/doris/analysis/IndexDef.java | 5 +-
.../org/apache/doris/analysis/InsertTarget.java | 17 +-
.../apache/doris/analysis/InvertedIndexUtil.java | 21 +-
.../org/apache/doris/analysis/LateralViewRef.java | 102 --
.../java/org/apache/doris/analysis/LockTable.java | 10 +-
.../apache/doris/analysis/ModifyColumnClause.java | 8 +-
.../analysis/ModifyTablePropertiesClause.java | 14 +-
.../org/apache/doris/analysis/OutFileClause.java | 17 +-
.../apache/doris/analysis/PartitionExprUtil.java | 28 +-
.../org/apache/doris/analysis/PartitionNames.java | 142 --
.../org/apache/doris/analysis/SchemaTableType.java | 6 +-
.../org/apache/doris/analysis/SearchPredicate.java | 24 +-
.../org/apache/doris/analysis/SlotDescriptor.java | 41 +-
.../java/org/apache/doris/analysis/SlotRef.java | 53 +-
.../java/org/apache/doris/analysis/StmtType.java | 1 +
.../org/apache/doris/analysis/StorageBackend.java | 25 +-
.../java/org/apache/doris/analysis/TableName.java | 195 ---
.../java/org/apache/doris/analysis/TableRef.java | 656 ---------
.../org/apache/doris/analysis/TableSnapshot.java | 8 +
.../doris/analysis/TableValuedFunctionRef.java | 126 --
.../org/apache/doris/analysis/TupleDescriptor.java | 5 +-
.../java/org/apache/doris/backup/Repository.java | 17 +-
.../apache/doris/binlog/AddPartitionRecord.java | 4 +-
.../doris/catalog/AutoIncrementGenerator.java | 2 +-
.../doris/catalog/BuiltinScalarFunctions.java | 67 +-
.../main/java/org/apache/doris/catalog/Column.java | 4 +
.../main/java/org/apache/doris/catalog/Env.java | 4 +-
.../main/java/org/apache/doris/catalog/Index.java | 9 +-
.../main/java/org/apache/doris/catalog/MTMV.java | 42 +-
.../org/apache/doris/catalog/MetadataViewer.java | 179 ---
.../java/org/apache/doris/catalog/OlapTable.java | 26 +-
.../org/apache/doris/catalog/RefreshManager.java | 11 +-
.../java/org/apache/doris/catalog/SchemaTable.java | 23 +
.../apache/doris/catalog/TabletInvertedIndex.java | 34 -
.../doris/clone/DynamicPartitionScheduler.java | 42 +-
.../apache/doris/cloud/backup/CloudRestoreJob.java | 1 +
.../org/apache/doris/cloud/catalog/CloudEnv.java | 4 +
.../cloud/catalog/CloudInstanceStatusChecker.java | 3 -
.../apache/doris/cloud/catalog/CloudReplica.java | 16 +
.../doris/cloud/catalog/CloudTabletRebalancer.java | 46 +-
.../cloud/datasource/CloudInternalCatalog.java | 6 +-
.../doris/cloud/master/CloudReportHandler.java | 4 +
.../apache/doris/cloud/storage/AzureRemote.java | 50 +-
.../apache/doris/cloud/storage/DefaultRemote.java | 116 +-
.../doris/cloud/system/CloudSystemInfoService.java | 11 +-
.../transaction/CloudGlobalTransactionMgr.java | 18 +-
.../org/apache/doris/common/profile/Profile.java | 3 +
.../doris/common/profile/ProfileManager.java | 3 +-
.../doris/common/util/AutoBucketCalculator.java | 205 +++
.../apache/doris/common/util/AutoBucketUtils.java | 10 +-
.../org/apache/doris/common/util/LocationPath.java | 15 +
.../apache/doris/common/util/PropertyAnalyzer.java | 12 +-
.../java/org/apache/doris/common/util/S3Util.java | 151 ++-
.../java/org/apache/doris/common/util/Util.java | 17 +-
.../apache/doris/datasource/CatalogFactory.java | 4 +-
.../org/apache/doris/datasource/CatalogIf.java | 4 +-
.../apache/doris/datasource/ExternalCatalog.java | 35 +-
.../apache/doris/datasource/ExternalDatabase.java | 22 +-
.../org/apache/doris/datasource/FileScanNode.java | 8 +-
.../apache/doris/datasource/InternalCatalog.java | 15 +-
.../doris/datasource/hive/HMSCachedClient.java | 4 +-
.../doris/datasource/hive/HMSExternalCatalog.java | 10 +-
.../doris/datasource/hive/HMSTransaction.java | 61 +-
.../doris/datasource/hive/HiveMetaStoreCache.java | 3 +-
.../doris/datasource/hive/HiveTransaction.java | 4 +-
.../datasource/hive/ThriftHMSCachedClient.java | 8 +-
.../hive/event/MetastoreEventsProcessor.java | 2 +-
.../datasource/iceberg/IcebergMetadataOps.java | 28 +-
.../doris/datasource/jdbc/source/JdbcScanNode.java | 2 +-
.../lakesoul/LakeSoulExternalCatalog.java | 40 +-
.../lakesoul/LakeSoulExternalDatabase.java | 4 +
.../datasource/lakesoul/LakeSoulExternalTable.java | 4 +
.../doris/datasource/lakesoul/LakeSoulUtils.java | 4 +
.../lakesoul/source/LakeSoulScanNode.java | 4 +
.../datasource/lakesoul/source/LakeSoulSplit.java | 4 +
.../maxcompute/MaxComputeExternalCatalog.java | 104 +-
.../maxcompute/MaxComputeExternalTable.java | 19 +-
.../maxcompute/MaxComputeSchemaCacheValue.java | 8 +-
.../datasource/maxcompute/McStructureHelper.java | 221 +++
.../maxcompute/source/MaxComputeScanNode.java | 4 +-
.../doris/datasource/odbc/source/OdbcScanNode.java | 2 +-
.../property/constants/MCProperties.java | 20 +
...Properties.java => AbstractHiveProperties.java} | 4 +-
.../metastore/AbstractPaimonProperties.java | 49 +-
.../property/metastore/HMSBaseProperties.java | 3 +
....java => HiveAliyunDLFMetaStoreProperties.java} | 4 +-
...rties.java => HiveGlueMetaStoreProperties.java} | 6 +-
.../{HMSProperties.java => HiveHMSProperties.java} | 4 +-
...tiesFactory.java => HivePropertiesFactory.java} | 18 +-
.../IcebergFileSystemMetaStoreProperties.java | 4 +-
.../metastore/IcebergHMSMetaStoreProperties.java | 3 -
.../property/metastore/IcebergRestProperties.java | 17 +-
.../property/metastore/MetastoreProperties.java | 2 +-
.../PaimonAliyunDLFMetaStoreProperties.java | 6 +-
.../PaimonFileSystemMetaStoreProperties.java | 7 +-
.../metastore/PaimonHMSMetaStoreProperties.java | 6 +-
.../metastore/PaimonRestMetaStoreProperties.java | 2 +-
.../storage/AbstractS3CompatibleProperties.java | 4 -
.../property/storage/AzureProperties.java | 60 +-
.../property/storage/AzurePropertyUtils.java | 189 +++
.../property/storage/BrokerProperties.java | 8 +
.../datasource/property/storage/COSProperties.java | 5 +
.../datasource/property/storage/GCSProperties.java | 5 +
.../property/storage/HdfsCompatibleProperties.java | 7 +
.../property/storage/LocalProperties.java | 6 +
.../property/storage/MinioProperties.java | 5 +
.../datasource/property/storage/OBSProperties.java | 5 +
.../datasource/property/storage/OSSProperties.java | 5 +
.../datasource/property/storage/S3Properties.java | 58 +-
.../property/storage/S3PropertyUtils.java | 8 +-
.../property/storage/StorageProperties.java | 63 +-
.../org/apache/doris/encryption/EncryptionKey.java | 17 +
.../apache/doris/encryption/KeyManagerStore.java | 11 +-
.../org/apache/doris/encryption/RootKeyInfo.java | 2 +-
.../java/org/apache/doris/fs/SchemaTypeMapper.java | 3 +-
.../org/apache/doris/fs/obj/AzureObjStorage.java | 97 +-
.../java/org/apache/doris/fs/obj/S3ObjStorage.java | 4 +-
.../apache/doris/fs/remote/AzureFileSystem.java | 19 +-
.../org/apache/doris/fs/remote/ObjFileSystem.java | 26 +-
.../org/apache/doris/fs/remote/S3FileSystem.java | 24 +
.../apache/doris/fs/remote/dfs/DFSFileSystem.java | 3 +-
.../httpv2/controller/HardwareInfoController.java | 1 +
.../doris/httpv2/rest/TableSchemaAction.java | 80 +-
...Validator.java => BasicTokenizerValidator.java} | 23 +-
...or.java => CharReplaceCharFilterValidator.java} | 30 +-
...erValidator.java => ICUTokenizerValidator.java} | 9 +-
.../org/apache/doris/indexpolicy/IndexPolicy.java | 8 +-
.../apache/doris/indexpolicy/IndexPolicyMgr.java | 69 +-
.../doris/indexpolicy/IndexPolicyTypeEnum.java | 3 +-
...zerValidator.java => NoOperationValidator.java} | 10 +-
.../org/apache/doris/info/PartitionNamesInfo.java | 3 +
.../java/org/apache/doris/info/TableNameInfo.java | 22 +-
.../org/apache/doris/job/base/AbstractJob.java | 1 +
.../org/apache/doris/job/common/FailureReason.java | 6 +
.../insert/streaming/StreamingInsertJob.java | 227 +++-
.../insert/streaming/StreamingInsertTask.java | 100 +-
.../insert/streaming/StreamingJobProperties.java | 72 +-
.../streaming/StreamingJobSchedulerTask.java | 28 +-
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 63 +-
.../org/apache/doris/job/manager/JobManager.java | 42 +-
.../java/org/apache/doris/job/offset/Offset.java | 5 +
.../doris/job/offset/SourceOffsetProvider.java | 6 +
.../org/apache/doris/job/offset/s3/S3Offset.java | 5 +
.../job/offset/s3/S3SourceOffsetProvider.java | 42 +-
.../job/scheduler/StreamingTaskScheduler.java | 31 +-
.../org/apache/doris/load/BrokerFileGroup.java | 10 +-
.../main/java/org/apache/doris/load/DeleteJob.java | 5 -
.../java/org/apache/doris/load/EtlJobType.java | 1 +
.../main/java/org/apache/doris/load/ExportJob.java | 4 +-
.../main/java/org/apache/doris/load/ExportMgr.java | 6 +-
.../org/apache/doris/load/RoutineLoadDesc.java | 12 +-
.../apache/doris/load/loadv2/InsertLoadJob.java | 6 +-
.../org/apache/doris/load/loadv2/LoadManager.java | 5 -
.../apache/doris/load/loadv2/MysqlLoadManager.java | 6 +-
.../load/routineload/KafkaRoutineLoadJob.java | 11 +-
.../doris/load/routineload/KafkaTaskInfo.java | 21 +-
.../doris/load/routineload/RoutineLoadJob.java | 38 +-
.../load/routineload/RoutineLoadTaskInfo.java | 12 +-
.../load/routineload/RoutineLoadTaskScheduler.java | 4 +-
.../java/org/apache/doris/metric/CloudMetrics.java | 26 +
.../java/org/apache/doris/metric/MetricRepo.java | 61 +
.../doris/metric/PrometheusMetricVisitor.java | 4 +-
.../MTMVAnalyzeQueryInfo.java} | 34 +-
.../org/apache/doris/mtmv/MTMVHookService.java | 14 +
.../java/org/apache/doris/mtmv/MTMVJobManager.java | 9 +
.../org/apache/doris/mtmv/MTMVPartitionInfo.java | 19 +
.../org/apache/doris/mtmv/MTMVPartitionUtil.java | 26 +-
.../java/org/apache/doris/mtmv/MTMVPlanUtil.java | 261 +++-
.../doris/mtmv/MTMVRefreshPartitionSnapshot.java | 6 +-
.../java/org/apache/doris/mtmv/MTMVRelation.java | 34 +-
.../org/apache/doris/mtmv/MTMVRelationManager.java | 99 +-
.../org/apache/doris/mtmv/MTMVRewriteUtil.java | 2 +-
.../java/org/apache/doris/mtmv/MTMVService.java | 26 +-
.../main/java/org/apache/doris/mtmv/MTMVUtil.java | 10 +-
.../java/org/apache/doris/mysql/MysqlChannel.java | 4 +-
.../mysql/privilege/AccessControllerManager.java | 6 -
.../org/apache/doris/nereids/CascadesContext.java | 4 +-
.../org/apache/doris/nereids/NereidsPlanner.java | 2 +-
.../org/apache/doris/nereids/StatementContext.java | 23 +
.../doris/nereids/analyzer/UnboundAlias.java | 10 +
.../nereids/analyzer/UnboundBlackholeSink.java | 158 +++
.../doris/nereids/analyzer/UnboundFunction.java | 14 +
.../doris/nereids/analyzer/UnboundInlineTable.java | 6 +
.../nereids/analyzer/UnboundOneRowRelation.java | 12 +
.../doris/nereids/analyzer/UnboundRelation.java | 23 +
.../doris/nereids/analyzer/UnboundResultSink.java | 5 +
.../apache/doris/nereids/analyzer/UnboundSlot.java | 5 +
.../apache/doris/nereids/analyzer/UnboundStar.java | 15 +
.../doris/nereids/analyzer/UnboundTVFRelation.java | 7 +
.../doris/nereids/analyzer/UnboundTableSink.java | 11 +
.../doris/nereids/analyzer/UnboundVariable.java | 5 +
.../doris/nereids/glue/LogicalPlanAdapter.java | 2 +-
.../glue/translator/PhysicalPlanTranslator.java | 13 +-
.../glue/translator/PlanTranslatorContext.java | 6 +-
.../doris/nereids/jobs/executor/Rewriter.java | 7 +-
.../executor/TableCollectAndHookInitializer.java | 16 +-
.../jobs/rewrite/TopDownVisitorRewriteJob.java | 2 -
.../doris/nereids/load/NereidsBrokerFileGroup.java | 10 +-
.../doris/nereids/load/NereidsBrokerLoadTask.java | 12 +-
.../doris/nereids/load/NereidsDataDescription.java | 68 +-
.../nereids/load/NereidsLoadPlanInfoCollector.java | 27 +-
.../doris/nereids/load/NereidsLoadTaskInfo.java | 4 +-
.../doris/nereids/load/NereidsLoadUtils.java | 10 +-
.../nereids/load/NereidsLoadingTaskPlanner.java | 14 +-
.../nereids/load/NereidsRoutineLoadTaskInfo.java | 12 +-
.../nereids/load/NereidsStreamLoadPlanner.java | 6 +-
.../doris/nereids/load/NereidsStreamLoadTask.java | 14 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 212 ++-
.../parser/LogicalPlanBuilderForEncryption.java | 20 +
.../apache/doris/nereids/properties/OrderKey.java | 18 +
.../nereids/properties/RequestPropertyDeriver.java | 8 +
.../org/apache/doris/nereids/rules/RuleSet.java | 2 +
.../org/apache/doris/nereids/rules/RuleType.java | 2 +
.../rules/analysis/AvgDistinctToSumDivCount.java | 3 +-
.../nereids/rules/analysis/BindExpression.java | 32 +-
.../doris/nereids/rules/analysis/BindSink.java | 31 +-
.../nereids/rules/analysis/CheckSearchUsage.java | 29 +-
.../rules/analysis/CollectOneLevelRelation.java | 4 +-
.../nereids/rules/analysis/CollectRelation.java | 19 +-
.../nereids/rules/analysis/ExpressionAnalyzer.java | 67 +-
.../rules/analysis/GetFormatFunctionBinder.java | 78 ++
.../LogicalSubQueryAliasToLogicalProject.java | 20 +-
.../nereids/rules/analysis/SubqueryToApply.java | 3 +-
.../mv/InitMaterializationContextHook.java | 24 +-
.../exploration/mv/MaterializedViewUtils.java | 13 +-
.../rules/expression/ExpressionNormalization.java | 4 +-
.../rules/expression/ExpressionOptimization.java | 6 +-
.../rules/expression/ExpressionRewrite.java | 58 +-
.../rules/expression/ExpressionRewriteContext.java | 28 +
.../rules/expression/ExpressionRuleType.java | 5 +-
.../nereids/rules/expression/check/CheckCast.java | 29 -
.../rules/CaseWhenToCompoundPredicate.java | 152 +++
.../expression/rules/CondReplaceNullWithFalse.java | 61 +
.../rules/expression/rules/ConditionRewrite.java | 183 +++
.../expression/rules/FoldConstantRuleOnFE.java | 58 +-
.../expression/rules/NullSafeEqualToEqual.java | 159 ++-
.../rules/OneListPartitionEvaluator.java | 5 +-
.../rules/ReplaceNullWithFalseForCond.java | 131 --
.../rules/SimplifyArithmeticComparisonRule.java | 2 +-
.../rules/SimplifyEqualBooleanLiteral.java | 74 +
...gicalBlackholeSinkToPhysicalBlackholeSink.java} | 26 +-
.../LogicalEsScanToPhysicalEsScan.java | 9 +-
.../LogicalJdbcScanToPhysicalJdbcScan.java | 10 +-
.../LogicalOdbcScanToPhysicalOdbcScan.java | 10 +-
.../LogicalOlapScanToPhysicalOlapScan.java | 5 +-
.../nereids/rules/rewrite/ConstantPropagation.java | 78 +-
.../nereids/rules/rewrite/EliminateFilter.java | 14 +-
.../rules/rewrite/PushProjectIntoUnion.java | 2 +-
.../doris/nereids/rules/rewrite/ReorderJoin.java | 4 +
.../rules/rewrite/RewriteSearchToSlots.java | 61 +-
.../org/apache/doris/nereids/trees/TreeNode.java | 4 +
.../doris/nereids/trees/expressions/Alias.java | 12 +
.../nereids/trees/expressions/BinaryOperator.java | 9 +
.../doris/nereids/trees/expressions/CaseWhen.java | 14 +
.../doris/nereids/trees/expressions/Cast.java | 11 +
.../trees/expressions/CompoundPredicate.java | 10 +
.../trees/expressions/DefaultValueSlot.java | 5 +
.../doris/nereids/trees/expressions/EqualTo.java | 2 +-
.../doris/nereids/trees/expressions/Exists.java | 10 +
.../trees/expressions/ExpressionEvaluator.java | 3 +-
.../nereids/trees/expressions/InPredicate.java | 25 +
.../nereids/trees/expressions/InSubquery.java | 10 +
.../doris/nereids/trees/expressions/IsNull.java | 8 +
.../doris/nereids/trees/expressions/Not.java | 7 +
.../nereids/trees/expressions/OrderExpression.java | 5 +
.../nereids/trees/expressions/Placeholder.java | 4 +
.../nereids/trees/expressions/Properties.java | 5 +
.../nereids/trees/expressions/ScalarSubquery.java | 9 +
.../trees/expressions/SearchExpression.java | 8 +-
.../trees/expressions/StringRegexPredicate.java | 11 +
.../doris/nereids/trees/expressions/Subtract.java | 13 +
.../trees/expressions/TimestampArithmetic.java | 15 +
.../nereids/trees/expressions/UnaryOperator.java | 8 +
.../nereids/trees/expressions/WhenClause.java | 7 +
.../trees/expressions/WindowExpression.java | 18 +
.../nereids/trees/expressions/WindowFrame.java | 38 +
.../trees/expressions/functions/BoundFunction.java | 11 +
.../functions/agg/AggregateFunction.java | 19 +
.../trees/expressions/functions/agg/Count.java | 8 +
.../executable/DateTimeExtractAndTransform.java | 168 ++-
.../functions/executable/StringArithmetic.java | 101 ++
.../expressions/functions/generator/Explode.java | 7 +-
.../functions/generator/ExplodeOuter.java | 7 +-
.../functions/generator/ExplodeVariantArray.java | 7 +-
.../generator/TableGeneratingFunction.java | 5 +
.../expressions/functions/scalar/ElementAt.java | 16 +-
.../scalar/{MaskFirstN.java => ExportSet.java} | 58 +-
.../scalar/{Length.java => FromBase64Binary.java} | 30 +-
.../functions/scalar/FromMicrosecond.java | 3 +-
.../functions/scalar/FromMillisecond.java | 4 +-
.../expressions/functions/scalar/FromSecond.java | 3 +-
.../scalar/{Length.java => GetFormat.java} | 35 +-
.../scalar/{Length.java => JsonHash.java} | 33 +-
.../trees/expressions/functions/scalar/Lambda.java | 12 +
.../trees/expressions/functions/scalar/Length.java | 5 +-
.../expressions/functions/scalar/MaskFirstN.java | 8 -
.../expressions/functions/scalar/MaskLastN.java | 8 -
.../scalar/{Length.java => MurmurHash364V2.java} | 31 +-
...ngth.java => NormalizeJsonNumbersToDouble.java} | 24 +-
.../scalar/{FromSecond.java => PeriodAdd.java} | 40 +-
.../scalar/{FromSecond.java => PeriodDiff.java} | 40 +-
.../trees/expressions/functions/scalar/Search.java | 46 +-
.../functions/scalar/SearchDslParser.java | 376 +++++-
.../{Length.java => SortJsonbObjectKeys.java} | 24 +-
.../scalar/{MaskFirstN.java => SubBinary.java} | 61 +-
.../scalar/{Length.java => ToBase64Binary.java} | 23 +-
.../trees/expressions/literal/ArrayLiteral.java | 9 +-
.../trees/expressions/literal/BooleanLiteral.java | 8 +
.../trees/expressions/literal/DateLiteral.java | 5 +
.../trees/expressions/literal/DateTimeLiteral.java | 5 +
.../trees/expressions/literal/DoubleLiteral.java | 12 -
.../trees/expressions/literal/FloatLiteral.java | 12 -
.../expressions/literal/FractionalLiteral.java | 2 +-
.../trees/expressions/literal/Interval.java | 9 +
.../nereids/trees/expressions/literal/Literal.java | 184 +--
.../trees/expressions/literal/MapLiteral.java | 6 +-
.../trees/expressions/literal/NullLiteral.java | 4 +-
.../trees/expressions/literal/StructLiteral.java | 2 +-
.../trees/expressions/literal/TimeV2Literal.java | 5 +
.../expressions/visitor/ScalarFunctionVisitor.java | 55 +
.../apache/doris/nereids/trees/plans/PlanType.java | 9 +
.../trees/plans/commands/AlterJobCommand.java | 116 +-
.../plans/commands/AlterJobStatusCommand.java | 23 +-
.../trees/plans/commands/AlterTableCommand.java | 2 +-
...mand.java => CreateIndexCharFilterCommand.java} | 64 +-
.../trees/plans/commands/CreateJobCommand.java | 6 +-
.../commands/CreateMaterializedViewCommand.java | 3 +
.../plans/commands/DropCachedStatsCommand.java | 4 +-
...ommand.java => DropIndexCharFilterCommand.java} | 37 +-
.../trees/plans/commands/DropJobCommand.java | 24 +-
.../trees/plans/commands/ExplainCommand.java | 8 +
.../trees/plans/commands/ExportCommand.java | 70 +-
.../nereids/trees/plans/commands/LoadCommand.java | 4 +-
.../trees/plans/commands/PauseJobCommand.java | 7 -
.../trees/plans/commands/ResumeJobCommand.java | 7 -
.../trees/plans/commands/ShowBackupCommand.java | 6 +
.../trees/plans/commands/ShowConfigCommand.java | 20 +-
...ommand.java => ShowIndexCharFilterCommand.java} | 36 +-
.../commands/ShowQueuedAnalyzeJobsCommand.java | 9 +-
.../trees/plans/commands/ShowRestoreCommand.java | 6 +
.../plans/commands/ShowWorkloadGroupsCommand.java | 10 +
.../trees/plans/commands/UpdateCommand.java | 22 +-
.../plans/commands/UpdateMvByPartitionCommand.java | 6 +-
.../trees/plans/commands/info/AlterViewInfo.java | 5 +-
.../trees/plans/commands/info/BuildIndexOp.java | 3 +-
.../trees/plans/commands/info/CopyFromDesc.java | 6 +-
.../trees/plans/commands/info/CopyIntoInfo.java | 22 +-
.../trees/plans/commands/info/CreateIndexOp.java | 3 +-
.../trees/plans/commands/info/CreateJobInfo.java | 7 +-
.../trees/plans/commands/info/CreateMTMVInfo.java | 174 +--
.../plans/commands/info/CreateRoutineLoadInfo.java | 10 +-
.../trees/plans/commands/info/CreateTableInfo.java | 8 +
.../trees/plans/commands/info/CreateViewInfo.java | 5 +-
.../trees/plans/commands/info/DropIndexOp.java | 3 +-
.../trees/plans/commands/info/IndexDefinition.java | 5 +-
.../trees/plans/commands/info/ModifyColumnOp.java | 4 -
.../trees/plans/commands/info/SetSessionVarOp.java | 10 +
.../commands/insert/AbstractInsertExecutor.java | 58 +-
.../insert/BaseExternalTableInsertExecutor.java | 4 +-
.../insert/BatchInsertIntoTableCommand.java | 7 +
...tExecutor.java => BlackholeInsertExecutor.java} | 41 +-
.../commands/insert/DictionaryInsertExecutor.java | 4 +-
.../plans/commands/insert/HiveInsertExecutor.java | 4 +-
.../commands/insert/IcebergInsertExecutor.java | 4 +-
.../commands/insert/InsertIntoTableCommand.java | 65 +-
.../insert/InsertOverwriteTableCommand.java | 12 +
.../trees/plans/commands/insert/InsertUtils.java | 5 +-
.../plans/commands/insert/JdbcInsertExecutor.java | 4 +-
.../plans/commands/insert/OlapInsertExecutor.java | 17 +-
.../plans/commands/insert/WarmupSelectCommand.java | 217 +++
.../plans/commands/merge/MergeIntoCommand.java | 534 ++++++++
.../plans/commands/merge/MergeMatchedClause.java | 55 +
.../commands/merge/MergeNotMatchedClause.java | 55 +
.../plans/distribute/worker/BackendWorker.java | 2 +-
.../trees/plans/logical/LogicalAggregate.java | 68 +-
.../trees/plans/logical/LogicalBlackholeSink.java | 88 ++
.../nereids/trees/plans/logical/LogicalCTE.java | 14 +
.../plans/logical/LogicalCatalogRelation.java | 49 +-
.../trees/plans/logical/LogicalCheckPolicy.java | 22 +
.../logical/LogicalDeferMaterializeOlapScan.java | 14 +-
.../nereids/trees/plans/logical/LogicalEsScan.java | 39 +-
.../nereids/trees/plans/logical/LogicalExcept.java | 9 +
.../trees/plans/logical/LogicalFileScan.java | 38 +-
.../trees/plans/logical/LogicalFileSink.java | 10 +
.../nereids/trees/plans/logical/LogicalFilter.java | 12 +
.../trees/plans/logical/LogicalGenerate.java | 22 +
.../nereids/trees/plans/logical/LogicalHaving.java | 13 +
.../trees/plans/logical/LogicalIntersect.java | 9 +
.../trees/plans/logical/LogicalJdbcScan.java | 29 +-
.../nereids/trees/plans/logical/LogicalJoin.java | 18 +
.../nereids/trees/plans/logical/LogicalLimit.java | 11 +
.../trees/plans/logical/LogicalOdbcScan.java | 25 +-
.../trees/plans/logical/LogicalOlapScan.java | 72 +-
.../trees/plans/logical/LogicalProject.java | 19 +
.../trees/plans/logical/LogicalQualify.java | 10 +
.../nereids/trees/plans/logical/LogicalRepeat.java | 45 +-
.../trees/plans/logical/LogicalSchemaScan.java | 53 +-
.../trees/plans/logical/LogicalSelectHint.java | 5 +
.../nereids/trees/plans/logical/LogicalSort.java | 12 +
.../trees/plans/logical/LogicalSubQueryAlias.java | 13 +
.../trees/plans/logical/LogicalTestScan.java | 25 +-
.../nereids/trees/plans/logical/LogicalUnion.java | 11 +-
.../trees/plans/logical/LogicalUsingJoin.java | 16 +
.../nereids/trees/plans/logical/LogicalWindow.java | 16 +
.../plans/physical/PhysicalBlackholeSink.java | 144 ++
.../plans/physical/PhysicalCatalogRelation.java | 33 +-
.../trees/plans/physical/PhysicalEsScan.java | 29 +-
.../trees/plans/physical/PhysicalJdbcScan.java | 33 +-
.../trees/plans/physical/PhysicalOdbcScan.java | 31 +-
.../trees/plans/physical/PhysicalOlapScan.java | 43 +-
.../trees/plans/visitor/CommandVisitor.java | 23 +
.../nereids/trees/plans/visitor/SinkVisitor.java | 17 +
.../apache/doris/nereids/util/ExpressionUtils.java | 15 +-
.../java/org/apache/doris/nereids/util/Utils.java | 5 +-
.../org/apache/doris/planner/BlackholeSink.java | 61 +
.../org/apache/doris/planner/HiveTableSink.java | 3 +-
.../org/apache/doris/planner/IcebergTableSink.java | 5 +-
.../org/apache/doris/planner/PlanFragment.java | 2 +-
.../java/org/apache/doris/planner/ResultSink.java | 2 +-
.../java/org/apache/doris/qe/ConnectContext.java | 2 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 1 +
.../org/apache/doris/qe/CoordinatorContext.java | 1 +
.../org/apache/doris/qe/NereidsCoordinator.java | 36 +-
.../java/org/apache/doris/qe/SessionVariable.java | 214 ++-
.../qe/runtime/RuntimeFiltersThriftBuilder.java | 15 +-
.../doris/qe/runtime/ThriftPlansBuilder.java | 44 +-
.../WorkloadRuntimeStatusMgr.java | 54 +-
.../apache/doris/service/FrontendServiceImpl.java | 250 ++--
.../sessions/FlightSqlConnectContext.java | 2 +-
.../apache/doris/statistics/AnalysisManager.java | 32 +-
.../doris/statistics/StatisticsAutoCollector.java | 20 +-
.../doris/statistics/StatisticsJobAppender.java | 17 +-
.../doris/statistics/util/StatisticsUtil.java | 24 +-
.../org/apache/doris/system/SystemInfoService.java | 14 +-
.../ExternalFileTableValuedFunction.java | 3 +-
.../tablefunction/HudiTableValuedFunction.java | 6 +-
.../tablefunction/IcebergTableValuedFunction.java | 6 +-
.../tablefunction/JobsTableValuedFunction.java | 7 -
.../doris/tablefunction/MetadataGenerator.java | 52 +-
.../tablefunction/PaimonTableValuedFunction.java | 6 +-
.../tablefunction/TasksTableValuedFunction.java | 7 -
.../java/org/apache/doris/task/LoadTaskInfo.java | 4 +-
.../doris/transaction/PublishVersionDaemon.java | 3 +
.../org/apache/doris/alter/CloudIndexTest.java | 12 +-
.../org/apache/doris/alter/IndexChangeJobTest.java | 25 +-
.../doris/analysis/PartitionExprUtilTest.java | 223 +++
.../doris/catalog/AutoIncrementGeneratorTest.java | 178 +++
.../doris/catalog/ListPartitionInfoTest.java | 6 +-
.../apache/doris/catalog/MetadataViewerTest.java | 4 +-
.../doris/catalog/RangePartitionInfoTest.java | 4 +-
.../doris/cloud/master/CloudReportHandlerTest.java | 63 +
.../cloud/system/CloudSystemInfoServiceTest.java | 645 ++++++++-
.../apache/doris/common/PropertyAnalyzerTest.java | 90 +-
.../doris/common/proc/IndexSchemaProcNodeTest.java | 6 +-
.../common/util/AutoBucketCalculatorTest.java | 106 ++
.../org/apache/doris/common/util/S3UtilTest.java | 252 ++++
.../doris/datasource/RefreshCatalogTest.java | 4 +-
.../doris/datasource/TestHMSCachedClient.java | 4 +-
.../datasource/iceberg/IcebergMetadataOpTest.java | 48 +
.../datasource/iceberg/IcebergPredicateTest.java | 20 +-
.../datasource/lakesoul/LakeSoulPredicateTest.java | 20 +-
.../metastore/AbstractPaimonPropertiesTest.java | 89 ++
.../HMSAliyunDLFMetaStorePropertiesTest.java | 4 +-
.../datasource/property/metastore/HMSGlueIT.java | 2 +-
.../metastore/HMSGlueMetaStorePropertiesTest.java | 16 +-
.../property/metastore/HMSPropertiesTest.java | 14 +-
.../metastore/IcebergRestPropertiesTest.java | 4 +-
.../PaimonRestMetaStorePropertiesTest.java | 14 +-
.../property/storage/AzurePropertiesTest.java | 28 +-
.../property/storage/AzurePropertyUtilsTest.java | 145 ++
.../property/storage/COSPropertiesTest.java | 16 +
.../property/storage/GCSPropertiesTest.java | 18 +
.../property/storage/OBSPropertyTest.java | 18 +
.../property/storage/OSSPropertiesTest.java | 18 +
.../property/storage/S3PropertiesTest.java | 24 +
.../TrinoConnectorPredicateTest.java | 36 +-
.../apache/doris/load/loadv2/ExportMgrTest.java | 4 +-
.../load/routineload/KafkaRoutineLoadJobTest.java | 6 +-
.../java/org/apache/doris/mtmv/AlterMTMVTest.java | 15 +-
.../apache/doris/mtmv/MTMVPartitionUtilTest.java | 54 +-
.../org/apache/doris/mtmv/MTMVPlanUtilTest.java | 226 +++-
.../apache/doris/mtmv/MTMVRelationManagerTest.java | 42 +-
.../org/apache/doris/mtmv/MTMVRelationTest.java | 117 ++
.../org/apache/doris/mtmv/MTMVRewriteUtilTest.java | 12 +-
.../apache/doris/mtmv/MTMVSchemaChangeTest.java | 94 ++
.../java/org/apache/doris/mtmv/MTMVTaskTest.java | 11 +-
.../test/java/org/apache/doris/mtmv/MTMVTest.java | 21 +-
.../doris/nereids/jobs/RewriteTopDownJobTest.java | 5 +
.../doris/nereids/parser/EncryptSQLTest.java | 60 +
.../nereids/parser/NereidsParserDigestTest.java | 300 +++++
.../doris/nereids/parser/NereidsParserTest.java | 126 ++
.../parser/UnboundFunctionWithUnboundStarTest.java | 106 ++
.../rules/analysis/ExpressionAnalyzerTest.java | 36 +
.../expression/ExpressionRewriteTestHelper.java | 6 +-
.../nereids/rules/expression/FoldConstantTest.java | 34 +-
.../rules/expression/check/CheckCastTest.java | 48 +-
.../rules/CaseWhenToCompoundPredicateTest.java | 87 ++
.../rules/CondReplaceNullWithFalseTest.java | 145 ++
.../expression/rules/ExpressionRewriteSqlTest.java | 2 +-
.../expression/rules/NullSafeEqualToEqualTest.java | 79 +-
.../rules/ReplaceNullWithFalseForCondTest.java | 110 --
.../rules/SimplifyEqualBooleanLiteralTest.java | 51 +
.../rules/rewrite/ConstantPropagationTest.java | 25 +-
.../GeneratePartitionTopnFromWindowTest.java | 40 +
.../nereids/rules/rewrite/ReorderJoinTest.java | 19 +
.../DateTimeExtractAndTransformTest.java | 28 +-
.../functions/scalar/SearchDslParserTest.java | 327 +++++
.../expressions/literal/BooleanLiteralTest.java | 24 +
.../trees/expressions/literal/DateLiteralTest.java | 11 +
.../expressions/literal/DateTimeLiteralTest.java | 21 +
.../expressions/literal/DecimalLiteralTest.java | 7 +
.../expressions/literal/DoubleLiteralTest.java | 6 +
.../expressions/literal/FloatLiteralTest.java | 5 +
.../expressions/literal/IntegerLiteralTest.java | 7 +
.../expressions/literal/LargeIntLiteralTest.java | 7 +
.../expressions/literal/TimeV2LiteralTest.java | 21 +
.../plans/commands/merge/MergeIntoCommandTest.java | 956 +++++++++++++
.../org/apache/doris/nereids/util/PlanChecker.java | 6 +-
.../org/apache/doris/planner/OlapScanNodeTest.java | 8 +-
.../statistics/StatisticsAutoCollectorTest.java | 14 +-
.../statistics/StatisticsJobAppenderTest.java | 58 +-
.../apache/doris/system/SystemInfoServiceTest.java | 61 +
.../apache/doris/utframe/TestWithFeService.java | 7 +
fe/pom.xml | 18 +
gensrc/proto/cloud.proto | 30 +
gensrc/proto/data.proto | 1 +
gensrc/proto/internal_service.proto | 17 +
gensrc/proto/segment_v2.proto | 5 +
gensrc/script/gen_build_version.sh | 5 +
gensrc/thrift/AgentService.thrift | 5 +-
gensrc/thrift/DataSinks.thrift | 9 +-
gensrc/thrift/Descriptors.thrift | 2 +
gensrc/thrift/Exprs.thrift | 5 +-
gensrc/thrift/FrontendService.thrift | 9 +-
gensrc/thrift/PaloInternalService.thrift | 13 +-
regression-test/conf/regression-conf.groovy | 6 +
.../product_quantization.out} | 6 +-
regression-test/data/cast_p0/datelike_print.out | 37 +
.../routine_load/test_routine_load.out | 33 +
.../test_schema_change_with_readd_cluster.out} | 6 +-
.../multi_cluster/stream_load}/all_types.csv | 0
.../multi_cluster/stream_load/stream_load.out} | 12 +-
.../multi_cluster/stream_load/stream_load_2pc.out | 25 +
.../multi_cluster/stream_load/stream_load_lb.out | 25 +
.../test_apsaradb_internal_stage.out} | 7 +-
.../data/correctness/test_compare_float.out | 8 -
.../data/correctness_p0/test_mask_function.out | 48 +-
.../test_multi_distinct.out} | 0
.../data/correctness_p0/test_probe_clean.out | 6 -
.../one_level_nestedtypes_with_s3data.out | 1020 +++++++-------
.../string_len/test_string_len_complex.out | 43 +
.../sql-functions/doc_date_functions_test.out | 351 +++++
.../outfile/test_outfile_parquet_schema.out | 11 +
.../cache/test_file_cache_statistics.out | 5 +-
.../data/external_table_p0/tvf/hdfs_data_1.txt | 1 +
.../data/external_table_p0/tvf/hdfs_data_2.txt | 1 +
.../data/external_table_p0/tvf/hdfs_data_3.txt | 1 +
.../tvf/test_s3_tvf_number_range.out | 57 +
.../maxcompute/test_max_compute_schema.out | 258 ++++
.../cloud/test_cloud_version_already_merged.out | 15 +
.../test_cast_to_string_from_decimalv3.out | 73 +
.../to_string/test_cast_to_string_from_int.out | 59 +
.../analyzer/test_custom_analyzer.out | 6 +
.../analyzer/test_custom_analyzer1.out | 6 +
.../analyzer/test_custom_analyzer2.out | 16 +
.../test_storage_format_default.out} | 3 +-
.../test_streaming_insert_job_alter.out | 23 +
.../test_streaming_insert_job_crud.out | 23 +
.../test_streaming_insert_job_offset.out | 45 +
regression-test/data/json_p0/simple_json.json | 8 +
regression-test/data/json_p0/test_json_root.out | 29 +
regression-test/data/jsonb_p0/test_json_hash.out | 49 +
.../test_json_normalize_json_numbers_to_double.out | 46 +
.../jsonb_p0/test_json_sort_json_object_keys.out | 37 +
.../data/load_p0/broker_load/test_array_load.out | 48 +
.../test_broker_load_with_partition.out | 11 +-
.../broker_load/test_broker_load_with_where.out | 4 +-
..._with_where.out => test_s3_load_with_where.out} | 4 +-
.../load_p0/insert/test_insert_move_memtable.out | 29 -
.../data/load_p0/merge_into/test_merge_into.out | 17 +
.../data/load_p0/stream_load/test_compress.csv.lz4 | Bin 80 -> 48 bytes
.../{stream_load => tvf}/test_compress.csv.lz4 | Bin
.../data/mtmv_p0/test_create_mtmv_with_view.out | 30 +
.../mtmv_p0/test_create_mtmv_with_view_alter.out | 19 +
.../test_create_mtmv_with_view_alter_table.out | 19 +
.../test_create_mtmv_with_view_commit.out} | 7 +-
.../test_create_mtmv_with_view_cte.out} | 7 +-
.../mtmv_p0/test_create_mtmv_with_view_mtmv.out | 20 +
.../mtmv_p0/test_create_mtmv_with_view_pct.out | 21 +
.../mtmv_p0/test_create_mtmv_with_view_rollup.out | 9 +
...mtmv.out => test_limit_partition_view_mtmv.out} | 0
.../data/mtmv_p0/test_view_hive_mtmv.out | 28 +
.../partition_topn/check_orderkey.out} | 3 +-
.../datetime_functions/test_date_function.out | 59 +
.../datetime_functions/test_date_function_v2.out | 81 ++
.../json_function/test_query_json_object.out | 16 +-
.../agg_skew_rewrite/agg_skew_rewrite.out | 10 +-
.../avg_distinct_to_sum_div_count.out} | 3 +-
.../nereids_rules_p0/eager_aggregate/basic.out | 32 +-
.../eager_aggregate/basic_one_side.out | 32 +-
...h_down_count_distinct_through_join_one_side.out | 20 +-
.../push_down_count_through_join.out | 352 ++---
.../push_down_count_through_join_one_side.out | 352 ++---
.../eager_aggregate/push_down_max_through_join.out | 196 +--
...ush_down_min_distinct_through_join_one_side.out | 20 +-
.../eager_aggregate/push_down_min_through_join.out | 196 +--
...ush_down_sum_distinct_through_join_one_side.out | 16 +-
.../eager_aggregate/push_down_sum_through_join.out | 196 +--
.../push_down_sum_through_join_one_side.out | 196 +--
.../eliminate_gby_key/eliminate_group_by.out | 84 +-
.../eliminate_group_by_key_by_uniform.out | 2 +-
.../eliminate_join_condition.out | 24 +-
.../eliminate_outer_join/eliminate_outer_join.out | 130 +-
...simplify_comparison_predicate_int_vs_double.out | 4 +-
.../extend_infer_equal_predicate.out | 308 ++---
.../infer_predicate/infer_intersect_except.out | 8 +-
.../infer_predicate/infer_unequal_predicates.out | 68 +-
.../infer_predicate/pull_up_predicate_agg.out | 4 +-
.../infer_predicate/pull_up_predicate_literal.out | 264 ++--
.../infer_predicate/pull_up_predicate_set_op.out | 84 +-
.../data/nereids_rules_p0/pkfk/eliminate_inner.out | 24 +-
.../predicate_infer/infer_predicate.out | 152 +--
.../pull_up_join_from_union.out | 82 +-
.../push_down_distinct_through_join.out | 10 +-
.../extract_from_disjunction_in_join.out | 32 +-
.../push_down_limit_distinct_through_join.out | 4 +-
.../push_down_top_n_distinct_through_join.out | 6 +-
.../push_down_top_n_distinct_through_union.out | 44 +-
.../push_down_top_n_through_union.out | 44 +-
.../data/nereids_rules_p0/salt_join/salt_join.out | 72 +-
.../data/query_p0/cache/condition_cache.out | 121 ++
.../join/eliminate_const_join_condition.out | 24 +-
.../conditional_functions/test_coalesce.out | 28 +
.../conditional_functions/test_query_in.out | 14 +
.../hash_functions/test_hash_function.out | 12 +
.../string_functions/test_string_all.out | 1414 ++++++++++++++++++++
.../string_functions/test_string_function.out | 217 +++
.../sql_functions/table_function/explode.out | 88 ++
.../system/test_query_sys_column_data_sizes.out | 15 +
.../data/query_p0/system/test_table_properties.out | 12 +-
.../adjust_virtual_slot_nullable.out | 4 +-
.../test_create_table_with_rollup.out} | 4 +-
.../search/test_search_vs_match_consistency.out | 6 +
.../data/shape_check/ssb_sf100/shape/flat.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query10.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query14.out | 12 +-
.../tpcds_sf100/noStatsRfPrune/query16.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query17.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query18.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query25.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query29.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query3.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query35.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query36.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query41.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query42.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query44.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query46.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query49.out | 12 +-
.../tpcds_sf100/noStatsRfPrune/query50.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query52.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query59.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query6.out | 12 +-
.../tpcds_sf100/noStatsRfPrune/query64.out | 22 +-
.../tpcds_sf100/noStatsRfPrune/query68.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query69.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query70.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query72.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query85.out | 4 +-
.../tpcds_sf100/noStatsRfPrune/query86.out | 2 +-
.../tpcds_sf100/noStatsRfPrune/query94.out | 6 +-
.../tpcds_sf100/noStatsRfPrune/query95.out | 6 +-
.../tpcds_sf100/no_stats_shape/query10.out | 4 +-
.../tpcds_sf100/no_stats_shape/query14.out | 12 +-
.../tpcds_sf100/no_stats_shape/query16.out | 6 +-
.../tpcds_sf100/no_stats_shape/query17.out | 6 +-
.../tpcds_sf100/no_stats_shape/query18.out | 4 +-
.../tpcds_sf100/no_stats_shape/query25.out | 6 +-
.../tpcds_sf100/no_stats_shape/query29.out | 6 +-
.../tpcds_sf100/no_stats_shape/query3.out | 2 +-
.../tpcds_sf100/no_stats_shape/query35.out | 4 +-
.../tpcds_sf100/no_stats_shape/query36.out | 2 +-
.../tpcds_sf100/no_stats_shape/query41.out | 2 +-
.../tpcds_sf100/no_stats_shape/query42.out | 2 +-
.../tpcds_sf100/no_stats_shape/query44.out | 4 +-
.../tpcds_sf100/no_stats_shape/query46.out | 2 +-
.../tpcds_sf100/no_stats_shape/query49.out | 12 +-
.../tpcds_sf100/no_stats_shape/query50.out | 4 +-
.../tpcds_sf100/no_stats_shape/query52.out | 2 +-
.../tpcds_sf100/no_stats_shape/query59.out | 4 +-
.../tpcds_sf100/no_stats_shape/query6.out | 12 +-
.../tpcds_sf100/no_stats_shape/query64.out | 22 +-
.../tpcds_sf100/no_stats_shape/query68.out | 2 +-
.../tpcds_sf100/no_stats_shape/query69.out | 4 +-
.../tpcds_sf100/no_stats_shape/query70.out | 2 +-
.../tpcds_sf100/no_stats_shape/query72.out | 6 +-
.../tpcds_sf100/no_stats_shape/query85.out | 4 +-
.../tpcds_sf100/no_stats_shape/query86.out | 2 +-
.../tpcds_sf100/no_stats_shape/query94.out | 6 +-
.../tpcds_sf100/no_stats_shape/query95.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query10.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query14.out | 12 +-
.../shape_check/tpcds_sf100/rf_prune/query16.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query17.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query18.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query25.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query29.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query3.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query35.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query36.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query41.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query42.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query44.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query46.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query49.out | 12 +-
.../shape_check/tpcds_sf100/rf_prune/query50.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query52.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query59.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query6.out | 12 +-
.../shape_check/tpcds_sf100/rf_prune/query64.out | 22 +-
.../shape_check/tpcds_sf100/rf_prune/query68.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query69.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query70.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query72.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query85.out | 4 +-
.../shape_check/tpcds_sf100/rf_prune/query86.out | 2 +-
.../shape_check/tpcds_sf100/rf_prune/query94.out | 6 +-
.../shape_check/tpcds_sf100/rf_prune/query95.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query10.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query14.out | 12 +-
.../data/shape_check/tpcds_sf100/shape/query16.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query17.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query18.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query25.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query29.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query3.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query35.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query36.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query41.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query42.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query44.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query46.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query49.out | 12 +-
.../data/shape_check/tpcds_sf100/shape/query50.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query52.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query59.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query6.out | 12 +-
.../data/shape_check/tpcds_sf100/shape/query64.out | 22 +-
.../data/shape_check/tpcds_sf100/shape/query68.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query69.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query70.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query72.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query85.out | 4 +-
.../data/shape_check/tpcds_sf100/shape/query86.out | 2 +-
.../data/shape_check/tpcds_sf100/shape/query94.out | 6 +-
.../data/shape_check/tpcds_sf100/shape/query95.out | 6 +-
.../tpcds_sf1000/bs_downgrade_shape/query44.out | 4 +-
.../tpcds_sf1000/bs_downgrade_shape/query6.out | 12 +-
.../tpcds_sf1000/bs_downgrade_shape/query68.out | 2 +-
.../tpcds_sf1000/bs_downgrade_shape/query95.out | 6 +-
.../tpcds_sf1000/eliminate_empty/query10_empty.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query10.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query14.out | 12 +-
.../data/shape_check/tpcds_sf1000/hint/query16.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query17.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query18.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query25.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query29.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query3.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query36.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query41.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query42.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query44.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query46.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query49.out | 12 +-
.../data/shape_check/tpcds_sf1000/hint/query50.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query52.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query59.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query6.out | 12 +-
.../data/shape_check/tpcds_sf1000/hint/query64.out | 22 +-
.../data/shape_check/tpcds_sf1000/hint/query68.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query69.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query70.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query72.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query85.out | 4 +-
.../data/shape_check/tpcds_sf1000/hint/query86.out | 2 +-
.../data/shape_check/tpcds_sf1000/hint/query94.out | 6 +-
.../data/shape_check/tpcds_sf1000/hint/query95.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query10.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query14.out | 12 +-
.../shape_check/tpcds_sf1000/shape/query16.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query17.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query18.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query25.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query29.out | 6 +-
.../data/shape_check/tpcds_sf1000/shape/query3.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query35.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query36.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query41.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query42.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query44.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query46.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query49.out | 12 +-
.../shape_check/tpcds_sf1000/shape/query50.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query52.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query59.out | 4 +-
.../data/shape_check/tpcds_sf1000/shape/query6.out | 12 +-
.../shape_check/tpcds_sf1000/shape/query64.out | 22 +-
.../shape_check/tpcds_sf1000/shape/query68.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query69.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query70.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query72.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query85.out | 4 +-
.../shape_check/tpcds_sf1000/shape/query86.out | 2 +-
.../shape_check/tpcds_sf1000/shape/query94.out | 6 +-
.../shape_check/tpcds_sf1000/shape/query95.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query10.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query14.out | 12 +-
.../shape_check/tpcds_sf10t_orc/shape/query16.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query17.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query18.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query25.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query29.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query3.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query35.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query36.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query41.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query42.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query44.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query46.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query49.out | 12 +-
.../shape_check/tpcds_sf10t_orc/shape/query50.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query52.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query59.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query6.out | 12 +-
.../shape_check/tpcds_sf10t_orc/shape/query64.out | 22 +-
.../shape_check/tpcds_sf10t_orc/shape/query68.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query69.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query70.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query72.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query85.out | 4 +-
.../shape_check/tpcds_sf10t_orc/shape/query86.out | 2 +-
.../shape_check/tpcds_sf10t_orc/shape/query94.out | 6 +-
.../shape_check/tpcds_sf10t_orc/shape/query95.out | 6 +-
.../data/shape_check/tpch_sf1000/hint/q7.out | 4 +-
.../data/shape_check/tpch_sf1000/hint/q8.out | 4 +-
.../tpch_sf1000/nostats_rf_prune/q21.out | 6 +-
.../tpch_sf1000/nostats_rf_prune/q7.out | 4 +-
.../tpch_sf1000/nostats_rf_prune/q8.out | 4 +-
.../data/shape_check/tpch_sf1000/rf_prune/q21.out | 6 +-
.../data/shape_check/tpch_sf1000/rf_prune/q7.out | 4 +-
.../data/shape_check/tpch_sf1000/rf_prune/q8.out | 4 +-
.../data/shape_check/tpch_sf1000/shape/q21.out | 6 +-
.../data/shape_check/tpch_sf1000/shape/q7.out | 4 +-
.../data/shape_check/tpch_sf1000/shape/q8.out | 4 +-
.../shape_check/tpch_sf1000/shape_no_stats/q21.out | 6 +-
.../shape_check/tpch_sf1000/shape_no_stats/q7.out | 4 +-
.../shape_check/tpch_sf1000/shape_no_stats/q8.out | 4 +-
.../test_show_create_table_and_views_nereids.out | 8 +-
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q2.1.out | 283 ----
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q2.2.out | 59 -
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q2.3.out | 10 -
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q3.1.out | 153 ---
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q3.2.out | 317 -----
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q3.3.out | 9 -
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q4.1.out | 38 -
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q4.2.out | 103 --
.../data/ssb_sf0.1_p1/sql/pipelinex_flat_q4.3.out | 205 ---
.../data/ssb_sf0.1_p1/sql/pipelinex_q1.3.out | 4 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q2.1.out | 43 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q2.2.out | 11 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q2.3.out | 4 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q3.1.out | 28 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q3.2.out | 51 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q3.3.out | 4 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q4.1.out | 8 -
.../data/ssb_sf0.1_p1/sql/pipelinex_q4.3.out | 3 -
regression-test/data/tpch_sf0.1_p1/sql/cse.out | 38 +-
.../sql/q01.out | 0
.../sql/q02.out | 0
.../sql/q03.out | 0
.../sql/q04.out | 0
.../sql/q05.out | 0
.../sql/q06.out | 0
.../sql/q07.out | 0
.../sql/q08.out | 0
.../sql/q09.out | 0
.../sql/q10.out | 0
.../sql/q11.out | 0
.../sql/q12.out | 0
.../sql/q13.out | 0
.../sql/q14.out | 0
.../sql/q15.out | 0
.../sql/q16.out | 0
.../sql/q17.out | 0
.../sql/q18.out | 0
.../sql/q19.out | 0
.../sql/q20.out | 0
.../sql/q21.out | 0
.../sql/q22.out | 0
.../sql/test_left_anti_join_batch_size.out | 0
.../variant_p0/test_variant_search_subcolumn.out | 32 +
.../org/apache/doris/regression/Config.groovy | 1 +
.../apache/doris/regression/RegressionTest.groovy | 31 +
.../org/apache/doris/regression/suite/Suite.groovy | 193 ++-
.../doris/regression/suite/SuiteCluster.groovy | 86 +-
.../doris/regression/suite/SuiteContext.groovy | 4 +
.../regression/util/RoutineLoadTestUtils.groovy | 152 +++
.../cloud_p0/conf/regression-conf-custom.groovy | 1 +
.../pipeline/cloud_p1/conf/be_custom.conf | 2 +
.../nonConcurrent/conf/regression-conf.groovy | 1 +
regression-test/pipeline/p0/conf/be.conf | 1 +
.../plugins/plugin_cluster_version.groovy | 46 +-
.../ann_index_p0/product_quantization.groovy | 107 ++
.../suites/ann_index_p0/scalar_quantization.groovy | 6 +-
.../suites/auth_call/test_ddl_backup_auth.groovy | 12 +-
.../suites/auth_call/test_ddl_job_auth.groovy | 22 +-
.../suites/auth_p0/test_query_tvf_auth.groovy | 2 +-
.../bloom_filter_p0/test_bloom_filter_hit.groovy | 2 +-
...est_bloom_filter_hit_with_renamed_column.groovy | 87 +-
.../suites/cast_p0/datelike_print.groovy | 44 +
.../cloud_p0/balance/test_balance_metrics.groovy | 110 ++
.../cloud_p0/balance/test_balance_warm_up.groovy | 86 +-
.../suites/cloud_p0/cache/test_load_cache.groovy | 350 +++++
.../async_copy_into/async_load.groovy | 275 ++++
.../async_copy_into}/ddl/customer.sql | 6 +-
.../async_copy_into}/ddl/customer_delete.sql | 0
.../async_copy_into}/ddl/lineitem.sql | 4 -
.../async_copy_into}/ddl/lineitem_delete.sql | 0
.../multi_cluster/async_copy_into}/ddl/nation.sql | 6 +-
.../async_copy_into}/ddl/nation_delete.sql | 0
.../multi_cluster/async_copy_into}/ddl/orders.sql | 6 +-
.../async_copy_into}/ddl/orders_delete.sql | 0
.../multi_cluster/async_copy_into}/ddl/part.sql | 6 +-
.../async_copy_into}/ddl/part_delete.sql | 0
.../async_copy_into}/ddl/partsupp.sql | 4 -
.../async_copy_into}/ddl/partsupp_delete.sql | 0
.../multi_cluster/async_copy_into}/ddl/region.sql | 6 +-
.../async_copy_into}/ddl/region_delete.sql | 0
.../async_copy_into}/ddl/revenue1.sql | 0
.../async_copy_into}/ddl/revenue1_delete.sql | 0
.../async_copy_into}/ddl/supplier.sql | 6 +-
.../async_copy_into}/ddl/supplier_delete.sql | 0
.../multi_cluster/copy_into}/ddl/customer.sql | 6 +-
.../copy_into}/ddl/customer_delete.sql | 0
.../multi_cluster/copy_into}/ddl/lineitem.sql | 4 -
.../copy_into}/ddl/lineitem_delete.sql | 0
.../multi_cluster/copy_into}/ddl/nation.sql | 6 +-
.../ddl/nation_delete.sql | 0
.../multi_cluster/copy_into}/ddl/orders.sql | 6 +-
.../multi_cluster/copy_into}/ddl/orders_delete.sql | 0
.../multi_cluster/copy_into}/ddl/part.sql | 6 +-
.../multi_cluster/copy_into}/ddl/part_delete.sql | 0
.../multi_cluster/copy_into}/ddl/partsupp.sql | 4 -
.../copy_into}/ddl/partsupp_delete.sql | 0
.../multi_cluster/copy_into}/ddl/region.sql | 6 +-
.../multi_cluster/copy_into}/ddl/region_delete.sql | 0
.../multi_cluster/copy_into}/ddl/revenue1.sql | 0
.../copy_into}/ddl/revenue1_delete.sql | 0
.../multi_cluster/copy_into}/ddl/supplier.sql | 6 +-
.../copy_into}/ddl/supplier_delete.sql | 0
.../multi_cluster/copy_into/sync_laod.groovy | 846 ++++++++++++
.../cloud_p0/multi_cluster/default_cluster.groovy | 172 +++
.../multi_cluster_s3_load}/ddl/customer.sql | 8 +-
.../multi_cluster_s3_load}/ddl/customer_delete.sql | 0
.../multi_cluster_s3_load}/ddl/customer_load.sql | 0
.../multi_cluster_s3_load}/ddl/lineitem.sql | 8 +-
.../multi_cluster_s3_load}/ddl/lineitem_delete.sql | 0
.../multi_cluster_s3_load/ddl/lineitem_load.sql | 6 +
.../multi_cluster_s3_load}/ddl/nation.sql | 7 +-
.../ddl/nation_delete.sql | 0
.../ddl/nation_load.sql | 0
.../multi_cluster_s3_load}/ddl/orders.sql | 9 +-
.../multi_cluster_s3_load}/ddl/orders_delete.sql | 0
.../multi_cluster_s3_load}/ddl/orders_load.sql | 0
.../multi_cluster_s3_load}/ddl/part.sql | 9 +-
.../multi_cluster_s3_load}/ddl/part_delete.sql | 0
.../multi_cluster_s3_load}/ddl/part_load.sql | 0
.../multi_cluster_s3_load}/ddl/partsupp.sql | 9 +-
.../multi_cluster_s3_load}/ddl/partsupp_delete.sql | 0
.../multi_cluster_s3_load}/ddl/partsupp_load.sql | 0
.../multi_cluster_s3_load}/ddl/region.sql | 7 +-
.../multi_cluster_s3_load}/ddl/region_delete.sql | 0
.../multi_cluster_s3_load}/ddl/region_load.sql | 0
.../multi_cluster_s3_load}/ddl/supplier.sql | 9 +-
.../multi_cluster_s3_load}/ddl/supplier_delete.sql | 0
.../multi_cluster_s3_load}/ddl/supplier_load.sql | 0
.../multi_cluster_s3_load/load.groovy | 166 +++
.../op/test_drop_and_add_cluster.groovy | 221 +++
.../cloud_p0/multi_cluster/rename_cluster.groovy | 115 ++
.../routine_load/test_routine_load.groovy | 189 +++
.../test_bitmap_index_with_drop_cluster.groovy | 143 ++
.../test_bitmap_index_with_readd_cluster.groovy | 154 +++
...test_materialized_view_with_drop_cluster.groovy | 123 ++
...est_materialized_view_with_readd_cluster.groovy | 155 +++
.../test_rollup_with_drop_cluster.groovy | 122 ++
.../test_rollup_with_readd_cluster.groovy | 156 +++
.../test_schema_change_with_drop_cluster.groovy | 120 ++
.../test_schema_change_with_readd_cluster.groovy | 169 +++
.../multi_cluster/smooth_upgrade/ddl/customer.sql | 15 +
.../smooth_upgrade}/ddl/customer_delete.sql | 0
.../smooth_upgrade}/ddl/customer_load.sql | 0
.../smooth_upgrade}/ddl/customer_load_sequence.sql | 0
.../smooth_upgrade}/ddl/customer_part_delete.sql | 0
.../smooth_upgrade/ddl/customer_sequence.sql | 17 +
.../multi_cluster/smooth_upgrade/ddl/lineitem.sql | 23 +
.../smooth_upgrade}/ddl/lineitem_delete.sql | 0
.../smooth_upgrade/ddl/lineitem_load.sql | 6 +
.../smooth_upgrade/ddl/lineitem_load_sequence.sql | 7 +
.../smooth_upgrade}/ddl/lineitem_part_delete.sql | 0
.../smooth_upgrade/ddl/lineitem_sequence.sql | 25 +
.../multi_cluster/smooth_upgrade/ddl/nation.sql | 11 +
.../smooth_upgrade}/ddl/nation_delete.sql | 0
.../smooth_upgrade}/ddl/nation_load.sql | 0
.../smooth_upgrade}/ddl/nation_load_sequence.sql | 0
.../smooth_upgrade}/ddl/nation_part_delete.sql | 0
.../smooth_upgrade/ddl/nation_sequence.sql | 13 +
.../multi_cluster/smooth_upgrade/ddl/orders.sql | 15 +
.../smooth_upgrade}/ddl/orders_delete.sql | 0
.../smooth_upgrade}/ddl/orders_load.sql | 0
.../smooth_upgrade}/ddl/orders_load_sequence.sql | 0
.../smooth_upgrade}/ddl/orders_part_delete.sql | 0
.../smooth_upgrade/ddl/orders_sequence.sql | 17 +
.../multi_cluster/smooth_upgrade/ddl/part.sql | 16 +
.../smooth_upgrade}/ddl/part_delete.sql | 0
.../smooth_upgrade}/ddl/part_load.sql | 0
.../smooth_upgrade}/ddl/part_load_sequence.sql | 0
.../smooth_upgrade}/ddl/part_part_delete.sql | 0
.../smooth_upgrade/ddl/part_sequence.sql | 18 +
.../multi_cluster/smooth_upgrade/ddl/partsupp.sql | 12 +
.../smooth_upgrade}/ddl/partsupp_delete.sql | 0
.../smooth_upgrade}/ddl/partsupp_load.sql | 0
.../smooth_upgrade}/ddl/partsupp_load_sequence.sql | 0
.../smooth_upgrade}/ddl/partsupp_part_delete.sql | 0
.../smooth_upgrade/ddl/partsupp_sequence.sql | 14 +
.../multi_cluster/smooth_upgrade/ddl/region.sql | 10 +
.../smooth_upgrade}/ddl/region_delete.sql | 0
.../smooth_upgrade}/ddl/region_load.sql | 0
.../smooth_upgrade}/ddl/region_load_sequence.sql | 0
.../smooth_upgrade}/ddl/region_part_delete.sql | 0
.../smooth_upgrade/ddl/region_sequence.sql | 12 +
.../multi_cluster/smooth_upgrade/ddl/supplier.sql | 13 +
.../smooth_upgrade}/ddl/supplier_delete.sql | 0
.../smooth_upgrade}/ddl/supplier_load.sql | 0
.../smooth_upgrade}/ddl/supplier_load_sequence.sql | 0
.../smooth_upgrade}/ddl/supplier_part_delete.sql | 0
.../smooth_upgrade/ddl/supplier_sequence.sql | 15 +
.../smooth_upgrade/smooth_upgrade.groovy | 165 +++
.../multi_cluster/stream_load/stream_load.groovy | 322 +++++
.../stream_load/stream_load_2pc.groovy | 396 ++++++
.../stream_load/stream_load_lb.groovy | 227 ++++
.../cloud_p0/multi_cluster/table_rebalance.groovy | 175 +++
.../test_apsaradb_internal_stage.groovy | 199 +++
.../multi_cluster/test_cloud_cluster.groovy | 202 +++
.../multi_cluster/test_drop_cluster.groovy | 96 ++
.../cloud_p0/multi_cluster/test_drop_userid.groovy | 121 ++
.../test_group_commit_multi_cluster.groovy | 129 ++
.../multi_cluster/test_overdue_instance.groovy | 243 ++++
.../cloud_p0/query_retry/test_retry_e-230.groovy | 1 -
.../warm_up_select/test_warmup_select.groovy | 113 ++
.../http/test_check_file_cache_consistency.groovy | 141 ++
.../cold_data_compaction.groovy | 6 +
.../test_single_replica_compaction.groovy | 3 +-
.../spark_connector/spark_connector.groovy | 41 -
.../spark_connector/spark_connector_arrow.groovy | 20 +-
.../spark_connector_read_type.groovy | 21 +-
.../suites/correctness/test_compare_float.groovy | 2 +-
...est_distinct_streaming_agg_local_shuffle.groovy | 3 -
.../correctness_p0/test_grouping_sets_empty.groovy | 5 -
.../correctness_p0/test_mask_function.groovy | 16 +-
.../correctness_p0/test_multi_distinct.groovy | 70 +
.../suites/correctness_p0/test_probe_clean.groovy | 21 -
.../suites/correctness_p0/test_time_round.groovy | 6 +
.../decimalv2/test_decimalv2_rqg.groovy | 59 +
.../one_level_nestedtypes_with_s3data.groovy | 6 +-
.../string_len/test_string_len_complex.groovy | 229 ++++
.../create_view_nereids_fix_null.groovy | 2 +-
.../suites/demo_p0/test_external_ms_cluster.groovy | 83 ++
.../data-operate/scheduler/job-scheduler.md.groovy | 2 +-
.../suites/doc/external/jdbc/mysql.md.groovy | 2 +-
.../basic-elements/data-types/array-md.groovy | 8 +-
.../basic-elements/data-types/map-md.groovy | 8 +-
.../basic-elements/data-types/struct-md.groovy | 8 +-
.../sql-functions/doc_date_functions_test.groovy | 454 +++++++
.../outfile/test_outfile_parquet_schema.groovy | 93 ++
.../cache/test_file_cache_features.groovy | 222 +++
.../cache/test_file_cache_statistics.groovy | 199 ++-
.../cache/test_hive_warmup_select.groovy | 95 ++
.../export/test_export_external_table.groovy | 2 +-
.../hive/test_hive_use_meta_cache_false.groovy | 2 +-
.../hive/test_parquet_nested_types.groovy | 8 +-
.../jdbc/test_doris_jdbc_catalog.groovy | 2 +-
.../test_doris_jdbc_catalog_query_bitmap.groovy | 2 +-
.../external_table_p0/jdbc/test_jdbc_call.groovy | 4 +-
.../jdbc/test_jdbc_catalog_push_cast.groovy | 2 +-
.../jdbc/test_jdbc_query_mysql.groovy | 2 +-
.../jdbc/test_jdbc_query_tvf.groovy | 2 +-
.../jdbc/test_jdbc_row_count.groovy | 2 +-
.../jdbc/test_jni_complex_type.groovy | 2 +-
.../jdbc/test_mariadb_jdbc_catalog.groovy | 2 +-
.../jdbc/test_mysql_jdbc_catalog.groovy | 16 +-
.../jdbc/test_mysql_jdbc_catalog_nereids.groovy | 2 +-
.../test_mysql_jdbc_catalog_table_comment.groovy | 6 +-
.../jdbc/test_mysql_jdbc_statistics.groovy | 2 +-
.../test_switch_catalog_and_delete_internal.groovy | 2 +-
.../ctas/test_mysql_all_types_ctas.groovy | 2 +-
.../select/test_doris_all_types_select.groovy | 2 +-
.../select/test_mysql_all_types_select.groovy | 2 +-
.../select/test_mysql_varbinary_with_udf.groovy | 2 +-
.../type_test/tvf/test_mysql_all_types_tvf.groovy | 2 +-
.../lower_case/test_conflict_name.groovy | 2 +-
.../lower_case/test_lower_case_meta_include.groovy | 2 +-
.../test_lower_case_meta_show_and_select.groovy | 2 +-
...er_case_meta_with_lower_table_conf_grant.groovy | 2 +-
...ta_with_lower_table_conf_show_and_select.groovy | 2 +-
.../lower_case/test_lower_case_mtmv.groovy | 2 +-
.../test_meta_cache_select_without_refresh.groovy | 2 +-
.../lower_case/test_meta_names_mapping.groovy | 2 +-
.../lower_case/test_timing_refresh_catalog.groovy | 2 +-
.../lower_case/upgrade/load.groovy | 2 +-
.../test_nereids_refresh_catalog.groovy | 2 +-
.../nereids_commands/test_use_database_stmt.groovy | 2 +-
.../polaris/test_iceberg_insert_refresh.groovy | 144 ++
.../tvf/test_s3_tvf_number_range.groovy | 398 ++++++
.../suites/external_table_p0/upgrade/load.groovy | 4 +-
.../maxcompute/test_max_compute_schema.groovy | 239 ++++
.../mysql/test_external_catalog_mysql.groovy | 2 +-
.../mysql/test_external_resource_mysql.groovy | 2 +-
.../cloud/test_cloud_version_already_merged.groovy | 142 ++
.../memtable/test_memtable_too_many_rows.groovy | 49 +
...st_cumu_compaction_delay_fault_injection.groovy | 5 +
...st_skip_index_compaction_fault_injection.groovy | 3 +
.../flink_connector_p0/flink_connector.groovy | 12 +-
.../flink_connector_syncdb.groovy | 19 +-
.../flink_connector_p0/flink_connector_type.groovy | 112 +-
.../test_cast_to_string_from_decimalv3.groovy | 53 +
.../to_string/test_cast_to_string_from_int.groovy | 51 +
.../test_ngram_bloomfilter_index_change.groovy | 3 +-
.../insert_overwrite_auto_detect.groovy | 16 +-
.../test_iot_auto_detect_fail.groovy | 4 +-
.../analyzer/test_custom_analyzer.groovy | 26 +-
.../analyzer/test_custom_analyzer1.groovy | 69 +-
.../analyzer/test_custom_analyzer2.groovy | 62 +
.../test_index_change_on_renamed_column.groovy | 2 +-
...est_cumulative_compaction_with_format_v2.groovy | 2 +-
.../test_mor_table_with_format_v2.groovy | 4 +-
.../test_mow_table_with_format_v2.groovy | 2 +-
.../test_storage_format_default.groovy | 91 ++
.../inverted_index_p0/test_inverted_is_null.groovy | 72 +
.../streaming_job/test_streaming_insert_job.groovy | 9 +-
.../test_streaming_insert_job_alter.groovy | 155 +++
.../test_streaming_insert_job_crud.groovy | 494 +++++++
.../test_streaming_insert_job_drop.groovy | 97 ++
.../test_streaming_insert_job_offset.groovy | 226 ++++
.../test_streaming_insert_job_priv.groovy | 204 +++
.../test_streaming_insert_job_task_retry.groovy | 92 ++
.../test_streaming_job_auto_resume.groovy | 113 ++
.../test_streaming_job_restart_fe.groovy | 123 ++
.../test_streaming_job_schedule_task_error.groovy | 94 ++
.../suites/json_p0/test_json_root.groovy | 159 +++
.../suites/jsonb_p0/test_json_hash.groovy | 116 ++
...st_json_normalize_json_numbers_to_double.groovy | 113 ++
.../test_json_sort_json_object_keys.groovy | 100 ++
.../load_p0/broker_load/test_array_load.groovy | 54 +-
.../test_broker_load_with_partition.groovy | 10 +-
.../broker_load/test_broker_load_with_where.groovy | 2 +-
.../test_broker_load_without_filepath.groovy | 2 +-
...where.groovy => test_s3_load_with_where.groovy} | 138 +-
.../test_ingestion_load_alter_column.groovy | 4 +-
.../insert/test_insert_move_memtable.groovy | 119 --
.../load_p0/insert/test_insert_statistic.groovy | 29 +-
.../load_p0/merge_into/test_merge_into.groovy | 254 ++++
.../test_routine_load_adaptive_param.groovy | 91 ++
.../test_routine_load_delay_schedule.groovy | 83 ++
.../test_map_load_and_compaction.groovy | 4 +-
.../load_p0/tvf/test_tvf_lz4_compress.groovy | 4 +-
.../suites/manager/test_manager_interface_1.groovy | 2 +-
.../suites/manager/test_manager_interface_3.groovy | 4 +-
.../suites/mtmv_p0/test_build_mtmv.groovy | 15 -
.../mtmv_p0/test_create_mtmv_with_view.groovy | 112 ++
.../test_create_mtmv_with_view_alter.groovy | 94 ++
.../test_create_mtmv_with_view_alter_table.groovy | 103 ++
.../test_create_mtmv_with_view_commit.groovy | 69 +
.../mtmv_p0/test_create_mtmv_with_view_cte.groovy | 92 ++
.../mtmv_p0/test_create_mtmv_with_view_mtmv.groovy | 126 ++
.../mtmv_p0/test_create_mtmv_with_view_pct.groovy | 103 ++
.../test_create_mtmv_with_view_rollup.groovy | 81 ++
.../mtmv_p0/test_limit_partition_view_mtmv.groovy | 254 ++++
.../suites/mtmv_p0/test_mysql_mtmv.groovy | 2 +-
.../suites/mtmv_p0/test_view_hive_mtmv.groovy | 98 ++
.../mv_p0/test_create_mv/test_create_mv.groovy | 12 +
.../test_mv_with_keyword.groovy | 67 +
.../nereids_p0/cache/mtmv_with_sql_cache.groovy | 971 ++++++++++++++
.../nereids_p0/cache/mv_with_sql_cache.groovy | 263 ++++
.../cache/parse_sql_from_sql_cache.groovy | 31 +
.../datatype/test_date_implicit_cast.groovy | 8 +-
.../nereids_p0/javaudf/test_alias_function.groovy | 2 +-
.../partition_topn/check_orderkey.groovy | 61 +
.../session_variable/test_default_limit.groovy | 8 +-
.../nereids_p0/show/test_nereids_show_load.groovy | 4 +-
.../show/test_nereids_show_resources.groovy | 2 +-
.../suites/nereids_p0/show/test_show_where.groovy | 2 +-
.../conditional_functions/test_nullif.groovy | 14 +-
.../datetime_functions/test_date_function.groovy | 34 +
.../test_date_function_v2.groovy | 68 +
.../json_function/test_query_json_object.groovy | 5 +-
.../nereids_p0/stats/partitionRowCount.groovy | 2 +-
.../avg_distinct_to_sum_div_count.groovy | 23 +-
.../expression/test_simplify_range.groovy | 2 +-
.../external/dml_query_has_external_table.groovy | 2 +-
.../partition_prune/one_key_list_part_test.groovy | 1 +
.../one_key_list_part_update_test.groovy | 1 +
.../partition_prune/one_key_range_part_test.groovy | 1 +
.../one_key_range_part_update_test.groovy | 3 +-
.../suites/nereids_syntax_p0/alias_conflict.groovy | 2 +-
.../suites/nereids_syntax_p0/explain.groovy | 2 +-
.../nereids_tpch_p0/tpch/push_topn_to_agg.groovy | 2 +-
.../auto_partition/test_auto_list_partition.groovy | 2 +-
.../test_auto_partition_behavior.groovy | 40 +-
...uto_partition_with_single_replica_insert.groovy | 4 +-
.../test_col_data_type_boundary.groovy | 20 +-
...t_dynamic_partition_mod_distribution_key.groovy | 1 +
.../suites/query_p0/cache/condition_cache.groovy | 290 ++++
.../suites/query_p0/explain/explain_alias.groovy | 56 +
.../query_p0/join/test_join_result_count.groovy | 3 -
.../join/test_partitioned_hash_join.groovy | 2 +-
.../schema_table/test_active_queries.groovy | 15 -
.../schema_table/test_backend_active_tasks.groovy | 15 -
.../session_variable/test_default_limit.groovy | 8 +-
.../query_p0/show/test_show_partitions.groovy | 2 +-
.../suites/query_p0/show/test_show_where.groovy | 2 +-
.../array_functions/test_array_map_function.groovy | 8 -
.../binary_functions/test_binary_function.groovy | 128 ++
.../bitmap_functions/test_bitmap_function.groovy | 26 -
.../test_cast_array_functions_by_literal.groovy | 36 +-
.../cast_function/test_cast_struct.groovy | 8 +-
.../conditional_functions/test_coalesce.groovy | 88 ++
.../conditional_functions/test_nullif.groovy | 14 +-
.../conditional_functions/test_query_in.groovy | 13 +
.../test_binary_for_digest.groovy | 2 +-
.../hash_functions/test_hash_function.groovy | 19 +
.../string_functions/test_string_all.groovy | 1095 +++++++++++++++
.../string_functions/test_string_function.groovy | 99 ++
.../struct_functions/test_struct_functions.groovy | 8 +-
.../sql_functions/table_function/explode.groovy | 47 +
.../system/test_query_sys_column_data_sizes.groovy | 130 ++
.../test_ranger_access_resource_catalog.groovy | 2 +-
.../test_ranger_access_resource_global.groovy | 2 +-
.../test_ranger_access_resource_resource.groovy | 2 +-
...test_ranger_access_role_resource_catalog.groovy | 2 +-
.../rollup_p0/test_create_table_with_rollup.groovy | 42 +
.../test_double_write_when_schema_change.groovy | 2 +-
.../test_alter_rollup_table.groovy | 9 +-
.../test_alter_table_modify_column.groovy | 2 +-
.../schema_change_p0/test_schema_change_agg.groovy | 4 +-
.../suites/search/test_search_mow_support.groovy | 78 ++
.../search/test_search_vs_match_consistency.groovy | 40 +
.../suites/show_p0/test_show_data_warehouse.groovy | 16 +-
.../ssb_sf0.1_p1/sql/pipelinex_flat_q1.1.sql | 24 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q1.2.sql | 24 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q1.3.sql | 25 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q2.1.sql | 24 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q2.2.sql | 27 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q2.3.sql | 26 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q3.1.sql | 29 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q3.2.sql | 29 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q3.3.sql | 29 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q3.4.sql | 29 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q4.1.sql | 27 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q4.2.sql | 33 -
.../ssb_sf0.1_p1/sql/pipelinex_flat_q4.3.sql | 29 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q1.1.sql | 24 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q1.2.sql | 24 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q1.3.sql | 25 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q2.1.sql | 26 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q2.2.sql | 27 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q2.3.sql | 26 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q3.1.sql | 28 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q3.2.sql | 28 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q3.3.sql | 30 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q3.4.sql | 30 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q4.1.sql | 30 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q4.2.sql | 31 -
.../suites/ssb_sf0.1_p1/sql/pipelinex_q4.3.sql | 29 -
.../test_temp_table_with_conn_timeout.groovy | 45 -
.../suites/tpch_sf0.1_p1/sql/cse.groovy | 2 +-
.../ddl/customer.sql | 0
.../ddl/customer_delete.sql | 0
.../ddl/lineitem.sql | 0
.../ddl/lineitem_delete.sql | 0
.../ddl/nation.sql | 0
.../ddl/nation_delete.sql | 0
.../ddl/orders.sql | 0
.../ddl/orders_delete.sql | 0
.../ddl/part.sql | 0
.../ddl/part_delete.sql | 0
.../ddl/partsupp.sql | 0
.../ddl/partsupp_delete.sql | 0
.../ddl/region.sql | 0
.../ddl/region_delete.sql | 0
.../ddl/revenue1.sql | 0
.../ddl/revenue1_delete.sql | 0
.../ddl/supplier.sql | 0
.../ddl/supplier_delete.sql | 0
.../load.groovy | 0
.../sql/q01.sql | 0
.../sql/q02.sql | 0
.../sql/q03.sql | 0
.../sql/q04.sql | 0
.../sql/q05.sql | 0
.../sql/q06.sql | 0
.../sql/q07.sql | 0
.../sql/q08.sql | 0
.../sql/q09.sql | 0
.../sql/q10.sql | 0
.../sql/q11.sql | 0
.../sql/q12.sql | 0
.../sql/q13.sql | 0
.../sql/q14.sql | 0
.../sql/q15.sql | 0
.../sql/q16.sql | 0
.../sql/q17.sql | 0
.../sql/q18.sql | 0
.../sql/q19.sql | 0
.../sql/q20.sql | 0
.../sql/q21.sql | 0
.../sql/q22.sql | 0
.../sql/test_left_anti_join_batch_size.sql | 0
.../test_key_bounds_truncation_basic.groovy | 12 +
...est_key_bounds_truncation_read_scenarios.groovy | 12 +
regression-test/suites/variant_p0/nested.groovy | 2 +-
.../predefine/test_multi_index_file.groovy | 2 +-
.../variant_p0/predefine/variant_with_mow.groovy | 4 +-
.../test_variant_search_subcolumn.groovy | 152 +++
run-regression-test.sh | 72 +-
samples/datalake/lakesoul/README.md | 2 +-
thirdparty/CHANGELOG.md | 7 +
thirdparty/build-thirdparty.sh | 120 +-
thirdparty/download-thirdparty.sh | 24 -
thirdparty/patches/googletest-release-1.11.0.patch | 71 -
thirdparty/patches/libevent-1532.patch | 188 ---
.../libevent-keepalive-accepted-socket.patch | 17 -
thirdparty/patches/libevent.patch | 193 ---
thirdparty/vars.sh | 39 +-
1921 files changed, 62173 insertions(+), 17429 deletions(-)
create mode 100644 be/benchmark/benchmark_block_bloom_filter.hpp
create mode 100644 be/benchmark/benchmark_string.hpp
create mode 100644 be/src/exec/schema_scanner/schema_column_data_sizes_scanner.cpp
create mode 100644 be/src/exec/schema_scanner/schema_column_data_sizes_scanner.h
copy be/src/olap/rowset/segment_v2/{binary_plain_page.h => binary_plain_page_v2.h} (56%)
create mode 100644 be/src/olap/rowset/segment_v2/condition_cache.cpp
create mode 100644 be/src/olap/rowset/segment_v2/condition_cache.h
copy be/src/olap/rowset/segment_v2/inverted_index/{tokenizer/tokenizer.h => char_filter/char_filter.h} (52%)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/char_filter/empty_char_filter_factory.h
copy be/src/olap/rowset/segment_v2/inverted_index/query_v2/{bitmap_query/bitmap_query.h => bit_set_query/bit_set_query.h} (73%)
rename be/src/olap/rowset/segment_v2/inverted_index/query_v2/{bitmap_query/bitmap_scorer.h => bit_set_query/bit_set_scorer.h} (87%)
copy be/src/olap/rowset/segment_v2/inverted_index/query_v2/{bitmap_query/bitmap_weight.h => bit_set_query/bit_set_weight.h} (81%)
copy be/src/olap/rowset/segment_v2/inverted_index/query_v2/{bitmap_query/bitmap_query.h => const_score_query/const_score_scorer.h} (53%)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/query_v2/intersection.cpp
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/query_v2/intersection.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/query_v2/phrase_query/phrase_query.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/query_v2/phrase_query/phrase_scorer.cpp
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/query_v2/phrase_query/phrase_scorer.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/query_v2/phrase_query/phrase_weight.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/query_v2/phrase_query/postings_with_offset.h
copy be/src/olap/rowset/segment_v2/inverted_index/query_v2/{bitmap_query/bitmap_query.h => regexp_query/regexp_query.h} (54%)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/query_v2/regexp_query/regexp_weight.cpp
rename be/src/olap/rowset/segment_v2/inverted_index/query_v2/{bitmap_query/bitmap_weight.h => regexp_query/regexp_weight.h} (51%)
rename be/src/olap/rowset/segment_v2/inverted_index/query_v2/{bitmap_query/bitmap_query.h => wildcard_query/wildcard_query.h} (53%)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/query_v2/wildcard_query/wildcard_weight.h
copy be/src/olap/rowset/segment_v2/inverted_index/{tokenizer/tokenizer.h => token_filter/empty_token_filter_factory.h} (51%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/basic/basic_tokenizer.cpp (75%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/basic/basic_tokenizer.h (69%)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer_factory.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/empty/empty_tokenizer_factory.h
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/break_iterator_wrapper.cpp (97%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/break_iterator_wrapper.h (94%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/composite_break_iterator.cpp (97%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/composite_break_iterator.h (94%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/default_icu_tokenizer_config.cpp (98%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/default_icu_tokenizer_config.h (94%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/icu_common.h (93%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/icu_tokenizer.cpp (84%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/icu_tokenizer.h (82%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/icu_tokenizer_config.h (93%)
copy be/src/olap/rowset/segment_v2/inverted_index/{token_filter/token_filter.h => tokenizer/icu/icu_tokenizer_factory.h} (61%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/script_iterator.cpp (97%)
rename be/src/olap/rowset/segment_v2/inverted_index/{analyzer => tokenizer}/icu/script_iterator.h (95%)
copy be/src/{vec/functions/function_hash.h => olap/rowset/segment_v2/inverted_index/util/reader.h} (78%)
create mode 100644 be/src/olap/rowset_version_mgr.cpp
create mode 100644 be/src/pipeline/exec/blackhole_sink_operator.cpp
create mode 100644 be/src/pipeline/exec/blackhole_sink_operator.h
create mode 100644 be/src/pipeline/revokable_task.h
create mode 100644 be/src/vec/exec/format/generic_reader.cpp
create mode 100644 be/src/vec/functions/function_json_hash.cpp
create mode 100644 be/src/vec/functions/function_jsonb_transform.cpp
create mode 100644 be/src/vec/functions/function_needs_to_handle_null.h
create mode 100644 be/src/vec/functions/function_varbinary.h
create mode 100644 be/src/vec/utils/varbinaryop_subbinary.h
copy be/{src/common/version_internal.h => test/common/compare_test.cpp} (61%)
create mode 100644 be/test/http/file_cache_action_test.cpp
create mode 100644 be/test/io/s3_client_factory_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/binary_plain_page_v2_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/analysis_factory_mgr_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/char_filter/char_filter_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/query_v2/intersection_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/query_v2/phrase_query_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/query_v2/regexp_query_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/query_v2/wildcard_query_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/tokenizer/basic_tokenizer_factory_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/tokenizer/icu_tokenizer_factory_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/util/reader_test.cpp
create mode 100644 be/test/util/test_data/deserialize_from_sparse_column_test.bin
create mode 100644 be/test/vec/function/function_varbinary_test.cpp
create mode 100644 cloud/src/recycler/snapshot_chain_compactor.cpp
create mode 100644 cloud/src/recycler/snapshot_chain_compactor.h
create mode 100644 cloud/src/recycler/snapshot_data_migrator.cpp
create mode 100644 cloud/src/recycler/snapshot_data_migrator.h
create mode 100644 common/cpp/custom_aws_credentials_provider_chain.cpp
copy be/src/vec/functions/function_hash.h => common/cpp/custom_aws_credentials_provider_chain.h (72%)
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/BaseTableRef.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/TableName.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketCalculator.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/McStructureHelper.java
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{AbstractHMSProperties.java => AbstractHiveProperties.java} (93%)
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{HMSAliyunDLFMetaStoreProperties.java => HiveAliyunDLFMetaStoreProperties.java} (94%)
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{HMSGlueMetaStoreProperties.java => HiveGlueMetaStoreProperties.java} (96%)
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{HMSProperties.java => HiveHMSProperties.java} (95%)
rename fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/{HMSPropertiesFactory.java => HivePropertiesFactory.java} (70%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AzurePropertyUtils.java
copy fe/fe-core/src/main/java/org/apache/doris/indexpolicy/{LowerCaseTokenFilterValidator.java => BasicTokenizerValidator.java} (59%)
copy fe/fe-core/src/main/java/org/apache/doris/indexpolicy/{KeywordTokenizerValidator.java => CharReplaceCharFilterValidator.java} (53%)
rename fe/fe-core/src/main/java/org/apache/doris/indexpolicy/{LowerCaseTokenFilterValidator.java => ICUTokenizerValidator.java} (83%)
rename fe/fe-core/src/main/java/org/apache/doris/indexpolicy/{KeywordTokenizerValidator.java => NoOperationValidator.java} (83%)
copy fe/fe-core/src/main/java/org/apache/doris/{nereids/rules/analysis/CollectOneLevelRelation.java => mtmv/MTMVAnalyzeQueryInfo.java} (50%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundBlackholeSink.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/GetFormatFunctionBinder.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/CaseWhenToCompoundPredicate.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/CondReplaceNullWithFalse.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConditionRewrite.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ReplaceNullWithFalseForCond.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyEqualBooleanLiteral.java
copy fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/{LogicalEsScanToPhysicalEsScan.java => LogicalBlackholeSinkToPhysicalBlackholeSink.java} (58%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{MaskFirstN.java => ExportSet.java} (53%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Length.java => FromBase64Binary.java} (74%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Length.java => GetFormat.java} (63%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Length.java => JsonHash.java} (64%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Length.java => MurmurHash364V2.java} (65%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Length.java => NormalizeJsonNumbersToDouble.java} (73%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{FromSecond.java => PeriodAdd.java} (64%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{FromSecond.java => PeriodDiff.java} (64%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Length.java => SortJsonbObjectKeys.java} (73%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{MaskFirstN.java => SubBinary.java} (56%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Length.java => ToBase64Binary.java} (75%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/{AlterJobStatusCommand.java => CreateIndexCharFilterCommand.java} (57%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/{DropJobCommand.java => DropIndexCharFilterCommand.java} (71%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/{PauseJobCommand.java => ShowIndexCharFilterCommand.java} (67%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/{DictionaryInsertExecutor.java => BlackholeInsertExecutor.java} (60%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/WarmupSelectCommand.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/merge/MergeIntoCommand.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/merge/MergeMatchedClause.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/merge/MergeNotMatchedClause.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalBlackholeSink.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalBlackholeSink.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/BlackholeSink.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionExprUtilTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/AutoIncrementGeneratorTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/cloud/master/CloudReportHandlerTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketCalculatorTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/AzurePropertyUtilsTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVSchemaChangeTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/parser/UnboundFunctionWithUnboundStarTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/CaseWhenToCompoundPredicateTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/CondReplaceNullWithFalseTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/ReplaceNullWithFalseForCondTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyEqualBooleanLiteralTest.java
rename fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/{ => executable}/DateTimeExtractAndTransformTest.java (87%)
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/merge/MergeIntoCommandTest.java
copy regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_flat_q3.4.out => ann_index_p0/product_quantization.out} (57%)
create mode 100644 regression-test/data/cast_p0/datelike_print.out
create mode 100644 regression-test/data/cloud_p0/multi_cluster/routine_load/test_routine_load.out
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_flat_q1.3.out => cloud_p0/multi_cluster/schema_change/test_schema_change_with_readd_cluster.out} (66%)
copy regression-test/data/{cloud/multi_cluster/vcluster => cloud_p0/multi_cluster/stream_load}/all_types.csv (100%)
copy regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_flat_q3.4.out => cloud_p0/multi_cluster/stream_load/stream_load.out} (58%)
create mode 100644 regression-test/data/cloud_p0/multi_cluster/stream_load/stream_load_2pc.out
create mode 100644 regression-test/data/cloud_p0/multi_cluster/stream_load/stream_load_lb.out
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_q1.2.out => cloud_p0/multi_cluster/test_apsaradb_internal_stage.out} (76%)
copy regression-test/data/{query_p0/aggregate/aggregate_count1.out => correctness_p0/test_multi_distinct.out} (100%)
create mode 100644 regression-test/data/datatype_p0/nested_types/string_len/test_string_len_complex.out
create mode 100644 regression-test/data/export_p0/outfile/test_outfile_parquet_schema.out
create mode 100644 regression-test/data/external_table_p0/tvf/hdfs_data_1.txt
create mode 100644 regression-test/data/external_table_p0/tvf/hdfs_data_2.txt
create mode 100644 regression-test/data/external_table_p0/tvf/hdfs_data_3.txt
create mode 100644 regression-test/data/external_table_p0/tvf/test_s3_tvf_number_range.out
create mode 100644 regression-test/data/external_table_p2/maxcompute/test_max_compute_schema.out
create mode 100644 regression-test/data/fault_injection_p0/cloud/test_cloud_version_already_merged.out
create mode 100644 regression-test/data/function_p0/cast/to_string/test_cast_to_string_from_decimalv3.out
create mode 100644 regression-test/data/function_p0/cast/to_string/test_cast_to_string_from_int.out
create mode 100644 regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer2.out
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_q3.4.out => inverted_index_p0/storage_format/test_storage_format_default.out} (81%)
create mode 100644 regression-test/data/job_p0/streaming_job/test_streaming_insert_job_alter.out
create mode 100644 regression-test/data/job_p0/streaming_job/test_streaming_insert_job_crud.out
create mode 100644 regression-test/data/job_p0/streaming_job/test_streaming_insert_job_offset.out
create mode 100644 regression-test/data/json_p0/simple_json.json
create mode 100644 regression-test/data/json_p0/test_json_root.out
create mode 100644 regression-test/data/jsonb_p0/test_json_hash.out
create mode 100644 regression-test/data/jsonb_p0/test_json_normalize_json_numbers_to_double.out
create mode 100644 regression-test/data/jsonb_p0/test_json_sort_json_object_keys.out
copy regression-test/data/load_p0/broker_load/{test_broker_load_with_where.out => test_s3_load_with_where.out} (91%)
delete mode 100644 regression-test/data/load_p0/insert/test_insert_move_memtable.out
create mode 100644 regression-test/data/load_p0/merge_into/test_merge_into.out
copy regression-test/data/load_p0/{stream_load => tvf}/test_compress.csv.lz4 (100%)
create mode 100644 regression-test/data/mtmv_p0/test_create_mtmv_with_view.out
create mode 100644 regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter.out
create mode 100644 regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter_table.out
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_flat_q1.1.out => mtmv_p0/test_create_mtmv_with_view_commit.out} (64%)
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_flat_q1.2.out => mtmv_p0/test_create_mtmv_with_view_cte.out} (63%)
create mode 100644 regression-test/data/mtmv_p0/test_create_mtmv_with_view_mtmv.out
create mode 100644 regression-test/data/mtmv_p0/test_create_mtmv_with_view_pct.out
create mode 100644 regression-test/data/mtmv_p0/test_create_mtmv_with_view_rollup.out
copy regression-test/data/mtmv_p0/{test_limit_partition_mtmv.out => test_limit_partition_view_mtmv.out} (100%)
create mode 100644 regression-test/data/mtmv_p0/test_view_hive_mtmv.out
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_q4.2.out => nereids_p0/partition_topn/check_orderkey.out} (81%)
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_flat_q3.4.out => nereids_rules_p0/avg_distinct_to_sum_div_count/avg_distinct_to_sum_div_count.out} (78%)
create mode 100644 regression-test/data/query_p0/cache/condition_cache.out
create mode 100644 regression-test/data/query_p0/sql_functions/string_functions/test_string_all.out
create mode 100644 regression-test/data/query_p0/system/test_query_sys_column_data_sizes.out
rename regression-test/data/{ssb_sf0.1_p1/sql/pipelinex_q1.1.out => rollup_p0/test_create_table_with_rollup.out} (75%)
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q2.1.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q2.2.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q2.3.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q3.1.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q3.2.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q3.3.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q4.1.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q4.2.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_flat_q4.3.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q1.3.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q2.1.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q2.2.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q2.3.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q3.1.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q3.2.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q3.3.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q4.1.out
delete mode 100644 regression-test/data/ssb_sf0.1_p1/sql/pipelinex_q4.3.out
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q01.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q02.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q03.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q04.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q05.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q06.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q07.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q08.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q09.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q10.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q11.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q12.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q13.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q14.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q15.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q16.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q17.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q18.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q19.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q20.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q21.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q22.out (100%)
rename regression-test/data/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/test_left_anti_join_batch_size.out (100%)
create mode 100644 regression-test/data/variant_p0/test_variant_search_subcolumn.out
create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/util/RoutineLoadTestUtils.groovy
create mode 100644 regression-test/suites/ann_index_p0/product_quantization.groovy
create mode 100644 regression-test/suites/cast_p0/datelike_print.groovy
create mode 100644 regression-test/suites/cloud_p0/balance/test_balance_metrics.groovy
create mode 100644 regression-test/suites/cloud_p0/cache/test_load_cache.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/async_copy_into/async_load.groovy
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/customer.sql (79%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/customer_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/lineitem.sql (91%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/lineitem_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/nation.sql (68%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/nation_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/orders.sql (81%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/orders_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/part.sql (81%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/part_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/partsupp.sql (82%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/partsupp_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/region.sql (64%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/region_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/revenue1.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/async_copy_into}/ddl/revenue1_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/supplier.sql (77%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/async_copy_into}/ddl/supplier_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/customer.sql (79%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/customer_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/lineitem.sql (91%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/lineitem_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/nation.sql (68%)
copy regression-test/suites/cloud_p0/multi_cluster/{virtual_compute_group => copy_into}/ddl/nation_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/orders.sql (81%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/orders_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/part.sql (81%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/part_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/partsupp.sql (82%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/partsupp_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/region.sql (64%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/region_delete.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/revenue1.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/revenue1_delete.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/copy_into}/ddl/supplier.sql (77%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/copy_into}/ddl/supplier_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/copy_into/sync_laod.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/default_cluster.groovy
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/customer.sql (71%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/customer_delete.sql (100%)
copy regression-test/suites/cloud_p0/{cache/multi_cluster/read_write => multi_cluster/multi_cluster_s3_load}/ddl/customer_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/lineitem.sql (78%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/lineitem_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/multi_cluster_s3_load/ddl/lineitem_load.sql
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/nation.sql (68%)
copy regression-test/suites/cloud_p0/multi_cluster/{virtual_compute_group => multi_cluster_s3_load}/ddl/nation_delete.sql (100%)
copy regression-test/suites/cloud_p0/multi_cluster/{virtual_compute_group => multi_cluster_s3_load}/ddl/nation_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/orders.sql (73%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/orders_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/orders_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/part.sql (72%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/part_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/part_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/partsupp.sql (61%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/partsupp_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/partsupp_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/region.sql (64%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/region_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/region_load.sql (100%)
copy regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/supplier.sql (68%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/supplier_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/multi_cluster_s3_load}/ddl/supplier_load.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/multi_cluster_s3_load/load.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/op/test_drop_and_add_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/rename_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/routine_load/test_routine_load.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_bitmap_index_with_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_bitmap_index_with_readd_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_materialized_view_with_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_materialized_view_with_readd_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_rollup_with_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_rollup_with_readd_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_schema_change_with_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/schema_change/test_schema_change_with_readd_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/customer.sql
copy regression-test/suites/{nereids_rules_p0/mv/tpch => cloud_p0/multi_cluster/smooth_upgrade}/ddl/customer_delete.sql (100%)
copy regression-test/suites/cloud_p0/{cache/multi_cluster/read_write => multi_cluster/smooth_upgrade}/ddl/customer_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/customer_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/customer_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/customer_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/lineitem.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/lineitem_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/lineitem_load.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/lineitem_load_sequence.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/lineitem_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/lineitem_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/nation.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/nation_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/nation_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/nation_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/nation_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/nation_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/orders.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/orders_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/orders_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/orders_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/orders_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/orders_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/part.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/part_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/part_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/part_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/part_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/part_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/partsupp.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/partsupp_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/partsupp_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/partsupp_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/partsupp_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/partsupp_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/region.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/region_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/region_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/region_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/region_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/region_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/supplier.sql
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/supplier_delete.sql (100%)
copy regression-test/suites/{tpch_sf100_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/supplier_load.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/supplier_load_sequence.sql (100%)
copy regression-test/suites/{tpch_sf100_unique_p2 => cloud_p0/multi_cluster/smooth_upgrade}/ddl/supplier_part_delete.sql (100%)
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/ddl/supplier_sequence.sql
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/smooth_upgrade/smooth_upgrade.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/stream_load/stream_load.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/stream_load/stream_load_2pc.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/stream_load/stream_load_lb.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/table_rebalance.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_apsaradb_internal_stage.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_cloud_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_drop_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_drop_userid.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_group_commit_multi_cluster.groovy
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_overdue_instance.groovy
create mode 100644 regression-test/suites/cloud_p0/warm_up_select/test_warmup_select.groovy
create mode 100644 regression-test/suites/cloud_p2/cache/http/test_check_file_cache_consistency.groovy
delete mode 100644 regression-test/suites/connector_p0/spark_connector/spark_connector.groovy
create mode 100644 regression-test/suites/correctness_p0/test_multi_distinct.groovy
create mode 100644 regression-test/suites/datatype_p0/decimalv2/test_decimalv2_rqg.groovy
create mode 100644 regression-test/suites/datatype_p0/nested_types/string_len/test_string_len_complex.groovy
create mode 100644 regression-test/suites/demo_p0/test_external_ms_cluster.groovy
create mode 100644 regression-test/suites/export_p0/outfile/test_outfile_parquet_schema.groovy
create mode 100644 regression-test/suites/external_table_p0/cache/test_file_cache_features.groovy
create mode 100644 regression-test/suites/external_table_p0/cache/test_hive_warmup_select.groovy
create mode 100644 regression-test/suites/external_table_p0/polaris/test_iceberg_insert_refresh.groovy
create mode 100644 regression-test/suites/external_table_p0/tvf/test_s3_tvf_number_range.groovy
create mode 100644 regression-test/suites/external_table_p2/maxcompute/test_max_compute_schema.groovy
create mode 100644 regression-test/suites/fault_injection_p0/cloud/test_cloud_version_already_merged.groovy
create mode 100644 regression-test/suites/fault_injection_p0/memtable/test_memtable_too_many_rows.groovy
create mode 100644 regression-test/suites/function_p0/cast/to_string/test_cast_to_string_from_decimalv3.groovy
create mode 100644 regression-test/suites/function_p0/cast/to_string/test_cast_to_string_from_int.groovy
create mode 100644 regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer2.groovy
create mode 100644 regression-test/suites/inverted_index_p0/storage_format/test_storage_format_default.groovy
create mode 100644 regression-test/suites/inverted_index_p0/test_inverted_is_null.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_alter.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_crud.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_drop.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_offset.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_priv.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_task_retry.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_job_auto_resume.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_job_restart_fe.groovy
create mode 100644 regression-test/suites/job_p0/streaming_job/test_streaming_job_schedule_task_error.groovy
create mode 100644 regression-test/suites/json_p0/test_json_root.groovy
create mode 100644 regression-test/suites/jsonb_p0/test_json_hash.groovy
create mode 100644 regression-test/suites/jsonb_p0/test_json_normalize_json_numbers_to_double.groovy
create mode 100644 regression-test/suites/jsonb_p0/test_json_sort_json_object_keys.groovy
copy regression-test/suites/load_p0/broker_load/{test_broker_load_with_where.groovy => test_s3_load_with_where.groovy} (54%)
delete mode 100644 regression-test/suites/load_p0/insert/test_insert_move_memtable.groovy
create mode 100644 regression-test/suites/load_p0/merge_into/test_merge_into.groovy
create mode 100644 regression-test/suites/load_p0/routine_load/test_routine_load_adaptive_param.groovy
create mode 100644 regression-test/suites/load_p0/routine_load/test_routine_load_delay_schedule.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter_table.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view_commit.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view_cte.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view_mtmv.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view_pct.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv_with_view_rollup.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_limit_partition_view_mtmv.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_view_hive_mtmv.groovy
create mode 100644 regression-test/suites/mv_p0/test_mv_with_keyword/test_mv_with_keyword.groovy
create mode 100644 regression-test/suites/nereids_p0/cache/mtmv_with_sql_cache.groovy
create mode 100644 regression-test/suites/nereids_p0/cache/mv_with_sql_cache.groovy
create mode 100644 regression-test/suites/nereids_p0/partition_topn/check_orderkey.groovy
copy fe/fe-core/src/main/java/org/apache/doris/load/EtlJobType.java => regression-test/suites/nereids_rules_p0/avg_distinct_to_sum_div_count/avg_distinct_to_sum_div_count.groovy (62%)
create mode 100755 regression-test/suites/query_p0/cache/condition_cache.groovy
create mode 100644 regression-test/suites/query_p0/explain/explain_alias.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/binary_functions/test_binary_function.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/string_functions/test_string_all.groovy
create mode 100644 regression-test/suites/query_p0/system/test_query_sys_column_data_sizes.groovy
create mode 100644 regression-test/suites/rollup_p0/test_create_table_with_rollup.groovy
create mode 100644 regression-test/suites/search/test_search_mow_support.groovy
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q1.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q1.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q1.3.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q2.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q2.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q2.3.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q3.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q3.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q3.3.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q3.4.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q4.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q4.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_flat_q4.3.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q1.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q1.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q1.3.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q2.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q2.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q2.3.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q3.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q3.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q3.3.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q3.4.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q4.1.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q4.2.sql
delete mode 100644 regression-test/suites/ssb_sf0.1_p1/sql/pipelinex_q4.3.sql
delete mode 100644 regression-test/suites/temp_table_p0/test_temp_table_with_conn_timeout.groovy
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/customer.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/customer_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/lineitem.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/lineitem_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/nation.sql (100%)
copy regression-test/suites/{cloud_p0/multi_cluster/virtual_compute_group => tpch_unique_sql_zstd_bucket1_p1}/ddl/nation_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/orders.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/orders_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/part.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/part_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/partsupp.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/partsupp_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/region.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/region_delete.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/revenue1.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/revenue1_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/ddl/supplier.sql (100%)
copy regression-test/suites/{nereids_rules_p0/mv/tpch => tpch_unique_sql_zstd_bucket1_p1}/ddl/supplier_delete.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/load.groovy (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q01.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q02.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q03.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q04.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q05.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q06.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q07.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q08.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q09.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q10.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q11.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q12.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q13.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q14.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q15.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q16.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q17.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q18.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q19.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q20.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q21.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/q22.sql (100%)
rename regression-test/suites/{tpch_unique_sql_zstd_bucket1_p0 => tpch_unique_sql_zstd_bucket1_p1}/sql/test_left_anti_join_batch_size.sql (100%)
create mode 100644 regression-test/suites/variant_p0/test_variant_search_subcolumn.groovy
delete mode 100644 thirdparty/patches/googletest-release-1.11.0.patch
delete mode 100644 thirdparty/patches/libevent-1532.patch
delete mode 100644 thirdparty/patches/libevent-keepalive-accepted-socket.patch
delete mode 100644 thirdparty/patches/libevent.patch
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/01: support prune nested columns
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 462492fe28b2939329d73c84f7cdcb11402d6f35
Author: 924060929 <la...@selectdb.com>
AuthorDate: Wed Sep 24 17:04:44 2025 +0800
support prune nested columns
---
.idea/vcs.xml | 52 +-
be/src/olap/iterators.h | 5 +
be/src/olap/rowset/beta_rowset_reader.cpp | 4 +
be/src/olap/rowset/rowset_reader_context.h | 5 +
be/src/olap/rowset/segment_v2/column_reader.cpp | 361 ++++++++++-
be/src/olap/rowset/segment_v2/column_reader.h | 70 ++-
be/src/olap/rowset/segment_v2/segment.cpp | 16 +
be/src/olap/tablet_reader.cpp | 3 +
be/src/olap/tablet_reader.h | 4 +
be/src/olap/tablet_schema.cpp | 16 +
be/src/olap/tablet_schema.h | 10 +
be/src/runtime/descriptors.cpp | 5 +
be/src/runtime/descriptors.h | 8 +
be/src/vec/columns/column_map.cpp | 1 +
be/src/vec/exec/scan/olap_scanner.cpp | 20 +-
.../olap/rowset/segment_v2/column_reader_test.cpp | 220 +++++++
.../org/apache/doris/analysis/AccessPathInfo.java | 38 ++
.../org/apache/doris/analysis/SlotDescriptor.java | 43 ++
.../org/apache/doris/datasource/FileScanNode.java | 10 +-
.../org/apache/doris/nereids/StatementContext.java | 10 +
.../glue/translator/PhysicalPlanTranslator.java | 5 +-
.../glue/translator/PlanTranslatorContext.java | 18 +
.../doris/nereids/jobs/executor/Rewriter.java | 6 +
.../org/apache/doris/nereids/rules/RuleSet.java | 2 +
.../org/apache/doris/nereids/rules/RuleType.java | 7 +
.../doris/nereids/rules/analysis/BindRelation.java | 9 +-
.../nereids/rules/analysis/ExpressionAnalyzer.java | 10 +
.../rules/expression/ExpressionNormalization.java | 2 +
.../rules/expression/ExpressionRuleType.java | 1 +
.../expression/rules/NormalizeStructElement.java | 66 ++
.../rewrite/AccessPathExpressionCollector.java | 511 ++++++++++++++++
.../rules/rewrite/AccessPathPlanCollector.java | 176 ++++++
.../nereids/rules/rewrite/NestedColumnPruning.java | 487 +++++++++++++++
.../rules/rewrite/PushDownNestedColumn.java | 315 ++++++++++
.../nereids/rules/rewrite/SlotTypeReplacer.java | 674 +++++++++++++++++++++
.../rules/rewrite/VariantSubPathPruning.java | 3 +-
.../trees/expressions/PreferPushDownProject.java | 41 ++
.../nereids/trees/expressions/SlotReference.java | 56 +-
.../expressions/functions/scalar/ArrayFirst.java | 6 +
.../expressions/functions/scalar/ArrayLast.java | 6 +
.../expressions/functions/scalar/ArrayMap.java | 3 +-
.../expressions/functions/scalar/ElementAt.java | 4 +-
.../functions/scalar/HighOrderFunction.java | 3 +-
.../functions/scalar/MapContainsEntry.java | 3 +-
.../functions/scalar/MapContainsKey.java | 3 +-
.../functions/scalar/MapContainsValue.java | 3 +-
.../expressions/functions/scalar/MapEntries.java | 3 +-
.../expressions/functions/scalar/MapKeys.java | 3 +-
.../expressions/functions/scalar/MapValues.java | 3 +-
.../functions/scalar/StructElement.java | 9 +-
.../expressions/visitor/ScalarFunctionVisitor.java | 14 +-
.../trees/plans/logical/LogicalFileScan.java | 44 +-
.../trees/plans/logical/LogicalHudiScan.java | 22 +-
.../trees/plans/logical/LogicalOlapScan.java | 17 +
.../trees/plans/logical/LogicalTVFRelation.java | 23 +-
.../org/apache/doris/nereids/types/ArrayType.java | 2 +-
.../org/apache/doris/nereids/types/MapType.java | 2 +-
.../doris/nereids/types/NestedColumnPrunable.java | 22 +
.../apache/doris/nereids/types/StructField.java | 3 +-
.../org/apache/doris/nereids/types/StructType.java | 10 +-
.../org/apache/doris/planner/OlapScanNode.java | 3 +
.../java/org/apache/doris/planner/PlanNode.java | 4 +-
.../java/org/apache/doris/planner/ScanNode.java | 109 ++++
.../java/org/apache/doris/qe/SessionVariable.java | 9 +
.../rules/rewrite/PruneNestedColumnTest.java | 625 +++++++++++++++++++
.../apache/doris/utframe/TestWithFeService.java | 7 +-
gensrc/thrift/Descriptors.thrift | 48 +-
.../complex_types/test_pruned_columns.out | 24 +
.../complex_types/test_pruned_columns.groovy | 61 ++
69 files changed, 4285 insertions(+), 103 deletions(-)
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
index 0a185d9bdc7..8b083db2f94 100644
--- a/.idea/vcs.xml
+++ b/.idea/vcs.xml
@@ -1,34 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
<project version="4">
- <component name="IssueNavigationConfiguration">
- <option name="links">
- <list>
- <IssueNavigationLink>
- <option name="issueRegexp" value="#(\d+)" />
- <option name="linkRegexp" value="https://github.com/apache/doris/pull/$1" />
- </IssueNavigationLink>
- </list>
- </option>
- </component>
- <component name="VcsDirectoryMappings">
- <mapping directory="$PROJECT_DIR$" vcs="Git" />
- <mapping directory="$PROJECT_DIR$/be/src/apache-orc" vcs="Git" />
- <mapping directory="$PROJECT_DIR$/be/src/clucene" vcs="Git" />
- </component>
-</project>
+ <component name="IssueNavigationConfiguration">
+ <option name="links">
+ <list>
+ <IssueNavigationLink>
+ <option name="issueRegexp" value="#(\d+)" />
+ <option name="linkRegexp" value="https://github.com/apache/doris/pull/$1" />
+ </IssueNavigationLink>
+ </list>
+ </option>
+ </component>
+ <component name="VcsDirectoryMappings">
+ <mapping directory="$PROJECT_DIR$" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/be/src/apache-orc" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/be/src/clucene" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/apache-orc" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/clucene" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/faiss" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/openblas" vcs="Git" />
+ </component>
+</project>
\ No newline at end of file
diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h
index 9a3ed4ea998..1b63aad4909 100644
--- a/be/src/olap/iterators.h
+++ b/be/src/olap/iterators.h
@@ -47,6 +47,8 @@ namespace segment_v2 {
struct SubstreamIterator;
}
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
class StorageReadOptions {
public:
struct KeyRange {
@@ -143,6 +145,9 @@ public:
std::map<ColumnId, size_t> vir_cid_to_idx_in_block;
std::map<size_t, vectorized::DataTypePtr> vir_col_idx_to_type;
+ std::map<int32_t, TColumnAccessPaths> all_access_paths;
+ std::map<int32_t, TColumnAccessPaths> predicate_access_paths;
+
std::shared_ptr<vectorized::ScoreRuntime> score_runtime;
CollectionStatisticsPtr collection_statistics;
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp
index 1de17e27eee..28683540365 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -103,6 +103,10 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
_read_options.remaining_conjunct_roots = _read_context->remaining_conjunct_roots;
_read_options.common_expr_ctxs_push_down = _read_context->common_expr_ctxs_push_down;
_read_options.virtual_column_exprs = _read_context->virtual_column_exprs;
+
+ _read_options.all_access_paths = _read_context->all_access_paths;
+ _read_options.predicate_access_paths = _read_context->predicate_access_paths;
+
_read_options.ann_topn_runtime = _read_context->ann_topn_runtime;
_read_options.vir_cid_to_idx_in_block = _read_context->vir_cid_to_idx_in_block;
_read_options.vir_col_idx_to_type = _read_context->vir_col_idx_to_type;
diff --git a/be/src/olap/rowset/rowset_reader_context.h b/be/src/olap/rowset/rowset_reader_context.h
index 951679f413b..fffa16d0f5c 100644
--- a/be/src/olap/rowset/rowset_reader_context.h
+++ b/be/src/olap/rowset/rowset_reader_context.h
@@ -18,6 +18,8 @@
#ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H
#define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H
+#include <vector>
+
#include "io/io_common.h"
#include "olap/column_predicate.h"
#include "olap/olap_common.h"
@@ -90,6 +92,9 @@ struct RowsetReaderContext {
std::map<ColumnId, size_t> vir_cid_to_idx_in_block;
std::map<size_t, vectorized::DataTypePtr> vir_col_idx_to_type;
+ std::map<int32_t, TColumnAccessPaths> all_access_paths;
+ std::map<int32_t, TColumnAccessPaths> predicate_access_paths;
+
std::shared_ptr<vectorized::ScoreRuntime> score_runtime;
CollectionStatisticsPtr collection_statistics;
std::shared_ptr<segment_v2::AnnTopNRuntime> ann_topn_runtime;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 528909c3389..e900e8aaa49 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -18,6 +18,7 @@
#include "olap/rowset/segment_v2/column_reader.h"
#include <assert.h>
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/segment_v2.pb.h>
#include <algorithm>
@@ -72,11 +73,11 @@
#include "vec/columns/column_map.h"
#include "vec/columns/column_nullable.h"
#include "vec/columns/column_struct.h"
-#include "vec/columns/column_variant.h"
#include "vec/columns/column_vector.h"
#include "vec/common/assert_cast.h"
#include "vec/common/schema_util.h"
#include "vec/common/string_ref.h"
+#include "vec/common/typeid_cast.h"
#include "vec/core/types.h"
#include "vec/data_types/data_type_agg_state.h"
#include "vec/data_types/data_type_factory.hpp"
@@ -805,11 +806,12 @@ Status ColumnReader::new_iterator(ColumnIteratorUPtr* iterator, const TabletColu
*iterator = std::make_unique<EmptyFileColumnIterator>();
return Status::OK();
}
- if (is_scalar_type((FieldType)_meta_type)) {
+ if (is_scalar_type(_meta_type)) {
*iterator = std::make_unique<FileColumnIterator>(shared_from_this());
+ (*iterator)->set_column_name(tablet_column ? tablet_column->name() : "");
return Status::OK();
} else {
- auto type = (FieldType)_meta_type;
+ auto type = _meta_type;
switch (type) {
case FieldType::OLAP_FIELD_TYPE_AGG_STATE: {
return new_agg_state_iterator(iterator);
@@ -843,6 +845,8 @@ Status ColumnReader::new_array_iterator(ColumnIteratorUPtr* iterator,
? &tablet_column->get_sub_column(0)
: nullptr));
+ item_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(0).name() : "");
+
ColumnIteratorUPtr offset_iterator;
RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&offset_iterator, nullptr));
auto* file_iter = static_cast<FileColumnIterator*>(offset_iterator.release());
@@ -866,11 +870,13 @@ Status ColumnReader::new_map_iterator(ColumnIteratorUPtr* iterator,
&key_iterator, tablet_column && tablet_column->get_subtype_count() > 1
? &tablet_column->get_sub_column(0)
: nullptr));
+ key_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(0).name() : "");
ColumnIteratorUPtr val_iterator;
RETURN_IF_ERROR(_sub_readers[1]->new_iterator(
&val_iterator, tablet_column && tablet_column->get_subtype_count() > 1
? &tablet_column->get_sub_column(1)
: nullptr));
+ val_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(1).name() : "");
ColumnIteratorUPtr offsets_iterator;
RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&offsets_iterator, nullptr));
auto* file_iter = static_cast<FileColumnIterator*>(offsets_iterator.release());
@@ -898,6 +904,8 @@ Status ColumnReader::new_struct_iterator(ColumnIteratorUPtr* iterator,
ColumnIteratorUPtr sub_column_iterator;
RETURN_IF_ERROR(_sub_readers[i]->new_iterator(
&sub_column_iterator, tablet_column ? &tablet_column->get_sub_column(i) : nullptr));
+ sub_column_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(i).name()
+ : "");
sub_column_iterators.emplace_back(std::move(sub_column_iterator));
}
@@ -918,6 +926,33 @@ Status ColumnReader::new_struct_iterator(ColumnIteratorUPtr* iterator,
return Status::OK();
}
+Result<TColumnAccessPaths> ColumnIterator::_get_sub_access_paths(
+ const TColumnAccessPaths& access_paths) {
+ TColumnAccessPaths sub_access_paths = access_paths;
+ for (auto it = sub_access_paths.begin(); it != sub_access_paths.end();) {
+ TColumnAccessPath& name_path = *it;
+ if (name_path.data_access_path.path.empty()) {
+ return ResultError(
+ Status::InternalError("Invalid access path for struct column: path is empty"));
+ }
+
+ if (name_path.data_access_path.path[0] != _column_name) {
+ return ResultError(Status::InternalError(
+ R"(Invalid access path for column: expected name "{}", got "{}")", _column_name,
+ name_path.data_access_path.path[0]));
+ }
+
+ name_path.data_access_path.path.erase(name_path.data_access_path.path.begin());
+ if (!name_path.data_access_path.path.empty()) {
+ ++it;
+ } else {
+ set_need_to_read();
+ it = sub_access_paths.erase(it);
+ }
+ }
+ return sub_access_paths;
+}
+
///====================== MapFileColumnIterator ============================////
MapFileColumnIterator::MapFileColumnIterator(std::shared_ptr<ColumnReader> reader,
ColumnIteratorUPtr null_iterator,
@@ -934,6 +969,10 @@ MapFileColumnIterator::MapFileColumnIterator(std::shared_ptr<ColumnReader> reade
}
Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ LOG(INFO) << "Map column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
RETURN_IF_ERROR(_key_iterator->init(opts));
RETURN_IF_ERROR(_val_iterator->init(opts));
RETURN_IF_ERROR(_offsets_iterator->init(opts));
@@ -944,6 +983,11 @@ Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
}
Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
if (_map_reader->is_nullable()) {
RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
}
@@ -958,10 +1002,16 @@ Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
bool* has_null) {
- const auto* column_map = vectorized::check_and_get_column<vectorized::ColumnMap>(
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading.";
+ dst->resize(*n);
+ return Status::OK();
+ }
+
+ auto& column_map = assert_cast<vectorized::ColumnMap&>(
dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
: *dst);
- auto column_offsets_ptr = column_map->get_offsets_column().assume_mutable();
+ auto column_offsets_ptr = column_map.get_offsets_column().assume_mutable();
bool offsets_has_null = false;
ssize_t start = column_offsets_ptr->size();
RETURN_IF_ERROR(_offsets_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null));
@@ -974,8 +1024,8 @@ Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr
DCHECK(column_offsets.get_data().back() >= column_offsets.get_data()[start - 1]);
size_t num_items =
column_offsets.get_data().back() - column_offsets.get_data()[start - 1]; // -1 is valid
- auto key_ptr = column_map->get_keys().assume_mutable();
- auto val_ptr = column_map->get_values().assume_mutable();
+ auto key_ptr = column_map.get_keys().assume_mutable();
+ auto val_ptr = column_map.get_values().assume_mutable();
if (num_items > 0) {
size_t num_read = num_items;
@@ -984,6 +1034,9 @@ Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr
RETURN_IF_ERROR(_key_iterator->next_batch(&num_read, key_ptr, &key_has_null));
RETURN_IF_ERROR(_val_iterator->next_batch(&num_read, val_ptr, &val_has_null));
DCHECK(num_read == num_items);
+
+ column_map.get_keys_ptr() = std::move(key_ptr);
+ column_map.get_values_ptr() = std::move(val_ptr);
}
if (dst->is_nullable()) {
@@ -1009,6 +1062,12 @@ Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr
Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
vectorized::MutableColumnPtr& dst) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ dst->resize(count);
+ return Status::OK();
+ }
+
for (size_t i = 0; i < count; ++i) {
RETURN_IF_ERROR(seek_to_ordinal(rowids[i]));
size_t num_read = 1;
@@ -1018,6 +1077,93 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t
return Status::OK();
}
+void MapFileColumnIterator::set_need_to_read() {
+ set_reading_flag(ReadingFlag::NEED_TO_READ);
+ _key_iterator->set_need_to_read();
+ _val_iterator->set_need_to_read();
+}
+
+void MapFileColumnIterator::remove_pruned_sub_iterators() {
+ _key_iterator->remove_pruned_sub_iterators();
+ _val_iterator->remove_pruned_sub_iterators();
+}
+
+Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) {
+ if (all_access_paths.empty()) {
+ return Status::OK();
+ }
+
+ if (!predicate_access_paths.empty()) {
+ set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
+ LOG(INFO) << "Map column iterator set sub-column " << _column_name
+ << " to READING_FOR_PREDICATE";
+ }
+
+ auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
+ auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
+
+ if (sub_all_access_paths.empty()) {
+ return Status::OK();
+ }
+
+ TColumnAccessPaths key_all_access_paths;
+ TColumnAccessPaths val_all_access_paths;
+ TColumnAccessPaths key_predicate_access_paths;
+ TColumnAccessPaths val_predicate_access_paths;
+
+ for (auto paths : sub_all_access_paths) {
+ if (paths.data_access_path.path[0] == "*") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_all_access_paths.emplace_back(paths);
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_all_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "KEYS") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_all_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "VALUES") {
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_all_access_paths.emplace_back(paths);
+ }
+ }
+ const auto need_read_keys = !key_all_access_paths.empty();
+ const auto need_read_values = !val_all_access_paths.empty();
+
+ for (auto paths : sub_predicate_access_paths) {
+ if (paths.data_access_path.path[0] == "*") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_predicate_access_paths.emplace_back(paths);
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_predicate_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "KEYS") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_predicate_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "VALUES") {
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_predicate_access_paths.emplace_back(paths);
+ }
+ }
+
+ if (need_read_keys) {
+ _key_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ);
+ RETURN_IF_ERROR(
+ _key_iterator->set_access_paths(key_all_access_paths, key_predicate_access_paths));
+ } else {
+ _key_iterator->set_reading_flag(ReadingFlag::SKIP_READING);
+ LOG(INFO) << "Map column iterator set key column to SKIP_READING";
+ }
+
+ if (need_read_values) {
+ _val_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ);
+ RETURN_IF_ERROR(
+ _val_iterator->set_access_paths(val_all_access_paths, val_predicate_access_paths));
+ } else {
+ _val_iterator->set_reading_flag(ReadingFlag::SKIP_READING);
+ LOG(INFO) << "Map column iterator set value column to SKIP_READING";
+ }
+ return Status::OK();
+}
+
////////////////////////////////////////////////////////////////////////////////
StructFileColumnIterator::StructFileColumnIterator(
@@ -1030,6 +1176,11 @@ StructFileColumnIterator::StructFileColumnIterator(
}
Status StructFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ LOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
for (auto& column_iterator : _sub_column_iterators) {
RETURN_IF_ERROR(column_iterator->init(opts));
}
@@ -1041,16 +1192,23 @@ Status StructFileColumnIterator::init(const ColumnIteratorOptions& opts) {
Status StructFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
bool* has_null) {
- const auto* column_struct = vectorized::check_and_get_column<vectorized::ColumnStruct>(
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
+ dst->resize(*n);
+ return Status::OK();
+ }
+
+ auto& column_struct = assert_cast<vectorized::ColumnStruct&>(
dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
: *dst);
- for (size_t i = 0; i < column_struct->tuple_size(); i++) {
+ for (size_t i = 0; i < column_struct.tuple_size(); i++) {
size_t num_read = *n;
- auto sub_column_ptr = column_struct->get_column(i).assume_mutable();
+ auto sub_column_ptr = column_struct.get_column(i).assume_mutable();
bool column_has_null = false;
RETURN_IF_ERROR(
_sub_column_iterators[i]->next_batch(&num_read, sub_column_ptr, &column_has_null));
DCHECK(num_read == *n);
+ column_struct.get_column_ptr(i) = std::move(sub_column_ptr);
}
if (dst->is_nullable()) {
@@ -1076,6 +1234,11 @@ Status StructFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumn
}
Status StructFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
for (auto& column_iterator : _sub_column_iterators) {
RETURN_IF_ERROR(column_iterator->seek_to_ordinal(ord));
}
@@ -1087,6 +1250,12 @@ Status StructFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
Status StructFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
vectorized::MutableColumnPtr& dst) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
+ dst->resize(count);
+ return Status::OK();
+ }
+
for (size_t i = 0; i < count; ++i) {
RETURN_IF_ERROR(seek_to_ordinal(rowids[i]));
size_t num_read = 1;
@@ -1096,6 +1265,83 @@ Status StructFileColumnIterator::read_by_rowids(const rowid_t* rowids, const siz
return Status::OK();
}
+void StructFileColumnIterator::set_need_to_read() {
+ set_reading_flag(ReadingFlag::NEED_TO_READ);
+ for (auto& sub_iterator : _sub_column_iterators) {
+ sub_iterator->set_need_to_read();
+ }
+}
+
+void StructFileColumnIterator::remove_pruned_sub_iterators() {
+ for (auto it = _sub_column_iterators.begin(); it != _sub_column_iterators.end();) {
+ auto& sub_iterator = *it;
+ if (sub_iterator->reading_flag() == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Struct column iterator remove pruned sub-column "
+ << sub_iterator->column_name();
+ it = _sub_column_iterators.erase(it);
+ } else {
+ sub_iterator->remove_pruned_sub_iterators();
+ ++it;
+ }
+ }
+}
+
+Status StructFileColumnIterator::set_access_paths(
+ const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) {
+ if (all_access_paths.empty()) {
+ return Status::OK();
+ }
+
+ if (!predicate_access_paths.empty()) {
+ set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
+ LOG(INFO) << "Struct column iterator set sub-column " << _column_name
+ << " to READING_FOR_PREDICATE";
+ }
+ auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
+ auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
+
+ const auto no_sub_column_to_skip = sub_all_access_paths.empty();
+ const auto no_predicate_sub_column = sub_predicate_access_paths.empty();
+
+ for (auto& sub_iterator : _sub_column_iterators) {
+ const auto name = sub_iterator->column_name();
+ bool need_to_read = no_sub_column_to_skip;
+ TColumnAccessPaths sub_all_access_paths_of_this;
+ if (!need_to_read) {
+ for (const auto& paths : sub_all_access_paths) {
+ if (paths.data_access_path.path[0] == name) {
+ sub_all_access_paths_of_this.emplace_back(paths);
+ }
+ }
+ need_to_read = !sub_all_access_paths_of_this.empty();
+ }
+
+ if (!need_to_read) {
+ set_reading_flag(ReadingFlag::SKIP_READING);
+ sub_iterator->set_reading_flag(ReadingFlag::SKIP_READING);
+ LOG(INFO) << "Struct column iterator set sub-column " << name << " to SKIP_READING";
+ continue;
+ }
+ set_reading_flag(ReadingFlag::NEED_TO_READ);
+ sub_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ);
+
+ TColumnAccessPaths sub_predicate_access_paths_of_this;
+
+ if (!no_predicate_sub_column) {
+ for (const auto& paths : sub_predicate_access_paths) {
+ if (paths.data_access_path.path[0] == name) {
+ sub_predicate_access_paths_of_this.emplace_back(paths);
+ }
+ }
+ }
+
+ RETURN_IF_ERROR(sub_iterator->set_access_paths(sub_all_access_paths_of_this,
+ sub_predicate_access_paths_of_this));
+ }
+ return Status::OK();
+}
+
////////////////////////////////////////////////////////////////////////////////
Status OffsetFileColumnIterator::init(const ColumnIteratorOptions& opts) {
RETURN_IF_ERROR(_offset_iterator->init(opts));
@@ -1167,6 +1413,11 @@ ArrayFileColumnIterator::ArrayFileColumnIterator(std::shared_ptr<ColumnReader> r
}
Status ArrayFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ LOG(INFO) << "Array column iterator column " << _column_name << " skip readking.";
+ return Status::OK();
+ }
+
RETURN_IF_ERROR(_offset_iterator->init(opts));
RETURN_IF_ERROR(_item_iterator->init(opts));
if (_array_reader->is_nullable()) {
@@ -1184,6 +1435,11 @@ Status ArrayFileColumnIterator::_seek_by_offsets(ordinal_t ord) {
}
Status ArrayFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
RETURN_IF_ERROR(_offset_iterator->seek_to_ordinal(ord));
if (_array_reader->is_nullable()) {
RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
@@ -1193,6 +1449,12 @@ Status ArrayFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
Status ArrayFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
bool* has_null) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading.";
+ dst->resize(*n);
+ return Status::OK();
+ }
+
const auto* column_array = vectorized::check_and_get_column<vectorized::ColumnArray>(
dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
: *dst);
@@ -1241,6 +1503,12 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnP
Status ArrayFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
vectorized::MutableColumnPtr& dst) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading.";
+ dst->resize(count);
+ return Status::OK();
+ }
+
for (size_t i = 0; i < count; ++i) {
// TODO(cambyzju): now read array one by one, need optimize later
RETURN_IF_ERROR(seek_to_ordinal(rowids[i]));
@@ -1251,11 +1519,67 @@ Status ArrayFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size
return Status::OK();
}
+void ArrayFileColumnIterator::set_need_to_read() {
+ set_reading_flag(ReadingFlag::NEED_TO_READ);
+ _item_iterator->set_need_to_read();
+}
+
+void ArrayFileColumnIterator::remove_pruned_sub_iterators() {
+ _item_iterator->remove_pruned_sub_iterators();
+}
+
+Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) {
+ if (all_access_paths.empty()) {
+ return Status::OK();
+ }
+
+ if (!predicate_access_paths.empty()) {
+ set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
+ LOG(INFO) << "Array column iterator set sub-column " << _column_name
+ << " to READING_FOR_PREDICATE";
+ }
+
+ auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
+ auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
+
+ const auto no_sub_column_to_skip = sub_all_access_paths.empty();
+ const auto no_predicate_sub_column = sub_predicate_access_paths.empty();
+
+ if (!no_sub_column_to_skip) {
+ for (auto& path : sub_all_access_paths) {
+ if (path.data_access_path.path[0] == "*") {
+ path.data_access_path.path[0] = _item_iterator->column_name();
+ }
+ }
+ }
+
+ if (!no_predicate_sub_column) {
+ for (auto& path : sub_predicate_access_paths) {
+ if (path.data_access_path.path[0] == "*") {
+ path.data_access_path.path[0] = _item_iterator->column_name();
+ }
+ }
+ }
+
+ if (!no_sub_column_to_skip || !no_predicate_sub_column) {
+ _item_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ);
+ RETURN_IF_ERROR(
+ _item_iterator->set_access_paths(sub_all_access_paths, sub_predicate_access_paths));
+ }
+ return Status::OK();
+}
+
////////////////////////////////////////////////////////////////////////////////
FileColumnIterator::FileColumnIterator(std::shared_ptr<ColumnReader> reader) : _reader(reader) {}
Status FileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ LOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
_opts = opts;
if (!_opts.use_page_cache) {
_reader->disable_index_meta_cache();
@@ -1287,6 +1611,11 @@ Status FileColumnIterator::init(const ColumnIteratorOptions& opts) {
FileColumnIterator::~FileColumnIterator() = default;
Status FileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
// if current page contains this row, we don't need to seek
if (!_page || !_page.contains(ord) || !_page_iter.valid()) {
RETURN_IF_ERROR(_reader->seek_at_or_before(ord, &_page_iter, _opts));
@@ -1337,6 +1666,12 @@ Status FileColumnIterator::next_batch_of_zone_map(size_t* n, vectorized::Mutable
Status FileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
bool* has_null) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ dst->resize(*n);
+ return Status::OK();
+ }
+
size_t curr_size = dst->byte_size();
dst->reserve(*n);
size_t remaining = *n;
@@ -1393,6 +1728,12 @@ Status FileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& d
Status FileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
vectorized::MutableColumnPtr& dst) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ dst->resize(count);
+ return Status::OK();
+ }
+
size_t remaining = count;
size_t total_read_count = 0;
size_t nrows_to_read = 0;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 91aadffbfce..fddbc94c213 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -17,6 +17,7 @@
#pragma once
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/segment_v2.pb.h>
#include <sys/types.h>
@@ -69,6 +70,8 @@ class FileReader;
struct Slice;
struct StringRef;
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
namespace segment_v2 {
class EncodingInfo;
@@ -271,7 +274,6 @@ private:
Status _calculate_row_ranges(const std::vector<uint32_t>& page_indexes, RowRanges* row_ranges,
const ColumnIteratorOptions& iter_opts);
-private:
int64_t _meta_length;
FieldType _meta_type;
FieldType _meta_children_column_type;
@@ -366,8 +368,54 @@ public:
virtual bool is_all_dict_encoding() const { return false; }
+ virtual Status set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) {
+ if (!predicate_access_paths.empty()) {
+ _reading_flag = ReadingFlag::READING_FOR_PREDICATE;
+ }
+ return Status::OK();
+ }
+
+ void set_column_name(const std::string& column_name) { _column_name = column_name; }
+
+ const std::string& column_name() const { return _column_name; }
+
+ // Since there may be multiple paths with conflicts or overlaps,
+ // we need to define several reading flags:
+ //
+ // NORMAL_READING — Default value, indicating that the column should be read.
+ // SKIP_READING — The column should not be read.
+ // NEED_TO_READ — The column must be read.
+ // READING_FOR_PREDICATE — The column is required for predicate evaluation.
+ //
+ // For example, suppose there are two paths:
+ // - Path 1 specifies that column A needs to be read, so it is marked as NEED_TO_READ.
+ // - Path 2 specifies that the column should not be read, but since it is already marked as NEED_TO_READ,
+ // it should not be changed to SKIP_READING.
+ enum class ReadingFlag : int {
+ NORMAL_READING,
+ SKIP_READING,
+ NEED_TO_READ,
+ READING_FOR_PREDICATE
+ };
+ void set_reading_flag(ReadingFlag flag) {
+ if (static_cast<int>(flag) > static_cast<int>(_reading_flag)) {
+ _reading_flag = flag;
+ }
+ }
+
+ ReadingFlag reading_flag() const { return _reading_flag; }
+
+ virtual void set_need_to_read() { set_reading_flag(ReadingFlag::NEED_TO_READ); }
+
+ virtual void remove_pruned_sub_iterators() {};
+
protected:
+ Result<TColumnAccessPaths> _get_sub_access_paths(const TColumnAccessPaths& access_paths);
ColumnIteratorOptions _opts;
+
+ ReadingFlag _reading_flag {ReadingFlag::NORMAL_READING};
+ std::string _column_name;
};
// This iterator is used to read column data from file
@@ -504,6 +552,13 @@ public:
return _offsets_iterator->get_current_ordinal();
}
+ Status set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) override;
+
+ void set_need_to_read() override;
+
+ void remove_pruned_sub_iterators() override;
+
private:
std::shared_ptr<ColumnReader> _map_reader = nullptr;
ColumnIteratorUPtr _null_iterator;
@@ -533,6 +588,13 @@ public:
return _sub_column_iterators[0]->get_current_ordinal();
}
+ Status set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) override;
+
+ void set_need_to_read() override;
+
+ void remove_pruned_sub_iterators() override;
+
private:
std::shared_ptr<ColumnReader> _struct_reader = nullptr;
ColumnIteratorUPtr _null_iterator;
@@ -561,6 +623,12 @@ public:
return _offset_iterator->get_current_ordinal();
}
+ Status set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) override;
+ void set_need_to_read() override;
+
+ void remove_pruned_sub_iterators() override;
+
private:
std::shared_ptr<ColumnReader> _array_reader = nullptr;
std::unique_ptr<OffsetFileColumnIterator> _offset_iterator;
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index 096596a9979..99b1092645c 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -17,12 +17,14 @@
#include "olap/rowset/segment_v2/segment.h"
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/PlanNodes_types.h>
#include <gen_cpp/olap_file.pb.h>
#include <gen_cpp/segment_v2.pb.h>
#include <cstring>
#include <memory>
+#include <sstream>
#include <utility>
#include "cloud/config.h"
@@ -765,6 +767,20 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column,
sparse_column_cache_ptr));
} else {
RETURN_IF_ERROR(reader->new_iterator(iter, &tablet_column, opt));
+ if (opt->all_access_paths.contains(unique_id) ||
+ opt->predicate_access_paths.contains(unique_id)) {
+ const auto& all_access_paths = opt->all_access_paths.contains(unique_id)
+ ? opt->all_access_paths.at(unique_id)
+ : TColumnAccessPaths {};
+ const auto& predicate_access_paths = opt->predicate_access_paths.contains(unique_id)
+ ? opt->predicate_access_paths.at(unique_id)
+ : TColumnAccessPaths {};
+
+ // set column name to apply access paths.
+ (*iter)->set_column_name(tablet_column.name());
+ RETURN_IF_ERROR((*iter)->set_access_paths(all_access_paths, predicate_access_paths));
+ (*iter)->remove_pruned_sub_iterators();
+ }
}
if (config::enable_column_type_check && !tablet_column.has_path_info() &&
diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
index 3660e709e22..3dc120ec4c4 100644
--- a/be/src/olap/tablet_reader.cpp
+++ b/be/src/olap/tablet_reader.cpp
@@ -263,6 +263,9 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) {
_reader_context.ann_topn_runtime = read_params.ann_topn_runtime;
_reader_context.condition_cache_digest = read_params.condition_cache_digest;
+ _reader_context.all_access_paths = read_params.all_access_paths;
+ _reader_context.predicate_access_paths = read_params.predicate_access_paths;
+
return Status::OK();
}
diff --git a/be/src/olap/tablet_reader.h b/be/src/olap/tablet_reader.h
index 8630c932052..593afa9dbe5 100644
--- a/be/src/olap/tablet_reader.h
+++ b/be/src/olap/tablet_reader.h
@@ -17,6 +17,7 @@
#pragma once
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/PaloInternalService_types.h>
#include <gen_cpp/PlanNodes_types.h>
#include <stddef.h>
@@ -146,6 +147,9 @@ public:
// slots that cast may be eliminated in storage layer
std::map<std::string, vectorized::DataTypePtr> target_cast_type_for_variants;
+ std::map<int32_t, TColumnAccessPaths> all_access_paths;
+ std::map<int32_t, TColumnAccessPaths> predicate_access_paths;
+
std::vector<RowSetSplits> rs_splits;
// For unique key table with merge-on-write
DeleteBitmapPtr delete_bitmap = nullptr;
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index bbeb0312dde..442774f1f00 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -1680,6 +1680,11 @@ vectorized::Block TabletSchema::create_block(
tablet_columns_need_convert_null->find(cid) !=
tablet_columns_need_convert_null->end());
auto data_type = vectorized::DataTypeFactory::instance().create_data_type(col, is_nullable);
+ if (col.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ if (_pruned_columns_data_type.contains(col.unique_id())) {
+ data_type = _pruned_columns_data_type.at(col.unique_id());
+ }
+ }
if (_vir_col_idx_to_unique_id.contains(cid)) {
block.insert({vectorized::ColumnNothing::create(0), data_type, col.name()});
VLOG_DEBUG << fmt::format(
@@ -1699,7 +1704,13 @@ vectorized::Block TabletSchema::create_block(bool ignore_dropped_col) const {
if (ignore_dropped_col && is_dropped_column(*col)) {
continue;
}
+
auto data_type = vectorized::DataTypeFactory::instance().create_data_type(*col);
+ if (col->type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ if (_pruned_columns_data_type.contains(col->unique_id())) {
+ data_type = _pruned_columns_data_type.at(col->unique_id());
+ }
+ }
block.insert({data_type->create_column(), data_type, col->name()});
}
return block;
@@ -1710,6 +1721,11 @@ vectorized::Block TabletSchema::create_block_by_cids(const std::vector<uint32_t>
for (const auto& cid : cids) {
const auto& col = *_cols[cid];
auto data_type = vectorized::DataTypeFactory::instance().create_data_type(col);
+ if (col.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ if (_pruned_columns_data_type.contains(col.unique_id())) {
+ data_type = _pruned_columns_data_type.at(col.unique_id());
+ }
+ }
block.insert({data_type->create_column(), data_type, col.name()});
}
return block;
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index a7b9cd34c8d..ee498818683 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -42,6 +42,7 @@
#include "runtime/define_primitive_type.h"
#include "runtime/descriptors.h"
#include "runtime/memory/lru_cache_policy.h"
+#include "udf/udf.h"
#include "util/debug_points.h"
#include "util/string_parser.hpp"
#include "util/string_util.h"
@@ -673,6 +674,14 @@ public:
return 0;
}
+ void add_pruned_columns_data_type(int32_t col_unique_id, vectorized::DataTypePtr data_type) {
+ _pruned_columns_data_type[col_unique_id] = std::move(data_type);
+ }
+
+ void clear_pruned_columns_data_type() { _pruned_columns_data_type.clear(); }
+
+ bool has_pruned_columns() const { return !_pruned_columns_data_type.empty(); }
+
private:
friend bool operator==(const TabletSchema& a, const TabletSchema& b);
friend bool operator!=(const TabletSchema& a, const TabletSchema& b);
@@ -743,6 +752,7 @@ private:
bool _enable_variant_flatten_nested = false;
std::map<size_t, int32_t> _vir_col_idx_to_unique_id;
+ std::map<int32_t, vectorized::DataTypePtr> _pruned_columns_data_type;
// value: extracted path set and sparse path set
std::unordered_map<int32_t, PathsSetInfo> _path_set_info_map;
diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp
index 7ccc69ed9ac..a17f0926752 100644
--- a/be/src/runtime/descriptors.cpp
+++ b/be/src/runtime/descriptors.cpp
@@ -63,6 +63,11 @@ SlotDescriptor::SlotDescriptor(const TSlotDescriptor& tdesc)
_is_materialized(tdesc.isMaterialized && tdesc.need_materialize),
_is_key(tdesc.is_key),
_column_paths(tdesc.column_paths),
+ _all_access_paths(tdesc.__isset.all_access_paths ? tdesc.all_access_paths
+ : TColumnAccessPaths {}),
+ _predicate_access_paths(tdesc.__isset.predicate_access_paths
+ ? tdesc.predicate_access_paths
+ : TColumnAccessPaths {}),
_is_auto_increment(tdesc.__isset.is_auto_increment ? tdesc.is_auto_increment : false),
_col_default_value(tdesc.__isset.col_default_value ? tdesc.col_default_value : "") {
if (tdesc.__isset.virtual_column_expr) {
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index 0d86844b226..c66ccd80b1d 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -53,6 +53,8 @@ class ObjectPool;
class PTupleDescriptor;
class PSlotDescriptor;
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
class SlotDescriptor {
public:
MOCK_DEFINE(virtual ~SlotDescriptor() = default;)
@@ -82,6 +84,9 @@ public:
bool is_key() const { return _is_key; }
const std::vector<std::string>& column_paths() const { return _column_paths; };
+ const TColumnAccessPaths& all_access_paths() const { return _all_access_paths; }
+ const TColumnAccessPaths& predicate_access_paths() const { return _predicate_access_paths; }
+
bool is_auto_increment() const { return _is_auto_increment; }
bool is_skip_bitmap_col() const { return _col_name == SKIP_BITMAP_COL; }
@@ -128,6 +133,9 @@ private:
const bool _is_key;
const std::vector<std::string> _column_paths;
+ const TColumnAccessPaths _all_access_paths;
+ const TColumnAccessPaths _predicate_access_paths;
+
const bool _is_auto_increment;
const std::string _col_default_value;
diff --git a/be/src/vec/columns/column_map.cpp b/be/src/vec/columns/column_map.cpp
index 8317aa5bb5e..7cd6bfe9f4a 100644
--- a/be/src/vec/columns/column_map.cpp
+++ b/be/src/vec/columns/column_map.cpp
@@ -59,6 +59,7 @@ ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values, Mutable
/// This will also prevent possible overflow in offset.
if (keys_column->size() != last_offset) {
+ DCHECK(0);
throw doris::Exception(
doris::ErrorCode::INTERNAL_ERROR,
"offsets_column size {} has data inconsistent with key_column {}", last_offset,
diff --git a/be/src/vec/exec/scan/olap_scanner.cpp b/be/src/vec/exec/scan/olap_scanner.cpp
index 2ff4820d28e..fe6e04961f7 100644
--- a/be/src/vec/exec/scan/olap_scanner.cpp
+++ b/be/src/vec/exec/scan/olap_scanner.cpp
@@ -83,6 +83,8 @@ OlapScanner::OlapScanner(pipeline::ScanLocalStateBase* parent, OlapScanner::Para
.function_filters {},
.delete_predicates {},
.target_cast_type_for_variants {},
+ .all_access_paths {},
+ .predicate_access_paths {},
.rs_splits {},
.return_columns {},
.output_columns {},
@@ -265,7 +267,7 @@ Status OlapScanner::prepare() {
// Add newly created tablet schema to schema cache if it does not have virtual columns.
if (cached_schema == nullptr && !schema_key.empty() &&
- tablet_schema->num_virtual_columns() == 0) {
+ tablet_schema->num_virtual_columns() == 0 && !tablet_schema->has_pruned_columns()) {
SchemaCache::instance()->insert_schema(schema_key, tablet_schema);
}
@@ -543,6 +545,22 @@ Status OlapScanner::_init_return_columns() {
_vir_col_idx_to_type[idx_in_block]->get_name());
}
+ const auto& column = tablet_schema->column(index);
+ if (!slot->all_access_paths().empty()) {
+ _tablet_reader_params.all_access_paths.insert(
+ {column.unique_id(), slot->all_access_paths()});
+ }
+
+ if (!slot->predicate_access_paths().empty()) {
+ _tablet_reader_params.predicate_access_paths.insert(
+ {column.unique_id(), slot->predicate_access_paths()});
+ }
+
+ if (slot->type()->get_primitive_type() == PrimitiveType::TYPE_STRUCT &&
+ !slot->all_access_paths().empty()) {
+ tablet_schema->add_pruned_columns_data_type(column.unique_id(), slot->type());
+ }
+
_return_columns.push_back(index);
if (slot->is_nullable() && !tablet_schema->column(index).is_nullable()) {
_tablet_columns_convert_to_null_set.emplace(index);
diff --git a/be/test/olap/rowset/segment_v2/column_reader_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_test.cpp
new file mode 100644
index 00000000000..98478df7016
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/column_reader_test.cpp
@@ -0,0 +1,220 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include "olap/rowset/segment_v2/column_reader.h"
+
+#include <gen_cpp/Descriptors_types.h>
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <chrono>
+#include <memory>
+#include <thread>
+#include <vector>
+
+#include "agent/be_exec_version_manager.h"
+#include "common/config.h"
+#include "gen_cpp/olap_file.pb.h"
+#include "gen_cpp/segment_v2.pb.h"
+#include "io/fs/file_reader.h"
+#include "mock/mock_segment.h"
+#include "olap/rowset/segment_v2/column_reader_cache.h"
+#include "olap/rowset/segment_v2/segment.h"
+#include "olap/rowset/segment_v2/variant/variant_column_reader.h"
+#include "olap/tablet_schema.h"
+#include "vec/json/path_in_data.h"
+
+namespace doris::segment_v2 {
+class ColumnReaderTest : public ::testing::Test {
+protected:
+ void SetUp() override {}
+ void TearDown() override {}
+};
+
+TEST_F(ColumnReaderTest, StructAccessPaths) {
+ auto create_struct_iterator = []() {
+ auto null_reader = std::make_shared<ColumnReader>();
+ auto null_iterator = std::make_unique<FileColumnIterator>(null_reader);
+
+ std::vector<ColumnIteratorUPtr> sub_column_iterators;
+ auto sub_reader1 = std::make_shared<ColumnReader>();
+ auto sub_iterator1 = std::make_unique<FileColumnIterator>(sub_reader1);
+ sub_iterator1->set_column_name("sub_col_1");
+ auto sub_reader2 = std::make_shared<ColumnReader>();
+ auto sub_iterator2 = std::make_unique<FileColumnIterator>(sub_reader2);
+ sub_iterator2->set_column_name("sub_col_2");
+
+ sub_column_iterators.emplace_back(std::move(sub_iterator1));
+ sub_column_iterators.emplace_back(std::move(sub_iterator2));
+ auto iterator = std::make_unique<StructFileColumnIterator>(std::make_shared<ColumnReader>(),
+ std::move(null_iterator),
+ std::move(sub_column_iterators));
+ return iterator;
+ };
+
+ auto iterator = create_struct_iterator();
+ auto st = iterator->set_access_paths(TColumnAccessPaths {}, TColumnAccessPaths {});
+
+ ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string();
+ ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::NORMAL_READING);
+
+ TColumnAccessPaths all_access_paths;
+ all_access_paths.emplace_back();
+
+ TColumnAccessPaths predicate_access_paths;
+ predicate_access_paths.emplace_back();
+
+ st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+ // empty paths leads to error
+ ASSERT_FALSE(st.ok());
+
+ // Only reading sub_col_1
+ // sub_col_2 should be set to SKIP_READING
+ all_access_paths[0].data_access_path.path = {"self", "sub_col_1"};
+
+ predicate_access_paths[0].data_access_path.path = {"self", "sub_col_1"};
+
+ st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+ // invalid name leads to error
+ ASSERT_FALSE(st.ok());
+
+ iterator->set_column_name("self");
+ // now column name is "self", should be ok
+ st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+ ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string();
+ ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE);
+
+ ASSERT_EQ(iterator->_sub_column_iterators[0]->_reading_flag,
+ ColumnIterator::ReadingFlag::READING_FOR_PREDICATE);
+ ASSERT_EQ(iterator->_sub_column_iterators[1]->_reading_flag,
+ ColumnIterator::ReadingFlag::SKIP_READING);
+
+ // Reading all sub columns
+ all_access_paths[0].data_access_path.path = {"self"};
+ iterator = create_struct_iterator();
+ iterator->set_column_name("self");
+ st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+
+ ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string();
+ ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE);
+
+ ASSERT_EQ(iterator->_sub_column_iterators[0]->_reading_flag,
+ ColumnIterator::ReadingFlag::READING_FOR_PREDICATE);
+ ASSERT_EQ(iterator->_sub_column_iterators[1]->_reading_flag,
+ ColumnIterator::ReadingFlag::NEED_TO_READ);
+}
+
+TEST_F(ColumnReaderTest, MultiAccessPaths) {
+ auto create_struct_iterator = []() {
+ auto null_reader = std::make_shared<ColumnReader>();
+ auto null_iterator = std::make_unique<FileColumnIterator>(null_reader);
+
+ std::vector<ColumnIteratorUPtr> sub_column_iterators;
+ auto sub_reader1 = std::make_shared<ColumnReader>();
+ auto sub_iterator1 = std::make_unique<FileColumnIterator>(sub_reader1);
+ sub_iterator1->set_column_name("sub_col_1");
+ auto sub_reader2 = std::make_shared<ColumnReader>();
+ auto sub_iterator2 = std::make_unique<FileColumnIterator>(sub_reader2);
+ sub_iterator2->set_column_name("sub_col_2");
+
+ sub_column_iterators.emplace_back(std::move(sub_iterator1));
+ sub_column_iterators.emplace_back(std::move(sub_iterator2));
+ auto iterator = std::make_unique<StructFileColumnIterator>(std::make_shared<ColumnReader>(),
+ std::move(null_iterator),
+ std::move(sub_column_iterators));
+ return iterator;
+ };
+
+ auto create_struct_iterator2 = [](ColumnIteratorUPtr&& nested_iterator) {
+ auto null_reader = std::make_shared<ColumnReader>();
+ auto null_iterator = std::make_unique<FileColumnIterator>(null_reader);
+
+ std::vector<ColumnIteratorUPtr> sub_column_iterators;
+ auto sub_reader1 = std::make_shared<ColumnReader>();
+ auto sub_iterator1 = std::make_unique<FileColumnIterator>(sub_reader1);
+ sub_iterator1->set_column_name("sub_col_1");
+
+ sub_column_iterators.emplace_back(std::move(sub_iterator1));
+ sub_column_iterators.emplace_back(std::move(nested_iterator));
+ auto iterator = std::make_unique<StructFileColumnIterator>(std::make_shared<ColumnReader>(),
+ std::move(null_iterator),
+ std::move(sub_column_iterators));
+ return iterator;
+ };
+
+ auto struct_iterator = create_struct_iterator();
+ struct_iterator->set_column_name("struct");
+
+ auto map_iterator = std::make_unique<MapFileColumnIterator>(
+ std::make_shared<ColumnReader>(),
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>()), // null iterator
+ std::make_unique<OffsetFileColumnIterator>(
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>())),
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>()),
+ std::move(struct_iterator));
+
+ auto array_iterator = std::make_unique<ArrayFileColumnIterator>(
+ std::make_shared<ColumnReader>(),
+ std::make_unique<OffsetFileColumnIterator>(
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>())),
+ std::move(map_iterator),
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>()));
+
+ // here create:
+ // struct<
+ // sub_col_1,
+ // sub_col_2: array<
+ // map<
+ // key,
+ // value: struct<
+ // sub_col_1,
+ // sub_col_2
+ // >
+ // >
+ // >
+ // >
+ array_iterator->set_column_name("sub_col_2");
+ auto iterator = create_struct_iterator2(std::move(array_iterator));
+ TColumnAccessPaths all_access_paths;
+ all_access_paths.emplace_back();
+
+ // all access paths:
+ // self.sub_col_2.*.KEYS
+ // predicates paths empty
+ all_access_paths[0].data_access_path.path = {"self", "sub_col_2", "*", "KEYS"};
+
+ TColumnAccessPaths predicate_access_paths;
+
+ iterator->set_column_name("self");
+ auto st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+
+ ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string();
+ ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ);
+
+ ASSERT_EQ(iterator->_sub_column_iterators[0]->_reading_flag,
+ ColumnIterator::ReadingFlag::SKIP_READING);
+ ASSERT_EQ(iterator->_sub_column_iterators[1]->_reading_flag,
+ ColumnIterator::ReadingFlag::NEED_TO_READ);
+
+ auto* array_iter =
+ static_cast<ArrayFileColumnIterator*>(iterator->_sub_column_iterators[1].get());
+ ASSERT_EQ(array_iter->_item_iterator->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ);
+
+ auto* map_iter = static_cast<MapFileColumnIterator*>(array_iter->_item_iterator.get());
+ ASSERT_EQ(map_iter->_key_iterator->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ);
+ ASSERT_EQ(map_iter->_val_iterator->_reading_flag, ColumnIterator::ReadingFlag::SKIP_READING);
+}
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AccessPathInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AccessPathInfo.java
new file mode 100644
index 00000000000..3df7baada4d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AccessPathInfo.java
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/TupleDescriptor.java
+// and modified by Doris
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.thrift.TColumnAccessPath;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.util.List;
+
+/** AccessPathInfo */
+@Data
+@AllArgsConstructor
+public class AccessPathInfo {
+ private DataType prunedType;
+ private List<TColumnAccessPath> allAccessPaths;
+ private List<TColumnAccessPath> predicateAccessPaths;
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
index 87843f721c9..c0b35bd8f96 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ColumnStats;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Type;
+import org.apache.doris.thrift.TColumnAccessPath;
import org.apache.doris.thrift.TSlotDescriptor;
import com.google.common.base.MoreObjects;
@@ -78,6 +79,10 @@ public class SlotDescriptor {
private boolean needMaterialize = true;
private boolean isAutoInc = false;
private Expr virtualColumn = null;
+ private List<TColumnAccessPath> allAccessPaths;
+ private List<TColumnAccessPath> predicateAccessPaths;
+ private List<TColumnAccessPath> displayAllAccessPaths;
+ private List<TColumnAccessPath> displayPredicateAccessPaths;
public SlotDescriptor(SlotId id, TupleDescriptor parent) {
@@ -132,6 +137,38 @@ public class SlotDescriptor {
return this.subColPath;
}
+ public List<TColumnAccessPath> getAllAccessPaths() {
+ return allAccessPaths;
+ }
+
+ public void setAllAccessPaths(List<TColumnAccessPath> allAccessPaths) {
+ this.allAccessPaths = allAccessPaths;
+ }
+
+ public List<TColumnAccessPath> getPredicateAccessPaths() {
+ return predicateAccessPaths;
+ }
+
+ public void setPredicateAccessPaths(List<TColumnAccessPath> predicateAccessPaths) {
+ this.predicateAccessPaths = predicateAccessPaths;
+ }
+
+ public List<TColumnAccessPath> getDisplayAllAccessPaths() {
+ return displayAllAccessPaths;
+ }
+
+ public void setDisplayAllAccessPaths(List<TColumnAccessPath> displayAllAccessPaths) {
+ this.displayAllAccessPaths = displayAllAccessPaths;
+ }
+
+ public List<TColumnAccessPath> getDisplayPredicateAccessPaths() {
+ return displayPredicateAccessPaths;
+ }
+
+ public void setDisplayPredicateAccessPaths(List<TColumnAccessPath> displayPredicateAccessPaths) {
+ this.displayPredicateAccessPaths = displayPredicateAccessPaths;
+ }
+
public TupleDescriptor getParent() {
return parent;
}
@@ -339,6 +376,12 @@ public class SlotDescriptor {
if (virtualColumn != null) {
tSlotDescriptor.setVirtualColumnExpr(virtualColumn.treeToThrift());
}
+ if (allAccessPaths != null) {
+ tSlotDescriptor.setAllAccessPaths(allAccessPaths);
+ }
+ if (predicateAccessPaths != null) {
+ tSlotDescriptor.setPredicateAccessPaths(predicateAccessPaths);
+ }
return tSlotDescriptor;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
index 8cfe183ebf0..5bc8d643b8f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
@@ -42,6 +42,7 @@ import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -171,6 +172,8 @@ public abstract class FileScanNode extends ExternalScanNode {
}
output.append(String.format("numNodes=%s", numNodes)).append("\n");
+ printNestedColumns(output, prefix);
+
// pushdown agg
output.append(prefix).append(String.format("pushdown agg=%s", pushDownAggNoGroupingOp));
if (pushDownAggNoGroupingOp.equals(TPushAggOp.COUNT)) {
@@ -196,7 +199,10 @@ public abstract class FileScanNode extends ExternalScanNode {
TExpr tExpr = new TExpr();
tExpr.setNodes(Lists.newArrayList());
+ int index = 0;
+ List<SlotDescriptor> slots = desc.getSlots();
for (Column column : getColumns()) {
+ SlotDescriptor slotDescriptor = slots.get(index++);
Expr expr;
if (column.getDefaultValue() != null) {
if (column.getDefaultValueExprDef() != null) {
@@ -210,7 +216,9 @@ public abstract class FileScanNode extends ExternalScanNode {
if (useVarcharAsNull) {
expr = NullLiteral.create(org.apache.doris.catalog.Type.VARCHAR);
} else {
- expr = NullLiteral.create(column.getType());
+ // the nested type(map/array/struct) maybe pruned,
+ // we should use the pruned type to avoid be core
+ expr = NullLiteral.create(slotDescriptor.getType());
}
} else {
expr = null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 5829a0c9b56..204cb852d08 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -285,6 +285,8 @@ public class StatementContext implements Closeable {
private Optional<Map<TableIf, Set<Expression>>> mvRefreshPredicates = Optional.empty();
+ private boolean hasNestedColumns;
+
public StatementContext() {
this(ConnectContext.get(), null, 0);
}
@@ -1017,4 +1019,12 @@ public class StatementContext implements Closeable {
Map<TableIf, Set<Expression>> mvRefreshPredicates) {
this.mvRefreshPredicates = Optional.of(mvRefreshPredicates);
}
+
+ public boolean hasNestedColumns() {
+ return hasNestedColumns;
+ }
+
+ public void setHasNestedColumns(boolean hasNestedColumns) {
+ this.hasNestedColumns = hasNestedColumns;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 44186fe3711..6b14de78e00 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -837,10 +837,11 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
OlapTable olapTable = olapScan.getTable();
// generate real output tuple
TupleDescriptor tupleDescriptor = generateTupleDesc(slots, olapTable, context);
+ List<SlotDescriptor> slotDescriptors = tupleDescriptor.getSlots();
// put virtual column expr into slot desc
Map<ExprId, Expression> slotToVirtualColumnMap = olapScan.getSlotToVirtualColumnMap();
- for (SlotDescriptor slotDescriptor : tupleDescriptor.getSlots()) {
+ for (SlotDescriptor slotDescriptor : slotDescriptors) {
ExprId exprId = context.findExprId(slotDescriptor.getId());
if (slotToVirtualColumnMap.containsKey(exprId)) {
slotDescriptor.setVirtualColumn(ExpressionTranslator.translate(
@@ -910,7 +911,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
&& !isComplexDataType(slot.getDataType())
&& !StatisticConstants.isSystemTable(olapTable)
&& !inVisibleCol) {
- context.addUnknownStatsColumn(olapScanNode, tupleDescriptor.getSlots().get(i).getId());
+ context.addUnknownStatsColumn(olapScanNode, slotDescriptors.get(i).getId());
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
index 7553a81d5b0..cbbd425c5b2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
@@ -29,6 +29,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.processor.post.TopnFilterContext;
import org.apache.doris.nereids.processor.post.runtimefilterv2.RuntimeFilterContextV2;
import org.apache.doris.nereids.trees.expressions.CTEId;
@@ -67,6 +68,7 @@ import javax.annotation.Nullable;
*/
public class PlanTranslatorContext {
private final ConnectContext connectContext;
+ private final StatementContext statementContext;
private final List<PlanFragment> planFragments = Lists.newArrayList();
private DescriptorTable descTable;
@@ -119,16 +121,20 @@ public class PlanTranslatorContext {
private final Set<SlotId> virtualColumnIds = Sets.newHashSet();
+ /** PlanTranslatorContext */
public PlanTranslatorContext(CascadesContext ctx) {
this.connectContext = ctx.getConnectContext();
+ this.statementContext = ctx.getStatementContext();
this.translator = new RuntimeFilterTranslator(ctx.getRuntimeFilterContext());
this.topnFilterContext = ctx.getTopnFilterContext();
this.runtimeFilterV2Context = ctx.getRuntimeFilterV2Context();
this.descTable = new DescriptorTable();
}
+ /** PlanTranslatorContext */
public PlanTranslatorContext(CascadesContext ctx, DescriptorTable descTable) {
this.connectContext = ctx.getConnectContext();
+ this.statementContext = ctx.getStatementContext();
this.translator = new RuntimeFilterTranslator(ctx.getRuntimeFilterContext());
this.topnFilterContext = ctx.getTopnFilterContext();
this.runtimeFilterV2Context = ctx.getRuntimeFilterV2Context();
@@ -141,6 +147,7 @@ public class PlanTranslatorContext {
@VisibleForTesting
public PlanTranslatorContext() {
this.connectContext = null;
+ this.statementContext = new StatementContext();
this.translator = null;
this.topnFilterContext = new TopnFilterContext();
IdGenerator<RuntimeFilterId> runtimeFilterIdGen = RuntimeFilterId.createGenerator();
@@ -180,6 +187,10 @@ public class PlanTranslatorContext {
return connectContext;
}
+ public StatementContext getStatementContext() {
+ return statementContext;
+ }
+
public Set<ScanNode> getScanNodeWithUnknownColumnStats() {
return statsUnknownColumnsMap.keySet();
}
@@ -329,6 +340,13 @@ public class PlanTranslatorContext {
}
this.addExprIdSlotRefPair(slotReference.getExprId(), slotRef);
slotDescriptor.setIsNullable(slotReference.nullable());
+
+ if (slotReference.getAllAccessPaths().isPresent()) {
+ slotDescriptor.setAllAccessPaths(slotReference.getAllAccessPaths().get());
+ slotDescriptor.setPredicateAccessPaths(slotReference.getPredicateAccessPaths().get());
+ slotDescriptor.setDisplayAllAccessPaths(slotReference.getDisplayAllAccessPaths().get());
+ slotDescriptor.setDisplayPredicateAccessPaths(slotReference.getDisplayPredicateAccessPaths().get());
+ }
return slotDescriptor;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 7eed68241f5..7b7485aba74 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -109,6 +109,7 @@ import org.apache.doris.nereids.rules.rewrite.MergeProjectable;
import org.apache.doris.nereids.rules.rewrite.MergeSetOperations;
import org.apache.doris.nereids.rules.rewrite.MergeSetOperationsExcept;
import org.apache.doris.nereids.rules.rewrite.MergeTopNs;
+import org.apache.doris.nereids.rules.rewrite.NestedColumnPruning;
import org.apache.doris.nereids.rules.rewrite.NormalizeSort;
import org.apache.doris.nereids.rules.rewrite.OperativeColumnDerive;
import org.apache.doris.nereids.rules.rewrite.OrExpansion;
@@ -911,6 +912,11 @@ public class Rewriter extends AbstractBatchJobExecutor {
)
));
}
+ rewriteJobs.add(
+ topic("nested column prune",
+ custom(RuleType.NESTED_COLUMN_PRUNING, NestedColumnPruning::new)
+ )
+ );
rewriteJobs.addAll(jobs(
topic("rewrite cte sub-tree after sub path push down",
custom(RuleType.CLEAR_CONTEXT_STATUS, ClearContextStatus::new),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 45cb22e6b52..630202de254 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -115,6 +115,7 @@ import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughSort;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughWindow;
import org.apache.doris.nereids.rules.rewrite.PushDownJoinOtherCondition;
import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughJoin;
+import org.apache.doris.nereids.rules.rewrite.PushDownNestedColumn;
import org.apache.doris.nereids.rules.rewrite.PushDownProjectThroughLimit;
import org.apache.doris.nereids.rules.rewrite.PushDownTopNDistinctThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughJoin;
@@ -166,6 +167,7 @@ public class RuleSet {
new PushDownFilterThroughSetOperation(),
new PushDownFilterThroughGenerate(),
new PushDownProjectThroughLimit(),
+ new PushDownNestedColumn(),
new EliminateOuterJoin(),
new MergeProjectable(),
new MergeFilters(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index a8bc2052348..5d4143fbb02 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -195,6 +195,12 @@ public enum RuleType {
PUSH_DOWN_PREDICATE_THROUGH_AGGREGATION(RuleTypeClass.REWRITE),
PUSH_DOWN_PREDICATE_THROUGH_REPEAT(RuleTypeClass.REWRITE),
PUSH_DOWN_EXPRESSIONS_IN_HASH_CONDITIONS(RuleTypeClass.REWRITE),
+ // Pushdown nested column project
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_JOIN(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_UNION(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_WINDOW(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_PARTITION_TOP_N(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_DEFER_MATERIALIZE_TOP_N(RuleTypeClass.REWRITE),
// Pushdown filter
PUSH_DOWN_FILTER_THROUGH_JOIN(RuleTypeClass.REWRITE),
PUSH_DOWN_FILTER_THROUGH_LEFT_SEMI_JOIN(RuleTypeClass.REWRITE),
@@ -219,6 +225,7 @@ public enum RuleType {
ADD_PROJECT_FOR_JOIN(RuleTypeClass.REWRITE),
ADD_PROJECT_FOR_UNIQUE_FUNCTION(RuleTypeClass.REWRITE),
VARIANT_SUB_PATH_PRUNING(RuleTypeClass.REWRITE),
+ NESTED_COLUMN_PRUNING(RuleTypeClass.REWRITE),
CLEAR_CONTEXT_STATUS(RuleTypeClass.REWRITE),
COLUMN_PRUNING(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index 84652b9eed0..36b7f0064ef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -424,7 +424,8 @@ public class BindRelation extends OneAnalysisRuleFactory {
if (hmsTable.getDlaType() == DLAType.HUDI) {
LogicalHudiScan hudiScan = new LogicalHudiScan(unboundRelation.getRelationId(), hmsTable,
qualifierWithoutTableName, ImmutableList.of(), Optional.empty(),
- unboundRelation.getTableSample(), unboundRelation.getTableSnapshot());
+ unboundRelation.getTableSample(), unboundRelation.getTableSnapshot(),
+ Optional.empty());
hudiScan = hudiScan.withScanParams(
hmsTable, Optional.ofNullable(unboundRelation.getScanParams()));
return hudiScan;
@@ -434,7 +435,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
ImmutableList.of(),
unboundRelation.getTableSample(),
unboundRelation.getTableSnapshot(),
- Optional.ofNullable(unboundRelation.getScanParams()));
+ Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty());
}
case ICEBERG_EXTERNAL_TABLE:
IcebergExternalTable icebergExternalTable = (IcebergExternalTable) table;
@@ -464,7 +465,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
qualifierWithoutTableName, ImmutableList.of(),
unboundRelation.getTableSample(),
unboundRelation.getTableSnapshot(),
- Optional.ofNullable(unboundRelation.getScanParams()));
+ Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty());
case PAIMON_EXTERNAL_TABLE:
case MAX_COMPUTE_EXTERNAL_TABLE:
case TRINO_CONNECTOR_EXTERNAL_TABLE:
@@ -473,7 +474,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
qualifierWithoutTableName, ImmutableList.of(),
unboundRelation.getTableSample(),
unboundRelation.getTableSnapshot(),
- Optional.ofNullable(unboundRelation.getScanParams()));
+ Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty());
case SCHEMA:
LogicalSchemaScan schemaScan = new LogicalSchemaScan(unboundRelation.getRelationId(), table,
qualifierWithoutTableName);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
index 35adb692be3..b90dc3adc81 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
@@ -93,6 +93,7 @@ import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.NestedColumnPrunable;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.TinyIntType;
import org.apache.doris.nereids.util.ExpressionUtils;
@@ -285,6 +286,9 @@ public class ExpressionAnalyzer extends SubExprAnalyzer<ExpressionRewriteContext
}
outerScope.get().getCorrelatedSlots().add((Slot) firstBound);
}
+ if (firstBound.getDataType() instanceof NestedColumnPrunable) {
+ context.cascadesContext.getStatementContext().setHasNestedColumns(true);
+ }
return firstBound;
default:
if (enableExactMatch) {
@@ -304,6 +308,9 @@ public class ExpressionAnalyzer extends SubExprAnalyzer<ExpressionRewriteContext
.filter(bound -> unboundSlot.getNameParts().size() == bound.getQualifier().size() + 1)
.collect(Collectors.toList());
if (exactMatch.size() == 1) {
+ if (exactMatch.get(0).getDataType() instanceof NestedColumnPrunable) {
+ context.cascadesContext.getStatementContext().setHasNestedColumns(true);
+ }
return exactMatch.get(0);
}
}
@@ -340,6 +347,9 @@ public class ExpressionAnalyzer extends SubExprAnalyzer<ExpressionRewriteContext
if (!(slot instanceof SlotReference) || (((SlotReference) slot).isVisible()) || showHidden) {
showSlots.add(slot);
}
+ if (slot.getDataType() instanceof NestedColumnPrunable) {
+ context.cascadesContext.getStatementContext().setHasNestedColumns(true);
+ }
}
ImmutableList<Slot> slots = showSlots.build();
switch (qualifier.size()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
index 2c957912ce9..c4302798e25 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
@@ -29,6 +29,7 @@ import org.apache.doris.nereids.rules.expression.rules.LogToLn;
import org.apache.doris.nereids.rules.expression.rules.MedianConvert;
import org.apache.doris.nereids.rules.expression.rules.MergeDateTrunc;
import org.apache.doris.nereids.rules.expression.rules.NormalizeBinaryPredicatesRule;
+import org.apache.doris.nereids.rules.expression.rules.NormalizeStructElement;
import org.apache.doris.nereids.rules.expression.rules.SimplifyArithmeticComparisonRule;
import org.apache.doris.nereids.rules.expression.rules.SimplifyArithmeticRule;
import org.apache.doris.nereids.rules.expression.rules.SimplifyCastRule;
@@ -68,6 +69,7 @@ public class ExpressionNormalization extends ExpressionRewrite {
SimplifyArithmeticComparisonRule.INSTANCE,
ConvertAggStateCast.INSTANCE,
MergeDateTrunc.INSTANCE,
+ NormalizeStructElement.INSTANCE,
CheckCast.INSTANCE,
SimplifyEqualBooleanLiteral.INSTANCE
)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
index 5a5daeb7143..7df6426acb5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
@@ -61,6 +61,7 @@ public enum ExpressionRuleType {
SIMPLIFY_RANGE,
SIMPLIFY_SELF_COMPARISON,
SUPPORT_JAVA_DATE_FORMATTER,
+ NORMALIZE_STRUCT_ELEMENT,
TOPN_TO_MAX;
public int type() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/NormalizeStructElement.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/NormalizeStructElement.java
new file mode 100644
index 00000000000..dc50921bc1e
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/NormalizeStructElement.java
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rules;
+
+import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
+import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
+import org.apache.doris.nereids.rules.expression.ExpressionRuleType;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
+import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * given column s has data type: struct<a: int, b: int>, if exists struct_element(s, 2), we will rewrite
+ * to struct_element(s, 'b')
+ */
+public class NormalizeStructElement implements ExpressionPatternRuleFactory {
+ public static final NormalizeStructElement INSTANCE = new NormalizeStructElement();
+
+ @Override
+ public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
+ return ImmutableList.of(
+ matchesType(StructElement.class).then(NormalizeStructElement::normalize)
+ .toRule(ExpressionRuleType.NORMALIZE_STRUCT_ELEMENT)
+ );
+ }
+
+ private static StructElement normalize(StructElement structElement) {
+ Expression field = structElement.getArgument(1);
+ if (field instanceof IntegerLikeLiteral) {
+ int fieldIndex = ((Number) ((IntegerLikeLiteral) field).getValue()).intValue();
+ StructType structType = (StructType) structElement.getArgument(0).getDataType();
+ List<StructField> fields = structType.getFields();
+ if (fieldIndex >= 0 && fieldIndex <= fields.size()) {
+ return structElement.withChildren(
+ ImmutableList.of(
+ structElement.child(0),
+ new StringLiteral(fields.get(fieldIndex - 1).getName())
+ )
+ );
+ }
+ }
+ return structElement;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
new file mode 100644
index 00000000000..4fe9e9e2f74
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
@@ -0,0 +1,511 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectorContext;
+import org.apache.doris.nereids.rules.rewrite.NestedColumnPruning.DataTypeAccessTree;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
+import org.apache.doris.nereids.trees.expressions.ArrayItemReference.ArrayItemSlot;
+import org.apache.doris.nereids.trees.expressions.Cast;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayCount;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExists;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFilter;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirst;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirstIndex;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLast;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLastIndex;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMap;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMatchAll;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMatchAny;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayReverseSplit;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySortBy;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySplit;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsEntry;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsKey;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsValue;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapKeys;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapValues;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.NestedColumnPrunable;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.thrift.TAccessPathType;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Stack;
+
+/**
+ * collect the access path, for example: `select struct_element(s, 'data')` has access path: ['s', 'data']
+ */
+public class AccessPathExpressionCollector extends DefaultExpressionVisitor<Void, CollectorContext> {
+ private StatementContext statementContext;
+ private boolean bottomPredicate;
+ private Multimap<Integer, CollectAccessPathResult> slotToAccessPaths;
+ private Stack<Map<String, Expression>> nameToLambdaArguments = new Stack<>();
+
+ public AccessPathExpressionCollector(
+ StatementContext statementContext, Multimap<Integer, CollectAccessPathResult> slotToAccessPaths,
+ boolean bottomPredicate) {
+ this.statementContext = statementContext;
+ this.slotToAccessPaths = slotToAccessPaths;
+ this.bottomPredicate = bottomPredicate;
+ }
+
+ public void collect(Expression expression) {
+ expression.accept(this, new CollectorContext(statementContext, bottomPredicate));
+ }
+
+ private Void continueCollectAccessPath(Expression expr, CollectorContext context) {
+ return expr.accept(this, context);
+ }
+
+ @Override
+ public Void visit(Expression expr, CollectorContext context) {
+ for (Expression child : expr.children()) {
+ child.accept(this, new CollectorContext(context.statementContext, context.bottomFilter));
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitSlotReference(SlotReference slotReference, CollectorContext context) {
+ DataType dataType = slotReference.getDataType();
+ if (dataType instanceof NestedColumnPrunable) {
+ context.accessPathBuilder.addPrefix(slotReference.getName().toLowerCase());
+ ImmutableList<String> path = Utils.fastToImmutableList(context.accessPathBuilder.accessPath);
+ int slotId = slotReference.getExprId().asInt();
+ slotToAccessPaths.put(slotId, new CollectAccessPathResult(path, context.bottomFilter, context.type));
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitArrayItemSlot(ArrayItemSlot arrayItemSlot, CollectorContext context) {
+ if (nameToLambdaArguments.isEmpty()) {
+ return null;
+ }
+ context.accessPathBuilder.addPrefix("*");
+ Expression argument = nameToLambdaArguments.peek().get(arrayItemSlot.getName());
+ if (argument == null) {
+ return null;
+ }
+ return continueCollectAccessPath(argument, context);
+ }
+
+ @Override
+ public Void visitAlias(Alias alias, CollectorContext context) {
+ return alias.child(0).accept(this, context);
+ }
+
+ @Override
+ public Void visitCast(Cast cast, CollectorContext context) {
+ if (!context.accessPathBuilder.isEmpty()
+ && cast.getDataType() instanceof NestedColumnPrunable
+ && cast.child().getDataType() instanceof NestedColumnPrunable) {
+
+ DataTypeAccessTree castTree = DataTypeAccessTree.of(cast.getDataType(), TAccessPathType.DATA);
+ DataTypeAccessTree originTree = DataTypeAccessTree.of(cast.child().getDataType(), TAccessPathType.DATA);
+
+ List<String> replacePath = new ArrayList<>(context.accessPathBuilder.getPathList());
+ if (originTree.replacePathByAnotherTree(castTree, replacePath, 0)) {
+ CollectorContext castContext = new CollectorContext(context.statementContext, context.bottomFilter);
+ castContext.accessPathBuilder.accessPath.addAll(replacePath);
+ return continueCollectAccessPath(cast.child(), castContext);
+ }
+ }
+ return cast.child(0).accept(this,
+ new CollectorContext(context.statementContext, context.bottomFilter)
+ );
+ }
+
+ // array element at
+ @Override
+ public Void visitElementAt(ElementAt elementAt, CollectorContext context) {
+ List<Expression> arguments = elementAt.getArguments();
+ Expression first = arguments.get(0);
+ if (first.getDataType().isArrayType() || first.getDataType().isMapType()) {
+ context.accessPathBuilder.addPrefix("*");
+ continueCollectAccessPath(first, context);
+
+ for (int i = 1; i < arguments.size(); i++) {
+ visit(arguments.get(i), context);
+ }
+ return null;
+ } else {
+ return visit(elementAt, context);
+ }
+ }
+
+ // struct element_at
+ @Override
+ public Void visitStructElement(StructElement structElement, CollectorContext context) {
+ List<Expression> arguments = structElement.getArguments();
+ Expression struct = arguments.get(0);
+ Expression fieldName = arguments.get(1);
+ DataType fieldType = fieldName.getDataType();
+
+ if (fieldName.isLiteral() && (fieldType.isIntegerLikeType() || fieldType.isStringLikeType())) {
+ if (fieldType.isIntegerLikeType()) {
+ int fieldIndex = ((Number) ((Literal) fieldName).getValue()).intValue();
+ List<StructField> fields = ((StructType) struct.getDataType()).getFields();
+ if (fieldIndex >= 1 && fieldIndex <= fields.size()) {
+ String realFieldName = fields.get(fieldIndex - 1).getName();
+ context.accessPathBuilder.addPrefix(realFieldName);
+ return continueCollectAccessPath(struct, context);
+ }
+ }
+ context.accessPathBuilder.addPrefix(((Literal) fieldName).getStringValue().toLowerCase());
+ return continueCollectAccessPath(struct, context);
+ }
+
+ for (Expression argument : arguments) {
+ visit(argument, context);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitMapKeys(MapKeys mapKeys, CollectorContext context) {
+ context = new CollectorContext(context.statementContext, context.bottomFilter);
+ context.accessPathBuilder.addPrefix("KEYS");
+ return continueCollectAccessPath(mapKeys.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitMapValues(MapValues mapValues, CollectorContext context) {
+ LinkedList<String> suffixPath = context.accessPathBuilder.accessPath;
+ if (!suffixPath.isEmpty() && suffixPath.get(0).equals("*")) {
+ CollectorContext removeStarContext
+ = new CollectorContext(context.statementContext, context.bottomFilter);
+ removeStarContext.accessPathBuilder.accessPath.addAll(suffixPath.subList(1, suffixPath.size()));
+ removeStarContext.accessPathBuilder.addPrefix("VALUES");
+ return continueCollectAccessPath(mapValues.getArgument(0), removeStarContext);
+ }
+ context.accessPathBuilder.addPrefix("VALUES");
+ return continueCollectAccessPath(mapValues.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitMapContainsKey(MapContainsKey mapContainsKey, CollectorContext context) {
+ context.accessPathBuilder.addPrefix("KEYS");
+ return continueCollectAccessPath(mapContainsKey.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitMapContainsValue(MapContainsValue mapContainsValue, CollectorContext context) {
+ context.accessPathBuilder.addPrefix("VALUES");
+ return continueCollectAccessPath(mapContainsValue.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitMapContainsEntry(MapContainsEntry mapContainsEntry, CollectorContext context) {
+ context.accessPathBuilder.addPrefix("*");
+ return continueCollectAccessPath(mapContainsEntry.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitArrayMap(ArrayMap arrayMap, CollectorContext context) {
+ // ARRAY_MAP(lambda, <arr> [ , <arr> ... ] )
+
+ Expression argument = arrayMap.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ return collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayMap, context);
+ }
+
+ @Override
+ public Void visitArrayCount(ArrayCount arrayCount, CollectorContext context) {
+ // ARRAY_COUNT(<lambda>, <arr>[, ... ])
+
+ Expression argument = arrayCount.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ return collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayCount, context);
+ }
+
+ @Override
+ public Void visitArrayExists(ArrayExists arrayExists, CollectorContext context) {
+ // ARRAY_EXISTS([ <lambda>, ] <arr1> [, <arr2> , ...] )
+
+ Expression argument = arrayExists.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ return collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayExists, context);
+ }
+
+ @Override
+ public Void visitArrayFilter(ArrayFilter arrayFilter, CollectorContext context) {
+ // ARRAY_FILTER(<lambda>, <arr>)
+
+ Expression argument = arrayFilter.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayFilter, context);
+ }
+
+ @Override
+ public Void visitArrayFirst(ArrayFirst arrayFirst, CollectorContext context) {
+ // ARRAY_FIRST(<lambda>, <arr>)
+
+ Expression argument = arrayFirst.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayFirst, context);
+ }
+
+ @Override
+ public Void visitArrayFirstIndex(ArrayFirstIndex arrayFirstIndex, CollectorContext context) {
+ // ARRAY_FIRST_INDEX(<lambda>, <arr> [, ...])
+
+ Expression argument = arrayFirstIndex.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayFirstIndex, context);
+ }
+
+ @Override
+ public Void visitArrayLast(ArrayLast arrayLast, CollectorContext context) {
+ // ARRAY_LAST(<lambda>, <arr>)
+
+ Expression argument = arrayLast.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayLast, context);
+ }
+
+ @Override
+ public Void visitArrayLastIndex(ArrayLastIndex arrayLastIndex, CollectorContext context) {
+ // ARRAY_LAST_INDEX(<lambda>, <arr> [, ...])
+
+ Expression argument = arrayLastIndex.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayLastIndex, context);
+ }
+
+ @Override
+ public Void visitArrayMatchAny(ArrayMatchAny arrayMatchAny, CollectorContext context) {
+ // array_match_any(lambda, <arr> [, <arr> ...])
+
+ Expression argument = arrayMatchAny.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayMatchAny, context);
+ }
+
+ @Override
+ public Void visitArrayMatchAll(ArrayMatchAll arrayMatchAll, CollectorContext context) {
+ // array_match_all(lambda, <arr> [, <arr> ...])
+
+ Expression argument = arrayMatchAll.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayMatchAll, context);
+ }
+
+ @Override
+ public Void visitArrayReverseSplit(ArrayReverseSplit arrayReverseSplit, CollectorContext context) {
+ // ARRAY_REVERSE_SPLIT(<lambda>, <arr> [, ...])
+
+ Expression argument = arrayReverseSplit.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayReverseSplit, context);
+ }
+
+ @Override
+ public Void visitArraySplit(ArraySplit arraySplit, CollectorContext context) {
+ // ARRAY_SPLIT(<lambda>, arr [, ...])
+
+ Expression argument = arraySplit.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arraySplit, context);
+ }
+
+ @Override
+ public Void visitArraySortBy(ArraySortBy arraySortBy, CollectorContext context) {
+ // ARRAY_SORTBY(<lambda>, <arr> [, ...])
+
+ Expression argument = arraySortBy.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arraySortBy, context);
+ }
+
+ // @Override
+ // public Void visitIsNull(IsNull isNull, CollectorContext context) {
+ // if (context.accessPathBuilder.isEmpty()) {
+ // context.setType(TAccessPathType.META);
+ // return continueCollectAccessPath(isNull.child(), context);
+ // }
+ // return visit(isNull, context);
+ // }
+
+ private Void collectArrayPathInLambda(Lambda lambda, CollectorContext context) {
+ List<Expression> arguments = lambda.getArguments();
+ Map<String, Expression> nameToArray = Maps.newLinkedHashMap();
+ for (Expression argument : arguments) {
+ if (argument instanceof ArrayItemReference) {
+ nameToArray.put(((ArrayItemReference) argument).getName(), argument.child(0));
+ }
+ }
+
+ List<String> path = context.accessPathBuilder.getPathList();
+ if (!path.isEmpty() && path.get(0).equals("*")) {
+ context.accessPathBuilder.removePrefix();
+ }
+
+ nameToLambdaArguments.push(nameToArray);
+ try {
+ continueCollectAccessPath(arguments.get(0), context);
+ } finally {
+ nameToLambdaArguments.pop();
+ }
+ return null;
+ }
+
+ /** CollectorContext */
+ public static class CollectorContext {
+ private StatementContext statementContext;
+ private AccessPathBuilder accessPathBuilder;
+ private boolean bottomFilter;
+ private TAccessPathType type;
+
+ public CollectorContext(StatementContext statementContext, boolean bottomFilter) {
+ this.statementContext = statementContext;
+ this.accessPathBuilder = new AccessPathBuilder();
+ this.bottomFilter = bottomFilter;
+ this.type = TAccessPathType.DATA;
+ }
+
+ public TAccessPathType getType() {
+ return type;
+ }
+
+ public void setType(TAccessPathType type) {
+ this.type = type;
+ }
+ }
+
+ private static class AccessPathBuilder {
+ private LinkedList<String> accessPath;
+
+ public AccessPathBuilder() {
+ accessPath = new LinkedList<>();
+ }
+
+ public AccessPathBuilder addPrefix(String prefix) {
+ accessPath.addFirst(prefix);
+ return this;
+ }
+
+ public AccessPathBuilder removePrefix() {
+ accessPath.removeFirst();
+ return this;
+ }
+
+ public List<String> getPathList() {
+ return accessPath;
+ }
+
+ public boolean isEmpty() {
+ return accessPath.isEmpty();
+ }
+
+ @Override
+ public String toString() {
+ return String.join(".", accessPath);
+ }
+ }
+
+ /** AccessPathIsPredicate */
+ public static class CollectAccessPathResult {
+ private final List<String> path;
+ private final boolean isPredicate;
+ private final TAccessPathType type;
+
+ public CollectAccessPathResult(List<String> path, boolean isPredicate, TAccessPathType type) {
+ this.path = path;
+ this.isPredicate = isPredicate;
+ this.type = type;
+ }
+
+ public TAccessPathType getType() {
+ return type;
+ }
+
+ public List<String> getPath() {
+ return path;
+ }
+
+ public boolean isPredicate() {
+ return isPredicate;
+ }
+
+ @Override
+ public String toString() {
+ return String.join(".", path) + ", " + isPredicate;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ CollectAccessPathResult that = (CollectAccessPathResult) o;
+ return isPredicate == that.isPredicate && Objects.equals(path, that.path);
+ }
+
+ @Override
+ public int hashCode() {
+ return path.hashCode();
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java
new file mode 100644
index 00000000000..514f7bb1e8c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectAccessPathResult;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
+import org.apache.doris.nereids.types.NestedColumnPrunable;
+
+import com.google.common.collect.LinkedHashMultimap;
+import com.google.common.collect.Multimap;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/** AccessPathPlanCollector */
+public class AccessPathPlanCollector extends DefaultPlanVisitor<Void, StatementContext> {
+ private Multimap<Integer, CollectAccessPathResult> allSlotToAccessPaths = LinkedHashMultimap.create();
+ private Map<Slot, List<CollectAccessPathResult>> scanSlotToAccessPaths = new LinkedHashMap<>();
+
+ public Map<Slot, List<CollectAccessPathResult>> collect(Plan root, StatementContext context) {
+ root.accept(this, context);
+ return scanSlotToAccessPaths;
+ }
+
+ @Override
+ public Void visitLogicalFilter(LogicalFilter<? extends Plan> filter, StatementContext context) {
+ boolean bottomFilter = filter.child().arity() == 0;
+ collectByExpressions(filter, context, bottomFilter);
+ return filter.child().accept(this, context);
+ }
+
+ @Override
+ public Void visitLogicalCTEAnchor(
+ LogicalCTEAnchor<? extends Plan, ? extends Plan> cteAnchor, StatementContext context) {
+ // first, collect access paths in the outer slots, and propagate outer slot's access path to inner slots
+ cteAnchor.right().accept(this, context);
+
+ // second, push down access path in the inner slots
+ cteAnchor.left().accept(this, context);
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalCTEConsumer(LogicalCTEConsumer cteConsumer, StatementContext context) {
+ // propagate outer slot's access path to inner slots
+ for (Entry<Slot, Slot> slots : cteConsumer.getConsumerToProducerOutputMap().entrySet()) {
+ Slot outerSlot = slots.getKey();
+
+ if (outerSlot.getDataType() instanceof NestedColumnPrunable) {
+ int outerSlotId = outerSlot.getExprId().asInt();
+ int innerSlotId = slots.getValue().getExprId().asInt();
+ allSlotToAccessPaths.putAll(innerSlotId, allSlotToAccessPaths.get(outerSlotId));
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalCTEProducer(LogicalCTEProducer<? extends Plan> cteProducer, StatementContext context) {
+ return cteProducer.child().accept(this, context);
+ }
+
+ @Override
+ public Void visitLogicalUnion(LogicalUnion union, StatementContext context) {
+ // now we will not prune complex type through union, because we can not prune the complex type's literal,
+ // for example, we can not prune the literal now: array(map(1, named_struct('a', 100, 'b', 100))),
+ // so we can not prune this sql:
+ // select struct_element(map_values(s[0]), 'a')
+ // from (
+ // select s from tbl
+ // union all
+ // select array(map(1, named_struct('a', 100, 'b', 100)))
+ // ) tbl;
+ //
+ // so we will not propagate access paths through the union
+ for (Plan child : union.children()) {
+ child.accept(this, context);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalOlapScan(LogicalOlapScan olapScan, StatementContext context) {
+ for (Slot slot : olapScan.getOutput()) {
+ if (!(slot.getDataType() instanceof NestedColumnPrunable)) {
+ continue;
+ }
+ Collection<CollectAccessPathResult> accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt());
+ if (!accessPaths.isEmpty()) {
+ scanSlotToAccessPaths.put(slot, new ArrayList<>(accessPaths));
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalFileScan(LogicalFileScan fileScan, StatementContext context) {
+ for (Slot slot : fileScan.getOutput()) {
+ if (!(slot.getDataType() instanceof NestedColumnPrunable)) {
+ continue;
+ }
+ Collection<CollectAccessPathResult> accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt());
+ if (!accessPaths.isEmpty()) {
+ scanSlotToAccessPaths.put(slot, new ArrayList<>(accessPaths));
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, StatementContext context) {
+ for (Slot slot : tvfRelation.getOutput()) {
+ if (!(slot.getDataType() instanceof NestedColumnPrunable)) {
+ continue;
+ }
+ Collection<CollectAccessPathResult> accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt());
+ if (!accessPaths.isEmpty()) {
+ scanSlotToAccessPaths.put(slot, new ArrayList<>(accessPaths));
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(Plan plan, StatementContext context) {
+ collectByExpressions(plan, context);
+
+ for (Plan child : plan.children()) {
+ child.accept(this, context);
+ }
+ return null;
+ }
+
+ private void collectByExpressions(Plan plan, StatementContext context) {
+ collectByExpressions(plan, context, false);
+ }
+
+ private void collectByExpressions(Plan plan, StatementContext context, boolean bottomPredicate) {
+ AccessPathExpressionCollector exprCollector
+ = new AccessPathExpressionCollector(context, allSlotToAccessPaths, bottomPredicate);
+ for (Expression expression : plan.getExpressions()) {
+ exprCollector.collect(expression);
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
new file mode 100644
index 00000000000..de22e9d2101
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
@@ -0,0 +1,487 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.analysis.AccessPathInfo;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.jobs.JobContext;
+import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectAccessPathResult;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
+import org.apache.doris.nereids.types.ArrayType;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.MapType;
+import org.apache.doris.nereids.types.NullType;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.thrift.TAccessPathType;
+import org.apache.doris.thrift.TColumnAccessPath;
+import org.apache.doris.thrift.TDataAccessPath;
+import org.apache.doris.thrift.TMetaAccessPath;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.TreeMultimap;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+
+/**
+ * <li> 1. prune the data type of struct/map
+ *
+ * <p> for example, column s is a struct<id: int, value: double>,
+ * and `select struct(s, 'id') from tbl` will return the data type: `struct<id: int>`
+ * </p>
+ * </li>
+ *
+ * <li> 2. collect the access paths
+ * <p> for example, select struct(s, 'id'), struct(s, 'data') from tbl` will collect the access path:
+ * [s.id, s.data]
+ * </p>
+ * </li>
+ */
+public class NestedColumnPruning implements CustomRewriter {
+ public static final Logger LOG = LogManager.getLogger(NestedColumnPruning.class);
+
+ @Override
+ public Plan rewriteRoot(Plan plan, JobContext jobContext) {
+ try {
+ StatementContext statementContext = jobContext.getCascadesContext().getStatementContext();
+ SessionVariable sessionVariable = statementContext.getConnectContext().getSessionVariable();
+ if (!sessionVariable.enablePruneNestedColumns || !statementContext.hasNestedColumns()) {
+ return plan;
+ }
+
+ AccessPathPlanCollector collector = new AccessPathPlanCollector();
+ Map<Slot, List<CollectAccessPathResult>> slotToAccessPaths = collector.collect(plan, statementContext);
+ Map<Integer, AccessPathInfo> slotToResult = pruneDataType(slotToAccessPaths);
+
+ if (!slotToResult.isEmpty()) {
+ Map<Integer, AccessPathInfo> slotIdToPruneType = Maps.newLinkedHashMap();
+ for (Entry<Integer, AccessPathInfo> kv : slotToResult.entrySet()) {
+ Integer slotId = kv.getKey();
+ AccessPathInfo accessPathInfo = kv.getValue();
+ slotIdToPruneType.put(slotId, accessPathInfo);
+ }
+ SlotTypeReplacer typeReplacer = new SlotTypeReplacer(slotIdToPruneType);
+ return plan.accept(typeReplacer, null);
+ }
+ return plan;
+ } catch (Throwable t) {
+ LOG.warn("NestedColumnPruning failed.", t);
+ return plan;
+ }
+ }
+
+ private static Map<Integer, AccessPathInfo> pruneDataType(
+ Map<Slot, List<CollectAccessPathResult>> slotToAccessPaths) {
+ Map<Integer, AccessPathInfo> result = new LinkedHashMap<>();
+ Map<Slot, DataTypeAccessTree> slotIdToAllAccessTree = new LinkedHashMap<>();
+ Map<Slot, DataTypeAccessTree> slotIdToPredicateAccessTree = new LinkedHashMap<>();
+
+ Comparator<Pair<TAccessPathType, List<String>>> pathComparator = Comparator.comparing(
+ l -> StringUtils.join(l.second, "."));
+
+ Multimap<Integer, Pair<TAccessPathType, List<String>>> allAccessPaths = TreeMultimap.create(
+ Comparator.naturalOrder(), pathComparator);
+ Multimap<Integer, Pair<TAccessPathType, List<String>>> predicateAccessPaths = TreeMultimap.create(
+ Comparator.naturalOrder(), pathComparator);
+
+ // first: build access data type tree
+ for (Entry<Slot, List<CollectAccessPathResult>> kv : slotToAccessPaths.entrySet()) {
+ Slot slot = kv.getKey();
+ List<CollectAccessPathResult> collectAccessPathResults = kv.getValue();
+ for (CollectAccessPathResult collectAccessPathResult : collectAccessPathResults) {
+ List<String> path = collectAccessPathResult.getPath();
+ TAccessPathType pathType = collectAccessPathResult.getType();
+ DataTypeAccessTree allAccessTree = slotIdToAllAccessTree.computeIfAbsent(
+ slot, i -> DataTypeAccessTree.ofRoot(slot, pathType)
+ );
+ allAccessTree.setAccessByPath(path, 0, pathType);
+ allAccessPaths.put(slot.getExprId().asInt(), Pair.of(pathType, path));
+
+ if (collectAccessPathResult.isPredicate()) {
+ DataTypeAccessTree predicateAccessTree = slotIdToPredicateAccessTree.computeIfAbsent(
+ slot, i -> DataTypeAccessTree.ofRoot(slot, pathType)
+ );
+ predicateAccessTree.setAccessByPath(path, 0, pathType);
+ predicateAccessPaths.put(
+ slot.getExprId().asInt(), Pair.of(pathType, path)
+ );
+ }
+ }
+ }
+
+ // second: build non-predicate access paths
+ for (Entry<Slot, DataTypeAccessTree> kv : slotIdToAllAccessTree.entrySet()) {
+ Slot slot = kv.getKey();
+ DataTypeAccessTree accessTree = kv.getValue();
+ DataType prunedDataType = accessTree.pruneDataType().orElse(slot.getDataType());
+
+ List<TColumnAccessPath> allPaths = new ArrayList<>();
+ boolean accessWholeColumn = false;
+ TAccessPathType accessWholeColumnType = TAccessPathType.META;
+ for (Pair<TAccessPathType, List<String>> pathInfo : allAccessPaths.get(slot.getExprId().asInt())) {
+ if (pathInfo.first == TAccessPathType.DATA) {
+ TDataAccessPath dataAccessPath = new TDataAccessPath();
+ dataAccessPath.setPath(new ArrayList<>(pathInfo.second));
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.DATA);
+ accessPath.setDataAccessPath(dataAccessPath);
+ allPaths.add(accessPath);
+ } else {
+ TMetaAccessPath dataAccessPath = new TMetaAccessPath();
+ dataAccessPath.setPath(new ArrayList<>(pathInfo.second));
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.META);
+ accessPath.setMetaAccessPath(dataAccessPath);
+ allPaths.add(accessPath);
+ }
+ // only retain access the whole root
+ if (pathInfo.second.size() == 1) {
+ accessWholeColumn = true;
+ if (pathInfo.first == TAccessPathType.DATA) {
+ accessWholeColumnType = TAccessPathType.DATA;
+ }
+ } else {
+ accessWholeColumnType = TAccessPathType.DATA;
+ }
+ }
+ if (accessWholeColumn) {
+ TColumnAccessPath accessPath = new TColumnAccessPath(accessWholeColumnType);
+ SlotReference slotReference = (SlotReference) kv.getKey();
+ String wholeColumnName = slotReference.getOriginalColumn().get().getName();
+ if (accessWholeColumnType == TAccessPathType.DATA) {
+ accessPath.setDataAccessPath(new TDataAccessPath(ImmutableList.of(wholeColumnName)));
+ } else {
+ accessPath.setMetaAccessPath(new TMetaAccessPath(ImmutableList.of(wholeColumnName)));
+ }
+ allPaths = ImmutableList.of(accessPath);
+ }
+ result.put(slot.getExprId().asInt(), new AccessPathInfo(prunedDataType, allPaths, new ArrayList<>()));
+ }
+
+ // third: build predicate access path
+ for (Entry<Slot, DataTypeAccessTree> kv : slotIdToPredicateAccessTree.entrySet()) {
+ Slot slot = kv.getKey();
+
+ List<TColumnAccessPath> predicatePaths = new ArrayList<>();
+ boolean accessWholeColumn = false;
+ TAccessPathType accessWholeColumnType = TAccessPathType.META;
+ for (Pair<TAccessPathType, List<String>> pathInfo : predicateAccessPaths.get(slot.getExprId().asInt())) {
+ if (pathInfo == null) {
+ throw new AnalysisException("This is a bug, please report this");
+ }
+
+ if (pathInfo.first == TAccessPathType.DATA) {
+ TDataAccessPath dataAccessPath = new TDataAccessPath();
+ dataAccessPath.setPath(new ArrayList<>(pathInfo.second));
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.DATA);
+ accessPath.setDataAccessPath(dataAccessPath);
+ predicatePaths.add(accessPath);
+ } else {
+ TMetaAccessPath dataAccessPath = new TMetaAccessPath();
+ dataAccessPath.setPath(new ArrayList<>(pathInfo.second));
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.META);
+ accessPath.setMetaAccessPath(dataAccessPath);
+ predicatePaths.add(accessPath);
+ }
+ // only retain access the whole root
+ if (pathInfo.second.size() == 1) {
+ accessWholeColumn = true;
+ if (pathInfo.first == TAccessPathType.DATA) {
+ accessWholeColumnType = TAccessPathType.DATA;
+ }
+ } else {
+ accessWholeColumnType = TAccessPathType.DATA;
+ }
+ }
+
+ if (accessWholeColumn) {
+ TColumnAccessPath accessPath = new TColumnAccessPath(accessWholeColumnType);
+ SlotReference slotReference = (SlotReference) kv.getKey();
+ String wholeColumnName = slotReference.getOriginalColumn().get().getName();
+ if (accessWholeColumnType == TAccessPathType.DATA) {
+ accessPath.setDataAccessPath(new TDataAccessPath(ImmutableList.of(wholeColumnName)));
+ } else {
+ accessPath.setMetaAccessPath(new TMetaAccessPath(ImmutableList.of(wholeColumnName)));
+ }
+ predicatePaths = ImmutableList.of(accessPath);
+ }
+
+ AccessPathInfo accessPathInfo = result.get(slot.getExprId().asInt());
+ accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths);
+ }
+
+ return result;
+ }
+
+ /** DataTypeAccessTree */
+ public static class DataTypeAccessTree {
+ private DataType type;
+ private boolean isRoot;
+ private boolean accessPartialChild;
+ private boolean accessAll;
+ private TAccessPathType pathType;
+ private Map<String, DataTypeAccessTree> children = new LinkedHashMap<>();
+
+ public DataTypeAccessTree(DataType type, TAccessPathType pathType) {
+ this(false, type, pathType);
+ }
+
+ public DataTypeAccessTree(boolean isRoot, DataType type, TAccessPathType pathType) {
+ this.isRoot = isRoot;
+ this.type = type;
+ this.pathType = pathType;
+ }
+
+ /** pruneCastType */
+ public DataType pruneCastType(DataTypeAccessTree origin, DataTypeAccessTree cast) {
+ if (type instanceof StructType) {
+ Map<String, String> nameMapping = new LinkedHashMap<>();
+ List<String> castNames = new ArrayList<>(cast.children.keySet());
+ int i = 0;
+ for (String s : origin.children.keySet()) {
+ nameMapping.put(s, castNames.get(i++));
+ }
+ List<StructField> mappingFields = new ArrayList<>();
+ StructType originCastStructType = (StructType) cast.type;
+ for (Entry<String, DataTypeAccessTree> kv : children.entrySet()) {
+ String originName = kv.getKey();
+ String mappingName = nameMapping.getOrDefault(originName, originName);
+ DataTypeAccessTree originPrunedTree = kv.getValue();
+ DataType mappingType = originPrunedTree.pruneCastType(
+ origin.children.get(originName),
+ cast.children.get(mappingName)
+ );
+ StructField originCastField = originCastStructType.getField(mappingName);
+ mappingFields.add(
+ new StructField(mappingName, mappingType,
+ originCastField.isNullable(), originCastField.getComment())
+ );
+ }
+ return new StructType(mappingFields);
+ } else if (type instanceof ArrayType) {
+ return ArrayType.of(
+ children.values().iterator().next().pruneCastType(
+ origin.children.values().iterator().next(),
+ cast.children.values().iterator().next()
+ ),
+ ((ArrayType) cast.type).containsNull()
+ );
+ } else if (type instanceof MapType) {
+ return MapType.of(
+ children.get("KEYS").pruneCastType(origin.children.get("KEYS"), cast.children.get("KEYS")),
+ children.get("VALUES").pruneCastType(origin.children.get("VALUES"), cast.children.get("VALUES"))
+ );
+ } else {
+ return cast.type;
+ }
+ }
+
+ /** replacePathByAnotherTree */
+ public boolean replacePathByAnotherTree(DataTypeAccessTree cast, List<String> path, int index) {
+ if (index >= path.size()) {
+ return true;
+ }
+ if (cast.type instanceof StructType) {
+ List<StructField> fields = ((StructType) cast.type).getFields();
+ for (int i = 0; i < fields.size(); i++) {
+ String castFieldName = path.get(index);
+ if (fields.get(i).getName().equalsIgnoreCase(castFieldName)) {
+ String originFieldName = ((StructType) type).getFields().get(i).getName();
+ path.set(index, originFieldName);
+ return children.get(originFieldName).replacePathByAnotherTree(
+ cast.children.get(castFieldName), path, index + 1
+ );
+ }
+ }
+ } else if (cast.type instanceof ArrayType) {
+ return children.values().iterator().next().replacePathByAnotherTree(
+ cast.children.values().iterator().next(), path, index + 1);
+ } else if (cast.type instanceof MapType) {
+ String fieldName = path.get(index);
+ return children.get("VALUES").replacePathByAnotherTree(
+ cast.children.get(fieldName), path, index + 1
+ );
+ }
+ return false;
+ }
+
+ /** setAccessByPath */
+ public void setAccessByPath(List<String> path, int accessIndex, TAccessPathType pathType) {
+ if (accessIndex >= path.size()) {
+ accessAll = true;
+ return;
+ } else {
+ accessPartialChild = true;
+ }
+
+ if (pathType == TAccessPathType.DATA) {
+ this.pathType = TAccessPathType.DATA;
+ }
+
+ if (this.type.isStructType()) {
+ String fieldName = path.get(accessIndex).toLowerCase();
+ DataTypeAccessTree child = children.get(fieldName);
+ if (child != null) {
+ child.setAccessByPath(path, accessIndex + 1, pathType);
+ } else {
+ // can not find the field
+ accessAll = true;
+ }
+ return;
+ } else if (this.type.isArrayType()) {
+ DataTypeAccessTree child = children.get("*");
+ if (path.get(accessIndex).equals("*")) {
+ // enter this array and skip next *
+ child.setAccessByPath(path, accessIndex + 1, pathType);
+ }
+ return;
+ } else if (this.type.isMapType()) {
+ String fieldName = path.get(accessIndex);
+ if (fieldName.equals("*")) {
+ // access value by the key, so we should access key and access value, then prune the value's type.
+ // e.g. map_column['id'] should access the keys, and access the values
+ DataTypeAccessTree keysChild = children.get("KEYS");
+ DataTypeAccessTree valuesChild = children.get("VALUES");
+ keysChild.accessAll = true;
+ valuesChild.setAccessByPath(path, accessIndex + 1, pathType);
+ return;
+ } else if (fieldName.equals("KEYS")) {
+ // only access the keys and not need enter keys, because it must be primitive type.
+ // e.g. map_keys(map_column)
+ DataTypeAccessTree keysChild = children.get("KEYS");
+ keysChild.accessAll = true;
+ return;
+ } else if (fieldName.equals("VALUES")) {
+ // only access the values without keys, and maybe prune the value's data type.
+ // e.g. map_values(map_columns)[0] will access the array of values first,
+ // and then access the array, so the access path is ['VALUES', '*']
+ DataTypeAccessTree valuesChild = children.get("VALUES");
+ valuesChild.setAccessByPath(path, accessIndex + 1, pathType);
+ return;
+ }
+ } else if (isRoot) {
+ children.get(path.get(accessIndex).toLowerCase()).setAccessByPath(path, accessIndex + 1, pathType);
+ return;
+ }
+ throw new AnalysisException("unsupported data type: " + this.type);
+ }
+
+ public static DataTypeAccessTree ofRoot(Slot slot, TAccessPathType pathType) {
+ DataTypeAccessTree child = of(slot.getDataType(), pathType);
+ DataTypeAccessTree root = new DataTypeAccessTree(true, NullType.INSTANCE, pathType);
+ root.children.put(slot.getName().toLowerCase(), child);
+ return root;
+ }
+
+ /** of */
+ public static DataTypeAccessTree of(DataType type, TAccessPathType pathType) {
+ DataTypeAccessTree root = new DataTypeAccessTree(type, pathType);
+ if (type instanceof StructType) {
+ StructType structType = (StructType) type;
+ for (Entry<String, StructField> kv : structType.getNameToFields().entrySet()) {
+ root.children.put(kv.getKey().toLowerCase(), of(kv.getValue().getDataType(), pathType));
+ }
+ } else if (type instanceof ArrayType) {
+ root.children.put("*", of(((ArrayType) type).getItemType(), pathType));
+ } else if (type instanceof MapType) {
+ root.children.put("KEYS", of(((MapType) type).getKeyType(), pathType));
+ root.children.put("VALUES", of(((MapType) type).getValueType(), pathType));
+ }
+ return root;
+ }
+
+ /** pruneDataType */
+ public Optional<DataType> pruneDataType() {
+ if (isRoot) {
+ return children.values().iterator().next().pruneDataType();
+ } else if (accessAll) {
+ return Optional.of(type);
+ } else if (!accessPartialChild) {
+ return Optional.empty();
+ }
+
+ List<Pair<String, DataType>> accessedChildren = new ArrayList<>();
+
+ if (type instanceof StructType) {
+ for (Entry<String, DataTypeAccessTree> kv : children.entrySet()) {
+ DataTypeAccessTree childTypeTree = kv.getValue();
+ Optional<DataType> childDataType = childTypeTree.pruneDataType();
+ if (childDataType.isPresent()) {
+ accessedChildren.add(Pair.of(kv.getKey(), childDataType.get()));
+ }
+ }
+ } else if (type instanceof ArrayType) {
+ Optional<DataType> childDataType = children.get("*").pruneDataType();
+ if (childDataType.isPresent()) {
+ accessedChildren.add(Pair.of("*", childDataType.get()));
+ }
+ } else if (type instanceof MapType) {
+ DataType prunedValueType = children.get("VALUES")
+ .pruneDataType()
+ .orElse(((MapType) type).getValueType());
+ // can not prune keys but can prune values
+ accessedChildren.add(Pair.of("KEYS", ((MapType) type).getKeyType()));
+ accessedChildren.add(Pair.of("VALUES", prunedValueType));
+ }
+ if (accessedChildren.isEmpty()) {
+ return Optional.of(type);
+ }
+
+ return Optional.of(pruneDataType(type, accessedChildren));
+ }
+
+ private DataType pruneDataType(DataType dataType, List<Pair<String, DataType>> newChildrenTypes) {
+ if (dataType instanceof StructType) {
+ // prune struct fields
+ StructType structType = (StructType) dataType;
+ Map<String, StructField> nameToFields = structType.getNameToFields();
+ List<StructField> newFields = new ArrayList<>();
+ for (Pair<String, DataType> kv : newChildrenTypes) {
+ String fieldName = kv.first;
+ StructField originField = nameToFields.get(fieldName);
+ DataType prunedType = kv.second;
+ newFields.add(new StructField(
+ originField.getName(), prunedType, originField.isNullable(), originField.getComment()
+ ));
+ }
+ return new StructType(newFields);
+ } else if (dataType instanceof ArrayType) {
+ return ArrayType.of(newChildrenTypes.get(0).second, ((ArrayType) dataType).containsNull());
+ } else if (dataType instanceof MapType) {
+ return MapType.of(newChildrenTypes.get(0).second, newChildrenTypes.get(1).second);
+ } else {
+ throw new AnalysisException("unsupported data type: " + dataType);
+ }
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java
new file mode 100644
index 00000000000..6ef42183c7b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java
@@ -0,0 +1,315 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.pattern.MatchingContext;
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+
+/** PushDownNestedColumnThroughJoin */
+public class PushDownNestedColumn implements RewriteRuleFactory {
+ @Override
+ public List<Rule> buildRules() {
+ return ImmutableList.of(
+ RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_JOIN.build(
+ logicalProject(logicalJoin()).thenApply(this::defaultPushDown)
+ ),
+ RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_WINDOW.build(
+ logicalProject(logicalWindow()).thenApply(this::defaultPushDown)
+ ),
+ RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_PARTITION_TOP_N.build(
+ logicalProject(logicalPartitionTopN()).thenApply(this::defaultPushDown)
+ ),
+ // RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_DEFER_MATERIALIZE_TOP_N.build(
+ // logicalProject(logicalDeferMaterializeTopN()).thenApply(this::defaultPushDown)
+ // ),
+ RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_UNION.build(
+ logicalProject(
+ logicalUnion().when(u -> u.getQualifier() == Qualifier.ALL)
+ ).thenApply(this::pushThroughUnion)
+ )
+ );
+ }
+
+ private <C extends LogicalPlan> Plan defaultPushDown(MatchingContext<LogicalProject<C>> ctx) {
+ if (!ctx.connectContext.getSessionVariable().enablePruneNestedColumns) {
+ return ctx.root;
+ }
+
+ LogicalProject<C> project = ctx.root;
+ C child = project.child();
+ PushdownProjectHelper pushdownProjectHelper
+ = new PushdownProjectHelper(ctx.statementContext, child);
+
+ Pair<Boolean, List<NamedExpression>> pushProjects
+ = pushdownProjectHelper.pushDownExpressions(project.getProjects());
+
+ if (pushProjects.first) {
+ List<Plan> newJoinChildren = pushdownProjectHelper.buildNewChildren();
+ return new LogicalProject<>(
+ pushProjects.second,
+ child.withChildren(newJoinChildren)
+ );
+ }
+ return project;
+ }
+
+ private Plan pushThroughUnion(MatchingContext<LogicalProject<LogicalUnion>> ctx) {
+ if (!ctx.connectContext.getSessionVariable().enablePruneNestedColumns) {
+ return ctx.root;
+ }
+ LogicalProject<LogicalUnion> project = ctx.root;
+ LogicalUnion union = project.child();
+ PushdownProjectHelper pushdownProjectHelper
+ = new PushdownProjectHelper(ctx.statementContext, project);
+
+ Pair<Boolean, List<NamedExpression>> pushProjects
+ = pushdownProjectHelper.pushDownExpressions(project.getProjects());
+ if (pushProjects.first) {
+ List<NamedExpression> unionOutputs = union.getOutputs();
+ Map<Slot, Integer> slotToColumnIndex = new LinkedHashMap<>();
+ for (int i = 0; i < unionOutputs.size(); i++) {
+ NamedExpression output = unionOutputs.get(i);
+ slotToColumnIndex.put(output.toSlot(), i);
+ }
+
+ Collection<NamedExpression> pushDownProjections
+ = pushdownProjectHelper.childToPushDownProjects.values();
+ List<Plan> newChildren = new ArrayList<>();
+ List<List<SlotReference>> newChildrenOutputs = new ArrayList<>();
+ for (Plan child : union.children()) {
+ List<NamedExpression> pushedOutput = replaceSlot(
+ ctx.statementContext,
+ pushDownProjections,
+ slot -> {
+ Integer sourceColumnIndex = slotToColumnIndex.get(slot);
+ if (sourceColumnIndex != null) {
+ return child.getOutput().get(sourceColumnIndex).toSlot();
+ }
+ return slot;
+ }
+ );
+
+ LogicalProject<Plan> newChild = new LogicalProject<>(
+ ImmutableList.<NamedExpression>builder()
+ .addAll(child.getOutput())
+ .addAll(pushedOutput)
+ .build(),
+ child
+ );
+
+ newChildrenOutputs.add((List) newChild.getOutput());
+ newChildren.add(newChild);
+ }
+
+ for (List<NamedExpression> originConstantExprs : union.getConstantExprsList()) {
+ List<NamedExpression> pushedOutput = replaceSlot(
+ ctx.statementContext,
+ pushDownProjections,
+ slot -> {
+ Integer sourceColumnIndex = slotToColumnIndex.get(slot);
+ if (sourceColumnIndex != null) {
+ return originConstantExprs.get(sourceColumnIndex).toSlot();
+ }
+ return slot;
+ }
+ );
+
+ LogicalOneRowRelation originOneRowRelation = new LogicalOneRowRelation(
+ ctx.statementContext.getNextRelationId(),
+ originConstantExprs
+ );
+
+ LogicalProject<Plan> newChild = new LogicalProject<>(
+ ImmutableList.<NamedExpression>builder()
+ .addAll(originOneRowRelation.getOutput())
+ .addAll(pushedOutput)
+ .build(),
+ originOneRowRelation
+ );
+
+ newChildrenOutputs.add((List) newChild.getOutput());
+ newChildren.add(newChild);
+ }
+
+ List<NamedExpression> newUnionOutputs = new ArrayList<>(union.getOutputs());
+ for (NamedExpression projection : pushDownProjections) {
+ newUnionOutputs.add(projection.toSlot());
+ }
+
+ return new LogicalProject<>(
+ pushProjects.second,
+ new LogicalUnion(
+ union.getQualifier(),
+ newUnionOutputs,
+ newChildrenOutputs,
+ ImmutableList.of(),
+ union.hasPushedFilter(),
+ newChildren
+ )
+ );
+ }
+ return project;
+ }
+
+ private List<NamedExpression> replaceSlot(
+ StatementContext statementContext,
+ Collection<NamedExpression> pushDownProjections,
+ Function<Slot, Slot> slotReplace) {
+ List<NamedExpression> pushedOutput = new ArrayList<>();
+ for (NamedExpression projection : pushDownProjections) {
+ NamedExpression newOutput = (NamedExpression) projection.rewriteUp(e -> {
+ if (e instanceof Slot) {
+ Slot newSlot = slotReplace.apply((Slot) e);
+ if (newSlot != null) {
+ return newSlot;
+ }
+ }
+ return e;
+ });
+ if (newOutput instanceof Alias) {
+ pushedOutput.add(new Alias(statementContext.getNextExprId(), newOutput.child(0)));
+ } else {
+ pushedOutput.add(new Alias(statementContext.getNextExprId(), newOutput));
+ }
+ }
+ return pushedOutput;
+ }
+
+ private static class PushdownProjectHelper {
+ private final Plan plan;
+ private final StatementContext statementContext;
+ private final Map<Expression, Pair<Slot, Plan>> exprToChildAndSlot;
+ private final Multimap<Plan, NamedExpression> childToPushDownProjects;
+
+ public PushdownProjectHelper(StatementContext statementContext, Plan plan) {
+ this.statementContext = statementContext;
+ this.plan = plan;
+ this.exprToChildAndSlot = new LinkedHashMap<>();
+ this.childToPushDownProjects = ArrayListMultimap.create();
+ }
+
+ public <C extends Collection<E>, E extends Expression> Pair<Boolean, C> pushDownExpressions(C expressions) {
+ ImmutableCollection.Builder<E> builder;
+ if (expressions instanceof List) {
+ builder = ImmutableList.builderWithExpectedSize(expressions.size());
+ } else {
+ builder = ImmutableSet.builderWithExpectedSize(expressions.size());
+ }
+
+ boolean extracted = false;
+ for (E expression : expressions) {
+ Optional<E> result = pushDownExpression(expression);
+ if (!result.isPresent()) {
+ builder.add(expression);
+ } else {
+ extracted = true;
+ builder.add(result.get());
+ }
+ }
+
+ if (extracted) {
+ return Pair.of(true, (C) builder.build());
+ } else {
+ return Pair.of(false, expressions);
+ }
+ }
+
+ public <E extends Expression> Optional<E> pushDownExpression(E expression) {
+ if (!(expression instanceof PreferPushDownProject
+ || (expression instanceof Alias && expression.child(0) instanceof PreferPushDownProject))) {
+ return Optional.empty();
+ }
+ Pair<Slot, Plan> existPushdown = exprToChildAndSlot.get(expression);
+ if (existPushdown != null) {
+ return Optional.of((E) existPushdown.first);
+ }
+
+ Alias pushDownAlias = null;
+ if (expression instanceof Alias) {
+ pushDownAlias = (Alias) expression;
+ } else {
+ pushDownAlias = new Alias(statementContext.getNextExprId(), expression);
+ }
+
+ Set<Slot> inputSlots = expression.getInputSlots();
+ for (Plan child : plan.children()) {
+ if (child.getOutputSet().containsAll(inputSlots)) {
+ Slot remaimSlot = pushDownAlias.toSlot();
+ exprToChildAndSlot.put(expression, Pair.of(remaimSlot, child));
+ childToPushDownProjects.put(child, pushDownAlias);
+ return Optional.of((E) remaimSlot);
+ }
+ }
+ return Optional.empty();
+ }
+
+ public List<Plan> buildNewChildren() {
+ if (childToPushDownProjects.isEmpty()) {
+ return plan.children();
+ }
+ ImmutableList.Builder<Plan> newChildren = ImmutableList.builderWithExpectedSize(plan.arity());
+ for (Plan child : plan.children()) {
+ Collection<NamedExpression> newProject = childToPushDownProjects.get(child);
+ if (newProject.isEmpty()) {
+ newChildren.add(child);
+ } else {
+ newChildren.add(
+ new LogicalProject<>(
+ ImmutableList.<NamedExpression>builder()
+ .addAll(child.getOutput())
+ .addAll(newProject)
+ .build(),
+ child
+ )
+ );
+ }
+ }
+ return newChildren.build();
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
new file mode 100644
index 00000000000..459c7362758
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
@@ -0,0 +1,674 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.analysis.AccessPathInfo;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.common.Pair;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.nereids.properties.OrderKey;
+import org.apache.doris.nereids.rules.rewrite.NestedColumnPruning.DataTypeAccessTree;
+import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
+import org.apache.doris.nereids.trees.expressions.Cast;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.OrderExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.Function;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
+import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN;
+import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
+import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
+import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
+import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
+import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
+import org.apache.doris.nereids.types.ArrayType;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.MapType;
+import org.apache.doris.nereids.types.NestedColumnPrunable;
+import org.apache.doris.nereids.types.StructType;
+import org.apache.doris.nereids.util.MoreFieldsThread;
+import org.apache.doris.thrift.TAccessPathType;
+import org.apache.doris.thrift.TColumnAccessPath;
+import org.apache.doris.thrift.TDataAccessPath;
+import org.apache.doris.thrift.TMetaAccessPath;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableMultimap.Builder;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.Set;
+
+/** SlotTypeReplacer */
+public class SlotTypeReplacer extends DefaultPlanRewriter<Void> {
+ private Map<Integer, AccessPathInfo> replacedDataTypes;
+
+ public SlotTypeReplacer(Map<Integer, AccessPathInfo> bottomReplacedDataTypes) {
+ this.replacedDataTypes = Maps.newLinkedHashMap(bottomReplacedDataTypes);
+ }
+
+ @Override
+ public Plan visitLogicalCTEProducer(LogicalCTEProducer<? extends Plan> cteProducer, Void context) {
+ return super.visitLogicalCTEProducer(cteProducer, context);
+ }
+
+ @Override
+ public Plan visitLogicalWindow(LogicalWindow<? extends Plan> window, Void context) {
+ window = visitChildren(this, window, context);
+ Pair<Boolean, ? extends List<? extends Expression>> replaced = replaceExpressions(
+ window.getExpressions(), false, false);
+ if (replaced.first) {
+ return window.withExpressionsAndChild((List) replaced.second, window.child());
+ }
+ return window;
+ }
+
+ @Override
+ public Plan visitLogicalCTEConsumer(LogicalCTEConsumer cteConsumer, Void context) {
+ Map<Slot, Slot> consumerToProducerOutputMap = cteConsumer.getConsumerToProducerOutputMap();
+ Multimap<Slot, Slot> producerToConsumerOutputMap = cteConsumer.getProducerToConsumerOutputMap();
+
+ Map<Slot, Slot> replacedConsumerToProducerOutputMap = new LinkedHashMap<>();
+ Builder<Slot, Slot> replacedProducerToConsumerOutputMap = ImmutableMultimap.builder();
+
+ boolean changed = false;
+ for (Entry<Slot, Slot> kv : consumerToProducerOutputMap.entrySet()) {
+ Slot consumerSlot = kv.getKey();
+ Slot producerSlot = kv.getValue();
+ AccessPathInfo accessPathInfo = replacedDataTypes.get(producerSlot.getExprId().asInt());
+ if (accessPathInfo != null) {
+ DataType prunedType = accessPathInfo.getPrunedType();
+ if (!prunedType.equals(producerSlot.getDataType())) {
+ replacedDataTypes.put(consumerSlot.getExprId().asInt(), accessPathInfo);
+ changed = true;
+ producerSlot = producerSlot.withNullableAndDataType(producerSlot.nullable(), prunedType);
+ consumerSlot = consumerSlot.withNullableAndDataType(consumerSlot.nullable(), prunedType);
+ }
+ }
+ replacedConsumerToProducerOutputMap.put(consumerSlot, producerSlot);
+ }
+
+ for (Entry<Slot, Collection<Slot>> kv : producerToConsumerOutputMap.asMap().entrySet()) {
+ Slot producerSlot = kv.getKey();
+ Collection<Slot> consumerSlots = kv.getValue();
+ AccessPathInfo accessPathInfo = replacedDataTypes.get(producerSlot.getExprId().asInt());
+ if (accessPathInfo != null && !accessPathInfo.getPrunedType().equals(producerSlot.getDataType())) {
+ DataType replacedDataType = accessPathInfo.getPrunedType();
+ changed = true;
+ producerSlot = producerSlot.withNullableAndDataType(producerSlot.nullable(), replacedDataType);
+ for (Slot consumerSlot : consumerSlots) {
+ consumerSlot = consumerSlot.withNullableAndDataType(consumerSlot.nullable(), replacedDataType);
+ replacedProducerToConsumerOutputMap.put(producerSlot, consumerSlot);
+ }
+ } else {
+ replacedProducerToConsumerOutputMap.putAll(producerSlot, consumerSlots);
+ }
+ }
+
+ if (changed) {
+ return new LogicalCTEConsumer(
+ cteConsumer.getRelationId(), cteConsumer.getCteId(), cteConsumer.getName(),
+ replacedConsumerToProducerOutputMap, replacedProducerToConsumerOutputMap.build()
+ );
+ }
+ return cteConsumer;
+ }
+
+ @Override
+ public Plan visitLogicalJoin(LogicalJoin<? extends Plan, ? extends Plan> join, Void context) {
+ join = visitChildren(this, join, context);
+ Pair<Boolean, List<Expression>> replacedHashJoinConjuncts
+ = replaceExpressions(join.getHashJoinConjuncts(), false, false);
+ Pair<Boolean, List<Expression>> replacedOtherJoinConjuncts
+ = replaceExpressions(join.getOtherJoinConjuncts(), false, false);
+
+ if (replacedHashJoinConjuncts.first || replacedOtherJoinConjuncts.first) {
+ return join.withJoinConjuncts(
+ replacedHashJoinConjuncts.second,
+ replacedOtherJoinConjuncts.second,
+ join.getJoinReorderContext());
+ }
+ return join;
+ }
+
+ @Override
+ public Plan visitLogicalProject(LogicalProject<? extends Plan> project, Void context) {
+ project = visitChildren(this, project, context);
+
+ Pair<Boolean, List<NamedExpression>> projects = replaceExpressions(project.getProjects(), true, false);
+ if (projects.first) {
+ return project.withProjects(projects.second);
+ }
+ return project;
+ }
+
+ @Override
+ public Plan visitLogicalPartitionTopN(LogicalPartitionTopN<? extends Plan> partitionTopN, Void context) {
+ partitionTopN = visitChildren(this, partitionTopN, context);
+
+ Pair<Boolean, List<Expression>> replacedPartitionKeys = replaceExpressions(
+ partitionTopN.getPartitionKeys(), false, false);
+ Pair<Boolean, List<OrderExpression>> replacedOrderExpressions
+ = replaceOrderExpressions(partitionTopN.getOrderKeys());
+ if (replacedPartitionKeys.first || replacedOrderExpressions.first) {
+ return partitionTopN.withPartitionKeysAndOrderKeys(
+ replacedPartitionKeys.second, replacedOrderExpressions.second);
+ }
+ return partitionTopN;
+ }
+
+ @Override
+ public Plan visitLogicalDeferMaterializeTopN(LogicalDeferMaterializeTopN<? extends Plan> topN, Void context) {
+ topN = visitChildren(this, topN, context);
+
+ LogicalTopN logicalTopN = (LogicalTopN) topN.getLogicalTopN().accept(this, context);
+ if (logicalTopN != topN.getLogicalTopN()) {
+ SlotReference replacedColumnIdSlot = replaceExpressions(
+ ImmutableList.of(topN.getColumnIdSlot()), false, false).second.get(0);
+ return new LogicalDeferMaterializeTopN(
+ logicalTopN, topN.getDeferMaterializeSlotIds(), replacedColumnIdSlot);
+ }
+
+ return topN;
+ }
+
+ @Override
+ public Plan visitLogicalExcept(LogicalExcept except, Void context) {
+ except = visitChildren(this, except, context);
+
+ Pair<Boolean, List<List<SlotReference>>> replacedRegularChildrenOutputs = replaceMultiExpressions(
+ except.getRegularChildrenOutputs());
+
+ Pair<Boolean, List<NamedExpression>> replacedOutputs
+ = replaceExpressions(except.getOutputs(), true, false);
+
+ if (replacedRegularChildrenOutputs.first || replacedOutputs.first) {
+ return new LogicalExcept(except.getQualifier(), except.getOutputs(),
+ except.getRegularChildrenOutputs(), except.children());
+ }
+
+ return except;
+ }
+
+ @Override
+ public Plan visitLogicalIntersect(LogicalIntersect intersect, Void context) {
+ intersect = visitChildren(this, intersect, context);
+
+ Pair<Boolean, List<List<SlotReference>>> replacedRegularChildrenOutputs = replaceMultiExpressions(
+ intersect.getRegularChildrenOutputs());
+
+ Pair<Boolean, List<NamedExpression>> replacedOutputs
+ = replaceExpressions(intersect.getOutputs(), true, false);
+
+ if (replacedRegularChildrenOutputs.first || replacedOutputs.first) {
+ return new LogicalIntersect(intersect.getQualifier(), intersect.getOutputs(),
+ intersect.getRegularChildrenOutputs(), intersect.children());
+ }
+ return intersect;
+ }
+
+ @Override
+ public Plan visitLogicalUnion(LogicalUnion union, Void context) {
+ union = visitChildren(this, union, context);
+
+ Pair<Boolean, List<List<SlotReference>>> replacedRegularChildrenOutputs = replaceMultiExpressions(
+ union.getRegularChildrenOutputs());
+
+ Pair<Boolean, List<NamedExpression>> replacedOutputs
+ = replaceExpressions(union.getOutputs(), true, false);
+
+ if (replacedRegularChildrenOutputs.first || replacedOutputs.first) {
+ return new LogicalUnion(
+ union.getQualifier(),
+ replacedOutputs.second,
+ replacedRegularChildrenOutputs.second,
+ union.getConstantExprsList(),
+ union.hasPushedFilter(),
+ union.children()
+ );
+ }
+
+ return union;
+ }
+
+ @Override
+ public Plan visitLogicalRepeat(LogicalRepeat<? extends Plan> repeat, Void context) {
+ repeat = visitChildren(this, repeat, context);
+
+ Pair<Boolean, List<List<Expression>>> replacedGroupingSets
+ = replaceMultiExpressions(repeat.getGroupingSets());
+ Pair<Boolean, List<NamedExpression>> replacedOutputs
+ = replaceExpressions(repeat.getOutputExpressions(), true, false);
+
+ if (replacedGroupingSets.first || replacedOutputs.first) {
+ return repeat.withGroupSetsAndOutput(replacedGroupingSets.second, replacedOutputs.second);
+ }
+ return repeat;
+ }
+
+ @Override
+ public Plan visitLogicalGenerate(LogicalGenerate<? extends Plan> generate, Void context) {
+ generate = visitChildren(this, generate, context);
+
+ Pair<Boolean, List<Function>> replacedGenerators
+ = replaceExpressions(generate.getGenerators(), false, false);
+ Pair<Boolean, List<Slot>> replacedGeneratorOutput
+ = replaceExpressions(generate.getGeneratorOutput(), false, false);
+ if (replacedGenerators.first || replacedGeneratorOutput.first) {
+ return new LogicalGenerate<>(replacedGenerators.second, replacedGeneratorOutput.second,
+ generate.getExpandColumnAlias(), generate.child());
+ }
+ return generate;
+ }
+
+ @Override
+ public Plan visitLogicalAggregate(LogicalAggregate<? extends Plan> aggregate, Void context) {
+ aggregate = visitChildren(this, aggregate, context);
+
+ Pair<Boolean, List<Expression>> replacedGroupBy = replaceExpressions(
+ aggregate.getGroupByExpressions(), false, false);
+ Pair<Boolean, List<NamedExpression>> replacedOutput = replaceExpressions(
+ aggregate.getOutputExpressions(), true, false);
+
+ if (replacedGroupBy.first || replacedOutput.first) {
+ return aggregate.withGroupByAndOutput(replacedGroupBy.second, replacedOutput.second);
+ }
+ return aggregate;
+ }
+
+ @Override
+ public Plan visitLogicalSort(LogicalSort<? extends Plan> sort, Void context) {
+ sort = visitChildren(this, sort, context);
+
+ Pair<Boolean, List<OrderKey>> replaced = replaceOrderKeys(sort.getOrderKeys());
+ if (replaced.first) {
+ return sort.withOrderKeys(replaced.second);
+ }
+ return sort;
+ }
+
+ @Override
+ public Plan visitLogicalTopN(LogicalTopN<? extends Plan> topN, Void context) {
+ topN = visitChildren(this, topN, context);
+
+ Pair<Boolean, List<OrderKey>> replaced = replaceOrderKeys(topN.getOrderKeys());
+ if (replaced.first) {
+ return topN.withOrderKeys(replaced.second);
+ }
+ return topN;
+ }
+
+ @Override
+ public Plan visitLogicalDeferMaterializeOlapScan(
+ LogicalDeferMaterializeOlapScan deferMaterializeOlapScan, Void context) {
+
+ LogicalOlapScan logicalOlapScan
+ = (LogicalOlapScan) deferMaterializeOlapScan.getLogicalOlapScan().accept(this, context);
+
+ if (logicalOlapScan != deferMaterializeOlapScan.getLogicalOlapScan()) {
+ SlotReference replacedColumnIdSlot = replaceExpressions(
+ ImmutableList.of(deferMaterializeOlapScan.getColumnIdSlot()), false, false).second.get(0);
+ return new LogicalDeferMaterializeOlapScan(
+ logicalOlapScan, deferMaterializeOlapScan.getDeferMaterializeSlotIds(), replacedColumnIdSlot
+ );
+ }
+ return deferMaterializeOlapScan;
+ }
+
+ @Override
+ public Plan visitLogicalFilter(LogicalFilter<? extends Plan> filter, Void context) {
+ filter = visitChildren(this, filter, context);
+
+ Pair<Boolean, Set<Expression>> replaced = replaceExpressions(filter.getConjuncts(), false, false);
+ if (replaced.first) {
+ return filter.withConjuncts(replaced.second);
+ }
+ return filter;
+ }
+
+ @Override
+ public Plan visitLogicalFileScan(LogicalFileScan fileScan, Void context) {
+ Pair<Boolean, List<Slot>> replaced = replaceExpressions(fileScan.getOutput(), false, true);
+ if (replaced.first) {
+ List<Slot> replaceSlots = new ArrayList<>(replaced.second);
+ if (fileScan.getTable() instanceof IcebergExternalTable) {
+ for (int i = 0; i < replaceSlots.size(); i++) {
+ Slot slot = replaceSlots.get(i);
+ if (!(slot instanceof SlotReference)) {
+ continue;
+ }
+ SlotReference slotReference = (SlotReference) slot;
+ Optional<List<TColumnAccessPath>> allAccessPaths = slotReference.getAllAccessPaths();
+ if (!allAccessPaths.isPresent() || !slotReference.getOriginalColumn().isPresent()) {
+ continue;
+ }
+ List<TColumnAccessPath> allAccessPathsWithId
+ = replaceIcebergAccessPathToId(allAccessPaths.get(), slotReference);
+ List<TColumnAccessPath> predicateAccessPathsWithId = replaceIcebergAccessPathToId(
+ slotReference.getPredicateAccessPaths().get(), slotReference);
+ replaceSlots.set(i, ((SlotReference) slot).withAccessPaths(
+ allAccessPathsWithId,
+ predicateAccessPathsWithId,
+ allAccessPaths.get(),
+ slotReference.getPredicateAccessPaths().get()
+ ));
+ }
+ }
+ return fileScan.withCachedOutput(replaceSlots);
+ }
+ return fileScan;
+ }
+
+ @Override
+ public Plan visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, Void context) {
+ Pair<Boolean, List<Slot>> replaced
+ = replaceExpressions(tvfRelation.getOutput(), false, true);
+ if (replaced.first) {
+ return tvfRelation.withCachedOutputs(replaced.second);
+ }
+ return tvfRelation;
+ }
+
+ @Override
+ public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Void context) {
+ Pair<Boolean, List<Slot>> replaced = replaceExpressions(olapScan.getOutput(), false, true);
+ if (replaced.first) {
+ return olapScan.withPrunedTypeSlots(replaced.second);
+ }
+ return olapScan;
+ }
+
+ @Override
+ public Plan visitLogicalEmptyRelation(LogicalEmptyRelation emptyRelation, Void context) {
+ Pair<Boolean, List<NamedExpression>> replacedProjects
+ = replaceExpressions(emptyRelation.getProjects(), true, false);
+
+ if (replacedProjects.first) {
+ return emptyRelation.withProjects(replacedProjects.second);
+ }
+ return emptyRelation;
+ }
+
+ @Override
+ public Plan visitLogicalOneRowRelation(LogicalOneRowRelation oneRowRelation, Void context) {
+ Pair<Boolean, List<NamedExpression>> replacedProjects
+ = replaceExpressions(oneRowRelation.getProjects(), true, false);
+
+ if (replacedProjects.first) {
+ return oneRowRelation.withProjects(replacedProjects.second);
+ }
+ return oneRowRelation;
+ }
+
+ @Override
+ public Plan visitLogicalResultSink(LogicalResultSink<? extends Plan> logicalResultSink, Void context) {
+ logicalResultSink = visitChildren(this, logicalResultSink, context);
+
+ Pair<Boolean, List<NamedExpression>> replacedOutput = replaceExpressions(logicalResultSink.getOutputExprs(),
+ false, false);
+ if (replacedOutput.first) {
+ return logicalResultSink.withOutputExprs(replacedOutput.second);
+ }
+ return logicalResultSink;
+ }
+
+ private Pair<Boolean, List<OrderExpression>> replaceOrderExpressions(List<OrderExpression> orderExpressions) {
+ ImmutableList.Builder<OrderExpression> newOrderKeys
+ = ImmutableList.builderWithExpectedSize(orderExpressions.size());
+ boolean changed = false;
+ for (OrderExpression orderExpression : orderExpressions) {
+ Expression newOrderKeyExpr = replaceSlot(orderExpression.getOrderKey().getExpr(), false);
+ if (newOrderKeyExpr != orderExpression.getOrderKey().getExpr()) {
+ newOrderKeys.add(new OrderExpression(orderExpression.getOrderKey().withExpression(newOrderKeyExpr)));
+ changed = true;
+ } else {
+ newOrderKeys.add(orderExpression);
+ }
+ }
+ return Pair.of(changed, newOrderKeys.build());
+ }
+
+ private Pair<Boolean, List<OrderKey>> replaceOrderKeys(List<OrderKey> orderKeys) {
+ ImmutableList.Builder<OrderKey> newOrderKeys = ImmutableList.builderWithExpectedSize(orderKeys.size());
+ boolean changed = false;
+ for (OrderKey orderKey : orderKeys) {
+ Expression newOrderKeyExpr = replaceSlot(orderKey.getExpr(), false);
+ if (newOrderKeyExpr != orderKey.getExpr()) {
+ newOrderKeys.add(orderKey.withExpression(newOrderKeyExpr));
+ changed = true;
+ } else {
+ newOrderKeys.add(orderKey);
+ }
+ }
+ return Pair.of(changed, newOrderKeys.build());
+ }
+
+ private <C extends Collection<E>, E extends Expression>
+ Pair<Boolean, List<C>> replaceMultiExpressions(List<C> expressionsList) {
+ ImmutableList.Builder<C> result = ImmutableList.builderWithExpectedSize(expressionsList.size());
+ boolean changed = false;
+ for (C expressions : expressionsList) {
+ Pair<Boolean, C> replaced = replaceExpressions(expressions, false, false);
+ changed |= replaced.first;
+ result.add(replaced.second);
+ }
+ return Pair.of(changed, result.build());
+ }
+
+ private <C extends Collection<E>, E extends Expression> Pair<Boolean, C> replaceExpressions(
+ C expressions, boolean propagateType, boolean fillAccessPaths) {
+ ImmutableCollection.Builder<E> newExprs;
+ if (expressions instanceof List) {
+ newExprs = ImmutableList.builder();
+ } else {
+ newExprs = ImmutableSet.builder();
+ }
+
+ boolean changed = false;
+ for (Expression oldExpr : expressions) {
+ Expression newExpr = replaceSlot(oldExpr, fillAccessPaths);
+ if (newExpr != oldExpr) {
+ newExprs.add((E) newExpr);
+ changed = true;
+
+ if (propagateType && oldExpr instanceof NamedExpression
+ && !oldExpr.getDataType().equals(newExpr.getDataType())) {
+ replacedDataTypes.put(
+ ((NamedExpression) oldExpr).getExprId().asInt(),
+ // not need access path in the upper slots
+ new AccessPathInfo(newExpr.getDataType(), null, null)
+ );
+ }
+ } else {
+ newExprs.add((E) oldExpr);
+ }
+ }
+ return Pair.of(changed, (C) newExprs.build());
+ }
+
+ private Expression replaceSlot(Expression e, boolean fillAccessPath) {
+ return MoreFieldsThread.keepFunctionSignature(false,
+ () -> doRewriteExpression(e, fillAccessPath)
+ );
+ }
+
+ private Expression doRewriteExpression(Expression e, boolean fillAccessPath) {
+ if (e instanceof Lambda) {
+ return rewriteLambda((Lambda) e, fillAccessPath);
+ } else if (e instanceof Cast) {
+ return rewriteCast((Cast) e, fillAccessPath);
+ } else if (e instanceof SlotReference) {
+ AccessPathInfo accessPathInfo = replacedDataTypes.get(((SlotReference) e).getExprId().asInt());
+ if (accessPathInfo != null) {
+ SlotReference newSlot = (SlotReference) ((SlotReference) e).withNullableAndDataType(
+ e.nullable(), accessPathInfo.getPrunedType());
+ if (fillAccessPath) {
+ newSlot = newSlot.withAccessPaths(
+ accessPathInfo.getAllAccessPaths(), accessPathInfo.getPredicateAccessPaths()
+ );
+ }
+ return newSlot;
+ }
+ }
+
+ ImmutableList.Builder<Expression> newChildren = ImmutableList.builderWithExpectedSize(e.arity());
+ boolean changed = false;
+ for (Expression child : e.children()) {
+ Expression newChild = doRewriteExpression(child, fillAccessPath);
+ changed |= child != newChild;
+ newChildren.add(newChild);
+ }
+ return changed ? e.withChildren(newChildren.build()) : e;
+ }
+
+ private Expression rewriteLambda(Lambda e, boolean fillAccessPath) {
+ // we should rewrite ArrayItemReference first, then we can replace the ArrayItemSlot int the lambda
+ Expression[] newChildren = new Expression[e.arity()];
+ for (int i = 0; i < e.arity(); i++) {
+ Expression child = e.child(i);
+ if (child instanceof ArrayItemReference) {
+ Expression newRef = child.withChildren(doRewriteExpression(child.child(0), fillAccessPath));
+ replacedDataTypes.put(((ArrayItemReference) child).getExprId().asInt(),
+ new AccessPathInfo(newRef.getDataType(), null, null));
+ newChildren[i] = newRef;
+ } else {
+ newChildren[i] = child;
+ }
+ }
+
+ for (int i = 0; i < newChildren.length; i++) {
+ Expression child = newChildren[i];
+ if (!(child instanceof ArrayItemReference)) {
+ newChildren[i] = doRewriteExpression(child, fillAccessPath);
+ }
+ }
+
+ return e.withChildren(newChildren);
+ }
+
+ private Expression rewriteCast(Cast cast, boolean fillAccessPath) {
+ Expression newChild = doRewriteExpression(cast.child(0), fillAccessPath);
+ if (newChild == cast.child(0)) {
+ return cast;
+ }
+
+ DataType newType = cast.getDataType();
+ if (cast.getDataType() instanceof NestedColumnPrunable
+ && newChild.getDataType() instanceof NestedColumnPrunable) {
+ DataTypeAccessTree originTree = DataTypeAccessTree.of(cast.child().getDataType(), TAccessPathType.DATA);
+ DataTypeAccessTree prunedTree = DataTypeAccessTree.of(newChild.getDataType(), TAccessPathType.DATA);
+ DataTypeAccessTree castTree = DataTypeAccessTree.of(cast.getDataType(), TAccessPathType.DATA);
+ newType = prunedTree.pruneCastType(originTree, castTree);
+ }
+
+ return new Cast(newChild, newType);
+ }
+
+ private List<TColumnAccessPath> replaceIcebergAccessPathToId(
+ List<TColumnAccessPath> originAccessPaths, SlotReference slotReference) {
+ Column column = slotReference.getOriginalColumn().get();
+ List<TColumnAccessPath> replacedAccessPaths = new ArrayList<>();
+ for (TColumnAccessPath accessPath : originAccessPaths) {
+ List<String> icebergColumnAccessPath = new ArrayList<>();
+ if (accessPath.type == TAccessPathType.DATA) {
+ icebergColumnAccessPath.addAll(accessPath.data_access_path.path);
+ replaceIcebergAccessPathToId(
+ icebergColumnAccessPath, 0, slotReference.getDataType(), column
+ );
+ TColumnAccessPath newAccessPath = new TColumnAccessPath(TAccessPathType.DATA);
+ newAccessPath.data_access_path = new TDataAccessPath(icebergColumnAccessPath);
+ replacedAccessPaths.add(newAccessPath);
+ } else {
+ icebergColumnAccessPath.addAll(accessPath.meta_access_path.path);
+ replaceIcebergAccessPathToId(
+ icebergColumnAccessPath, 0, slotReference.getDataType(), column
+ );
+ TColumnAccessPath newAccessPath = new TColumnAccessPath(TAccessPathType.META);
+ newAccessPath.meta_access_path = new TMetaAccessPath(icebergColumnAccessPath);
+ replacedAccessPaths.add(newAccessPath);
+ }
+ }
+ return replacedAccessPaths;
+ }
+
+ private void replaceIcebergAccessPathToId(List<String> originPath, int index, DataType type, Column column) {
+ if (index >= originPath.size()) {
+ return;
+ }
+ if (index == 0) {
+ originPath.set(index, String.valueOf(column.getUniqueId()));
+ replaceIcebergAccessPathToId(originPath, index + 1, type, column);
+ } else {
+ String fieldName = originPath.get(index);
+ if (type instanceof ArrayType) {
+ // skip replace *
+ replaceIcebergAccessPathToId(
+ originPath, index + 1, ((ArrayType) type).getItemType(), column.getChildren().get(0)
+ );
+ } else if (type instanceof MapType) {
+ if (fieldName.equals("*") || fieldName.equals("VALUES")) {
+ replaceIcebergAccessPathToId(
+ originPath, index + 1, ((MapType) type).getValueType(), column.getChildren().get(1)
+ );
+ }
+ } else if (type instanceof StructType) {
+ for (Column child : column.getChildren()) {
+ if (child.getName().equals(fieldName)) {
+ originPath.set(index, String.valueOf(child.getUniqueId()));
+ DataType childType = ((StructType) type).getNameToFields().get(fieldName).getDataType();
+ replaceIcebergAccessPathToId(originPath, index + 1, childType, child);
+ break;
+ }
+ }
+ } else {
+ originPath.set(index, String.valueOf(column.getUniqueId()));
+ }
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
index 1ed8d151270..abb65a283ed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
@@ -22,7 +22,6 @@ import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.properties.OrderKey;
-import org.apache.doris.nereids.rules.rewrite.ColumnPruning.PruneContext;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -87,7 +86,7 @@ import java.util.Set;
* generating the slots for the required sub path on scan, union, and cte consumer.
* Then, it replaces the element_at with the corresponding slot.
*/
-public class VariantSubPathPruning extends DefaultPlanRewriter<PruneContext> implements CustomRewriter {
+public class VariantSubPathPruning implements CustomRewriter {
public static final Logger LOG = LogManager.getLogger(VariantSubPathPruning.class);
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PreferPushDownProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PreferPushDownProject.java
new file mode 100644
index 00000000000..4c9c3bebe5a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PreferPushDownProject.java
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions;
+
+import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
+
+/**
+ * some expressions prefer push down project under the plan,
+ *
+ * e.g. the element_at function prefer push down to a project in under the plan:
+ * Project(element_at(left.column, 'id') as a)
+ * |
+ * Join
+ * / |
+ * left right
+ *
+ * we will push down project through the Join, so we can prune the column which has complex type:
+ *
+ * Join
+ * / \
+ * Project(element_at(left.column, 'id') as a, ...) right
+ * |
+ * left
+ */
+public interface PreferPushDownProject extends ExpressionTrait {
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
index ce2d3dbb3e5..1c77dc669ac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
@@ -24,6 +24,7 @@ import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.thrift.TColumnAccessPath;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -56,6 +57,10 @@ public class SlotReference extends Slot {
// that need return original table and name for view not its original table if u query a view
private final TableIf oneLevelTable;
private final Column oneLevelColumn;
+ private final Optional<List<TColumnAccessPath>> allAccessPaths;
+ private final Optional<List<TColumnAccessPath>> predicateAccessPaths;
+ private final Optional<List<TColumnAccessPath>> displayAllAccessPaths;
+ private final Optional<List<TColumnAccessPath>> displayPredicateAccessPaths;
public SlotReference(String name, DataType dataType) {
this(StatementScopeIdGenerator.newExprId(), name, dataType, true, ImmutableList.of(),
@@ -92,6 +97,15 @@ public class SlotReference extends Slot {
subPath, Optional.empty());
}
+ public SlotReference(ExprId exprId, Supplier<String> name, DataType dataType, boolean nullable,
+ List<String> qualifier, @Nullable TableIf originalTable, @Nullable Column originalColumn,
+ @Nullable TableIf oneLevelTable, Column oneLevelColumn,
+ List<String> subPath, Optional<Pair<Integer, Integer>> indexInSql) {
+ this(exprId, name, dataType, nullable, qualifier, originalTable, originalColumn, oneLevelTable,
+ oneLevelColumn, subPath, indexInSql, Optional.empty(), Optional.empty(),
+ Optional.empty(), Optional.empty());
+ }
+
/**
* Constructor for SlotReference.
*
@@ -106,7 +120,10 @@ public class SlotReference extends Slot {
public SlotReference(ExprId exprId, Supplier<String> name, DataType dataType, boolean nullable,
List<String> qualifier, @Nullable TableIf originalTable, @Nullable Column originalColumn,
@Nullable TableIf oneLevelTable, Column oneLevelColumn,
- List<String> subPath, Optional<Pair<Integer, Integer>> indexInSql) {
+ List<String> subPath, Optional<Pair<Integer, Integer>> indexInSql,
+ Optional<List<TColumnAccessPath>> allAccessPaths, Optional<List<TColumnAccessPath>> predicateAccessPaths,
+ Optional<List<TColumnAccessPath>> displayAllAccessPaths,
+ Optional<List<TColumnAccessPath>> displayPredicateAccessPaths) {
super(indexInSql);
this.exprId = exprId;
this.name = name;
@@ -119,6 +136,10 @@ public class SlotReference extends Slot {
this.oneLevelTable = oneLevelTable;
this.oneLevelColumn = oneLevelColumn;
this.subPath = Objects.requireNonNull(subPath, "subPath can not be null");
+ this.allAccessPaths = allAccessPaths;
+ this.predicateAccessPaths = predicateAccessPaths;
+ this.displayAllAccessPaths = displayAllAccessPaths;
+ this.displayPredicateAccessPaths = displayPredicateAccessPaths;
}
public static SlotReference of(String name, DataType type) {
@@ -342,4 +363,37 @@ public class SlotReference extends Slot {
public boolean hasAutoInc() {
return originalColumn != null ? originalColumn.isAutoInc() : false;
}
+
+ public SlotReference withAccessPaths(
+ List<TColumnAccessPath> allAccessPaths, List<TColumnAccessPath> predicateAccessPaths) {
+ return new SlotReference(exprId, name, dataType, nullable, qualifier,
+ originalTable, originalColumn, oneLevelTable, oneLevelColumn,
+ subPath, indexInSqlString, Optional.of(allAccessPaths), Optional.of(predicateAccessPaths),
+ Optional.of(allAccessPaths), Optional.of(predicateAccessPaths));
+ }
+
+ public SlotReference withAccessPaths(
+ List<TColumnAccessPath> allAccessPaths, List<TColumnAccessPath> predicateAccessPaths,
+ List<TColumnAccessPath> displayAllAccessPaths, List<TColumnAccessPath> displayPredicateAccessPaths) {
+ return new SlotReference(exprId, name, dataType, nullable, qualifier,
+ originalTable, originalColumn, oneLevelTable, oneLevelColumn,
+ subPath, indexInSqlString, Optional.of(allAccessPaths), Optional.of(predicateAccessPaths),
+ Optional.of(displayAllAccessPaths), Optional.of(displayPredicateAccessPaths));
+ }
+
+ public Optional<List<TColumnAccessPath>> getAllAccessPaths() {
+ return allAccessPaths;
+ }
+
+ public Optional<List<TColumnAccessPath>> getPredicateAccessPaths() {
+ return predicateAccessPaths;
+ }
+
+ public Optional<List<TColumnAccessPath>> getDisplayAllAccessPaths() {
+ return displayAllAccessPaths;
+ }
+
+ public Optional<List<TColumnAccessPath>> getDisplayPredicateAccessPaths() {
+ return displayPredicateAccessPaths;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirst.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirst.java
index 8c0babc39ca..5410de371a7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirst.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirst.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import java.util.List;
@@ -51,4 +52,9 @@ public class ArrayFirst extends ElementAt
public List<FunctionSignature> getImplSignature() {
return SIGNATURES;
}
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitArrayFirst(this, context);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLast.java
index e1ed4f4d27b..b9f5650156f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLast.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLast.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import java.util.List;
@@ -51,4 +52,9 @@ public class ArrayLast extends ElementAt
public ElementAt withChildren(List<Expression> children) {
return new ArrayLast(getFunctionParams(children));
}
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitArrayLast(this, context);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
index 5d74f5e1457..1a1a246cea6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ImplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -37,7 +38,7 @@ import java.util.List;
* ScalarFunction 'array_map'.
*/
public class ArrayMap extends ScalarFunction
- implements ImplicitlyCastableSignature, PropagateNullable {
+ implements ImplicitlyCastableSignature, PropagateNullable, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(new FollowToAnyDataType(0)).args(LambdaType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java
index b24d46c98a4..cd2cb974364 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.RewriteWhenAnalyze;
@@ -42,7 +43,8 @@ import java.util.List;
* ScalarFunction 'element_at'. This class is generated by GenerateFunction.
*/
public class ElementAt extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, RewriteWhenAnalyze {
+ implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable,
+ RewriteWhenAnalyze, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(new FollowToAnyDataType(0))
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java
index a929f2dd5d5..e7a054fc864 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.types.LambdaType;
import org.apache.doris.nereids.types.coercion.AnyDataType;
@@ -29,7 +30,7 @@ import java.util.List;
/**
* Interface of HighOrderFunction, provide default FunctionSignature of LambdaType argument
*/
-public interface HighOrderFunction extends ExplicitlyCastableSignature {
+public interface HighOrderFunction extends ExplicitlyCastableSignature, PreferPushDownProject {
@Override
default List<FunctionSignature> getSignatures() {
return new ImmutableList.Builder<FunctionSignature>()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java
index b8ecc8667c7..517bfaa7748 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -36,7 +37,7 @@ import java.util.List;
* ScalarFunction 'map_contains_entry'.
*/
public class MapContainsEntry extends ScalarFunction
- implements TernaryExpression, ExplicitlyCastableSignature {
+ implements TernaryExpression, ExplicitlyCastableSignature, PreferPushDownProject {
/**
* Basic signature, forcing search key/value type to convert to the same type as map's key/value type.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
index 4fc4c32f233..845e84f71b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -36,7 +37,7 @@ import java.util.List;
* ScalarFunction 'map_contains_key'. This class is generated by GenerateFunction.
*/
public class MapContainsKey extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature {
+ implements BinaryExpression, ExplicitlyCastableSignature, PreferPushDownProject {
public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
FunctionSignature.ret(BooleanType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
index a60f6653cde..fd42c42fb65 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -36,7 +37,7 @@ import java.util.List;
* ScalarFunction 'map_contains_value'. This class is generated by GenerateFunction.
*/
public class MapContainsValue extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature {
+ implements BinaryExpression, ExplicitlyCastableSignature, PreferPushDownProject {
public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
FunctionSignature.ret(BooleanType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java
index 4be48638e5e..22a14c31df3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.functions.SearchSignature;
@@ -41,7 +42,7 @@ import java.util.List;
* fields 'key' and 'value'.
*/
public class MapEntries extends ScalarFunction
- implements UnaryExpression, CustomSignature, PropagateNullable {
+ implements UnaryExpression, CustomSignature, PropagateNullable, PreferPushDownProject {
/**
* constructor with 1 argument.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java
index 957dae2844a..a3123d29cb0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -37,7 +38,7 @@ import java.util.List;
* ScalarFunction 'map_keys'.
*/
public class MapKeys extends ScalarFunction
- implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable {
+ implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(ArrayType.of(new FollowToAnyDataType(0))).args(MapType.of(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java
index e7ca7f48dd3..1622b277156 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -37,7 +38,7 @@ import java.util.List;
* ScalarFunction 'map_values'.
*/
public class MapValues extends ScalarFunction
- implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable {
+ implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(ArrayType.of(new FollowToAnyDataType(0))).args(MapType.of(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java
index 85bb424de19..ed29818b688 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral;
@@ -29,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.NullType;
+import org.apache.doris.nereids.types.StructField;
import org.apache.doris.nereids.types.StructType;
import com.google.common.base.Preconditions;
@@ -40,7 +42,7 @@ import java.util.List;
* ScalarFunction 'struct_element'.
*/
public class StructElement extends ScalarFunction
- implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral {
+ implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral, PreferPushDownProject {
/**
* constructor with 0 or more arguments.
@@ -98,10 +100,11 @@ public class StructElement extends ScalarFunction
}
} else if (child(1) instanceof StringLikeLiteral) {
String name = ((StringLikeLiteral) child(1)).getStringValue();
- if (!structArgType.getNameToFields().containsKey(name.toLowerCase())) {
+ StructField field = structArgType.getField(name);
+ if (field == null) {
throw new AnalysisException("the specified field name " + name + " was not found: " + this.toSql());
} else {
- retType = structArgType.getNameToFields().get(name).getDataType();
+ retType = field.getDataType();
}
} else {
throw new AnalysisException("struct_element only allows"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
index 361144d2a5f..6997063d95e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
@@ -54,10 +54,12 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayEnumerat
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExcept;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExists;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFilter;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirst;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirstIndex;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFlatten;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayIntersect;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayJoin;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLast;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLastIndex;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMap;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMatchAll;
@@ -635,6 +637,10 @@ public interface ScalarFunctionVisitor<R, C> {
return visitScalarFunction(arrayFilter, context);
}
+ default R visitArrayFirst(ArrayFirst arrayFirst, C context) {
+ return visitElementAt(arrayFirst, context);
+ }
+
default R visitArrayFirstIndex(ArrayFirstIndex arrayFirstIndex, C context) {
return visitScalarFunction(arrayFirstIndex, context);
}
@@ -647,6 +653,10 @@ public interface ScalarFunctionVisitor<R, C> {
return visitScalarFunction(arrayJoin, context);
}
+ default R visitArrayLast(ArrayLast arrayLast, C context) {
+ return visitElementAt(arrayLast, context);
+ }
+
default R visitArrayLastIndex(ArrayLastIndex arrayLastIndex, C context) {
return visitScalarFunction(arrayLastIndex, context);
}
@@ -751,8 +761,8 @@ public interface ScalarFunctionVisitor<R, C> {
return visitScalarFunction(arrayFlatten, context);
}
- default R visitArrayMap(ArrayMap arraySort, C context) {
- return visitScalarFunction(arraySort, context);
+ default R visitArrayMap(ArrayMap arrayMap, C context) {
+ return visitScalarFunction(arrayMap, context);
}
default R visitArrayMatchAll(ArrayMatchAll arrayMatchAll, C context) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
index 6ae9e5dd32a..860fbee192e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
@@ -51,16 +51,17 @@ public class LogicalFileScan extends LogicalCatalogRelation {
protected final Optional<TableSample> tableSample;
protected final Optional<TableSnapshot> tableSnapshot;
protected final Optional<TableScanParams> scanParams;
+ protected final Optional<List<Slot>> cachedOutputs;
public LogicalFileScan(RelationId id, ExternalTable table, List<String> qualifier,
Collection<Slot> operativeSlots,
Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot,
- Optional<TableScanParams> scanParams) {
+ Optional<TableScanParams> scanParams, Optional<List<Slot>> cachedOutputs) {
this(id, table, qualifier,
table.initSelectedPartitions(MvccUtil.getSnapshotFromContext(table)),
operativeSlots, ImmutableList.of(),
tableSample, tableSnapshot,
- scanParams, Optional.empty(), Optional.empty());
+ scanParams, Optional.empty(), Optional.empty(), cachedOutputs);
}
/**
@@ -71,22 +72,24 @@ public class LogicalFileScan extends LogicalCatalogRelation {
List<NamedExpression> virtualColumns, Optional<TableSample> tableSample,
Optional<TableSnapshot> tableSnapshot, Optional<TableScanParams> scanParams,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
- String tableAlias) {
+ String tableAlias, Optional<List<Slot>> cachedSlots) {
super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, operativeSlots, virtualColumns,
groupExpression, logicalProperties, tableAlias);
this.selectedPartitions = selectedPartitions;
this.tableSample = tableSample;
this.tableSnapshot = tableSnapshot;
this.scanParams = scanParams;
+ this.cachedOutputs = cachedSlots;
}
protected LogicalFileScan(RelationId id, ExternalTable table, List<String> qualifier,
SelectedPartitions selectedPartitions, Collection<Slot> operativeSlots,
List<NamedExpression> virtualColumns, Optional<TableSample> tableSample,
Optional<TableSnapshot> tableSnapshot, Optional<TableScanParams> scanParams,
- Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties) {
+ Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
+ Optional<List<Slot>> cachedOutputs) {
this(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, groupExpression, logicalProperties, "");
+ scanParams, groupExpression, logicalProperties, "", cachedOutputs);
}
public SelectedPartitions getSelectedPartitions() {
@@ -126,7 +129,7 @@ public class LogicalFileScan extends LogicalCatalogRelation {
public LogicalFileScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, groupExpression, Optional.of(getLogicalProperties()), tableAlias);
+ scanParams, groupExpression, Optional.of(getLogicalProperties()), tableAlias, cachedOutputs);
}
@Override
@@ -134,26 +137,26 @@ public class LogicalFileScan extends LogicalCatalogRelation {
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, groupExpression, logicalProperties, tableAlias);
+ scanParams, groupExpression, logicalProperties, tableAlias, cachedOutputs);
}
public LogicalFileScan withSelectedPartitions(SelectedPartitions selectedPartitions) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, Optional.empty(), Optional.of(getLogicalProperties()), tableAlias);
+ scanParams, Optional.empty(), Optional.of(getLogicalProperties()), tableAlias, cachedOutputs);
}
@Override
public LogicalFileScan withRelationId(RelationId relationId) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, Optional.empty(), Optional.empty(), tableAlias);
+ scanParams, Optional.empty(), Optional.empty(), tableAlias, cachedOutputs);
}
public LogicalFileScan withTableAlias(String tableAlias) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, Optional.empty(), Optional.of(getLogicalProperties()), tableAlias);
+ scanParams, Optional.empty(), Optional.of(getLogicalProperties()), tableAlias, cachedOutputs);
}
@Override
@@ -166,6 +169,19 @@ public class LogicalFileScan extends LogicalCatalogRelation {
return super.equals(o) && Objects.equals(selectedPartitions, ((LogicalFileScan) o).selectedPartitions);
}
+ @Override
+ public List<Slot> computeOutput() {
+ if (cachedOutputs.isPresent()) {
+ return cachedOutputs.get();
+ }
+ return super.computeOutput();
+ }
+
+ @Override
+ public List<Slot> computeAsteriskOutput() {
+ return super.computeAsteriskOutput();
+ }
+
/**
* SelectedPartitions contains the selected partitions and the total partition number.
* Mainly for hive table partition pruning.
@@ -223,7 +239,13 @@ public class LogicalFileScan extends LogicalCatalogRelation {
public LogicalFileScan withOperativeSlots(Collection<Slot> operativeSlots) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, groupExpression, Optional.of(getLogicalProperties()));
+ scanParams, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
+ }
+
+ public LogicalFileScan withCachedOutput(List<Slot> cachedOutputs) {
+ return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
+ selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
+ scanParams, groupExpression, Optional.empty(), Optional.of(cachedOutputs));
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
index 9a123a04d2b..134f48309d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
@@ -77,9 +77,10 @@ public class LogicalHudiScan extends LogicalFileScan {
Collection<Slot> operativeSlots,
List<NamedExpression> virtualColumns,
Optional<GroupExpression> groupExpression,
- Optional<LogicalProperties> logicalProperties) {
+ Optional<LogicalProperties> logicalProperties,
+ Optional<List<Slot>> cachedOutputs) {
super(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns,
- tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties);
+ tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties, cachedOutputs);
Objects.requireNonNull(scanParams, "scanParams should not null");
Objects.requireNonNull(incrementalRelation, "incrementalRelation should not null");
this.incrementalRelation = incrementalRelation;
@@ -87,10 +88,11 @@ public class LogicalHudiScan extends LogicalFileScan {
public LogicalHudiScan(RelationId id, ExternalTable table, List<String> qualifier,
Collection<Slot> operativeSlots, Optional<TableScanParams> scanParams,
- Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot) {
+ Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot,
+ Optional<List<Slot>> cachedOutputs) {
this(id, table, qualifier, ((HMSExternalTable) table).initHudiSelectedPartitions(tableSnapshot),
tableSample, tableSnapshot, scanParams, Optional.empty(), operativeSlots, ImmutableList.of(),
- Optional.empty(), Optional.empty());
+ Optional.empty(), Optional.empty(), cachedOutputs);
}
public Optional<TableScanParams> getScanParams() {
@@ -140,7 +142,7 @@ public class LogicalHudiScan extends LogicalFileScan {
public LogicalHudiScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
@Override
@@ -148,20 +150,20 @@ public class LogicalHudiScan extends LogicalFileScan {
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, logicalProperties);
+ operativeSlots, virtualColumns, groupExpression, logicalProperties, cachedOutputs);
}
public LogicalHudiScan withSelectedPartitions(SelectedPartitions selectedPartitions) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
@Override
public LogicalHudiScan withRelationId(RelationId relationId) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
@Override
@@ -173,7 +175,7 @@ public class LogicalHudiScan extends LogicalFileScan {
public LogicalFileScan withOperativeSlots(Collection<Slot> operativeSlots) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
/**
@@ -226,6 +228,6 @@ public class LogicalHudiScan extends LogicalFileScan {
}
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, newIncrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
index 4278c22f123..394e414cc2e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
@@ -851,4 +851,21 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
}
return replaceMap;
}
+
+ /** withPrunedTypeSlots */
+ public LogicalOlapScan withPrunedTypeSlots(List<Slot> outputSlots) {
+ Map<Pair<Long, String>, Slot> replaceSlotMap = new HashMap<>();
+ for (Slot outputSlot : outputSlots) {
+ Pair<Long, String> key = Pair.of(selectedIndexId, outputSlot.getName());
+ replaceSlotMap.put(key, outputSlot);
+ }
+
+ return new LogicalOlapScan(relationId, (Table) table, qualifier,
+ groupExpression, Optional.empty(),
+ selectedPartitionIds, partitionPruned, selectedTabletIds,
+ selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
+ hints, replaceSlotMap, tableSample, directMvScan, colToSubPathsMap,
+ manuallySpecifiedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit,
+ annOrderKeys, annLimit, tableAlias);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java
index a434f81b231..c6d0241f393 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java
@@ -47,20 +47,31 @@ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation,
private final TableValuedFunction function;
private final ImmutableList<String> qualifier;
private final ImmutableList<Slot> operativeSlots;
+ private final Optional<List<Slot>> cachedOutputs;
public LogicalTVFRelation(RelationId id, TableValuedFunction function, ImmutableList<Slot> operativeSlots) {
super(id, PlanType.LOGICAL_TVF_RELATION);
this.operativeSlots = operativeSlots;
this.function = function;
- qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName());
+ this.qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName());
+ this.cachedOutputs = Optional.empty();
}
public LogicalTVFRelation(RelationId id, TableValuedFunction function, ImmutableList<Slot> operativeSlots,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties) {
+ this(id, function, operativeSlots, Optional.empty(), groupExpression, logicalProperties);
+ }
+
+ public LogicalTVFRelation(RelationId id, TableValuedFunction function,
+ ImmutableList<Slot> operativeSlots,
+ Optional<List<Slot>> cachedOutputs,
+ Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties) {
super(id, PlanType.LOGICAL_TVF_RELATION, groupExpression, logicalProperties);
this.operativeSlots = operativeSlots;
this.function = function;
- qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName());
+ this.cachedOutputs = Objects.requireNonNull(cachedOutputs, "cachedOutputs can not be null");
+ this.qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName());
}
@Override
@@ -120,6 +131,9 @@ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation,
@Override
public List<Slot> computeOutput() {
+ if (cachedOutputs.isPresent()) {
+ return cachedOutputs.get();
+ }
IdGenerator<ExprId> exprIdGenerator = StatementScopeIdGenerator.getExprIdGenerator();
return function.getTable().getBaseSchema()
.stream()
@@ -138,4 +152,9 @@ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation,
public TableValuedFunction getFunction() {
return function;
}
+
+ public LogicalTVFRelation withCachedOutputs(List<Slot> replaceSlots) {
+ return new LogicalTVFRelation(relationId, function, Utils.fastToImmutableList(operativeSlots),
+ Optional.of(replaceSlots), Optional.empty(), Optional.empty());
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java
index 9a34ab740d3..6d4ec539ff3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java
@@ -25,7 +25,7 @@ import java.util.Objects;
/**
* Array type in Nereids.
*/
-public class ArrayType extends DataType implements ComplexDataType {
+public class ArrayType extends DataType implements ComplexDataType, NestedColumnPrunable {
public static final ArrayType SYSTEM_DEFAULT = new ArrayType(NullType.INSTANCE, true);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java
index fb8346987f7..176c1db1d0d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java
@@ -27,7 +27,7 @@ import java.util.Objects;
* Struct type in Nereids.
*/
@Developing
-public class MapType extends DataType implements ComplexDataType {
+public class MapType extends DataType implements ComplexDataType, NestedColumnPrunable {
public static final MapType SYSTEM_DEFAULT = new MapType();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/NestedColumnPrunable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/NestedColumnPrunable.java
new file mode 100644
index 00000000000..2d5e464f042
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/NestedColumnPrunable.java
@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.types;
+
+/** NestedColumnPrunable */
+public interface NestedColumnPrunable {
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java
index 88317682c4c..63e94bc369b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java
@@ -19,7 +19,6 @@ package org.apache.doris.nereids.types;
import org.apache.doris.nereids.util.Utils;
-import java.util.Locale;
import java.util.Objects;
/**
@@ -41,7 +40,7 @@ public class StructField {
* @param nullable Indicates if values of this field can be `null` values
*/
public StructField(String name, DataType dataType, boolean nullable, String comment) {
- this.name = Objects.requireNonNull(name, "name should not be null").toLowerCase(Locale.ROOT);
+ this.name = Objects.requireNonNull(name, "name should not be null").toLowerCase();
this.dataType = Objects.requireNonNull(dataType, "dataType should not be null");
this.nullable = nullable;
this.comment = Objects.requireNonNull(comment, "comment should not be null");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java
index ffbff7c61e1..8296ac28281 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java
@@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -35,7 +36,7 @@ import java.util.stream.Collectors;
* Struct type in Nereids.
*/
@Developing
-public class StructType extends DataType implements ComplexDataType {
+public class StructType extends DataType implements ComplexDataType, NestedColumnPrunable {
public static final StructType SYSTEM_DEFAULT = new StructType();
@@ -55,7 +56,8 @@ public class StructType extends DataType implements ComplexDataType {
public StructType(List<StructField> fields) {
this.fields = ImmutableList.copyOf(Objects.requireNonNull(fields, "fields should not be null"));
// field name should be lowercase and check the same or not
- this.nameToFields = new HashMap<>();
+ // ATTN: should use LinkedHashMap to keep order
+ this.nameToFields = new LinkedHashMap<>();
for (StructField field : this.fields) {
String fieldName = field.getName().toLowerCase();
StructField existingField = this.nameToFields.put(fieldName, field);
@@ -73,6 +75,10 @@ public class StructType extends DataType implements ComplexDataType {
return nameToFields;
}
+ public StructField getField(String name) {
+ return nameToFields.get(name.toLowerCase());
+ }
+
@Override
public DataType conversion() {
return new StructType(fields.stream().map(StructField::conversion).collect(Collectors.toList()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index ca3edc13c50..bd7b006f3ea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -1088,6 +1088,9 @@ public class OlapScanNode extends ScanNode {
output.append(prefix).append("rewrittenProjectList: ").append(
getExplainString(rewrittenProjectList)).append("\n");
}
+
+ printNestedColumns(output, prefix);
+
return output.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index 27369e40e18..382782c393a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -518,7 +518,7 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
return expBuilder.toString();
}
- private String getplanNodeExplainString(String prefix, TExplainLevel detailLevel) {
+ private String getPlanNodeExplainString(String prefix, TExplainLevel detailLevel) {
StringBuilder expBuilder = new StringBuilder();
expBuilder.append(getNodeExplainString(prefix, detailLevel));
if (limit != -1) {
@@ -533,7 +533,7 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
}
public void getExplainStringMap(TExplainLevel detailLevel, Map<Integer, String> planNodeMap) {
- planNodeMap.put(id.asInt(), getplanNodeExplainString("", detailLevel));
+ planNodeMap.put(id.asInt(), getPlanNodeExplainString("", detailLevel));
for (int i = 0; i < children.size(); ++i) {
children.get(i).getExplainStringMap(detailLevel, planNodeMap);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
index 4a9a1945d9b..465a11d152c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
@@ -48,10 +48,13 @@ import org.apache.doris.datasource.FederationBackendPolicy;
import org.apache.doris.datasource.SplitAssignment;
import org.apache.doris.datasource.SplitGenerator;
import org.apache.doris.datasource.SplitSource;
+import org.apache.doris.datasource.iceberg.source.IcebergScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rpc.RpcException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.system.Backend;
+import org.apache.doris.thrift.TAccessPathType;
+import org.apache.doris.thrift.TColumnAccessPath;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TScanRange;
@@ -66,6 +69,7 @@ import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeRangeSet;
import org.apache.commons.collections.map.CaseInsensitiveMap;
+import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -557,6 +561,111 @@ public abstract class ScanNode extends PlanNode implements SplitGenerator {
.addValue(super.debugString()).toString();
}
+ protected void printNestedColumns(StringBuilder output, String prefix) {
+ boolean printNestedColumnsHeader = true;
+ for (SlotDescriptor slot : getTupleDesc().getSlots()) {
+ String prunedType = null;
+ if (!slot.getType().equals(slot.getColumn().getType())) {
+ prunedType = slot.getType().toString();
+ }
+ String displayAllAccessPathsString = null;
+ if (slot.getDisplayAllAccessPaths() != null
+ && slot.getDisplayAllAccessPaths() != null
+ && !slot.getDisplayAllAccessPaths().isEmpty()) {
+ if (this instanceof IcebergScanNode) {
+ displayAllAccessPathsString = mergeIcebergAccessPathsWithId(
+ slot.getAllAccessPaths(),
+ slot.getDisplayAllAccessPaths()
+ );
+ } else {
+ displayAllAccessPathsString = slot.getDisplayAllAccessPaths()
+ .stream()
+ .map(a -> {
+ if (a.type == TAccessPathType.DATA) {
+ return StringUtils.join(a.data_access_path.path, ".");
+ } else {
+ return StringUtils.join(a.meta_access_path.path, ".");
+ }
+ })
+ .collect(Collectors.joining(", "));
+ }
+ }
+ String displayPredicateAccessPathsString = null;
+ if (slot.getDisplayPredicateAccessPaths() != null
+ && slot.getDisplayPredicateAccessPaths() != null
+ && !slot.getDisplayPredicateAccessPaths().isEmpty()) {
+ if (this instanceof IcebergScanNode) {
+ displayPredicateAccessPathsString = mergeIcebergAccessPathsWithId(
+ slot.getPredicateAccessPaths(),
+ slot.getDisplayPredicateAccessPaths()
+ );
+ } else {
+ displayPredicateAccessPathsString = slot.getPredicateAccessPaths()
+ .stream()
+ .map(a -> {
+ if (a.type == TAccessPathType.DATA) {
+ return StringUtils.join(a.data_access_path.path, ".");
+ } else {
+ return StringUtils.join(a.meta_access_path.path, ".");
+ }
+ })
+ .collect(Collectors.joining(", "));
+ }
+ }
+
+
+ if (prunedType == null
+ && displayAllAccessPathsString == null
+ && displayPredicateAccessPathsString == null) {
+ continue;
+ }
+
+ if (printNestedColumnsHeader) {
+ output.append(prefix).append("nested columns:\n");
+ printNestedColumnsHeader = false;
+ }
+ output.append(prefix).append(" ").append(slot.getColumn().getName()).append(":\n");
+ output.append(prefix).append(" origin type: ").append(slot.getColumn().getType()).append("\n");
+ if (prunedType != null) {
+ output.append(prefix).append(" pruned type: ").append(prunedType).append("\n");
+ }
+ if (displayAllAccessPathsString != null) {
+ output.append(prefix).append(" all access paths: [")
+ .append(displayAllAccessPathsString).append("]\n");
+ }
+ if (displayPredicateAccessPathsString != null) {
+ output.append(prefix).append(" predicate access paths: [")
+ .append(displayPredicateAccessPathsString).append("]\n");
+ }
+ }
+ }
+
+ private String mergeIcebergAccessPathsWithId(
+ List<TColumnAccessPath> accessPaths, List<TColumnAccessPath> displayAccessPaths) {
+ List<String> mergeDisplayAccessPaths = Lists.newArrayList();
+ for (int i = 0; i < displayAccessPaths.size(); i++) {
+ TColumnAccessPath displayAccessPath = displayAccessPaths.get(i);
+ TColumnAccessPath idAccessPath = accessPaths.get(i);
+ List<String> nameAccessPathStrings = displayAccessPath.type == TAccessPathType.DATA
+ ? displayAccessPath.data_access_path.path : displayAccessPath.meta_access_path.path;
+ List<String> idAccessPathStrings = idAccessPath.type == TAccessPathType.DATA
+ ? idAccessPath.data_access_path.path : idAccessPath.meta_access_path.path;
+
+ List<String> mergedPath = new ArrayList<>();
+ for (int j = 0; j < idAccessPathStrings.size(); j++) {
+ String name = nameAccessPathStrings.get(j);
+ String id = idAccessPathStrings.get(j);
+ if (name.equals(id)) {
+ mergedPath.add(name);
+ } else {
+ mergedPath.add(name + "(" + id + ")");
+ }
+ }
+ mergeDisplayAccessPaths.add(StringUtils.join(mergedPath, "."));
+ }
+ return StringUtils.join(mergeDisplayAccessPaths, ", ");
+ }
+
public List<TupleId> getOutputTupleIds() {
if (outputTupleDesc != null) {
return Lists.newArrayList(outputTupleDesc.getId());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 7582a7b6b6d..d5820dc6a0c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -408,6 +408,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_RUNTIME_FILTER_PARTITION_PRUNE =
"enable_runtime_filter_partition_prune";
+ public static final String ENABLE_PRUNE_NESTED_COLUMN = "enable_prune_nested_column";
+
static final String SESSION_CONTEXT = "session_context";
public static final String DEFAULT_ORDER_BY_LIMIT = "default_order_by_limit";
@@ -1545,6 +1547,13 @@ public class SessionVariable implements Serializable, Writable {
varType = VariableAnnotation.EXPERIMENTAL)
public int topNLazyMaterializationThreshold = 1024;
+ @VariableMgr.VarAttr(name = ENABLE_PRUNE_NESTED_COLUMN, needForward = true,
+ fuzzy = false,
+ varType = VariableAnnotation.EXPERIMENTAL,
+ description = {"是否裁剪map/struct类型", "Whether to prune the type of map/struct"}
+ )
+ public boolean enablePruneNestedColumns = true;
+
public boolean enableTopnLazyMaterialization() {
return ConnectContext.get() != null
&& ConnectContext.get().getSessionVariable().topNLazyMaterializationThreshold > 0;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
new file mode 100644
index 00000000000..8e8d7c44d16
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -0,0 +1,625 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.analysis.SlotDescriptor;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.Pair;
+import org.apache.doris.common.Triple;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.util.MemoPatternMatchSupported;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.planner.OlapScanNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.thrift.TAccessPathType;
+import org.apache.doris.thrift.TColumnAccessPath;
+import org.apache.doris.thrift.TDataAccessPath;
+import org.apache.doris.thrift.TMetaAccessPath;
+import org.apache.doris.utframe.TestWithFeService;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeSet;
+import java.util.function.Consumer;
+
+public class PruneNestedColumnTest extends TestWithFeService implements MemoPatternMatchSupported {
+ @BeforeAll
+ public void createTable() throws Exception {
+ createDatabase("test");
+ useDatabase("test");
+
+ createTable("create table tbl(\n"
+ + " id int,\n"
+ + " s struct<\n"
+ + " city: string,\n"
+ + " data: array<map<\n"
+ + " int,\n"
+ + " struct<a: int, b: double>\n"
+ + " >>\n"
+ + ">)\n"
+ + "properties ('replication_num'='1')");
+
+ createTable("create table tbl2(\n"
+ + " id2 int,\n"
+ + " s2 struct<\n"
+ + " city2: string,\n"
+ + " data2: array<map<\n"
+ + " int,\n"
+ + " struct<a2: int, b2: double>\n"
+ + " >>\n"
+ + ">)\n"
+ + "properties ('replication_num'='1')");
+
+ connectContext.getSessionVariable().setDisableNereidsRules(RuleType.PRUNE_EMPTY_PARTITION.name());
+ connectContext.getSessionVariable().enableNereidsTimeout = false;
+ }
+
+ @Test
+ public void testCaseInsensitive() throws Exception {
+ assertColumn("select struct_element(MAP_VALUES(struct_element(S, 'DATA')[1])[1], 'B') from tbl",
+ "struct<data:array<map<int,struct<b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "b")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testMap() throws Exception {
+ assertColumn("select MAP_KEYS(struct_element(s, 'data')[0])[1] from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "KEYS")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select MAP_VALUES(struct_element(s, 'data')[0])[1] from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testStruct() throws Throwable {
+ assertColumn("select struct_element(s, 1) from tbl",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select struct_element(map_values(struct_element(s, 'data')[0])[0], 1) from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testPruneCast() throws Exception {
+ assertColumn("select struct_element(cast(s as struct<k:text,l:array<map<int,struct<x:int,y:double>>>>), 'k') from tbl",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select struct_element(map_values(struct_element(cast(s as struct<k:text,l:array<map<int,struct<x:double,y:double>>>>), 'l')[0])[0], 'x') from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of()
+ );
+
+ assertColumns("select struct_element(s, 'city') from (select * from tbl union all select * from tbl2)t",
+ ImmutableList.of(
+ Triple.of(
+ "struct<city2:text>",
+ ImmutableList.of(path("s2", "city2")),
+ ImmutableList.of()
+ ),
+ Triple.of(
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ )
+ )
+ );
+
+ assertColumns("select struct_element(s, 'city'), struct_element(map_values(struct_element(s, 'data')[0])[0], 'b') from (select * from tbl union all select * from tbl2)t",
+ ImmutableList.of(
+ Triple.of(
+ "struct<city2:text,data2:array<map<int,struct<b2:double>>>>",
+ ImmutableList.of(path("s2", "city2"), path("s2", "data2", "*", "VALUES", "b2")),
+ ImmutableList.of()
+ ),
+ Triple.of(
+ "struct<city:text,data:array<map<int,struct<b:double>>>>",
+ ImmutableList.of(path("s", "city"), path("s", "data", "*", "VALUES", "b")),
+ ImmutableList.of()
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testPruneArrayLambda() throws Exception {
+ // map_values(struct_element(s, 'data').*)[0].a
+ assertColumn("select struct_element(array_map(x -> map_values(x)[0], struct_element(s, 'data'))[0], 'a') from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select array_map((x, y) -> struct_element(map_values(x)[0], 'a') + struct_element(map_values(y)[0], 'b'), struct_element(s, 'data'), struct_element(s, 'data')) from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a"), path("s", "data", "*", "VALUES", "b")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testProject() throws Exception {
+ assertColumn("select 100 from tbl", null, null, null);
+ assertColumn("select * from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select tbl.* from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select test.tbl.* from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select internal.test.tbl.* from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select s from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'city'), s from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'city') from tbl",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'data') from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'data')[1] from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*")),
+ ImmutableList.of()
+ );
+ assertColumn("select map_keys(struct_element(s, 'data')[1]) from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "KEYS")),
+ ImmutableList.of()
+ );
+ assertColumn("select map_values(struct_element(s, 'data')[1]) from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(map_values(struct_element(s, 'data')[1])[1], 'a') from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'data')[1][1] from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "*")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(struct_element(s, 'data')[1][1], 'a') from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "*", "a")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(struct_element(s, 'data')[1][1], 'b') from tbl",
+ "struct<data:array<map<int,struct<b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "*", "b")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testFilter() throws Throwable {
+ assertColumn("select 100 from tbl where s is not null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of(path("s"))
+ );
+
+ assertColumn("select 100 from tbl where if(id = 1, null, s) is not null and struct_element(s, 'city') = 'beijing'",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of(path("s"))
+ );
+
+ assertColumn("select 100 from tbl where struct_element(s, 'city') is not null",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of(path("s", "city"))
+ );
+
+ assertColumn("select 100 from tbl where struct_element(s, 'data') is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data")),
+ ImmutableList.of(path("s", "data"))
+ );
+ assertColumn("select 100 from tbl where struct_element(s, 'data')[1] is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*")),
+ ImmutableList.of(path("s", "data", "*"))
+ );
+ assertColumn("select 100 from tbl where map_keys(struct_element(s, 'data')[1]) is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "KEYS")),
+ ImmutableList.of(path("s", "data", "*", "KEYS"))
+ );
+ assertColumn("select 100 from tbl where map_values(struct_element(s, 'data')[1]) is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES")),
+ ImmutableList.of(path("s", "data", "*", "VALUES"))
+ );
+ assertColumn("select 100 from tbl where struct_element(map_values(struct_element(s, 'data')[1])[1], 'a') is not null",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a"))
+ );
+ assertColumn("select 100 from tbl where struct_element(s, 'data')[1][1] is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "*")),
+ ImmutableList.of(path("s", "data", "*", "*"))
+ );
+ assertColumn("select 100 from tbl where struct_element(struct_element(s, 'data')[1][1], 'a') is not null",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "*", "a")),
+ ImmutableList.of(path("s", "data", "*", "*", "a"))
+ );
+ assertColumn("select 100 from tbl where struct_element(struct_element(s, 'data')[1][1], 'b') is not null",
+ "struct<data:array<map<int,struct<b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "*", "b")),
+ ImmutableList.of(path("s", "data", "*", "*", "b"))
+ );
+ }
+
+ @Test
+ public void testProjectFilter() throws Throwable {
+ assertColumn("select s from tbl where struct_element(s, 'city') is not null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of(path("s", "city"))
+ );
+
+ assertColumn("select struct_element(s, 'data') from tbl where struct_element(s, 'city') is not null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data"), path("s", "city")),
+ ImmutableList.of(path("s", "city"))
+ );
+
+ assertColumn("select struct_element(s, 'data') from tbl where struct_element(s, 'city') is not null and struct_element(s, 'data') is not null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data"), path("s", "city")),
+ ImmutableList.of(path("s", "data"), path("s", "city"))
+ );
+ }
+
+ @Test
+ public void testCte() throws Throwable {
+ assertColumn("with t as (select id, s from tbl) select struct_element(t1.s, 'city') from t t1 join t t2 on t1.id = t2.id",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("with t as (select id, struct_element(s, 'city') as c from tbl) select t1.c from t t1 join t t2 on t1.id = t2.id",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testUnion() throws Throwable {
+ assertColumn("select struct_element(s, 'city') from (select s from tbl union all select null)a",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select * from (select struct_element(s, 'city') from tbl union all select null)a",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testCteAndUnion() throws Throwable {
+ assertColumn("with t as (select id, s from tbl) select struct_element(s, 'city') from (select * from t union all select 1, null) tmp",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("with t as (select id, s from tbl) select * from (select struct_element(s, 'city') from t union all select null) tmp",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testPushDownThroughJoin() {
+ PlanChecker.from(connectContext)
+ .analyze("select struct_element(s, 'city') from (select * from tbl)a join (select 100 id, 'f1' name)b on a.id=b.id")
+ .rewrite()
+ .matches(
+ logicalResultSink(
+ logicalProject(
+ logicalJoin(
+ logicalProject(
+ logicalFilter(
+ logicalOlapScan()
+ )
+ ).when(p -> {
+ Assertions.assertEquals(2, p.getProjects().size());
+ Assertions.assertTrue(p.getProjects().stream()
+ .anyMatch(o -> o instanceof Alias && o.child(0) instanceof StructElement));
+ return true;
+ }),
+ logicalOneRowRelation()
+ )
+ ).when(p -> {
+ Assertions.assertTrue(p.getProjects().size() == 1 && p.getProjects().get(0) instanceof SlotReference);
+ return true;
+ })
+ )
+ );
+ }
+
+ @Test
+ public void testPushDownThroughWindow() {
+ PlanChecker.from(connectContext)
+ .analyze("select struct_element(s, 'city'), r from (select s, rank() over(partition by id) r from tbl t)a")
+ .rewrite()
+ .matches(
+ logicalResultSink(
+ logicalProject(
+ logicalWindow(
+ logicalProject(
+ logicalOlapScan()
+ ).when(p -> {
+ Assertions.assertEquals(2, p.getProjects().size());
+ Assertions.assertTrue(p.getProjects().stream()
+ .anyMatch(o -> o instanceof Alias && o.child(0) instanceof StructElement));
+ return true;
+ })
+ )
+ ).when(p -> {
+ Assertions.assertTrue(p.getProjects().size() == 2 && p.getProjects().get(0) instanceof SlotReference);
+ return true;
+ })
+ )
+ );
+ }
+
+ @Test
+ public void testPushDownThroughPartitionTopN() {
+ PlanChecker.from(connectContext)
+ .analyze("select struct_element(s, 'city'), r from (select s, rank() over(partition by id) r from tbl t limit 10)a")
+ .rewrite()
+ .matches(
+ logicalResultSink(
+ logicalLimit(
+ logicalLimit(
+ logicalProject(
+ logicalWindow(
+ logicalPartitionTopN(
+ logicalProject(
+ logicalOlapScan()
+ ).when(p -> {
+ Assertions.assertEquals(2, p.getProjects().size());
+ Assertions.assertTrue(p.getProjects().stream()
+ .anyMatch(o -> o instanceof Alias && o.child(0) instanceof StructElement));
+ return true;
+ })
+ )
+ )
+ ).when(p -> {
+ Assertions.assertTrue(p.getProjects().size() == 2 && p.getProjects().get(0) instanceof SlotReference);
+ return true;
+ })
+ )
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testPushDownThroughUnion() {
+ PlanChecker.from(connectContext)
+ .analyze("select struct_element(s, 'city') from (select id, s from tbl union all select 1, null) tmp")
+ .rewrite()
+ .matches(
+ logicalResultSink(
+ logicalUnion(
+ logicalProject(
+ logicalOlapScan()
+ ).when(p -> {
+ Assertions.assertEquals(1, p.getProjects().size());
+ Assertions.assertInstanceOf(StructElement.class, p.getProjects().get(0).child(0));
+ return true;
+ })
+ ).when(u -> {
+ Assertions.assertEquals(1, u.getConstantExprsList().size());
+ Assertions.assertInstanceOf(NullLiteral.class, u.getConstantExprsList().get(0).get(0).child(0));
+ return true;
+ })
+ )
+ );
+ }
+
+ private void assertColumn(String sql, String expectType,
+ List<TColumnAccessPath> expectAllAccessPaths,
+ List<TColumnAccessPath> expectPredicateAccessPaths) throws Exception {
+ assertColumns(sql, expectType == null ? null : ImmutableList.of(Triple.of(expectType, expectAllAccessPaths, expectPredicateAccessPaths)));
+ }
+
+ private void assertColumns(String sql,
+ List<Triple<String, List<TColumnAccessPath>, List<TColumnAccessPath>>> expectResults) throws Exception {
+ Pair<PhysicalPlan, List<SlotDescriptor>> result = collectComplexSlots(sql);
+ PhysicalPlan physicalPlan = result.first;
+ List<SlotDescriptor> slotDescriptors = result.second;
+ if (expectResults == null) {
+ Assertions.assertEquals(0, slotDescriptors.size());
+ return;
+ }
+
+ Assertions.assertEquals(expectResults.size(), slotDescriptors.size());
+ int slotIndex = 0;
+ for (Triple<String, List<TColumnAccessPath>, List<TColumnAccessPath>> expectResult : expectResults) {
+ String expectType = expectResult.left;
+ List<TColumnAccessPath> expectAllAccessPaths = expectResult.middle;
+ List<TColumnAccessPath> expectPredicateAccessPaths = expectResult.right;
+ SlotDescriptor slotDescriptor = slotDescriptors.get(slotIndex++);
+ Assertions.assertEquals(expectType, slotDescriptor.getType().toString());
+
+ TreeSet<TColumnAccessPath> expectAllAccessPathSet = new TreeSet<>(expectAllAccessPaths);
+ TreeSet<TColumnAccessPath> actualAllAccessPaths
+ = new TreeSet<>(slotDescriptor.getAllAccessPaths());
+ Assertions.assertEquals(expectAllAccessPathSet, actualAllAccessPaths);
+
+ TreeSet<TColumnAccessPath> expectPredicateAccessPathSet = new TreeSet<>(expectPredicateAccessPaths);
+ TreeSet<TColumnAccessPath> actualPredicateAccessPaths
+ = new TreeSet<>(slotDescriptor.getPredicateAccessPaths());
+ Assertions.assertEquals(expectPredicateAccessPathSet, actualPredicateAccessPaths);
+
+ Map<Integer, DataType> slotIdToDataTypes = new LinkedHashMap<>();
+ Consumer<Expression> assertHasSameType = e -> {
+ if (e instanceof NamedExpression) {
+ DataType dataType = slotIdToDataTypes.get(((NamedExpression) e).getExprId().asInt());
+ if (dataType != null) {
+ Assertions.assertEquals(dataType, e.getDataType());
+ } else {
+ slotIdToDataTypes.put(((NamedExpression) e).getExprId().asInt(), e.getDataType());
+ }
+ }
+ };
+
+ // assert same slot id has same type
+ physicalPlan.foreachUp(plan -> {
+ List<? extends Expression> expressions = ((PhysicalPlan) plan).getExpressions();
+ for (Expression expression : expressions) {
+ expression.foreach(e -> {
+ assertHasSameType.accept((Expression) e);
+ if (e instanceof Alias && e.child(0) instanceof Slot) {
+ assertHasSameType.accept((Alias) e);
+ } else if (e instanceof ArrayItemReference) {
+ assertHasSameType.accept((ArrayItemReference) e);
+ }
+ });
+ }
+
+ if (plan instanceof PhysicalCTEConsumer) {
+ for (Entry<Slot, Collection<Slot>> kv : ((PhysicalCTEConsumer) plan).getProducerToConsumerSlotMap()
+ .asMap().entrySet()) {
+ Slot producerSlot = kv.getKey();
+ for (Slot consumerSlot : kv.getValue()) {
+ Assertions.assertEquals(producerSlot.getDataType(), consumerSlot.getDataType());
+ }
+ }
+ } else if (plan instanceof PhysicalUnion) {
+ List<Slot> output = ((PhysicalUnion) plan).getOutput();
+ for (List<SlotReference> regularChildrenOutput : ((PhysicalUnion) plan).getRegularChildrenOutputs()) {
+ Assertions.assertEquals(output.size(), regularChildrenOutput.size());
+ for (int i = 0; i < output.size(); i++) {
+ Assertions.assertEquals(output.get(i).getDataType(), regularChildrenOutput.get(i).getDataType());
+ }
+ }
+ }
+ });
+ }
+ }
+
+ private Pair<PhysicalPlan, List<SlotDescriptor>> collectComplexSlots(String sql) throws Exception {
+ NereidsPlanner planner = (NereidsPlanner) executeNereidsSql(sql).planner();
+ List<SlotDescriptor> complexSlots = new ArrayList<>();
+ PhysicalPlan physicalPlan = planner.getPhysicalPlan();
+ for (PlanFragment fragment : planner.getFragments()) {
+ List<OlapScanNode> olapScanNodes = fragment.getPlanRoot().collectInCurrentFragment(OlapScanNode.class::isInstance);
+ for (OlapScanNode olapScanNode : olapScanNodes) {
+ List<SlotDescriptor> slots = olapScanNode.getTupleDesc().getSlots();
+ for (SlotDescriptor slot : slots) {
+ Type type = slot.getType();
+ if (type.isComplexType() || type.isVariantType()) {
+ complexSlots.add(slot);
+ }
+ }
+ }
+ }
+ return Pair.of(physicalPlan, complexSlots);
+ }
+
+ private TColumnAccessPath path(String... path) {
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.DATA);
+ accessPath.data_access_path = new TDataAccessPath(ImmutableList.copyOf(path));
+ return accessPath;
+ }
+
+ private TColumnAccessPath metaPath(String... path) {
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.META);
+ accessPath.meta_access_path = new TMetaAccessPath(ImmutableList.copyOf(path));
+ return accessPath;
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
index 21c45c7135c..55d8e70d50d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
@@ -559,8 +559,8 @@ public abstract class TestWithFeService {
&& connectContext.getState().getErrorCode() == null) {
return stmtExecutor;
} else {
- // throw new IllegalStateException(connectContext.getState().getErrorMessage());
- return null;
+ throw new IllegalStateException(connectContext.getState().getErrorMessage());
+ // return null;
}
}
@@ -574,7 +574,7 @@ public abstract class TestWithFeService {
}
}
- public void executeNereidsSql(String queryStr) throws Exception {
+ public StmtExecutor executeNereidsSql(String queryStr) throws Exception {
connectContext.getState().reset();
StatementContext statementContext = new StatementContext(connectContext, new OriginStatement(queryStr, 0));
@@ -590,6 +590,7 @@ public abstract class TestWithFeService {
|| connectContext.getState().getErrorCode() != null) {
throw new IllegalStateException(connectContext.getState().getErrorMessage());
}
+ return stmtExecutor;
}
public void createDatabase(String db) throws Exception {
diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift
index f7c1e32daad..35f276c006c 100644
--- a/gensrc/thrift/Descriptors.thrift
+++ b/gensrc/thrift/Descriptors.thrift
@@ -27,6 +27,49 @@ enum TPatternType {
MATCH_NAME_GLOB = 2
}
+enum TAccessPathType {
+ DATA = 1,
+ META = 2 // use to prune `where s.data is not null` by only scan the meta of s.data
+}
+
+struct TDataAccessPath {
+ // the specification of special path:
+ // <empty>: access the whole complex column
+ // *:
+ // 1. access every items when the type is array
+ // 2. access key and value when the type is map
+ // KEYS: only access the keys of map
+ // VALUES: only access the keys of map
+ //
+ // example:
+ // s: struct<
+ // data: array<
+ // map<
+ // int,
+ // struct<
+ // a: id
+ // b: double
+ // >
+ // >
+ // >
+ // >
+ // if we want to access `map_keys(s.data[0])`, the path will be: ['s', 'data', '*', 'KEYS'],
+ // if we want to access `map_values(s.data[0])[0].b`, the path will be: ['s', 'data', '*', 'VALUES', 'b'],
+ // if we want to access `s.data[0]['k'].b`, the path will be ['s', 'data', '*', '*', 'b']
+ // if we want to access the whole struct of s, the path will be: ['s'],
+ 1: required list<string> path
+}
+
+struct TMetaAccessPath {
+ 1: required list<string> path
+}
+
+struct TColumnAccessPath {
+ 1: required TAccessPathType type
+ 2: optional TDataAccessPath data_access_path
+ 3: optional TMetaAccessPath meta_access_path
+}
+
struct TColumn {
1: required string column_name
2: required Types.TColumnType column_type
@@ -68,14 +111,17 @@ struct TSlotDescriptor {
11: optional i32 col_unique_id = -1
12: optional bool is_key = false
// If set to false, then such slots will be ignored during
- // materialize them.Used to optmize to read less data and less memory usage
+ // materialize them.Used to optimize to read less data and less memory usage
13: optional bool need_materialize = true
14: optional bool is_auto_increment = false;
// subcolumn path info list for semi structure column(variant)
+ // deprecated: will be replaced to column_access_paths
15: optional list<string> column_paths
16: optional string col_default_value
17: optional Types.TPrimitiveType primitive_type = Types.TPrimitiveType.INVALID_TYPE
18: optional Exprs.TExpr virtual_column_expr
+ 19: optional list<TColumnAccessPath> all_access_paths
+ 20: optional list<TColumnAccessPath> predicate_access_paths
}
struct TTupleDescriptor {
diff --git a/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out b/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out
new file mode 100644
index 00000000000..74f336d27e5
--- /dev/null
+++ b/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out
@@ -0,0 +1,24 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+1 {"city":"beijing", "data":[{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}]}
+2 {"city":"shanghai", "data":[{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}]}
+
+-- !sql1 --
+1 [10]
+
+-- !sql2 --
+1 beijing
+2 shanghai
+
+-- !sql3 --
+1 [{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}]
+2 [{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}]
+
+-- !sql4 --
+1 [{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}]
+2 [{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}]
+
+-- !sql5 --
+1 beijing
+2 shanghai
+
diff --git a/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy b/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy
new file mode 100644
index 00000000000..ad032fab8f2
--- /dev/null
+++ b/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_pruned_columns") {
+ sql """DROP TABLE IF EXISTS `tbl_test_pruned_columns`"""
+ sql """
+ CREATE TABLE `tbl_test_pruned_columns` (
+ `id` int NULL,
+ `s` struct<city:text,data:array<map<int,struct<a:int,b:double>>>> NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY RANDOM BUCKETS AUTO
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """
+ insert into `tbl_test_pruned_columns` values
+ (1, named_struct('city', 'beijing', 'data', array(map(1, named_struct('a', 10, 'b', 20.0), 2, named_struct('a', 30, 'b', 40))))),
+ (2, named_struct('city', 'shanghai', 'data', array(map(2, named_struct('a', 50, 'b', 40.0), 1, named_struct('a', 70, 'b', 80)))));
+ """
+
+ qt_sql """
+ select * from `tbl_test_pruned_columns` order by 1;
+ """
+
+ qt_sql1 """
+ select b.id, array_map(x -> struct_element(map_values(x)[1], 'a'), struct_element(s, 'data')) from `tbl_test_pruned_columns` t join (select 1 id) b on t.id = b.id order by 1;
+ """
+
+ qt_sql2 """
+ select id, struct_element(s, 'city') from `tbl_test_pruned_columns` order by 1;
+ """
+
+ qt_sql3 """
+ select id, struct_element(s, 'data') from `tbl_test_pruned_columns` order by 1;
+ """
+
+ qt_sql4 """
+ select id, struct_element(s, 'data') from `tbl_test_pruned_columns` where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 order by 1;
+ """
+
+ qt_sql5 """
+ select id, struct_element(s, 'city') from `tbl_test_pruned_columns` where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 order by 1;
+ """
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org