You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by hu...@apache.org on 2025/11/06 03:25:13 UTC
(doris) branch nested_column_prune updated (4da3edd0274 -> 5119904664c)
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a change to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
discard 4da3edd0274 remove debug log
discard 5bbdab65a13 fix: missed pruned type with map/array types
discard 46b503aee5f Clean external table implementation's code.
omit 9628af43edb use selected column ids for parquet merge io ranges.
omit a8f60b2ea8e update orc thirdparty lib commit.
omit de04e5b102d should not prune project nested column if the bottom table can not prune, e.g. hive(textfile)
omit 490886b37b2 fix npe
omit 462d4657c7b Fix some tests's bugs of external table.
omit e63fed2f49c only support prune olapTable/iceberg/hive(parquet/orc)
omit e807410f7f2 fix be ut
omit 0af94c7abc6 support push project in aggregate/join
omit 2151bcdcdf0 fix topN late materialzation access_path.
omit 78a46f59acc fix case error in BE
omit 3943ee61664 fix "Index 2 out of bounds for length 2" when use tvf
omit e038fb267de use slot->col_name_lower_case().
omit 392aafdd1a8 Fix parquet_thrift_test unit test.
omit ae92dbce841 [Feature](catalog) Prune nested columns for hive/iceberg parquet and orc reader.
omit 6757c8db8c2 support prune nested columns
add 6fe6656d955 [enhancement](parquet)Optimize the performance of parquet reader when decode RLE_DICTIONARY encoding (#57208)
add e4c029b5227 [opt](nereids)push down assert one row join (#57414)
add ad6c7f3bb27 [fix](nereids)varchar(-1) and varchar(65533) should be treated as the same data type (#57176)
add 13c0449d2e1 [fix](maxcompute)fix maxcompute push down predicates null pointer. (#57474)
add 0a1905fdaa4 [Opt](function) Optimize the performance of FROM_UNIXTIME (#57423)
add 3912e523fb0 [enhancemment](planner)check sql regex block rule before plan phase (#57477)
add 3eb94a14e4e [Enhancement](schema_table)Improve the field information of the information_schema.table_options table. (#56122)
add 33b2efa86a8 [refactor](expr) remove castTo in Catalog's Expr (#57560)
add 1e77d345e5b [fix] (parquet-reader) Fix parquet all row groups are filtered. (#57490)
add eacaa4b5926 [enhancement](s3)Add TotalGetRequestTime profile metric to the S3 reader. (#57519)
add 7b9948ddeee [chore](ci) fix bad resource (#57585)
add bf0b87c1cb7 [fix](scanner) Make Scanner close() method thread-safe using atomic operations (#57436)
add 4187476d77a [chore](third party) bump librdkafka from 1.9.2 to 2.11.0 for Kafka higher version compatibility (#57565)
add 748e8679956 [function](agg) support agg function of group_array_union (#57166)
add f3ac7db1ef7 [Bug](Jni)should delete local ref after convert to global ref (#57513)
add c757cb88053 [fix](mysql) Fix SSL unwrap infinite loop on handshake failure (#57364)
add 11eabf02b8a [opt](inverted index) support builtin analyzer names in custom analyzer field (#57512)
add 2742eacad75 [fix](schema-change) Prevent coredump when reading non-overlapping segments from a single rowset during heavy schema change (#57191)
add 77380ff1e2d [fix](cloud)Fix read from peer use thread pool not asyncio (#57587)
add f11570b9614 [Opt](storage) Add pre decoder for BinaryPlainPageV2 and BinaryDictPage (#57327)
add 781f428a276 [chore](dep)upgrade netty and hadoop (#57226)
add f9411472943 [improve](glue) Glue endpoint is required. (#57543)
add 0c2597997e1 [fix](docker) fix iceberg docker error: unable to prepare context: path "spark/" not found (#57417)
add f2e75cbb156 [improve](job) Adding FailureReason after manually changing the status of a streaming job (#57551)
add 4cef3b1223c [fix](mtmv) Fix mv rewrite fail when mv contains group sets and filter above scan (#57343)
add 192225bcedd [chore](cast) Make the result of casting a struct to string consistent with its output to MySQL. (#57451)
add 56fdf39c65e [fix](cloud) Avoid recycle orphan partitions if data migrating is not finished (#57550)
add 0d304f1738c [feat](cloud) Ignore the staled clusters of the rollbacked instance (#57478)
add 91465afaf8e [fix](cloud) Remove global snapshot data migrating context (#57553)
add dc3bb256021 [fix](cloud) Recycle operation logs if multi version status is WRITE_ONLY (#57552)
add fdef53596b2 [feature](partition prune) support remove predicates that are always true after partition prune in list partition (#57169)
add 909cdfde989 [fix](variant) fix deserialize array from sparse column (#57561)
add e676738e171 [feat](hdfs) upgrade libhdfs to 3.4.2 (#57621)
add 022dce9d9bd [chore](dep)Exclude AWS SDK bundle to reduce dependency size and include only required S3 modules (#57629)
add 312be21f205 [test](inverted index) add test for query type and add test for searcher (#57603)
add ef4993b0336 [fix](faiss) fix faiss compile on noavx2 enviroment (#57642)
add f5adb679e7f [chore](cloud) Add thread name for snapshot data migrator and compactor (#57627)
add f6b9a7d7be4 [fix](cloud) Avoid to enable snapshot switch when multi version is disabled (#57626)
add a82699ba087 [fix](auto-partition) Replace std::mutex with bthread::Mutex in VTabletWriter (#57508)
add 194cbe36dd7 [enhancement](load)add LogicalPostProject to cast outputs according to dest table's schema (#57579)
add 177061201e2 Revert "[fix](varbinary) Rename StringView to StringContainer" (#57656)
add 32c91fd17a2 [fix](insert) fix insert job duplicate registration (#57572)
add 43cea886f3e [Improvement](Variant) hash-shard sparse subcolumns via `variant_sparse_hash_shard_count` (#56502)
add efbfa950d9a [fix](cloud) fix wrong private endpoint when only configured port number (#57429)
add 576d0e6101f [chore](ci) cloud_p0 add regression conf (#57677)
add 3bf4db3a0f7 [chore](UT) Add --gdb option to run-be-ut.sh and run-cloud-ut.sh (#57655)
add 25bfafc119a [opt](coercion) let matchType do right thing (#57586)
add 75c8291e5c4 [chore](UT) Add UT for SegmentWriter::_full_encode_keys (#57653)
add a22f24a85c4 [feat](function) Add new function of strip_null_value (#57386)
add eb771a485fe [fix](spill) Memory leak in the arena used by the aggregation operator (#57657)
add 78a0d78016f [fix](function) str_to_date and from_iso8601_date return null instead of raise error (#57669)
add aef112c95bd [chore](ci) fix typo (#57594)
add 4809cf596d9 [feat](snapshot) clone snapshot support absolute path file (#57685)
add 6f4468bd9d0 [be](comment) add some comment to page zonemap (#57703)
add ae72d5d1919 [Fix](Variant) handle spase column type correctly (#57701)
add daf013ad93f [fix](cloud) Do not recycle operation logs referenced by active snapshots (#57694)
add a9093cea761 [feat](cloud) Write and recycle restore job log (#57405)
add 2dee657a397 [Feature](agg) add sem agg function (#57545)
add 7a3ff665668 [Chore](exchange) reduce create wait_channel_timer (#57683)
add 925b7e70b82 [refactor](variant) remove unused function `same_with_storage_type` (#57624)
add 4dd828ec15c [fix](cosn) fix be jni read on cosn failed (#57241)
add 7631f2b8de5 [feature](partition prune) Add variable skip_prune_predicate to skip prune predicate after partition prune (#57688)
add 94fe6aae389 [feature](inverted index) add custom analyzer support with pinyin tokenzer and pinyin filter (#57097)
add e870f7bbb5e [fix](jdbc catalog) Compatible with different versions of `zeroDateTimeBehavior=convertToNull` writing method (#57682)
add 7a77c085982 [fix](catalog) Clean up refresh thread when deleting catalog (#57680)
add 5285135c126 [Fix](funtion) Fix utc_time result when input null (#57716)
add a77e7c587a3 [fix](function) Align function signatures between backend and frontend and remove datev1 types (#56807)
add 30406d6d827 [enhancement](filecache) use rocksdb to persist cache block meta (#57072)
add a5e4e1c7018 [fix](nereids) Fix not in aggregate's output err after eliminate by uniform when group sets exist (#56942)
add ef81e72aae0 [fix](cloud) Make tablet stats compatible between single/multi version keys (#57690)
add 9b5a2132d3f [ann] Ann index only scan (#57243)
add 9429add71b8 [feat](profile) add automatic profile archiving to compressed ZIP files (#57575)
add da41f40ee90 [fix](tablet report)Replace tablet report with ForkJoinPool (#57382)
add 54720872b13 [fix](cloud) Add more prompt information when compute group is invalid (#56946)
new 6ccc8ca260a support prune nested columns
new 1e5c9614055 [Feature](catalog) Prune nested columns for hive/iceberg parquet and orc reader.
new 28ba969f5b5 Fix parquet_thrift_test unit test.
new 3dfbdea9e40 use slot->col_name_lower_case().
new b7f17ea8820 fix "Index 2 out of bounds for length 2" when use tvf
new 9b00e6fa107 fix case error in BE
new 4000802e5fb fix topN late materialzation access_path.
new bbc61b4cdab support push project in aggregate/join
new cd95b8eed1d fix be ut
new 37beb93061f only support prune olapTable/iceberg/hive(parquet/orc)
new 0279d9c5758 Fix some tests's bugs of external table.
new b2a048deff9 fix npe
new 9efa81907b5 should not prune project nested column if the bottom table can not prune, e.g. hive(textfile)
new 6fdbdbd020d update orc thirdparty lib commit.
new 4f4b423b6ad use selected column ids for parquet merge io ranges.
new 6572e52de83 Clean external table implementation's code.
new 05c28879f0b fix: missed pruned type with map/array types
new 5119904664c remove debug log
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (4da3edd0274)
\
N -- N -- N refs/heads/nested_column_prune (5119904664c)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/comment-to-trigger-teamcity.yml | 2 +-
.licenserc.yaml | 1 +
be/CMakeLists.txt | 7 +-
be/dict/pinyin/pinyin.txt | 20902 +
be/dict/pinyin/pinyin_alphabet.dict | 442 +
be/dict/pinyin/polyphone.txt | 462490 ++++++++++++++++++
be/src/cloud/cloud_internal_service.cpp | 225 +-
be/src/cloud/config.cpp | 2 +-
be/src/common/config.cpp | 10 +-
be/src/common/config.h | 3 +
be/src/common/consts.h | 1 +
.../schema_table_options_scanner.cpp | 16 +-
.../schema_scanner/schema_table_options_scanner.h | 1 +
be/src/exec/tablet_info.cpp | 6 +
be/src/exprs/hybrid_set.h | 53 +-
be/src/io/cache/block_file_cache.cpp | 34 +-
be/src/io/cache/block_file_cache.h | 2 +
be/src/io/cache/block_file_cache_downloader.cpp | 8 +
be/src/io/cache/cache_block_meta_store.cpp | 582 +
be/src/io/cache/cache_block_meta_store.h | 142 +
be/src/io/cache/cached_remote_file_reader.cpp | 3 +
be/src/io/cache/file_block.cpp | 42 +-
be/src/io/cache/file_block.h | 9 +-
be/src/io/cache/file_cache_common.cpp | 84 +-
be/src/io/cache/file_cache_common.h | 11 +-
be/src/io/cache/file_cache_storage.h | 9 +-
be/src/io/cache/fs_file_cache_storage.cpp | 676 +-
be/src/io/cache/fs_file_cache_storage.h | 40 +-
be/src/io/cache/mem_file_cache_storage.cpp | 9 +-
be/src/io/cache/mem_file_cache_storage.h | 8 +-
be/src/io/fs/hdfs.h | 2 +-
be/src/io/fs/hdfs_file_writer.cpp | 3 +-
be/src/io/fs/local_file_system.cpp | 1 +
be/src/io/fs/s3_file_reader.cpp | 4 +
be/src/io/fs/s3_file_reader.h | 1 +
be/src/io/fs/s3_file_writer.cpp | 10 +-
be/src/olap/compaction.cpp | 4 +
.../ann_index/cmake-protect/CMakeLists.txt | 9 +
be/src/olap/rowset/segment_v2/binary_dict_page.h | 26 +-
.../segment_v2/binary_dict_page_pre_decoder.h | 134 +
.../olap/rowset/segment_v2/binary_plain_page_v2.h | 183 +-
.../segment_v2/binary_plain_page_v2_pre_decoder.h | 166 +
.../segment_v2/bitshuffle_page_pre_decoder.h | 53 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 10 +-
be/src/olap/rowset/segment_v2/column_reader.h | 2 +-
be/src/olap/rowset/segment_v2/column_writer.cpp | 4 +-
be/src/olap/rowset/segment_v2/encoding_info.cpp | 12 +-
be/src/olap/rowset/segment_v2/encoding_info.h | 4 +-
.../inverted_index/analysis_factory_mgr.cpp | 6 +
.../inverted_index/analyzer/analyzer.cpp | 141 +-
.../segment_v2/inverted_index/analyzer/analyzer.h | 11 +-
.../inverted_index/token_filter/pinyin_filter.cpp | 441 +
.../inverted_index/token_filter/pinyin_filter.h | 105 +
.../token_filter/pinyin_filter_factory.cpp | 76 +
.../token_filter/pinyin_filter_factory.h} | 37 +-
.../tokenizer/pinyin/chinese_util.cpp | 45 +-
.../tokenizer/pinyin/chinese_util.h} | 20 +-
.../tokenizer/pinyin/pinyin_alphabet_tokenizer.cpp | 224 +
.../tokenizer/pinyin/pinyin_alphabet_tokenizer.h | 49 +
.../tokenizer/pinyin/pinyin_config.h} | 44 +-
.../tokenizer/pinyin/pinyin_format.cpp | 70 +
.../tokenizer/pinyin/pinyin_format.h | 78 +
.../tokenizer/pinyin/pinyin_formatter.cpp | 214 +
.../tokenizer/pinyin/pinyin_formatter.h} | 24 +-
.../tokenizer/pinyin/pinyin_tokenizer.cpp | 339 +
.../tokenizer/pinyin/pinyin_tokenizer.h | 84 +
.../tokenizer/pinyin/pinyin_tokenizer_factory.cpp | 43 +
.../tokenizer/pinyin/pinyin_tokenizer_factory.h} | 25 +-
.../tokenizer/pinyin/pinyin_util.cpp | 292 +
.../inverted_index/tokenizer/pinyin/pinyin_util.h | 55 +
.../inverted_index/tokenizer/pinyin/rune.h} | 29 +-
.../tokenizer/pinyin/smart_forest.cpp | 195 +
.../inverted_index/tokenizer/pinyin/smart_forest.h | 143 +
.../tokenizer/pinyin/smart_get_word.cpp | 219 +
.../tokenizer/pinyin/smart_get_word.h | 95 +
.../inverted_index/tokenizer/pinyin/term_item.h | 73 +
be/src/olap/rowset/segment_v2/page_io.cpp | 23 +-
be/src/olap/rowset/segment_v2/page_io.h | 5 +
be/src/olap/rowset/segment_v2/segment.cpp | 18 +-
be/src/olap/rowset/segment_v2/segment.h | 2 -
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 100 +-
be/src/olap/rowset/segment_v2/segment_iterator.h | 2 +
be/src/olap/rowset/segment_v2/segment_writer.cpp | 5 +-
be/src/olap/rowset/segment_v2/segment_writer.h | 2 +-
.../variant/sparse_column_merge_iterator.cpp | 3 +-
.../segment_v2/variant/variant_column_reader.cpp | 251 +-
.../segment_v2/variant/variant_column_reader.h | 48 +-
.../variant/variant_column_writer_impl.cpp | 278 +-
.../variant/variant_column_writer_impl.h | 72 +-
.../rowset/segment_v2/variant_stats_calculator.cpp | 3 +-
.../rowset/segment_v2/virtual_column_iterator.cpp | 12 +-
be/src/olap/rowset/segment_v2/zone_map_index.cpp | 2 +-
be/src/olap/rowset/segment_v2/zone_map_index.h | 4 +-
be/src/olap/schema_change.cpp | 4 +-
be/src/olap/tablet_meta.cpp | 3 +
be/src/olap/tablet_schema.cpp | 4 +
be/src/olap/tablet_schema.h | 8 +
be/src/pipeline/exec/aggregation_sink_operator.cpp | 1 +
be/src/pipeline/exec/exchange_sink_operator.cpp | 27 +-
be/src/pipeline/exec/olap_scan_operator.cpp | 2 +-
be/src/pipeline/exec/olap_scan_operator.h | 2 +-
.../exec/partitioned_aggregation_sink_operator.cpp | 1 +
be/src/runtime/primitive_type.h | 14 +-
be/src/util/jni-util.cpp | 1 +
.../aggregate_function_group_array_intersect.cpp | 66 -
.../aggregate_function_group_array_intersect.h | 507 -
.../aggregate_function_group_array_set_op.cpp | 158 +
.../aggregate_function_group_array_set_op.h | 480 +
.../aggregate_function_map_v2.cpp | 2 -
.../aggregate_functions/aggregate_function_sem.cpp | 41 +
.../aggregate_functions/aggregate_function_sem.h | 149 +
.../aggregate_function_simple_factory.cpp | 6 +-
.../aggregate_function_window.cpp | 22 -
be/src/vec/columns/column_varbinary.cpp | 6 +-
be/src/vec/columns/column_varbinary.h | 18 +-
be/src/vec/columns/column_variant.cpp | 4 +-
be/src/vec/common/schema_util.cpp | 43 +-
be/src/vec/common/schema_util.h | 6 +
.../{string_container.cpp => string_view.cpp} | 6 +-
.../common/{string_container.h => string_view.h} | 40 +-
be/src/vec/core/field.h | 17 +-
be/src/vec/data_types/data_type_varbinary.cpp | 6 +-
be/src/vec/data_types/data_type_varbinary.h | 7 +-
.../data_types/serde/data_type_struct_serde.cpp | 2 +
be/src/vec/exec/format/parquet/decoder.cpp | 12 +-
.../format/parquet/fix_length_dict_decoder.hpp | 65 +-
be/src/vec/exec/scan/es_scanner.cpp | 2 +-
be/src/vec/exec/scan/file_scanner.cpp | 10 +-
be/src/vec/exec/scan/jdbc_scanner.cpp | 3 +
be/src/vec/exec/scan/meta_scanner.cpp | 3 +
be/src/vec/exec/scan/olap_scanner.cpp | 4 +-
be/src/vec/exec/scan/scan_node.h | 2 +-
be/src/vec/exec/scan/scanner.cpp | 9 +-
be/src/vec/exec/scan/scanner.h | 8 +-
be/src/vec/exprs/vectorized_fn_call.cpp | 24 +-
be/src/vec/exprs/vexpr.cpp | 6 +-
be/src/vec/exprs/vexpr.h | 11 +-
be/src/vec/exprs/vexpr_context.cpp | 43 +-
be/src/vec/exprs/vexpr_context.h | 3 +
.../vec/functions/array/function_array_apply.cpp | 4 -
be/src/vec/functions/array/function_array_index.h | 16 +-
be/src/vec/functions/array/function_array_remove.h | 11 -
be/src/vec/functions/function_bit.cpp | 16 +-
be/src/vec/functions/function_bit_count.cpp | 19 +-
.../function_date_or_datetime_computation.cpp | 167 +-
.../function_date_or_datetime_computation.h | 13 +-
.../function_date_or_datetime_computation_v2.cpp | 165 -
.../function_date_or_datetime_to_string.cpp | 4 -
.../vec/functions/function_datetime_floor_ceil.cpp | 31 +-
.../function_datetime_string_to_string.cpp | 4 -
be/src/vec/functions/function_jsonb.cpp | 67 +
.../vec/functions/function_other_types_to_date.cpp | 58 +-
be/src/vec/functions/function_unary_arithmetic.h | 54 +-
be/src/vec/functions/function_varbinary.cpp | 8 +-
be/src/vec/functions/functions_comparison.h | 4 -
be/src/vec/functions/least_greast.cpp | 14 -
be/src/vec/functions/math.cpp | 150 +-
be/src/vec/functions/round.cpp | 1 -
be/src/vec/functions/simple_function_factory.h | 2 -
be/src/vec/functions/time_of_function.cpp | 17 -
be/src/vec/functions/to_time_function.cpp | 31 -
be/src/vec/olap/vcollect_iterator.cpp | 20 +
be/src/vec/sink/writer/vtablet_writer.cpp | 4 +-
be/src/vec/sink/writer/vtablet_writer.h | 4 +-
be/src/vec/utils/varbinaryop_subbinary.h | 8 +-
be/test/io/cache/block_file_cache_test.cpp | 87 +-
.../io/cache/block_file_cache_test_meta_store.cpp | 421 +
be/test/io/cache/cache_block_meta_store_test.cpp | 860 +
.../rowset/segment_v2/binary_dict_page_test.cpp | 78 +-
.../segment_v2/binary_plain_page_v2_test.cpp | 57 +-
.../rowset/segment_v2/bitshuffle_page_test.cpp | 14 +-
.../olap/rowset/segment_v2/encoding_info_test.cpp | 171 +
.../inverted_index/ananlyzer/analyzer_test.cpp | 348 +
.../inverted_index/query_v2/doc_set_test.cpp | 154 +
.../token_filter/pinyin_filter_test.cpp | 409 +
.../tokenizer/pinyin_alphabet_tokenizer_test.cpp | 248 +
.../tokenizer/pinyin_analysis_test.cpp | 1372 +
.../inverted_index/tokenizer/pinyin_util_test.cpp | 685 +
.../inverted_index/tokenizer/smart_forest_test.cpp | 204 +
.../tokenizer/smart_get_word_test.cpp | 361 +
.../segment_v2/inverted_index_query_type_test.cpp | 125 +
.../segment_v2/inverted_index_searcher_test.cpp | 242 +
.../variant_column_writer_reader_test.cpp | 31 +-
.../olap/segment_writer_full_encode_keys_test.cpp | 101 +
.../olap/vector_search/ann_range_search_test.cpp | 17 +-
.../vec/aggregate_functions/agg_collect_test.cpp | 15 +-
.../agg_group_array_intersect_test.cpp | 38 +-
be/test/vec/aggregate_functions/agg_test.cpp | 161 +
be/test/vec/columns/column_varbinary_test.cpp | 66 +-
be/test/vec/columns/column_variant_test.cpp | 94 +
be/test/vec/common/schema_util_rowset_test.cpp | 29 +
...ing_container_test.cpp => string_view_test.cpp} | 86 +-
be/test/vec/data_types/data_type_struct_test.cpp | 2 +-
.../vec/data_types/data_type_varbinary_test.cpp | 8 +-
.../serde/data_type_from_string_test.cpp | 4 +-
.../data_types/serde/data_type_to_string_test.cpp | 2 +-
.../serde/data_type_write_to_jsonb_test.cpp | 14 +-
.../parquet/fix_length_dict_decoder_test.cpp | 4 +-
be/test/vec/exec/orc/orc_reader_fill_data_test.cpp | 152 +-
.../vec/function/function_array_element_test.cpp | 36 -
be/test/vec/function/function_array_index_test.cpp | 57 -
.../vec/function/function_arrays_overlap_test.cpp | 31 -
be/test/vec/function/function_math_test.cpp | 40 -
be/test/vec/function/function_test_util.h | 6 +-
be/test/vec/function/function_time_test.cpp | 58 +-
be/test/vec/function/table_function_test.cpp | 24 -
cloud/script/run_all_tests.sh | 20 +-
cloud/src/meta-service/meta_service.cpp | 81 +-
cloud/src/meta-service/meta_service_job.cpp | 176 +-
cloud/src/meta-service/meta_service_resource.cpp | 17 +-
.../src/meta-service/meta_service_tablet_stats.cpp | 50 +-
cloud/src/meta-service/meta_service_tablet_stats.h | 20 +-
cloud/src/meta-service/meta_service_txn.cpp | 8 +-
cloud/src/meta-service/txn_lazy_committer.cpp | 4 +-
cloud/src/recycler/recycler.cpp | 8 +-
cloud/src/recycler/recycler.h | 7 +-
cloud/src/recycler/recycler_operation_log.cpp | 21 +-
cloud/src/recycler/snapshot_chain_compactor.cpp | 1 +
cloud/src/recycler/snapshot_data_migrator.cpp | 15 +-
cloud/src/recycler/snapshot_data_migrator.h | 24 +-
cloud/src/resource-manager/resource_manager.cpp | 17 +-
cloud/test/meta_service_operation_log_test.cpp | 1394 -
cloud/test/meta_service_test.cpp | 36 +-
cloud/test/recycler_operation_log_test.cpp | 20 +-
cloud/test/resource_test.cpp | 97 +
contrib/faiss | 2 +-
dist/LICENSE-dist.txt | 2 +-
.../docker-compose/iceberg/iceberg.yaml.tpl | 1 -
.../iceberg-metadata-scanner/pom.xml | 1 -
fe/be-java-extensions/preload-extensions/pom.xml | 21 +-
.../java/org/apache/doris/catalog/ScalarType.java | 33 +-
.../main/java/org/apache/doris/catalog/Type.java | 8 +
.../java/org/apache/doris/catalog/VariantType.java | 20 +-
.../main/java/org/apache/doris/common/Config.java | 46 +-
fe/fe-core/pom.xml | 19 +-
.../src/main/java/org/apache/doris/DorisFE.java | 7 +-
.../org/apache/doris/analysis/ArrayLiteral.java | 72 -
.../java/org/apache/doris/analysis/CastExpr.java | 96 -
.../org/apache/doris/analysis/DateLiteral.java | 62 -
.../org/apache/doris/analysis/DecimalLiteral.java | 31 -
.../apache/doris/analysis/DefaultValueExprDef.java | 30 +-
.../main/java/org/apache/doris/analysis/Expr.java | 144 -
.../org/apache/doris/analysis/FloatLiteral.java | 28 -
.../apache/doris/analysis/FunctionCallExpr.java | 22 -
.../java/org/apache/doris/analysis/IntLiteral.java | 44 -
.../org/apache/doris/analysis/JsonLiteral.java | 6 -
.../org/apache/doris/analysis/LargeIntLiteral.java | 18 -
.../org/apache/doris/analysis/LiteralExpr.java | 34 -
.../java/org/apache/doris/analysis/MapLiteral.java | 86 -
.../org/apache/doris/analysis/NullLiteral.java | 13 -
.../org/apache/doris/analysis/StringLiteral.java | 154 -
.../org/apache/doris/analysis/StructLiteral.java | 32 -
.../org/apache/doris/analysis/TryCastExpr.java | 11 -
.../doris/catalog/BuiltinAggregateFunctions.java | 4 +
.../doris/catalog/BuiltinScalarFunctions.java | 6 +
.../main/java/org/apache/doris/catalog/Column.java | 31 +-
.../java/org/apache/doris/catalog/SchemaTable.java | 4 +
.../apache/doris/catalog/TabletInvertedIndex.java | 518 +-
.../doris/cloud/catalog/CloudTabletRebalancer.java | 4 +-
.../doris/cloud/load/CloudBrokerLoadJob.java | 4 +-
.../apache/doris/cloud/qe/CloudCoordinator.java | 7 +-
.../common/profile/ProfileArchiveManager.java | 682 +
.../doris/common/profile/ProfileManager.java | 157 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 21 +
.../org/apache/doris/datasource/CatalogMgr.java | 1 +
.../org/apache/doris/datasource/FileScanNode.java | 15 +-
.../doris/datasource/jdbc/JdbcExternalCatalog.java | 9 +-
.../datasource/jdbc/client/JdbcMySQLClient.java | 6 +-
.../maxcompute/MaxComputeExternalTable.java | 11 +-
.../maxcompute/MaxComputeSchemaCacheValue.java | 9 +-
.../maxcompute/source/MaxComputeScanNode.java | 10 +
.../metastore/AWSGlueMetaStoreBaseProperties.java | 6 +-
.../org/apache/doris/httpv2/rest/LoadAction.java | 3 +-
.../org/apache/doris/indexpolicy/IndexPolicy.java | 7 +-
.../apache/doris/indexpolicy/IndexPolicyMgr.java | 31 +-
.../indexpolicy/PinyinTokenFilterValidator.java | 159 +
.../indexpolicy/PinyinTokenizerValidator.java | 157 +
.../insert/streaming/StreamingInsertJob.java | 10 +
.../streaming/StreamingJobSchedulerTask.java | 1 +
.../org/apache/doris/job/manager/JobManager.java | 7 +-
.../org/apache/doris/load/loadv2/LoadManager.java | 2 +-
.../java/org/apache/doris/mysql/MysqlChannel.java | 6 +-
.../org/apache/doris/mysql/MysqlSslContext.java | 7 +-
.../org/apache/doris/mysql/SslEngineHelper.java | 75 +
.../org/apache/doris/nereids/StatementContext.java | 9 +
.../doris/nereids/jobs/executor/Rewriter.java | 2 +
.../nereids/load/NereidsLoadPlanInfoCollector.java | 105 +-
.../nereids/load/NereidsLoadScanProvider.java | 16 +-
.../doris/nereids/load/NereidsLoadUtils.java | 48 +
.../doris/nereids/parser/LogicalPlanBuilder.java | 10 +-
.../org/apache/doris/nereids/rules/RuleSet.java | 2 +-
.../org/apache/doris/nereids/rules/RuleType.java | 3 +
.../doris/nereids/rules/analysis/BindSink.java | 36 +-
.../nereids/rules/exploration/mv/StructInfo.java | 15 +-
.../rules/expression/ExpressionRuleType.java | 1 +
.../rules/expression/rules/PartitionPruner.java | 131 +-
.../rewrite/EliminateGroupByKeyByUniform.java | 6 +
.../nereids/rules/rewrite/ExprIdRewriter.java | 59 +-
.../rules/rewrite/PruneFileScanPartition.java | 8 +-
.../rules/rewrite/PruneOlapScanPartition.java | 41 +-
.../rules/rewrite/PushDownJoinOnAssertNumRows.java | 229 +
.../trees/expressions/VirtualSlotReference.java | 7 +
.../functions/ComputeSignatureHelper.java | 111 +
.../GroupArrayUnion.java} | 55 +-
.../{scalar/SecToTime.java => agg/Sem.java} | 50 +-
.../expressions/functions/agg/SequenceCount.java | 2 +-
.../expressions/functions/agg/SequenceMatch.java | 2 +-
.../expressions/functions/agg/TopNWeighted.java | 25 +-
.../expressions/functions/agg/WindowFunnel.java | 2 +-
.../functions/executable/DateTimeArithmetic.java | 13 +-
.../executable/DateTimeExtractAndTransform.java | 261 +-
.../expressions/functions/scalar/ArrayRange.java | 8 +-
.../functions/scalar/ArrayRangeDayUnit.java | 4 +-
.../functions/scalar/ArrayRangeHourUnit.java | 4 +-
.../functions/scalar/ArrayRangeMinuteUnit.java | 4 +-
.../functions/scalar/ArrayRangeMonthUnit.java | 4 +-
.../functions/scalar/ArrayRangeQuarterUnit.java | 4 +-
.../functions/scalar/ArrayRangeSecondUnit.java | 4 +-
.../functions/scalar/ArrayRangeWeekUnit.java | 4 +-
.../functions/scalar/ArrayRangeYearUnit.java | 4 +-
.../expressions/functions/scalar/ConvertTz.java | 4 +-
.../functions/scalar/Crc32Internal.java | 26 +-
.../expressions/functions/scalar/CurrentTime.java | 4 +-
.../trees/expressions/functions/scalar/Date.java | 5 +-
.../expressions/functions/scalar/DateDiff.java | 7 +-
.../expressions/functions/scalar/DateFormat.java | 8 +-
.../expressions/functions/scalar/DateTrunc.java | 20 +-
.../expressions/functions/scalar/DayCeil.java | 14 +-
.../expressions/functions/scalar/DayFloor.java | 14 +-
.../expressions/functions/scalar/DayName.java | 5 +-
.../expressions/functions/scalar/DayOfMonth.java | 5 +-
.../expressions/functions/scalar/DayOfWeek.java | 5 +-
.../expressions/functions/scalar/DayOfYear.java | 5 +-
.../expressions/functions/scalar/DaySecondAdd.java | 2 +-
.../expressions/functions/scalar/DaysAdd.java | 2 +-
.../expressions/functions/scalar/DaysDiff.java | 8 +-
.../expressions/functions/scalar/DaysSub.java | 3 +-
.../trees/expressions/functions/scalar/Field.java | 3 +-
.../functions/scalar/FromIso8601Date.java | 4 +-
.../trees/expressions/functions/scalar/Hour.java | 6 +-
.../expressions/functions/scalar/HourCeil.java | 14 +-
.../expressions/functions/scalar/HourFloor.java | 14 +-
.../expressions/functions/scalar/HoursAdd.java | 2 +-
.../expressions/functions/scalar/HoursDiff.java | 7 +-
.../expressions/functions/scalar/HoursSub.java | 2 +-
.../expressions/functions/scalar/LastDay.java | 2 +-
.../expressions/functions/scalar/MakeTime.java | 2 +-
.../functions/scalar/MicroSecondTimestamp.java | 2 +-
.../functions/scalar/MicroSecondsAdd.java | 2 +-
.../functions/scalar/MicroSecondsDiff.java | 2 +-
.../functions/scalar/MicroSecondsSub.java | 2 +-
.../expressions/functions/scalar/Microsecond.java | 2 +-
.../functions/scalar/MilliSecondTimestamp.java | 2 +-
.../functions/scalar/MilliSecondsAdd.java | 2 +-
.../functions/scalar/MilliSecondsDiff.java | 2 +-
.../functions/scalar/MilliSecondsSub.java | 2 +-
.../trees/expressions/functions/scalar/Minute.java | 6 +-
.../expressions/functions/scalar/MinuteCeil.java | 14 +-
.../expressions/functions/scalar/MinuteFloor.java | 14 +-
.../expressions/functions/scalar/MinutesAdd.java | 2 +-
.../expressions/functions/scalar/MinutesDiff.java | 7 +-
.../expressions/functions/scalar/MinutesSub.java | 2 +-
.../trees/expressions/functions/scalar/Month.java | 4 +-
.../expressions/functions/scalar/MonthCeil.java | 14 +-
.../expressions/functions/scalar/MonthFloor.java | 14 +-
.../expressions/functions/scalar/MonthName.java | 5 +-
.../expressions/functions/scalar/MonthsAdd.java | 2 +-
.../expressions/functions/scalar/MonthsDiff.java | 5 +-
.../expressions/functions/scalar/MonthsSub.java | 2 +-
.../trees/expressions/functions/scalar/Now.java | 2 +-
.../expressions/functions/scalar/Quarter.java | 7 +-
.../expressions/functions/scalar/QuarterCeil.java | 14 +-
.../expressions/functions/scalar/QuarterFloor.java | 14 +-
.../expressions/functions/scalar/QuartersAdd.java | 2 +-
.../expressions/functions/scalar/QuartersDiff.java | 2 +-
.../expressions/functions/scalar/QuartersSub.java | 2 +-
.../expressions/functions/scalar/SecToTime.java | 2 +-
.../trees/expressions/functions/scalar/Second.java | 6 +-
.../expressions/functions/scalar/SecondCeil.java | 14 +-
.../expressions/functions/scalar/SecondFloor.java | 14 +-
.../functions/scalar/SecondTimestamp.java | 2 +-
.../expressions/functions/scalar/SecondsAdd.java | 2 +-
.../expressions/functions/scalar/SecondsDiff.java | 7 +-
.../expressions/functions/scalar/SecondsSub.java | 2 +-
.../expressions/functions/scalar/StrToDate.java | 4 +-
.../scalar/{TimeToSec.java => StripNullValue.java} | 30 +-
.../trees/expressions/functions/scalar/Time.java | 2 +-
.../expressions/functions/scalar/TimeDiff.java | 6 +-
.../expressions/functions/scalar/TimeToSec.java | 2 +-
.../expressions/functions/scalar/Timestamp.java | 16 +-
.../trees/expressions/functions/scalar/ToDate.java | 6 +-
.../expressions/functions/scalar/ToDateV2.java | 2 +-
.../trees/expressions/functions/scalar/ToDays.java | 4 +-
.../expressions/functions/scalar/ToIso8601.java | 9 +-
.../trees/expressions/functions/scalar/ToJson.java | 4 +-
.../expressions/functions/scalar/ToMonday.java | 2 +-
.../functions/scalar/UnixTimestamp.java | 8 +-
.../expressions/functions/scalar/UtcTime.java | 13 +-
.../expressions/functions/scalar/UtcTimestamp.java | 9 +-
.../trees/expressions/functions/scalar/Week.java | 8 +-
.../expressions/functions/scalar/WeekCeil.java | 14 +-
.../expressions/functions/scalar/WeekFloor.java | 14 +-
.../expressions/functions/scalar/WeekOfYear.java | 5 +-
.../expressions/functions/scalar/Weekday.java | 5 +-
.../expressions/functions/scalar/WeeksAdd.java | 2 +-
.../expressions/functions/scalar/WeeksDiff.java | 5 +-
.../expressions/functions/scalar/WeeksSub.java | 2 +-
.../trees/expressions/functions/scalar/Year.java | 5 +-
.../expressions/functions/scalar/YearCeil.java | 14 +-
.../expressions/functions/scalar/YearFloor.java | 14 +-
.../expressions/functions/scalar/YearWeek.java | 8 +-
.../expressions/functions/scalar/YearsAdd.java | 2 +-
.../expressions/functions/scalar/YearsDiff.java | 5 +-
.../expressions/functions/scalar/YearsSub.java | 2 +-
.../functions/window/RequireTrivialTypes.java | 4 +-
.../visitor/AggregateFunctionVisitor.java | 10 +
.../expressions/visitor/ScalarFunctionVisitor.java | 5 +
.../trees/plans/commands/DeleteFromCommand.java | 7 +-
.../trees/plans/commands/PauseJobCommand.java | 5 +-
.../trees/plans/commands/ResumeJobCommand.java | 4 +-
.../trees/plans/commands/UpdateCommand.java | 3 +-
.../trees/plans/commands/insert/InsertUtils.java | 44 +-
.../plans/commands/merge/MergeIntoCommand.java | 19 +-
.../BackendDistributedPlanWorkerManager.java | 5 +-
.../trees/plans/logical/LogicalAggregate.java | 6 +
...LogicalProject.java => LogicalPostProject.java} | 148 +-
.../trees/plans/logical/LogicalProject.java | 19 +
.../nereids/trees/plans/visitor/PlanVisitor.java | 5 +
.../org/apache/doris/nereids/types/DataType.java | 5 +-
.../apache/doris/nereids/types/DateTimeV2Type.java | 4 +-
.../apache/doris/nereids/types/DecimalV2Type.java | 2 +-
.../org/apache/doris/nereids/types/TimeV2Type.java | 7 +-
.../apache/doris/nereids/types/VarcharType.java | 4 +-
.../apache/doris/nereids/types/VariantType.java | 23 +-
.../doris/nereids/util/TypeCoercionUtils.java | 13 +-
.../org/apache/doris/planner/OlapScanNode.java | 23 +-
.../java/org/apache/doris/qe/ConnectContext.java | 43 +-
.../java/org/apache/doris/qe/SessionVariable.java | 238 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 11 +-
.../resource/computegroup/ComputeGroupMgr.java | 22 +-
.../java/org/apache/doris/rewrite/FEFunctions.java | 50 -
.../ExternalFileTableValuedFunction.java | 9 +-
.../doris/tablefunction/MetadataGenerator.java | 83 +-
.../tablefunction/NumbersTableValuedFunction.java | 9 +-
.../common/profile/ProfileArchiveManagerTest.java | 1107 +
.../doris/common/profile/ProfileManagerTest.java | 230 +
.../AWSGlueMetaStoreBasePropertiesTest.java | 2 +-
.../external/elasticsearch/QueryBuildersTest.java | 38 +-
.../nereids/mv/OptimizeGetAvailableMvsTest.java | 4 +-
.../nereids/rules/expression/FoldConstantTest.java | 95 +-
.../rules/expression/check/CheckCastTest.java | 112 +-
.../nereids/rules/rewrite/PartitionPrunerTest.java | 570 +
.../rewrite/PushDownJoinOnAssertNumRowsTest.java | 454 +
.../doris/nereids/trees/expressions/CastTest.java | 26 +-
.../nereids/trees/expressions/TryCastTest.java | 2 +-
.../functions/ComputeSignatureHelperTest.java | 58 +-
.../plans/commands/CreateResourceCommandTest.java | 2 +-
.../GrantResourcePrivilegeCommandTest.java | 2 +-
.../RevokeResourcePrivilegeCommandTest.java | 2 +-
.../plans/commands/merge/MergeIntoCommandTest.java | 2 +-
.../apache/doris/nereids/types/DataTypeTest.java | 10 +-
.../doris/nereids/util/TypeCoercionUtilsTest.java | 11 +-
.../org/apache/doris/rewrite/FEFunctionsTest.java | 840 -
.../doris/rewrite/RewriteDateLiteralRuleTest.java | 255 -
fe/pom.xml | 28 +-
gensrc/proto/file_cache.proto | 14 +
gensrc/proto/olap_file.proto | 2 +
gensrc/thrift/Descriptors.thrift | 1 +
.../data/ann_index_p0/ann_index_only_scan.out | 19 +
.../data/correctness/test_str_to_date.out | 16 +
.../test_function_signature_all_types.out | 553 +
.../sql-functions/doc_date_functions_test.out | 36 +
.../paimon/paimon_base_filesystem.out | 3 +
.../test_builtin_analyzer_in_custom_analyzer.out | 49 +
.../analyzer/test_custom_analyzer.out | 174 +
.../data/nereids_function_p0/scalar_function/S.out | 58 +
.../datetime_functions/test_date_function.out | 1 +
.../infer_predicate/infer_unequal_predicates.out | 19 +-
.../mv/grouping_sets/grouping_sets.out | 10 +
.../partition_prune/list_prune_predicate.out | 4 +
.../query_p0/aggregate/group_array_intersect.out | 95 +
.../test_aggregate_all_functions2.out | 54 +
.../test_strip_null_value.out | 28 +
.../datetime_functions/test_date_function.out | 6 +
.../datetime_functions/test_from_iso8601_date.out | 114 +
.../data/query_p0/system/test_table_options.out | 40 +-
.../test_non_overlap_seg_heavy_sc.out | 7 +
.../tpcds_sf100/noStatsRfPrune/query54.out | 76 +-
.../tpcds_sf100/no_stats_shape/query54.out | 76 +-
.../shape_check/tpcds_sf100/rf_prune/query54.out | 82 +-
.../data/shape_check/tpcds_sf100/shape/query54.out | 82 +-
.../tpcds_sf1000/bs_downgrade_shape/query54.out | 82 +-
.../data/shape_check/tpcds_sf1000/hint/query54.out | 82 +-
.../shape_check/tpcds_sf1000/shape/query54.out | 82 +-
.../shape_check/tpcds_sf10t_orc/shape/query54.out | 76 +-
.../predefine/test_all_prdefine_type_to_sparse.out | 144 +-
.../pipeline/cloud_p0/conf/be_custom.conf | 2 +
regression-test/pipeline/cloud_p0/run.sh | 2 +
regression-test/pipeline/external/conf/be.conf | 2 +
.../pipeline/external/conf/regression-conf.groovy | 2 -
.../pipeline/nonConcurrent/conf/be.conf | 1 +
.../nonConcurrent/conf/regression-conf.groovy | 2 -
regression-test/pipeline/p0/conf/be.conf | 2 +
.../pipeline/p0/conf/regression-conf.groovy | 2 -
regression-test/pipeline/p1/conf/be.conf | 2 +
.../suites/ann_index_p0/ann_index_only_scan.groovy | 450 +
.../ann_index_only_scan_distance_expr.groovy | 206 +
.../ann_index_only_scan_metric_direction.groovy | 228 +
.../cloud_p0/balance/test_balance_metrics.groovy | 2 +-
.../balance/test_peer_read_async_warmup.groovy | 1 +
.../test_drop_cluster_clean_metrics.groovy | 2 +-
.../test_fe_tablet_same_backend.groovy | 2 +-
.../multi_cluster/test_no_cluster_hits.groovy | 34 +-
.../cloud_p0/multi_cluster/test_rebalance.groovy | 6 +-
.../suites/correctness/test_str_to_date.groovy | 40 +-
.../test_function_signature_all_types.groovy | 1044 +
regression-test/suites/ddl_p0/test_ctas.groovy | 7 -
.../sql-manual/sql-functions/doc_date_error.groovy | 12 -
.../sql-functions/doc_date_functions_test.groovy | 92 +-
.../test_hive_metadata_refresh_interval.groovy | 101 +
.../paimon/paimon_base_filesystem.groovy | 3 +-
...test_builtin_analyzer_in_custom_analyzer.groovy | 140 +
.../analyzer/test_custom_analyzer.groovy | 347 +-
.../load_p0/insert/test_insert_statistic.groovy | 10 +-
.../nereids_function_p0/scalar_function/S.groovy | 20 +-
.../eliminate_group_by_key_by_uniform.groovy | 95 +
.../mv/grouping_sets/grouping_sets.groovy | 37 +
.../partition_prune/hive_partition_prune.groovy | 2 +-
.../partition_prune/list_prune_predicate.groovy | 328 +
.../aggregate/group_array_intersect.groovy | 32 +
.../test_aggregate_all_functions2.groovy | 22 +
.../test_strip_null_value.groovy | 88 +
.../datetime_functions/test_date_function.groovy | 19 +-
.../test_from_iso8601_date.groovy | 167 +-
.../query_p0/system/test_table_options.groovy | 10 +-
.../test_non_overlap_seg_heavy_sc.groovy | 74 +
regression-test/suites/variant_p0/desc.groovy | 1 +
.../test_all_prdefine_type_to_sparse.groovy | 3 +-
.../suites/variant_sparse_benchmark_p2/load.groovy | 208 +
run-be-ut.sh | 21 +-
run-cloud-ut.sh | 19 +-
thirdparty/CHANGELOG.md | 4 +
thirdparty/download-thirdparty.sh | 14 +-
thirdparty/patches/cctz-civil-cache.patch | 119 +
thirdparty/patches/librdkafka-1.9.2.patch | 189 -
thirdparty/patches/librdkafka-2.11.0.patch | 76 +
thirdparty/vars.sh | 10 +-
547 files changed, 509351 insertions(+), 8767 deletions(-)
create mode 100644 be/dict/pinyin/pinyin.txt
create mode 100644 be/dict/pinyin/pinyin_alphabet.dict
create mode 100644 be/dict/pinyin/polyphone.txt
create mode 100644 be/src/io/cache/cache_block_meta_store.cpp
create mode 100644 be/src/io/cache/cache_block_meta_store.h
create mode 100644 be/src/olap/rowset/segment_v2/binary_dict_page_pre_decoder.h
create mode 100644 be/src/olap/rowset/segment_v2/binary_plain_page_v2_pre_decoder.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/token_filter/pinyin_filter.cpp
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/token_filter/pinyin_filter.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/token_filter/pinyin_filter_factory.cpp
copy be/src/{vec/exec/scan/scan_node.h => olap/rowset/segment_v2/inverted_index/token_filter/pinyin_filter_factory.h} (52%)
copy gensrc/proto/file_cache.proto => be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/chinese_util.cpp (52%)
copy be/src/{io/fs/hdfs.h => olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/chinese_util.h} (66%)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/pinyin_alphabet_tokenizer.cpp
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/pinyin_alphabet_tokenizer.h
copy be/src/{vec/exec/scan/scan_node.h => olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/pinyin_config.h} (51%)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/pinyin_format.cpp
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/pinyin_format.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/pinyin_formatter.cpp
copy be/src/{io/fs/hdfs.h => olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/pinyin_formatter.h} (62%)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/pinyin_tokenizer.cpp
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/pinyin_tokenizer.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/pinyin_tokenizer_factory.cpp
copy be/src/{io/fs/hdfs.h => olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/pinyin_tokenizer_factory.h} (55%)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/pinyin_util.cpp
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/pinyin_util.h
copy be/src/{io/fs/hdfs.h => olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/rune.h} (55%)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/smart_forest.cpp
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/smart_forest.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/smart_get_word.cpp
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/smart_get_word.h
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin/term_item.h
delete mode 100644 be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.cpp
delete mode 100644 be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_group_array_set_op.cpp
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_group_array_set_op.h
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_sem.cpp
create mode 100644 be/src/vec/aggregate_functions/aggregate_function_sem.h
rename be/src/vec/common/{string_container.cpp => string_view.cpp} (91%)
rename be/src/vec/common/{string_container.h => string_view.h} (80%)
delete mode 100644 be/src/vec/functions/function_date_or_datetime_computation_v2.cpp
create mode 100644 be/test/io/cache/block_file_cache_test_meta_store.cpp
create mode 100644 be/test/io/cache/cache_block_meta_store_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/ananlyzer/analyzer_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/token_filter/pinyin_filter_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin_alphabet_tokenizer_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin_analysis_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/tokenizer/pinyin_util_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/tokenizer/smart_forest_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/tokenizer/smart_get_word_test.cpp
create mode 100644 be/test/olap/rowset/segment_v2/inverted_index_query_type_test.cpp
create mode 100644 be/test/olap/segment_writer_full_encode_keys_test.cpp
rename be/test/vec/common/{string_container_test.cpp => string_view_test.cpp} (78%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileArchiveManager.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/indexpolicy/PinyinTokenFilterValidator.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/indexpolicy/PinyinTokenizerValidator.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mysql/SslEngineHelper.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOnAssertNumRows.java
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/{scalar/FromIso8601Date.java => agg/GroupArrayUnion.java} (59%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/{scalar/SecToTime.java => agg/Sem.java} (62%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{TimeToSec.java => StripNullValue.java} (67%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/{LogicalProject.java => LogicalPostProject.java} (53%)
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfileArchiveManagerTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOnAssertNumRowsTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java
create mode 100644 regression-test/data/ann_index_p0/ann_index_only_scan.out
create mode 100644 regression-test/data/correctness_p0/test_function_signature_all_types.out
create mode 100644 regression-test/data/inverted_index_p0/analyzer/test_builtin_analyzer_in_custom_analyzer.out
create mode 100644 regression-test/data/nereids_rules_p0/partition_prune/list_prune_predicate.out
create mode 100644 regression-test/data/query_p0/sql_functions/conditional_functions/test_strip_null_value.out
create mode 100644 regression-test/data/schema_change_p0/test_non_overlap_seg_heavy_sc.out
create mode 100644 regression-test/suites/ann_index_p0/ann_index_only_scan.groovy
create mode 100644 regression-test/suites/ann_index_p0/ann_index_only_scan_distance_expr.groovy
create mode 100644 regression-test/suites/ann_index_p0/ann_index_only_scan_metric_direction.groovy
create mode 100644 regression-test/suites/correctness_p0/test_function_signature_all_types.groovy
create mode 100644 regression-test/suites/external_table_p0/hive/test_hive_metadata_refresh_interval.groovy
create mode 100644 regression-test/suites/inverted_index_p0/analyzer/test_builtin_analyzer_in_custom_analyzer.groovy
create mode 100644 regression-test/suites/nereids_rules_p0/partition_prune/list_prune_predicate.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/conditional_functions/test_strip_null_value.groovy
create mode 100644 regression-test/suites/schema_change_p0/test_non_overlap_seg_heavy_sc.groovy
create mode 100644 regression-test/suites/variant_sparse_benchmark_p2/load.groovy
create mode 100644 thirdparty/patches/cctz-civil-cache.patch
delete mode 100644 thirdparty/patches/librdkafka-1.9.2.patch
create mode 100644 thirdparty/patches/librdkafka-2.11.0.patch
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/18: support prune nested columns
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6ccc8ca260ad1ac6a1ba67d157a41923aa2cde69
Author: 924060929 <la...@selectdb.com>
AuthorDate: Wed Sep 24 17:04:44 2025 +0800
support prune nested columns
---
.idea/vcs.xml | 52 +-
be/src/olap/iterators.h | 5 +
be/src/olap/rowset/beta_rowset_reader.cpp | 4 +
be/src/olap/rowset/rowset_reader_context.h | 5 +
be/src/olap/rowset/segment_v2/column_reader.cpp | 361 ++++++++++-
be/src/olap/rowset/segment_v2/column_reader.h | 70 ++-
be/src/olap/rowset/segment_v2/segment.cpp | 16 +
be/src/olap/tablet_reader.cpp | 3 +
be/src/olap/tablet_reader.h | 4 +
be/src/olap/tablet_schema.cpp | 16 +
be/src/olap/tablet_schema.h | 10 +
be/src/runtime/descriptors.cpp | 5 +
be/src/runtime/descriptors.h | 8 +
be/src/vec/columns/column_map.cpp | 1 +
be/src/vec/exec/scan/olap_scanner.cpp | 20 +-
.../olap/rowset/segment_v2/column_reader_test.cpp | 220 +++++++
.../org/apache/doris/analysis/AccessPathInfo.java | 38 ++
.../org/apache/doris/analysis/SlotDescriptor.java | 43 ++
.../org/apache/doris/datasource/FileScanNode.java | 10 +-
.../org/apache/doris/nereids/StatementContext.java | 10 +
.../glue/translator/PhysicalPlanTranslator.java | 5 +-
.../glue/translator/PlanTranslatorContext.java | 18 +
.../doris/nereids/jobs/executor/Rewriter.java | 6 +
.../org/apache/doris/nereids/rules/RuleSet.java | 2 +
.../org/apache/doris/nereids/rules/RuleType.java | 7 +
.../doris/nereids/rules/analysis/BindRelation.java | 9 +-
.../nereids/rules/analysis/ExpressionAnalyzer.java | 10 +
.../rules/expression/ExpressionNormalization.java | 2 +
.../rules/expression/ExpressionRuleType.java | 1 +
.../expression/rules/NormalizeStructElement.java | 66 ++
.../rewrite/AccessPathExpressionCollector.java | 511 ++++++++++++++++
.../rules/rewrite/AccessPathPlanCollector.java | 176 ++++++
.../nereids/rules/rewrite/NestedColumnPruning.java | 487 +++++++++++++++
.../rules/rewrite/PushDownNestedColumn.java | 315 ++++++++++
.../nereids/rules/rewrite/SlotTypeReplacer.java | 674 +++++++++++++++++++++
.../rules/rewrite/VariantSubPathPruning.java | 3 +-
.../trees/expressions/PreferPushDownProject.java | 41 ++
.../nereids/trees/expressions/SlotReference.java | 56 +-
.../expressions/functions/scalar/ArrayFirst.java | 6 +
.../expressions/functions/scalar/ArrayLast.java | 6 +
.../expressions/functions/scalar/ArrayMap.java | 3 +-
.../expressions/functions/scalar/ElementAt.java | 4 +-
.../functions/scalar/HighOrderFunction.java | 3 +-
.../functions/scalar/MapContainsEntry.java | 3 +-
.../functions/scalar/MapContainsKey.java | 3 +-
.../functions/scalar/MapContainsValue.java | 3 +-
.../expressions/functions/scalar/MapEntries.java | 3 +-
.../expressions/functions/scalar/MapKeys.java | 3 +-
.../expressions/functions/scalar/MapValues.java | 3 +-
.../functions/scalar/StructElement.java | 9 +-
.../expressions/visitor/ScalarFunctionVisitor.java | 14 +-
.../trees/plans/logical/LogicalFileScan.java | 44 +-
.../trees/plans/logical/LogicalHudiScan.java | 22 +-
.../trees/plans/logical/LogicalOlapScan.java | 17 +
.../trees/plans/logical/LogicalTVFRelation.java | 23 +-
.../org/apache/doris/nereids/types/ArrayType.java | 2 +-
.../org/apache/doris/nereids/types/MapType.java | 2 +-
.../doris/nereids/types/NestedColumnPrunable.java | 22 +
.../apache/doris/nereids/types/StructField.java | 3 +-
.../org/apache/doris/nereids/types/StructType.java | 10 +-
.../org/apache/doris/planner/OlapScanNode.java | 3 +
.../java/org/apache/doris/planner/PlanNode.java | 4 +-
.../java/org/apache/doris/planner/ScanNode.java | 109 ++++
.../java/org/apache/doris/qe/SessionVariable.java | 9 +
.../rules/rewrite/PruneNestedColumnTest.java | 625 +++++++++++++++++++
.../apache/doris/utframe/TestWithFeService.java | 7 +-
gensrc/thrift/Descriptors.thrift | 48 +-
.../complex_types/test_pruned_columns.out | 24 +
.../complex_types/test_pruned_columns.groovy | 61 ++
69 files changed, 4285 insertions(+), 103 deletions(-)
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
index 0a185d9bdc7..8b083db2f94 100644
--- a/.idea/vcs.xml
+++ b/.idea/vcs.xml
@@ -1,34 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
<project version="4">
- <component name="IssueNavigationConfiguration">
- <option name="links">
- <list>
- <IssueNavigationLink>
- <option name="issueRegexp" value="#(\d+)" />
- <option name="linkRegexp" value="https://github.com/apache/doris/pull/$1" />
- </IssueNavigationLink>
- </list>
- </option>
- </component>
- <component name="VcsDirectoryMappings">
- <mapping directory="$PROJECT_DIR$" vcs="Git" />
- <mapping directory="$PROJECT_DIR$/be/src/apache-orc" vcs="Git" />
- <mapping directory="$PROJECT_DIR$/be/src/clucene" vcs="Git" />
- </component>
-</project>
+ <component name="IssueNavigationConfiguration">
+ <option name="links">
+ <list>
+ <IssueNavigationLink>
+ <option name="issueRegexp" value="#(\d+)" />
+ <option name="linkRegexp" value="https://github.com/apache/doris/pull/$1" />
+ </IssueNavigationLink>
+ </list>
+ </option>
+ </component>
+ <component name="VcsDirectoryMappings">
+ <mapping directory="$PROJECT_DIR$" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/be/src/apache-orc" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/be/src/clucene" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/apache-orc" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/clucene" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/faiss" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/openblas" vcs="Git" />
+ </component>
+</project>
\ No newline at end of file
diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h
index 9a3ed4ea998..1b63aad4909 100644
--- a/be/src/olap/iterators.h
+++ b/be/src/olap/iterators.h
@@ -47,6 +47,8 @@ namespace segment_v2 {
struct SubstreamIterator;
}
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
class StorageReadOptions {
public:
struct KeyRange {
@@ -143,6 +145,9 @@ public:
std::map<ColumnId, size_t> vir_cid_to_idx_in_block;
std::map<size_t, vectorized::DataTypePtr> vir_col_idx_to_type;
+ std::map<int32_t, TColumnAccessPaths> all_access_paths;
+ std::map<int32_t, TColumnAccessPaths> predicate_access_paths;
+
std::shared_ptr<vectorized::ScoreRuntime> score_runtime;
CollectionStatisticsPtr collection_statistics;
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp
index 1de17e27eee..28683540365 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -103,6 +103,10 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
_read_options.remaining_conjunct_roots = _read_context->remaining_conjunct_roots;
_read_options.common_expr_ctxs_push_down = _read_context->common_expr_ctxs_push_down;
_read_options.virtual_column_exprs = _read_context->virtual_column_exprs;
+
+ _read_options.all_access_paths = _read_context->all_access_paths;
+ _read_options.predicate_access_paths = _read_context->predicate_access_paths;
+
_read_options.ann_topn_runtime = _read_context->ann_topn_runtime;
_read_options.vir_cid_to_idx_in_block = _read_context->vir_cid_to_idx_in_block;
_read_options.vir_col_idx_to_type = _read_context->vir_col_idx_to_type;
diff --git a/be/src/olap/rowset/rowset_reader_context.h b/be/src/olap/rowset/rowset_reader_context.h
index 951679f413b..fffa16d0f5c 100644
--- a/be/src/olap/rowset/rowset_reader_context.h
+++ b/be/src/olap/rowset/rowset_reader_context.h
@@ -18,6 +18,8 @@
#ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H
#define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H
+#include <vector>
+
#include "io/io_common.h"
#include "olap/column_predicate.h"
#include "olap/olap_common.h"
@@ -90,6 +92,9 @@ struct RowsetReaderContext {
std::map<ColumnId, size_t> vir_cid_to_idx_in_block;
std::map<size_t, vectorized::DataTypePtr> vir_col_idx_to_type;
+ std::map<int32_t, TColumnAccessPaths> all_access_paths;
+ std::map<int32_t, TColumnAccessPaths> predicate_access_paths;
+
std::shared_ptr<vectorized::ScoreRuntime> score_runtime;
CollectionStatisticsPtr collection_statistics;
std::shared_ptr<segment_v2::AnnTopNRuntime> ann_topn_runtime;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 16d87a1d425..9c68eee2fea 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -18,6 +18,7 @@
#include "olap/rowset/segment_v2/column_reader.h"
#include <assert.h>
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/segment_v2.pb.h>
#include <algorithm>
@@ -72,11 +73,11 @@
#include "vec/columns/column_map.h"
#include "vec/columns/column_nullable.h"
#include "vec/columns/column_struct.h"
-#include "vec/columns/column_variant.h"
#include "vec/columns/column_vector.h"
#include "vec/common/assert_cast.h"
#include "vec/common/schema_util.h"
#include "vec/common/string_ref.h"
+#include "vec/common/typeid_cast.h"
#include "vec/core/types.h"
#include "vec/data_types/data_type_agg_state.h"
#include "vec/data_types/data_type_factory.hpp"
@@ -804,11 +805,12 @@ Status ColumnReader::new_iterator(ColumnIteratorUPtr* iterator, const TabletColu
*iterator = std::make_unique<EmptyFileColumnIterator>();
return Status::OK();
}
- if (is_scalar_type((FieldType)_meta_type)) {
+ if (is_scalar_type(_meta_type)) {
*iterator = std::make_unique<FileColumnIterator>(shared_from_this());
+ (*iterator)->set_column_name(tablet_column ? tablet_column->name() : "");
return Status::OK();
} else {
- auto type = (FieldType)_meta_type;
+ auto type = _meta_type;
switch (type) {
case FieldType::OLAP_FIELD_TYPE_AGG_STATE: {
return new_agg_state_iterator(iterator);
@@ -842,6 +844,8 @@ Status ColumnReader::new_array_iterator(ColumnIteratorUPtr* iterator,
? &tablet_column->get_sub_column(0)
: nullptr));
+ item_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(0).name() : "");
+
ColumnIteratorUPtr offset_iterator;
RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&offset_iterator, nullptr));
auto* file_iter = static_cast<FileColumnIterator*>(offset_iterator.release());
@@ -865,11 +869,13 @@ Status ColumnReader::new_map_iterator(ColumnIteratorUPtr* iterator,
&key_iterator, tablet_column && tablet_column->get_subtype_count() > 1
? &tablet_column->get_sub_column(0)
: nullptr));
+ key_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(0).name() : "");
ColumnIteratorUPtr val_iterator;
RETURN_IF_ERROR(_sub_readers[1]->new_iterator(
&val_iterator, tablet_column && tablet_column->get_subtype_count() > 1
? &tablet_column->get_sub_column(1)
: nullptr));
+ val_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(1).name() : "");
ColumnIteratorUPtr offsets_iterator;
RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&offsets_iterator, nullptr));
auto* file_iter = static_cast<FileColumnIterator*>(offsets_iterator.release());
@@ -897,6 +903,8 @@ Status ColumnReader::new_struct_iterator(ColumnIteratorUPtr* iterator,
ColumnIteratorUPtr sub_column_iterator;
RETURN_IF_ERROR(_sub_readers[i]->new_iterator(
&sub_column_iterator, tablet_column ? &tablet_column->get_sub_column(i) : nullptr));
+ sub_column_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(i).name()
+ : "");
sub_column_iterators.emplace_back(std::move(sub_column_iterator));
}
@@ -917,6 +925,33 @@ Status ColumnReader::new_struct_iterator(ColumnIteratorUPtr* iterator,
return Status::OK();
}
+Result<TColumnAccessPaths> ColumnIterator::_get_sub_access_paths(
+ const TColumnAccessPaths& access_paths) {
+ TColumnAccessPaths sub_access_paths = access_paths;
+ for (auto it = sub_access_paths.begin(); it != sub_access_paths.end();) {
+ TColumnAccessPath& name_path = *it;
+ if (name_path.data_access_path.path.empty()) {
+ return ResultError(
+ Status::InternalError("Invalid access path for struct column: path is empty"));
+ }
+
+ if (name_path.data_access_path.path[0] != _column_name) {
+ return ResultError(Status::InternalError(
+ R"(Invalid access path for column: expected name "{}", got "{}")", _column_name,
+ name_path.data_access_path.path[0]));
+ }
+
+ name_path.data_access_path.path.erase(name_path.data_access_path.path.begin());
+ if (!name_path.data_access_path.path.empty()) {
+ ++it;
+ } else {
+ set_need_to_read();
+ it = sub_access_paths.erase(it);
+ }
+ }
+ return sub_access_paths;
+}
+
///====================== MapFileColumnIterator ============================////
MapFileColumnIterator::MapFileColumnIterator(std::shared_ptr<ColumnReader> reader,
ColumnIteratorUPtr null_iterator,
@@ -933,6 +968,10 @@ MapFileColumnIterator::MapFileColumnIterator(std::shared_ptr<ColumnReader> reade
}
Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ LOG(INFO) << "Map column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
RETURN_IF_ERROR(_key_iterator->init(opts));
RETURN_IF_ERROR(_val_iterator->init(opts));
RETURN_IF_ERROR(_offsets_iterator->init(opts));
@@ -943,6 +982,11 @@ Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
}
Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
if (_map_reader->is_nullable()) {
RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
}
@@ -957,10 +1001,16 @@ Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
bool* has_null) {
- const auto* column_map = vectorized::check_and_get_column<vectorized::ColumnMap>(
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading.";
+ dst->resize(*n);
+ return Status::OK();
+ }
+
+ auto& column_map = assert_cast<vectorized::ColumnMap&>(
dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
: *dst);
- auto column_offsets_ptr = column_map->get_offsets_column().assume_mutable();
+ auto column_offsets_ptr = column_map.get_offsets_column().assume_mutable();
bool offsets_has_null = false;
ssize_t start = column_offsets_ptr->size();
RETURN_IF_ERROR(_offsets_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null));
@@ -973,8 +1023,8 @@ Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr
DCHECK(column_offsets.get_data().back() >= column_offsets.get_data()[start - 1]);
size_t num_items =
column_offsets.get_data().back() - column_offsets.get_data()[start - 1]; // -1 is valid
- auto key_ptr = column_map->get_keys().assume_mutable();
- auto val_ptr = column_map->get_values().assume_mutable();
+ auto key_ptr = column_map.get_keys().assume_mutable();
+ auto val_ptr = column_map.get_values().assume_mutable();
if (num_items > 0) {
size_t num_read = num_items;
@@ -983,6 +1033,9 @@ Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr
RETURN_IF_ERROR(_key_iterator->next_batch(&num_read, key_ptr, &key_has_null));
RETURN_IF_ERROR(_val_iterator->next_batch(&num_read, val_ptr, &val_has_null));
DCHECK(num_read == num_items);
+
+ column_map.get_keys_ptr() = std::move(key_ptr);
+ column_map.get_values_ptr() = std::move(val_ptr);
}
if (dst->is_nullable()) {
@@ -1008,6 +1061,12 @@ Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr
Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
vectorized::MutableColumnPtr& dst) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ dst->resize(count);
+ return Status::OK();
+ }
+
for (size_t i = 0; i < count; ++i) {
RETURN_IF_ERROR(seek_to_ordinal(rowids[i]));
size_t num_read = 1;
@@ -1017,6 +1076,93 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t
return Status::OK();
}
+void MapFileColumnIterator::set_need_to_read() {
+ set_reading_flag(ReadingFlag::NEED_TO_READ);
+ _key_iterator->set_need_to_read();
+ _val_iterator->set_need_to_read();
+}
+
+void MapFileColumnIterator::remove_pruned_sub_iterators() {
+ _key_iterator->remove_pruned_sub_iterators();
+ _val_iterator->remove_pruned_sub_iterators();
+}
+
+Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) {
+ if (all_access_paths.empty()) {
+ return Status::OK();
+ }
+
+ if (!predicate_access_paths.empty()) {
+ set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
+ LOG(INFO) << "Map column iterator set sub-column " << _column_name
+ << " to READING_FOR_PREDICATE";
+ }
+
+ auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
+ auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
+
+ if (sub_all_access_paths.empty()) {
+ return Status::OK();
+ }
+
+ TColumnAccessPaths key_all_access_paths;
+ TColumnAccessPaths val_all_access_paths;
+ TColumnAccessPaths key_predicate_access_paths;
+ TColumnAccessPaths val_predicate_access_paths;
+
+ for (auto paths : sub_all_access_paths) {
+ if (paths.data_access_path.path[0] == "*") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_all_access_paths.emplace_back(paths);
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_all_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "KEYS") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_all_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "VALUES") {
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_all_access_paths.emplace_back(paths);
+ }
+ }
+ const auto need_read_keys = !key_all_access_paths.empty();
+ const auto need_read_values = !val_all_access_paths.empty();
+
+ for (auto paths : sub_predicate_access_paths) {
+ if (paths.data_access_path.path[0] == "*") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_predicate_access_paths.emplace_back(paths);
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_predicate_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "KEYS") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_predicate_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "VALUES") {
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_predicate_access_paths.emplace_back(paths);
+ }
+ }
+
+ if (need_read_keys) {
+ _key_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ);
+ RETURN_IF_ERROR(
+ _key_iterator->set_access_paths(key_all_access_paths, key_predicate_access_paths));
+ } else {
+ _key_iterator->set_reading_flag(ReadingFlag::SKIP_READING);
+ LOG(INFO) << "Map column iterator set key column to SKIP_READING";
+ }
+
+ if (need_read_values) {
+ _val_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ);
+ RETURN_IF_ERROR(
+ _val_iterator->set_access_paths(val_all_access_paths, val_predicate_access_paths));
+ } else {
+ _val_iterator->set_reading_flag(ReadingFlag::SKIP_READING);
+ LOG(INFO) << "Map column iterator set value column to SKIP_READING";
+ }
+ return Status::OK();
+}
+
////////////////////////////////////////////////////////////////////////////////
StructFileColumnIterator::StructFileColumnIterator(
@@ -1029,6 +1175,11 @@ StructFileColumnIterator::StructFileColumnIterator(
}
Status StructFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ LOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
for (auto& column_iterator : _sub_column_iterators) {
RETURN_IF_ERROR(column_iterator->init(opts));
}
@@ -1040,16 +1191,23 @@ Status StructFileColumnIterator::init(const ColumnIteratorOptions& opts) {
Status StructFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
bool* has_null) {
- const auto* column_struct = vectorized::check_and_get_column<vectorized::ColumnStruct>(
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
+ dst->resize(*n);
+ return Status::OK();
+ }
+
+ auto& column_struct = assert_cast<vectorized::ColumnStruct&>(
dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
: *dst);
- for (size_t i = 0; i < column_struct->tuple_size(); i++) {
+ for (size_t i = 0; i < column_struct.tuple_size(); i++) {
size_t num_read = *n;
- auto sub_column_ptr = column_struct->get_column(i).assume_mutable();
+ auto sub_column_ptr = column_struct.get_column(i).assume_mutable();
bool column_has_null = false;
RETURN_IF_ERROR(
_sub_column_iterators[i]->next_batch(&num_read, sub_column_ptr, &column_has_null));
DCHECK(num_read == *n);
+ column_struct.get_column_ptr(i) = std::move(sub_column_ptr);
}
if (dst->is_nullable()) {
@@ -1075,6 +1233,11 @@ Status StructFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumn
}
Status StructFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
for (auto& column_iterator : _sub_column_iterators) {
RETURN_IF_ERROR(column_iterator->seek_to_ordinal(ord));
}
@@ -1086,6 +1249,12 @@ Status StructFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
Status StructFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
vectorized::MutableColumnPtr& dst) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
+ dst->resize(count);
+ return Status::OK();
+ }
+
for (size_t i = 0; i < count; ++i) {
RETURN_IF_ERROR(seek_to_ordinal(rowids[i]));
size_t num_read = 1;
@@ -1095,6 +1264,83 @@ Status StructFileColumnIterator::read_by_rowids(const rowid_t* rowids, const siz
return Status::OK();
}
+void StructFileColumnIterator::set_need_to_read() {
+ set_reading_flag(ReadingFlag::NEED_TO_READ);
+ for (auto& sub_iterator : _sub_column_iterators) {
+ sub_iterator->set_need_to_read();
+ }
+}
+
+void StructFileColumnIterator::remove_pruned_sub_iterators() {
+ for (auto it = _sub_column_iterators.begin(); it != _sub_column_iterators.end();) {
+ auto& sub_iterator = *it;
+ if (sub_iterator->reading_flag() == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Struct column iterator remove pruned sub-column "
+ << sub_iterator->column_name();
+ it = _sub_column_iterators.erase(it);
+ } else {
+ sub_iterator->remove_pruned_sub_iterators();
+ ++it;
+ }
+ }
+}
+
+Status StructFileColumnIterator::set_access_paths(
+ const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) {
+ if (all_access_paths.empty()) {
+ return Status::OK();
+ }
+
+ if (!predicate_access_paths.empty()) {
+ set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
+ LOG(INFO) << "Struct column iterator set sub-column " << _column_name
+ << " to READING_FOR_PREDICATE";
+ }
+ auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
+ auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
+
+ const auto no_sub_column_to_skip = sub_all_access_paths.empty();
+ const auto no_predicate_sub_column = sub_predicate_access_paths.empty();
+
+ for (auto& sub_iterator : _sub_column_iterators) {
+ const auto name = sub_iterator->column_name();
+ bool need_to_read = no_sub_column_to_skip;
+ TColumnAccessPaths sub_all_access_paths_of_this;
+ if (!need_to_read) {
+ for (const auto& paths : sub_all_access_paths) {
+ if (paths.data_access_path.path[0] == name) {
+ sub_all_access_paths_of_this.emplace_back(paths);
+ }
+ }
+ need_to_read = !sub_all_access_paths_of_this.empty();
+ }
+
+ if (!need_to_read) {
+ set_reading_flag(ReadingFlag::SKIP_READING);
+ sub_iterator->set_reading_flag(ReadingFlag::SKIP_READING);
+ LOG(INFO) << "Struct column iterator set sub-column " << name << " to SKIP_READING";
+ continue;
+ }
+ set_reading_flag(ReadingFlag::NEED_TO_READ);
+ sub_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ);
+
+ TColumnAccessPaths sub_predicate_access_paths_of_this;
+
+ if (!no_predicate_sub_column) {
+ for (const auto& paths : sub_predicate_access_paths) {
+ if (paths.data_access_path.path[0] == name) {
+ sub_predicate_access_paths_of_this.emplace_back(paths);
+ }
+ }
+ }
+
+ RETURN_IF_ERROR(sub_iterator->set_access_paths(sub_all_access_paths_of_this,
+ sub_predicate_access_paths_of_this));
+ }
+ return Status::OK();
+}
+
////////////////////////////////////////////////////////////////////////////////
Status OffsetFileColumnIterator::init(const ColumnIteratorOptions& opts) {
RETURN_IF_ERROR(_offset_iterator->init(opts));
@@ -1166,6 +1412,11 @@ ArrayFileColumnIterator::ArrayFileColumnIterator(std::shared_ptr<ColumnReader> r
}
Status ArrayFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ LOG(INFO) << "Array column iterator column " << _column_name << " skip readking.";
+ return Status::OK();
+ }
+
RETURN_IF_ERROR(_offset_iterator->init(opts));
RETURN_IF_ERROR(_item_iterator->init(opts));
if (_array_reader->is_nullable()) {
@@ -1183,6 +1434,11 @@ Status ArrayFileColumnIterator::_seek_by_offsets(ordinal_t ord) {
}
Status ArrayFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
RETURN_IF_ERROR(_offset_iterator->seek_to_ordinal(ord));
if (_array_reader->is_nullable()) {
RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
@@ -1192,6 +1448,12 @@ Status ArrayFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
Status ArrayFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
bool* has_null) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading.";
+ dst->resize(*n);
+ return Status::OK();
+ }
+
const auto* column_array = vectorized::check_and_get_column<vectorized::ColumnArray>(
dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
: *dst);
@@ -1240,6 +1502,12 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnP
Status ArrayFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
vectorized::MutableColumnPtr& dst) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading.";
+ dst->resize(count);
+ return Status::OK();
+ }
+
for (size_t i = 0; i < count; ++i) {
// TODO(cambyzju): now read array one by one, need optimize later
RETURN_IF_ERROR(seek_to_ordinal(rowids[i]));
@@ -1250,11 +1518,67 @@ Status ArrayFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size
return Status::OK();
}
+void ArrayFileColumnIterator::set_need_to_read() {
+ set_reading_flag(ReadingFlag::NEED_TO_READ);
+ _item_iterator->set_need_to_read();
+}
+
+void ArrayFileColumnIterator::remove_pruned_sub_iterators() {
+ _item_iterator->remove_pruned_sub_iterators();
+}
+
+Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) {
+ if (all_access_paths.empty()) {
+ return Status::OK();
+ }
+
+ if (!predicate_access_paths.empty()) {
+ set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
+ LOG(INFO) << "Array column iterator set sub-column " << _column_name
+ << " to READING_FOR_PREDICATE";
+ }
+
+ auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
+ auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
+
+ const auto no_sub_column_to_skip = sub_all_access_paths.empty();
+ const auto no_predicate_sub_column = sub_predicate_access_paths.empty();
+
+ if (!no_sub_column_to_skip) {
+ for (auto& path : sub_all_access_paths) {
+ if (path.data_access_path.path[0] == "*") {
+ path.data_access_path.path[0] = _item_iterator->column_name();
+ }
+ }
+ }
+
+ if (!no_predicate_sub_column) {
+ for (auto& path : sub_predicate_access_paths) {
+ if (path.data_access_path.path[0] == "*") {
+ path.data_access_path.path[0] = _item_iterator->column_name();
+ }
+ }
+ }
+
+ if (!no_sub_column_to_skip || !no_predicate_sub_column) {
+ _item_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ);
+ RETURN_IF_ERROR(
+ _item_iterator->set_access_paths(sub_all_access_paths, sub_predicate_access_paths));
+ }
+ return Status::OK();
+}
+
////////////////////////////////////////////////////////////////////////////////
FileColumnIterator::FileColumnIterator(std::shared_ptr<ColumnReader> reader) : _reader(reader) {}
Status FileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ LOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
_opts = opts;
if (!_opts.use_page_cache) {
_reader->disable_index_meta_cache();
@@ -1286,6 +1610,11 @@ Status FileColumnIterator::init(const ColumnIteratorOptions& opts) {
FileColumnIterator::~FileColumnIterator() = default;
Status FileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
// if current page contains this row, we don't need to seek
if (!_page || !_page.contains(ord) || !_page_iter.valid()) {
RETURN_IF_ERROR(_reader->seek_at_or_before(ord, &_page_iter, _opts));
@@ -1336,6 +1665,12 @@ Status FileColumnIterator::next_batch_of_zone_map(size_t* n, vectorized::Mutable
Status FileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
bool* has_null) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ dst->resize(*n);
+ return Status::OK();
+ }
+
size_t curr_size = dst->byte_size();
dst->reserve(*n);
size_t remaining = *n;
@@ -1392,6 +1727,12 @@ Status FileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& d
Status FileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
vectorized::MutableColumnPtr& dst) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ dst->resize(count);
+ return Status::OK();
+ }
+
size_t remaining = count;
size_t total_read_count = 0;
size_t nrows_to_read = 0;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 2ffdb2ee699..8e96e2f3faa 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -17,6 +17,7 @@
#pragma once
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/segment_v2.pb.h>
#include <sys/types.h>
@@ -69,6 +70,8 @@ class FileReader;
struct Slice;
struct StringRef;
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
namespace segment_v2 {
class EncodingInfo;
@@ -271,7 +274,6 @@ private:
Status _calculate_row_ranges(const std::vector<uint32_t>& page_indexes, RowRanges* row_ranges,
const ColumnIteratorOptions& iter_opts);
-private:
int64_t _meta_length;
FieldType _meta_type;
FieldType _meta_children_column_type;
@@ -366,8 +368,54 @@ public:
virtual bool is_all_dict_encoding() const { return false; }
+ virtual Status set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) {
+ if (!predicate_access_paths.empty()) {
+ _reading_flag = ReadingFlag::READING_FOR_PREDICATE;
+ }
+ return Status::OK();
+ }
+
+ void set_column_name(const std::string& column_name) { _column_name = column_name; }
+
+ const std::string& column_name() const { return _column_name; }
+
+ // Since there may be multiple paths with conflicts or overlaps,
+ // we need to define several reading flags:
+ //
+ // NORMAL_READING — Default value, indicating that the column should be read.
+ // SKIP_READING — The column should not be read.
+ // NEED_TO_READ — The column must be read.
+ // READING_FOR_PREDICATE — The column is required for predicate evaluation.
+ //
+ // For example, suppose there are two paths:
+ // - Path 1 specifies that column A needs to be read, so it is marked as NEED_TO_READ.
+ // - Path 2 specifies that the column should not be read, but since it is already marked as NEED_TO_READ,
+ // it should not be changed to SKIP_READING.
+ enum class ReadingFlag : int {
+ NORMAL_READING,
+ SKIP_READING,
+ NEED_TO_READ,
+ READING_FOR_PREDICATE
+ };
+ void set_reading_flag(ReadingFlag flag) {
+ if (static_cast<int>(flag) > static_cast<int>(_reading_flag)) {
+ _reading_flag = flag;
+ }
+ }
+
+ ReadingFlag reading_flag() const { return _reading_flag; }
+
+ virtual void set_need_to_read() { set_reading_flag(ReadingFlag::NEED_TO_READ); }
+
+ virtual void remove_pruned_sub_iterators() {};
+
protected:
+ Result<TColumnAccessPaths> _get_sub_access_paths(const TColumnAccessPaths& access_paths);
ColumnIteratorOptions _opts;
+
+ ReadingFlag _reading_flag {ReadingFlag::NORMAL_READING};
+ std::string _column_name;
};
// This iterator is used to read column data from file
@@ -504,6 +552,13 @@ public:
return _offsets_iterator->get_current_ordinal();
}
+ Status set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) override;
+
+ void set_need_to_read() override;
+
+ void remove_pruned_sub_iterators() override;
+
private:
std::shared_ptr<ColumnReader> _map_reader = nullptr;
ColumnIteratorUPtr _null_iterator;
@@ -533,6 +588,13 @@ public:
return _sub_column_iterators[0]->get_current_ordinal();
}
+ Status set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) override;
+
+ void set_need_to_read() override;
+
+ void remove_pruned_sub_iterators() override;
+
private:
std::shared_ptr<ColumnReader> _struct_reader = nullptr;
ColumnIteratorUPtr _null_iterator;
@@ -561,6 +623,12 @@ public:
return _offset_iterator->get_current_ordinal();
}
+ Status set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) override;
+ void set_need_to_read() override;
+
+ void remove_pruned_sub_iterators() override;
+
private:
std::shared_ptr<ColumnReader> _array_reader = nullptr;
std::unique_ptr<OffsetFileColumnIterator> _offset_iterator;
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index d95d049c7cf..211ba10bbbb 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -17,12 +17,14 @@
#include "olap/rowset/segment_v2/segment.h"
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/PlanNodes_types.h>
#include <gen_cpp/olap_file.pb.h>
#include <gen_cpp/segment_v2.pb.h>
#include <cstring>
#include <memory>
+#include <sstream>
#include <utility>
#include "cloud/config.h"
@@ -765,6 +767,20 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column,
sparse_column_cache_ptr));
} else {
RETURN_IF_ERROR(reader->new_iterator(iter, &tablet_column, opt));
+ if (opt->all_access_paths.contains(unique_id) ||
+ opt->predicate_access_paths.contains(unique_id)) {
+ const auto& all_access_paths = opt->all_access_paths.contains(unique_id)
+ ? opt->all_access_paths.at(unique_id)
+ : TColumnAccessPaths {};
+ const auto& predicate_access_paths = opt->predicate_access_paths.contains(unique_id)
+ ? opt->predicate_access_paths.at(unique_id)
+ : TColumnAccessPaths {};
+
+ // set column name to apply access paths.
+ (*iter)->set_column_name(tablet_column.name());
+ RETURN_IF_ERROR((*iter)->set_access_paths(all_access_paths, predicate_access_paths));
+ (*iter)->remove_pruned_sub_iterators();
+ }
}
if (config::enable_column_type_check && !tablet_column.has_path_info() &&
diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
index 3660e709e22..3dc120ec4c4 100644
--- a/be/src/olap/tablet_reader.cpp
+++ b/be/src/olap/tablet_reader.cpp
@@ -263,6 +263,9 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) {
_reader_context.ann_topn_runtime = read_params.ann_topn_runtime;
_reader_context.condition_cache_digest = read_params.condition_cache_digest;
+ _reader_context.all_access_paths = read_params.all_access_paths;
+ _reader_context.predicate_access_paths = read_params.predicate_access_paths;
+
return Status::OK();
}
diff --git a/be/src/olap/tablet_reader.h b/be/src/olap/tablet_reader.h
index 8630c932052..593afa9dbe5 100644
--- a/be/src/olap/tablet_reader.h
+++ b/be/src/olap/tablet_reader.h
@@ -17,6 +17,7 @@
#pragma once
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/PaloInternalService_types.h>
#include <gen_cpp/PlanNodes_types.h>
#include <stddef.h>
@@ -146,6 +147,9 @@ public:
// slots that cast may be eliminated in storage layer
std::map<std::string, vectorized::DataTypePtr> target_cast_type_for_variants;
+ std::map<int32_t, TColumnAccessPaths> all_access_paths;
+ std::map<int32_t, TColumnAccessPaths> predicate_access_paths;
+
std::vector<RowSetSplits> rs_splits;
// For unique key table with merge-on-write
DeleteBitmapPtr delete_bitmap = nullptr;
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 079cf9278ff..cee5f53ae6b 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -1684,6 +1684,11 @@ vectorized::Block TabletSchema::create_block(
tablet_columns_need_convert_null->find(cid) !=
tablet_columns_need_convert_null->end());
auto data_type = vectorized::DataTypeFactory::instance().create_data_type(col, is_nullable);
+ if (col.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ if (_pruned_columns_data_type.contains(col.unique_id())) {
+ data_type = _pruned_columns_data_type.at(col.unique_id());
+ }
+ }
if (_vir_col_idx_to_unique_id.contains(cid)) {
block.insert({vectorized::ColumnNothing::create(0), data_type, col.name()});
VLOG_DEBUG << fmt::format(
@@ -1703,7 +1708,13 @@ vectorized::Block TabletSchema::create_block(bool ignore_dropped_col) const {
if (ignore_dropped_col && is_dropped_column(*col)) {
continue;
}
+
auto data_type = vectorized::DataTypeFactory::instance().create_data_type(*col);
+ if (col->type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ if (_pruned_columns_data_type.contains(col->unique_id())) {
+ data_type = _pruned_columns_data_type.at(col->unique_id());
+ }
+ }
block.insert({data_type->create_column(), data_type, col->name()});
}
return block;
@@ -1714,6 +1725,11 @@ vectorized::Block TabletSchema::create_block_by_cids(const std::vector<uint32_t>
for (const auto& cid : cids) {
const auto& col = *_cols[cid];
auto data_type = vectorized::DataTypeFactory::instance().create_data_type(col);
+ if (col.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ if (_pruned_columns_data_type.contains(col.unique_id())) {
+ data_type = _pruned_columns_data_type.at(col.unique_id());
+ }
+ }
block.insert({data_type->create_column(), data_type, col.name()});
}
return block;
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 57463316cdf..9382a68c3a1 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -42,6 +42,7 @@
#include "runtime/define_primitive_type.h"
#include "runtime/descriptors.h"
#include "runtime/memory/lru_cache_policy.h"
+#include "udf/udf.h"
#include "util/debug_points.h"
#include "util/string_parser.hpp"
#include "util/string_util.h"
@@ -681,6 +682,14 @@ public:
return 0;
}
+ void add_pruned_columns_data_type(int32_t col_unique_id, vectorized::DataTypePtr data_type) {
+ _pruned_columns_data_type[col_unique_id] = std::move(data_type);
+ }
+
+ void clear_pruned_columns_data_type() { _pruned_columns_data_type.clear(); }
+
+ bool has_pruned_columns() const { return !_pruned_columns_data_type.empty(); }
+
private:
friend bool operator==(const TabletSchema& a, const TabletSchema& b);
friend bool operator!=(const TabletSchema& a, const TabletSchema& b);
@@ -751,6 +760,7 @@ private:
bool _enable_variant_flatten_nested = false;
std::map<size_t, int32_t> _vir_col_idx_to_unique_id;
+ std::map<int32_t, vectorized::DataTypePtr> _pruned_columns_data_type;
// value: extracted path set and sparse path set
std::unordered_map<int32_t, PathsSetInfo> _path_set_info_map;
diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp
index 7ccc69ed9ac..a17f0926752 100644
--- a/be/src/runtime/descriptors.cpp
+++ b/be/src/runtime/descriptors.cpp
@@ -63,6 +63,11 @@ SlotDescriptor::SlotDescriptor(const TSlotDescriptor& tdesc)
_is_materialized(tdesc.isMaterialized && tdesc.need_materialize),
_is_key(tdesc.is_key),
_column_paths(tdesc.column_paths),
+ _all_access_paths(tdesc.__isset.all_access_paths ? tdesc.all_access_paths
+ : TColumnAccessPaths {}),
+ _predicate_access_paths(tdesc.__isset.predicate_access_paths
+ ? tdesc.predicate_access_paths
+ : TColumnAccessPaths {}),
_is_auto_increment(tdesc.__isset.is_auto_increment ? tdesc.is_auto_increment : false),
_col_default_value(tdesc.__isset.col_default_value ? tdesc.col_default_value : "") {
if (tdesc.__isset.virtual_column_expr) {
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index 0d86844b226..c66ccd80b1d 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -53,6 +53,8 @@ class ObjectPool;
class PTupleDescriptor;
class PSlotDescriptor;
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
class SlotDescriptor {
public:
MOCK_DEFINE(virtual ~SlotDescriptor() = default;)
@@ -82,6 +84,9 @@ public:
bool is_key() const { return _is_key; }
const std::vector<std::string>& column_paths() const { return _column_paths; };
+ const TColumnAccessPaths& all_access_paths() const { return _all_access_paths; }
+ const TColumnAccessPaths& predicate_access_paths() const { return _predicate_access_paths; }
+
bool is_auto_increment() const { return _is_auto_increment; }
bool is_skip_bitmap_col() const { return _col_name == SKIP_BITMAP_COL; }
@@ -128,6 +133,9 @@ private:
const bool _is_key;
const std::vector<std::string> _column_paths;
+ const TColumnAccessPaths _all_access_paths;
+ const TColumnAccessPaths _predicate_access_paths;
+
const bool _is_auto_increment;
const std::string _col_default_value;
diff --git a/be/src/vec/columns/column_map.cpp b/be/src/vec/columns/column_map.cpp
index 8317aa5bb5e..7cd6bfe9f4a 100644
--- a/be/src/vec/columns/column_map.cpp
+++ b/be/src/vec/columns/column_map.cpp
@@ -59,6 +59,7 @@ ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values, Mutable
/// This will also prevent possible overflow in offset.
if (keys_column->size() != last_offset) {
+ DCHECK(0);
throw doris::Exception(
doris::ErrorCode::INTERNAL_ERROR,
"offsets_column size {} has data inconsistent with key_column {}", last_offset,
diff --git a/be/src/vec/exec/scan/olap_scanner.cpp b/be/src/vec/exec/scan/olap_scanner.cpp
index b1b67e8300d..51fc4daa669 100644
--- a/be/src/vec/exec/scan/olap_scanner.cpp
+++ b/be/src/vec/exec/scan/olap_scanner.cpp
@@ -83,6 +83,8 @@ OlapScanner::OlapScanner(pipeline::ScanLocalStateBase* parent, OlapScanner::Para
.function_filters {},
.delete_predicates {},
.target_cast_type_for_variants {},
+ .all_access_paths {},
+ .predicate_access_paths {},
.rs_splits {},
.return_columns {},
.output_columns {},
@@ -265,7 +267,7 @@ Status OlapScanner::prepare() {
// Add newly created tablet schema to schema cache if it does not have virtual columns.
if (cached_schema == nullptr && !schema_key.empty() &&
- tablet_schema->num_virtual_columns() == 0) {
+ tablet_schema->num_virtual_columns() == 0 && !tablet_schema->has_pruned_columns()) {
SchemaCache::instance()->insert_schema(schema_key, tablet_schema);
}
@@ -543,6 +545,22 @@ Status OlapScanner::_init_return_columns() {
_vir_col_idx_to_type[idx_in_block]->get_name());
}
+ const auto& column = tablet_schema->column(index);
+ if (!slot->all_access_paths().empty()) {
+ _tablet_reader_params.all_access_paths.insert(
+ {column.unique_id(), slot->all_access_paths()});
+ }
+
+ if (!slot->predicate_access_paths().empty()) {
+ _tablet_reader_params.predicate_access_paths.insert(
+ {column.unique_id(), slot->predicate_access_paths()});
+ }
+
+ if (slot->type()->get_primitive_type() == PrimitiveType::TYPE_STRUCT &&
+ !slot->all_access_paths().empty()) {
+ tablet_schema->add_pruned_columns_data_type(column.unique_id(), slot->type());
+ }
+
_return_columns.push_back(index);
if (slot->is_nullable() && !tablet_schema->column(index).is_nullable()) {
_tablet_columns_convert_to_null_set.emplace(index);
diff --git a/be/test/olap/rowset/segment_v2/column_reader_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_test.cpp
new file mode 100644
index 00000000000..98478df7016
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/column_reader_test.cpp
@@ -0,0 +1,220 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include "olap/rowset/segment_v2/column_reader.h"
+
+#include <gen_cpp/Descriptors_types.h>
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <chrono>
+#include <memory>
+#include <thread>
+#include <vector>
+
+#include "agent/be_exec_version_manager.h"
+#include "common/config.h"
+#include "gen_cpp/olap_file.pb.h"
+#include "gen_cpp/segment_v2.pb.h"
+#include "io/fs/file_reader.h"
+#include "mock/mock_segment.h"
+#include "olap/rowset/segment_v2/column_reader_cache.h"
+#include "olap/rowset/segment_v2/segment.h"
+#include "olap/rowset/segment_v2/variant/variant_column_reader.h"
+#include "olap/tablet_schema.h"
+#include "vec/json/path_in_data.h"
+
+namespace doris::segment_v2 {
+class ColumnReaderTest : public ::testing::Test {
+protected:
+ void SetUp() override {}
+ void TearDown() override {}
+};
+
+TEST_F(ColumnReaderTest, StructAccessPaths) {
+ auto create_struct_iterator = []() {
+ auto null_reader = std::make_shared<ColumnReader>();
+ auto null_iterator = std::make_unique<FileColumnIterator>(null_reader);
+
+ std::vector<ColumnIteratorUPtr> sub_column_iterators;
+ auto sub_reader1 = std::make_shared<ColumnReader>();
+ auto sub_iterator1 = std::make_unique<FileColumnIterator>(sub_reader1);
+ sub_iterator1->set_column_name("sub_col_1");
+ auto sub_reader2 = std::make_shared<ColumnReader>();
+ auto sub_iterator2 = std::make_unique<FileColumnIterator>(sub_reader2);
+ sub_iterator2->set_column_name("sub_col_2");
+
+ sub_column_iterators.emplace_back(std::move(sub_iterator1));
+ sub_column_iterators.emplace_back(std::move(sub_iterator2));
+ auto iterator = std::make_unique<StructFileColumnIterator>(std::make_shared<ColumnReader>(),
+ std::move(null_iterator),
+ std::move(sub_column_iterators));
+ return iterator;
+ };
+
+ auto iterator = create_struct_iterator();
+ auto st = iterator->set_access_paths(TColumnAccessPaths {}, TColumnAccessPaths {});
+
+ ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string();
+ ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::NORMAL_READING);
+
+ TColumnAccessPaths all_access_paths;
+ all_access_paths.emplace_back();
+
+ TColumnAccessPaths predicate_access_paths;
+ predicate_access_paths.emplace_back();
+
+ st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+ // empty paths leads to error
+ ASSERT_FALSE(st.ok());
+
+ // Only reading sub_col_1
+ // sub_col_2 should be set to SKIP_READING
+ all_access_paths[0].data_access_path.path = {"self", "sub_col_1"};
+
+ predicate_access_paths[0].data_access_path.path = {"self", "sub_col_1"};
+
+ st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+ // invalid name leads to error
+ ASSERT_FALSE(st.ok());
+
+ iterator->set_column_name("self");
+ // now column name is "self", should be ok
+ st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+ ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string();
+ ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE);
+
+ ASSERT_EQ(iterator->_sub_column_iterators[0]->_reading_flag,
+ ColumnIterator::ReadingFlag::READING_FOR_PREDICATE);
+ ASSERT_EQ(iterator->_sub_column_iterators[1]->_reading_flag,
+ ColumnIterator::ReadingFlag::SKIP_READING);
+
+ // Reading all sub columns
+ all_access_paths[0].data_access_path.path = {"self"};
+ iterator = create_struct_iterator();
+ iterator->set_column_name("self");
+ st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+
+ ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string();
+ ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE);
+
+ ASSERT_EQ(iterator->_sub_column_iterators[0]->_reading_flag,
+ ColumnIterator::ReadingFlag::READING_FOR_PREDICATE);
+ ASSERT_EQ(iterator->_sub_column_iterators[1]->_reading_flag,
+ ColumnIterator::ReadingFlag::NEED_TO_READ);
+}
+
+TEST_F(ColumnReaderTest, MultiAccessPaths) {
+ auto create_struct_iterator = []() {
+ auto null_reader = std::make_shared<ColumnReader>();
+ auto null_iterator = std::make_unique<FileColumnIterator>(null_reader);
+
+ std::vector<ColumnIteratorUPtr> sub_column_iterators;
+ auto sub_reader1 = std::make_shared<ColumnReader>();
+ auto sub_iterator1 = std::make_unique<FileColumnIterator>(sub_reader1);
+ sub_iterator1->set_column_name("sub_col_1");
+ auto sub_reader2 = std::make_shared<ColumnReader>();
+ auto sub_iterator2 = std::make_unique<FileColumnIterator>(sub_reader2);
+ sub_iterator2->set_column_name("sub_col_2");
+
+ sub_column_iterators.emplace_back(std::move(sub_iterator1));
+ sub_column_iterators.emplace_back(std::move(sub_iterator2));
+ auto iterator = std::make_unique<StructFileColumnIterator>(std::make_shared<ColumnReader>(),
+ std::move(null_iterator),
+ std::move(sub_column_iterators));
+ return iterator;
+ };
+
+ auto create_struct_iterator2 = [](ColumnIteratorUPtr&& nested_iterator) {
+ auto null_reader = std::make_shared<ColumnReader>();
+ auto null_iterator = std::make_unique<FileColumnIterator>(null_reader);
+
+ std::vector<ColumnIteratorUPtr> sub_column_iterators;
+ auto sub_reader1 = std::make_shared<ColumnReader>();
+ auto sub_iterator1 = std::make_unique<FileColumnIterator>(sub_reader1);
+ sub_iterator1->set_column_name("sub_col_1");
+
+ sub_column_iterators.emplace_back(std::move(sub_iterator1));
+ sub_column_iterators.emplace_back(std::move(nested_iterator));
+ auto iterator = std::make_unique<StructFileColumnIterator>(std::make_shared<ColumnReader>(),
+ std::move(null_iterator),
+ std::move(sub_column_iterators));
+ return iterator;
+ };
+
+ auto struct_iterator = create_struct_iterator();
+ struct_iterator->set_column_name("struct");
+
+ auto map_iterator = std::make_unique<MapFileColumnIterator>(
+ std::make_shared<ColumnReader>(),
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>()), // null iterator
+ std::make_unique<OffsetFileColumnIterator>(
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>())),
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>()),
+ std::move(struct_iterator));
+
+ auto array_iterator = std::make_unique<ArrayFileColumnIterator>(
+ std::make_shared<ColumnReader>(),
+ std::make_unique<OffsetFileColumnIterator>(
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>())),
+ std::move(map_iterator),
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>()));
+
+ // here create:
+ // struct<
+ // sub_col_1,
+ // sub_col_2: array<
+ // map<
+ // key,
+ // value: struct<
+ // sub_col_1,
+ // sub_col_2
+ // >
+ // >
+ // >
+ // >
+ array_iterator->set_column_name("sub_col_2");
+ auto iterator = create_struct_iterator2(std::move(array_iterator));
+ TColumnAccessPaths all_access_paths;
+ all_access_paths.emplace_back();
+
+ // all access paths:
+ // self.sub_col_2.*.KEYS
+ // predicates paths empty
+ all_access_paths[0].data_access_path.path = {"self", "sub_col_2", "*", "KEYS"};
+
+ TColumnAccessPaths predicate_access_paths;
+
+ iterator->set_column_name("self");
+ auto st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+
+ ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string();
+ ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ);
+
+ ASSERT_EQ(iterator->_sub_column_iterators[0]->_reading_flag,
+ ColumnIterator::ReadingFlag::SKIP_READING);
+ ASSERT_EQ(iterator->_sub_column_iterators[1]->_reading_flag,
+ ColumnIterator::ReadingFlag::NEED_TO_READ);
+
+ auto* array_iter =
+ static_cast<ArrayFileColumnIterator*>(iterator->_sub_column_iterators[1].get());
+ ASSERT_EQ(array_iter->_item_iterator->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ);
+
+ auto* map_iter = static_cast<MapFileColumnIterator*>(array_iter->_item_iterator.get());
+ ASSERT_EQ(map_iter->_key_iterator->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ);
+ ASSERT_EQ(map_iter->_val_iterator->_reading_flag, ColumnIterator::ReadingFlag::SKIP_READING);
+}
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AccessPathInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AccessPathInfo.java
new file mode 100644
index 00000000000..3df7baada4d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AccessPathInfo.java
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/TupleDescriptor.java
+// and modified by Doris
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.thrift.TColumnAccessPath;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.util.List;
+
+/** AccessPathInfo */
+@Data
+@AllArgsConstructor
+public class AccessPathInfo {
+ private DataType prunedType;
+ private List<TColumnAccessPath> allAccessPaths;
+ private List<TColumnAccessPath> predicateAccessPaths;
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
index 87843f721c9..c0b35bd8f96 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ColumnStats;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Type;
+import org.apache.doris.thrift.TColumnAccessPath;
import org.apache.doris.thrift.TSlotDescriptor;
import com.google.common.base.MoreObjects;
@@ -78,6 +79,10 @@ public class SlotDescriptor {
private boolean needMaterialize = true;
private boolean isAutoInc = false;
private Expr virtualColumn = null;
+ private List<TColumnAccessPath> allAccessPaths;
+ private List<TColumnAccessPath> predicateAccessPaths;
+ private List<TColumnAccessPath> displayAllAccessPaths;
+ private List<TColumnAccessPath> displayPredicateAccessPaths;
public SlotDescriptor(SlotId id, TupleDescriptor parent) {
@@ -132,6 +137,38 @@ public class SlotDescriptor {
return this.subColPath;
}
+ public List<TColumnAccessPath> getAllAccessPaths() {
+ return allAccessPaths;
+ }
+
+ public void setAllAccessPaths(List<TColumnAccessPath> allAccessPaths) {
+ this.allAccessPaths = allAccessPaths;
+ }
+
+ public List<TColumnAccessPath> getPredicateAccessPaths() {
+ return predicateAccessPaths;
+ }
+
+ public void setPredicateAccessPaths(List<TColumnAccessPath> predicateAccessPaths) {
+ this.predicateAccessPaths = predicateAccessPaths;
+ }
+
+ public List<TColumnAccessPath> getDisplayAllAccessPaths() {
+ return displayAllAccessPaths;
+ }
+
+ public void setDisplayAllAccessPaths(List<TColumnAccessPath> displayAllAccessPaths) {
+ this.displayAllAccessPaths = displayAllAccessPaths;
+ }
+
+ public List<TColumnAccessPath> getDisplayPredicateAccessPaths() {
+ return displayPredicateAccessPaths;
+ }
+
+ public void setDisplayPredicateAccessPaths(List<TColumnAccessPath> displayPredicateAccessPaths) {
+ this.displayPredicateAccessPaths = displayPredicateAccessPaths;
+ }
+
public TupleDescriptor getParent() {
return parent;
}
@@ -339,6 +376,12 @@ public class SlotDescriptor {
if (virtualColumn != null) {
tSlotDescriptor.setVirtualColumnExpr(virtualColumn.treeToThrift());
}
+ if (allAccessPaths != null) {
+ tSlotDescriptor.setAllAccessPaths(allAccessPaths);
+ }
+ if (predicateAccessPaths != null) {
+ tSlotDescriptor.setPredicateAccessPaths(predicateAccessPaths);
+ }
return tSlotDescriptor;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
index 8578b094897..55564d89f04 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
@@ -52,6 +52,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -185,6 +186,8 @@ public abstract class FileScanNode extends ExternalScanNode {
}
output.append(String.format("numNodes=%s", numNodes)).append("\n");
+ printNestedColumns(output, prefix);
+
// pushdown agg
output.append(prefix).append(String.format("pushdown agg=%s", pushDownAggNoGroupingOp));
if (pushDownAggNoGroupingOp.equals(TPushAggOp.COUNT)) {
@@ -210,7 +213,10 @@ public abstract class FileScanNode extends ExternalScanNode {
TExpr tExpr = new TExpr();
tExpr.setNodes(Lists.newArrayList());
+ int index = 0;
+ List<SlotDescriptor> slots = desc.getSlots();
for (Column column : getColumns()) {
+ SlotDescriptor slotDescriptor = slots.get(index++);
Expr expr;
Expression expression;
if (column.getDefaultValue() != null) {
@@ -225,7 +231,9 @@ public abstract class FileScanNode extends ExternalScanNode {
if (useVarcharAsNull) {
expression = new NullLiteral(VarcharType.SYSTEM_DEFAULT);
} else {
- expression = new NullLiteral(DataType.fromCatalogType(column.getType()));
+ // the nested type(map/array/struct) maybe pruned,
+ // we should use the pruned type to avoid be core
+ expression = new NullLiteral(DataType.fromCatalogType(slotDescriptor.getType()));
}
} else {
expression = null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 0a841e14dad..38fa6aedb91 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -286,6 +286,8 @@ public class StatementContext implements Closeable {
private Optional<Map<TableIf, Set<Expression>>> mvRefreshPredicates = Optional.empty();
+ private boolean hasNestedColumns;
+
public StatementContext() {
this(ConnectContext.get(), null, 0);
}
@@ -1026,4 +1028,12 @@ public class StatementContext implements Closeable {
public void setSkipPrunePredicate(boolean skipPrunePredicate) {
this.skipPrunePredicate = skipPrunePredicate;
}
+
+ public boolean hasNestedColumns() {
+ return hasNestedColumns;
+ }
+
+ public void setHasNestedColumns(boolean hasNestedColumns) {
+ this.hasNestedColumns = hasNestedColumns;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 44186fe3711..6b14de78e00 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -837,10 +837,11 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
OlapTable olapTable = olapScan.getTable();
// generate real output tuple
TupleDescriptor tupleDescriptor = generateTupleDesc(slots, olapTable, context);
+ List<SlotDescriptor> slotDescriptors = tupleDescriptor.getSlots();
// put virtual column expr into slot desc
Map<ExprId, Expression> slotToVirtualColumnMap = olapScan.getSlotToVirtualColumnMap();
- for (SlotDescriptor slotDescriptor : tupleDescriptor.getSlots()) {
+ for (SlotDescriptor slotDescriptor : slotDescriptors) {
ExprId exprId = context.findExprId(slotDescriptor.getId());
if (slotToVirtualColumnMap.containsKey(exprId)) {
slotDescriptor.setVirtualColumn(ExpressionTranslator.translate(
@@ -910,7 +911,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
&& !isComplexDataType(slot.getDataType())
&& !StatisticConstants.isSystemTable(olapTable)
&& !inVisibleCol) {
- context.addUnknownStatsColumn(olapScanNode, tupleDescriptor.getSlots().get(i).getId());
+ context.addUnknownStatsColumn(olapScanNode, slotDescriptors.get(i).getId());
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
index 7553a81d5b0..cbbd425c5b2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
@@ -29,6 +29,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.processor.post.TopnFilterContext;
import org.apache.doris.nereids.processor.post.runtimefilterv2.RuntimeFilterContextV2;
import org.apache.doris.nereids.trees.expressions.CTEId;
@@ -67,6 +68,7 @@ import javax.annotation.Nullable;
*/
public class PlanTranslatorContext {
private final ConnectContext connectContext;
+ private final StatementContext statementContext;
private final List<PlanFragment> planFragments = Lists.newArrayList();
private DescriptorTable descTable;
@@ -119,16 +121,20 @@ public class PlanTranslatorContext {
private final Set<SlotId> virtualColumnIds = Sets.newHashSet();
+ /** PlanTranslatorContext */
public PlanTranslatorContext(CascadesContext ctx) {
this.connectContext = ctx.getConnectContext();
+ this.statementContext = ctx.getStatementContext();
this.translator = new RuntimeFilterTranslator(ctx.getRuntimeFilterContext());
this.topnFilterContext = ctx.getTopnFilterContext();
this.runtimeFilterV2Context = ctx.getRuntimeFilterV2Context();
this.descTable = new DescriptorTable();
}
+ /** PlanTranslatorContext */
public PlanTranslatorContext(CascadesContext ctx, DescriptorTable descTable) {
this.connectContext = ctx.getConnectContext();
+ this.statementContext = ctx.getStatementContext();
this.translator = new RuntimeFilterTranslator(ctx.getRuntimeFilterContext());
this.topnFilterContext = ctx.getTopnFilterContext();
this.runtimeFilterV2Context = ctx.getRuntimeFilterV2Context();
@@ -141,6 +147,7 @@ public class PlanTranslatorContext {
@VisibleForTesting
public PlanTranslatorContext() {
this.connectContext = null;
+ this.statementContext = new StatementContext();
this.translator = null;
this.topnFilterContext = new TopnFilterContext();
IdGenerator<RuntimeFilterId> runtimeFilterIdGen = RuntimeFilterId.createGenerator();
@@ -180,6 +187,10 @@ public class PlanTranslatorContext {
return connectContext;
}
+ public StatementContext getStatementContext() {
+ return statementContext;
+ }
+
public Set<ScanNode> getScanNodeWithUnknownColumnStats() {
return statsUnknownColumnsMap.keySet();
}
@@ -329,6 +340,13 @@ public class PlanTranslatorContext {
}
this.addExprIdSlotRefPair(slotReference.getExprId(), slotRef);
slotDescriptor.setIsNullable(slotReference.nullable());
+
+ if (slotReference.getAllAccessPaths().isPresent()) {
+ slotDescriptor.setAllAccessPaths(slotReference.getAllAccessPaths().get());
+ slotDescriptor.setPredicateAccessPaths(slotReference.getPredicateAccessPaths().get());
+ slotDescriptor.setDisplayAllAccessPaths(slotReference.getDisplayAllAccessPaths().get());
+ slotDescriptor.setDisplayPredicateAccessPaths(slotReference.getDisplayPredicateAccessPaths().get());
+ }
return slotDescriptor;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 29f0b0c79a8..07339e6dd63 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -109,6 +109,7 @@ import org.apache.doris.nereids.rules.rewrite.MergeProjectable;
import org.apache.doris.nereids.rules.rewrite.MergeSetOperations;
import org.apache.doris.nereids.rules.rewrite.MergeSetOperationsExcept;
import org.apache.doris.nereids.rules.rewrite.MergeTopNs;
+import org.apache.doris.nereids.rules.rewrite.NestedColumnPruning;
import org.apache.doris.nereids.rules.rewrite.NormalizeSort;
import org.apache.doris.nereids.rules.rewrite.OperativeColumnDerive;
import org.apache.doris.nereids.rules.rewrite.OrExpansion;
@@ -913,6 +914,11 @@ public class Rewriter extends AbstractBatchJobExecutor {
)
));
}
+ rewriteJobs.add(
+ topic("nested column prune",
+ custom(RuleType.NESTED_COLUMN_PRUNING, NestedColumnPruning::new)
+ )
+ );
rewriteJobs.addAll(jobs(
topic("rewrite cte sub-tree after sub path push down",
custom(RuleType.CLEAR_CONTEXT_STATUS, ClearContextStatus::new),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 7e3e710eee2..a2268581a47 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -115,6 +115,7 @@ import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughSort;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughWindow;
import org.apache.doris.nereids.rules.rewrite.PushDownJoinOtherCondition;
import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughJoin;
+import org.apache.doris.nereids.rules.rewrite.PushDownNestedColumn;
import org.apache.doris.nereids.rules.rewrite.PushDownProjectThroughLimit;
import org.apache.doris.nereids.rules.rewrite.PushDownTopNDistinctThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughJoin;
@@ -166,6 +167,7 @@ public class RuleSet {
new PushDownFilterThroughSetOperation(),
new PushDownFilterThroughGenerate(),
new PushDownProjectThroughLimit(),
+ new PushDownNestedColumn(),
new EliminateOuterJoin(),
new MergeProjectable(),
new MergeFilters(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 7b32bb710e0..70bedd1b85b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -195,6 +195,12 @@ public enum RuleType {
PUSH_DOWN_PREDICATE_THROUGH_AGGREGATION(RuleTypeClass.REWRITE),
PUSH_DOWN_PREDICATE_THROUGH_REPEAT(RuleTypeClass.REWRITE),
PUSH_DOWN_EXPRESSIONS_IN_HASH_CONDITIONS(RuleTypeClass.REWRITE),
+ // Pushdown nested column project
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_JOIN(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_UNION(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_WINDOW(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_PARTITION_TOP_N(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_DEFER_MATERIALIZE_TOP_N(RuleTypeClass.REWRITE),
// Pushdown filter
PUSH_DOWN_FILTER_THROUGH_JOIN(RuleTypeClass.REWRITE),
PUSH_DOWN_FILTER_THROUGH_LEFT_SEMI_JOIN(RuleTypeClass.REWRITE),
@@ -219,6 +225,7 @@ public enum RuleType {
ADD_PROJECT_FOR_JOIN(RuleTypeClass.REWRITE),
ADD_PROJECT_FOR_UNIQUE_FUNCTION(RuleTypeClass.REWRITE),
VARIANT_SUB_PATH_PRUNING(RuleTypeClass.REWRITE),
+ NESTED_COLUMN_PRUNING(RuleTypeClass.REWRITE),
CLEAR_CONTEXT_STATUS(RuleTypeClass.REWRITE),
COLUMN_PRUNING(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index 84652b9eed0..36b7f0064ef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -424,7 +424,8 @@ public class BindRelation extends OneAnalysisRuleFactory {
if (hmsTable.getDlaType() == DLAType.HUDI) {
LogicalHudiScan hudiScan = new LogicalHudiScan(unboundRelation.getRelationId(), hmsTable,
qualifierWithoutTableName, ImmutableList.of(), Optional.empty(),
- unboundRelation.getTableSample(), unboundRelation.getTableSnapshot());
+ unboundRelation.getTableSample(), unboundRelation.getTableSnapshot(),
+ Optional.empty());
hudiScan = hudiScan.withScanParams(
hmsTable, Optional.ofNullable(unboundRelation.getScanParams()));
return hudiScan;
@@ -434,7 +435,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
ImmutableList.of(),
unboundRelation.getTableSample(),
unboundRelation.getTableSnapshot(),
- Optional.ofNullable(unboundRelation.getScanParams()));
+ Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty());
}
case ICEBERG_EXTERNAL_TABLE:
IcebergExternalTable icebergExternalTable = (IcebergExternalTable) table;
@@ -464,7 +465,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
qualifierWithoutTableName, ImmutableList.of(),
unboundRelation.getTableSample(),
unboundRelation.getTableSnapshot(),
- Optional.ofNullable(unboundRelation.getScanParams()));
+ Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty());
case PAIMON_EXTERNAL_TABLE:
case MAX_COMPUTE_EXTERNAL_TABLE:
case TRINO_CONNECTOR_EXTERNAL_TABLE:
@@ -473,7 +474,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
qualifierWithoutTableName, ImmutableList.of(),
unboundRelation.getTableSample(),
unboundRelation.getTableSnapshot(),
- Optional.ofNullable(unboundRelation.getScanParams()));
+ Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty());
case SCHEMA:
LogicalSchemaScan schemaScan = new LogicalSchemaScan(unboundRelation.getRelationId(), table,
qualifierWithoutTableName);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
index 35adb692be3..b90dc3adc81 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
@@ -93,6 +93,7 @@ import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.NestedColumnPrunable;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.TinyIntType;
import org.apache.doris.nereids.util.ExpressionUtils;
@@ -285,6 +286,9 @@ public class ExpressionAnalyzer extends SubExprAnalyzer<ExpressionRewriteContext
}
outerScope.get().getCorrelatedSlots().add((Slot) firstBound);
}
+ if (firstBound.getDataType() instanceof NestedColumnPrunable) {
+ context.cascadesContext.getStatementContext().setHasNestedColumns(true);
+ }
return firstBound;
default:
if (enableExactMatch) {
@@ -304,6 +308,9 @@ public class ExpressionAnalyzer extends SubExprAnalyzer<ExpressionRewriteContext
.filter(bound -> unboundSlot.getNameParts().size() == bound.getQualifier().size() + 1)
.collect(Collectors.toList());
if (exactMatch.size() == 1) {
+ if (exactMatch.get(0).getDataType() instanceof NestedColumnPrunable) {
+ context.cascadesContext.getStatementContext().setHasNestedColumns(true);
+ }
return exactMatch.get(0);
}
}
@@ -340,6 +347,9 @@ public class ExpressionAnalyzer extends SubExprAnalyzer<ExpressionRewriteContext
if (!(slot instanceof SlotReference) || (((SlotReference) slot).isVisible()) || showHidden) {
showSlots.add(slot);
}
+ if (slot.getDataType() instanceof NestedColumnPrunable) {
+ context.cascadesContext.getStatementContext().setHasNestedColumns(true);
+ }
}
ImmutableList<Slot> slots = showSlots.build();
switch (qualifier.size()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
index 2c957912ce9..c4302798e25 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
@@ -29,6 +29,7 @@ import org.apache.doris.nereids.rules.expression.rules.LogToLn;
import org.apache.doris.nereids.rules.expression.rules.MedianConvert;
import org.apache.doris.nereids.rules.expression.rules.MergeDateTrunc;
import org.apache.doris.nereids.rules.expression.rules.NormalizeBinaryPredicatesRule;
+import org.apache.doris.nereids.rules.expression.rules.NormalizeStructElement;
import org.apache.doris.nereids.rules.expression.rules.SimplifyArithmeticComparisonRule;
import org.apache.doris.nereids.rules.expression.rules.SimplifyArithmeticRule;
import org.apache.doris.nereids.rules.expression.rules.SimplifyCastRule;
@@ -68,6 +69,7 @@ public class ExpressionNormalization extends ExpressionRewrite {
SimplifyArithmeticComparisonRule.INSTANCE,
ConvertAggStateCast.INSTANCE,
MergeDateTrunc.INSTANCE,
+ NormalizeStructElement.INSTANCE,
CheckCast.INSTANCE,
SimplifyEqualBooleanLiteral.INSTANCE
)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
index f0d1b93ed12..0265b7bf381 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
@@ -62,6 +62,7 @@ public enum ExpressionRuleType {
SIMPLIFY_RANGE,
SIMPLIFY_SELF_COMPARISON,
SUPPORT_JAVA_DATE_FORMATTER,
+ NORMALIZE_STRUCT_ELEMENT,
TOPN_TO_MAX;
public int type() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/NormalizeStructElement.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/NormalizeStructElement.java
new file mode 100644
index 00000000000..dc50921bc1e
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/NormalizeStructElement.java
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rules;
+
+import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
+import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
+import org.apache.doris.nereids.rules.expression.ExpressionRuleType;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
+import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * given column s has data type: struct<a: int, b: int>, if exists struct_element(s, 2), we will rewrite
+ * to struct_element(s, 'b')
+ */
+public class NormalizeStructElement implements ExpressionPatternRuleFactory {
+ public static final NormalizeStructElement INSTANCE = new NormalizeStructElement();
+
+ @Override
+ public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
+ return ImmutableList.of(
+ matchesType(StructElement.class).then(NormalizeStructElement::normalize)
+ .toRule(ExpressionRuleType.NORMALIZE_STRUCT_ELEMENT)
+ );
+ }
+
+ private static StructElement normalize(StructElement structElement) {
+ Expression field = structElement.getArgument(1);
+ if (field instanceof IntegerLikeLiteral) {
+ int fieldIndex = ((Number) ((IntegerLikeLiteral) field).getValue()).intValue();
+ StructType structType = (StructType) structElement.getArgument(0).getDataType();
+ List<StructField> fields = structType.getFields();
+ if (fieldIndex >= 0 && fieldIndex <= fields.size()) {
+ return structElement.withChildren(
+ ImmutableList.of(
+ structElement.child(0),
+ new StringLiteral(fields.get(fieldIndex - 1).getName())
+ )
+ );
+ }
+ }
+ return structElement;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
new file mode 100644
index 00000000000..4fe9e9e2f74
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
@@ -0,0 +1,511 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectorContext;
+import org.apache.doris.nereids.rules.rewrite.NestedColumnPruning.DataTypeAccessTree;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
+import org.apache.doris.nereids.trees.expressions.ArrayItemReference.ArrayItemSlot;
+import org.apache.doris.nereids.trees.expressions.Cast;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayCount;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExists;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFilter;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirst;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirstIndex;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLast;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLastIndex;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMap;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMatchAll;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMatchAny;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayReverseSplit;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySortBy;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySplit;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsEntry;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsKey;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsValue;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapKeys;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapValues;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.NestedColumnPrunable;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.thrift.TAccessPathType;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Stack;
+
+/**
+ * collect the access path, for example: `select struct_element(s, 'data')` has access path: ['s', 'data']
+ */
+public class AccessPathExpressionCollector extends DefaultExpressionVisitor<Void, CollectorContext> {
+ private StatementContext statementContext;
+ private boolean bottomPredicate;
+ private Multimap<Integer, CollectAccessPathResult> slotToAccessPaths;
+ private Stack<Map<String, Expression>> nameToLambdaArguments = new Stack<>();
+
+ public AccessPathExpressionCollector(
+ StatementContext statementContext, Multimap<Integer, CollectAccessPathResult> slotToAccessPaths,
+ boolean bottomPredicate) {
+ this.statementContext = statementContext;
+ this.slotToAccessPaths = slotToAccessPaths;
+ this.bottomPredicate = bottomPredicate;
+ }
+
+ public void collect(Expression expression) {
+ expression.accept(this, new CollectorContext(statementContext, bottomPredicate));
+ }
+
+ private Void continueCollectAccessPath(Expression expr, CollectorContext context) {
+ return expr.accept(this, context);
+ }
+
+ @Override
+ public Void visit(Expression expr, CollectorContext context) {
+ for (Expression child : expr.children()) {
+ child.accept(this, new CollectorContext(context.statementContext, context.bottomFilter));
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitSlotReference(SlotReference slotReference, CollectorContext context) {
+ DataType dataType = slotReference.getDataType();
+ if (dataType instanceof NestedColumnPrunable) {
+ context.accessPathBuilder.addPrefix(slotReference.getName().toLowerCase());
+ ImmutableList<String> path = Utils.fastToImmutableList(context.accessPathBuilder.accessPath);
+ int slotId = slotReference.getExprId().asInt();
+ slotToAccessPaths.put(slotId, new CollectAccessPathResult(path, context.bottomFilter, context.type));
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitArrayItemSlot(ArrayItemSlot arrayItemSlot, CollectorContext context) {
+ if (nameToLambdaArguments.isEmpty()) {
+ return null;
+ }
+ context.accessPathBuilder.addPrefix("*");
+ Expression argument = nameToLambdaArguments.peek().get(arrayItemSlot.getName());
+ if (argument == null) {
+ return null;
+ }
+ return continueCollectAccessPath(argument, context);
+ }
+
+ @Override
+ public Void visitAlias(Alias alias, CollectorContext context) {
+ return alias.child(0).accept(this, context);
+ }
+
+ @Override
+ public Void visitCast(Cast cast, CollectorContext context) {
+ if (!context.accessPathBuilder.isEmpty()
+ && cast.getDataType() instanceof NestedColumnPrunable
+ && cast.child().getDataType() instanceof NestedColumnPrunable) {
+
+ DataTypeAccessTree castTree = DataTypeAccessTree.of(cast.getDataType(), TAccessPathType.DATA);
+ DataTypeAccessTree originTree = DataTypeAccessTree.of(cast.child().getDataType(), TAccessPathType.DATA);
+
+ List<String> replacePath = new ArrayList<>(context.accessPathBuilder.getPathList());
+ if (originTree.replacePathByAnotherTree(castTree, replacePath, 0)) {
+ CollectorContext castContext = new CollectorContext(context.statementContext, context.bottomFilter);
+ castContext.accessPathBuilder.accessPath.addAll(replacePath);
+ return continueCollectAccessPath(cast.child(), castContext);
+ }
+ }
+ return cast.child(0).accept(this,
+ new CollectorContext(context.statementContext, context.bottomFilter)
+ );
+ }
+
+ // array element at
+ @Override
+ public Void visitElementAt(ElementAt elementAt, CollectorContext context) {
+ List<Expression> arguments = elementAt.getArguments();
+ Expression first = arguments.get(0);
+ if (first.getDataType().isArrayType() || first.getDataType().isMapType()) {
+ context.accessPathBuilder.addPrefix("*");
+ continueCollectAccessPath(first, context);
+
+ for (int i = 1; i < arguments.size(); i++) {
+ visit(arguments.get(i), context);
+ }
+ return null;
+ } else {
+ return visit(elementAt, context);
+ }
+ }
+
+ // struct element_at
+ @Override
+ public Void visitStructElement(StructElement structElement, CollectorContext context) {
+ List<Expression> arguments = structElement.getArguments();
+ Expression struct = arguments.get(0);
+ Expression fieldName = arguments.get(1);
+ DataType fieldType = fieldName.getDataType();
+
+ if (fieldName.isLiteral() && (fieldType.isIntegerLikeType() || fieldType.isStringLikeType())) {
+ if (fieldType.isIntegerLikeType()) {
+ int fieldIndex = ((Number) ((Literal) fieldName).getValue()).intValue();
+ List<StructField> fields = ((StructType) struct.getDataType()).getFields();
+ if (fieldIndex >= 1 && fieldIndex <= fields.size()) {
+ String realFieldName = fields.get(fieldIndex - 1).getName();
+ context.accessPathBuilder.addPrefix(realFieldName);
+ return continueCollectAccessPath(struct, context);
+ }
+ }
+ context.accessPathBuilder.addPrefix(((Literal) fieldName).getStringValue().toLowerCase());
+ return continueCollectAccessPath(struct, context);
+ }
+
+ for (Expression argument : arguments) {
+ visit(argument, context);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitMapKeys(MapKeys mapKeys, CollectorContext context) {
+ context = new CollectorContext(context.statementContext, context.bottomFilter);
+ context.accessPathBuilder.addPrefix("KEYS");
+ return continueCollectAccessPath(mapKeys.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitMapValues(MapValues mapValues, CollectorContext context) {
+ LinkedList<String> suffixPath = context.accessPathBuilder.accessPath;
+ if (!suffixPath.isEmpty() && suffixPath.get(0).equals("*")) {
+ CollectorContext removeStarContext
+ = new CollectorContext(context.statementContext, context.bottomFilter);
+ removeStarContext.accessPathBuilder.accessPath.addAll(suffixPath.subList(1, suffixPath.size()));
+ removeStarContext.accessPathBuilder.addPrefix("VALUES");
+ return continueCollectAccessPath(mapValues.getArgument(0), removeStarContext);
+ }
+ context.accessPathBuilder.addPrefix("VALUES");
+ return continueCollectAccessPath(mapValues.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitMapContainsKey(MapContainsKey mapContainsKey, CollectorContext context) {
+ context.accessPathBuilder.addPrefix("KEYS");
+ return continueCollectAccessPath(mapContainsKey.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitMapContainsValue(MapContainsValue mapContainsValue, CollectorContext context) {
+ context.accessPathBuilder.addPrefix("VALUES");
+ return continueCollectAccessPath(mapContainsValue.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitMapContainsEntry(MapContainsEntry mapContainsEntry, CollectorContext context) {
+ context.accessPathBuilder.addPrefix("*");
+ return continueCollectAccessPath(mapContainsEntry.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitArrayMap(ArrayMap arrayMap, CollectorContext context) {
+ // ARRAY_MAP(lambda, <arr> [ , <arr> ... ] )
+
+ Expression argument = arrayMap.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ return collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayMap, context);
+ }
+
+ @Override
+ public Void visitArrayCount(ArrayCount arrayCount, CollectorContext context) {
+ // ARRAY_COUNT(<lambda>, <arr>[, ... ])
+
+ Expression argument = arrayCount.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ return collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayCount, context);
+ }
+
+ @Override
+ public Void visitArrayExists(ArrayExists arrayExists, CollectorContext context) {
+ // ARRAY_EXISTS([ <lambda>, ] <arr1> [, <arr2> , ...] )
+
+ Expression argument = arrayExists.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ return collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayExists, context);
+ }
+
+ @Override
+ public Void visitArrayFilter(ArrayFilter arrayFilter, CollectorContext context) {
+ // ARRAY_FILTER(<lambda>, <arr>)
+
+ Expression argument = arrayFilter.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayFilter, context);
+ }
+
+ @Override
+ public Void visitArrayFirst(ArrayFirst arrayFirst, CollectorContext context) {
+ // ARRAY_FIRST(<lambda>, <arr>)
+
+ Expression argument = arrayFirst.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayFirst, context);
+ }
+
+ @Override
+ public Void visitArrayFirstIndex(ArrayFirstIndex arrayFirstIndex, CollectorContext context) {
+ // ARRAY_FIRST_INDEX(<lambda>, <arr> [, ...])
+
+ Expression argument = arrayFirstIndex.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayFirstIndex, context);
+ }
+
+ @Override
+ public Void visitArrayLast(ArrayLast arrayLast, CollectorContext context) {
+ // ARRAY_LAST(<lambda>, <arr>)
+
+ Expression argument = arrayLast.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayLast, context);
+ }
+
+ @Override
+ public Void visitArrayLastIndex(ArrayLastIndex arrayLastIndex, CollectorContext context) {
+ // ARRAY_LAST_INDEX(<lambda>, <arr> [, ...])
+
+ Expression argument = arrayLastIndex.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayLastIndex, context);
+ }
+
+ @Override
+ public Void visitArrayMatchAny(ArrayMatchAny arrayMatchAny, CollectorContext context) {
+ // array_match_any(lambda, <arr> [, <arr> ...])
+
+ Expression argument = arrayMatchAny.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayMatchAny, context);
+ }
+
+ @Override
+ public Void visitArrayMatchAll(ArrayMatchAll arrayMatchAll, CollectorContext context) {
+ // array_match_all(lambda, <arr> [, <arr> ...])
+
+ Expression argument = arrayMatchAll.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayMatchAll, context);
+ }
+
+ @Override
+ public Void visitArrayReverseSplit(ArrayReverseSplit arrayReverseSplit, CollectorContext context) {
+ // ARRAY_REVERSE_SPLIT(<lambda>, <arr> [, ...])
+
+ Expression argument = arrayReverseSplit.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayReverseSplit, context);
+ }
+
+ @Override
+ public Void visitArraySplit(ArraySplit arraySplit, CollectorContext context) {
+ // ARRAY_SPLIT(<lambda>, arr [, ...])
+
+ Expression argument = arraySplit.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arraySplit, context);
+ }
+
+ @Override
+ public Void visitArraySortBy(ArraySortBy arraySortBy, CollectorContext context) {
+ // ARRAY_SORTBY(<lambda>, <arr> [, ...])
+
+ Expression argument = arraySortBy.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arraySortBy, context);
+ }
+
+ // @Override
+ // public Void visitIsNull(IsNull isNull, CollectorContext context) {
+ // if (context.accessPathBuilder.isEmpty()) {
+ // context.setType(TAccessPathType.META);
+ // return continueCollectAccessPath(isNull.child(), context);
+ // }
+ // return visit(isNull, context);
+ // }
+
+ private Void collectArrayPathInLambda(Lambda lambda, CollectorContext context) {
+ List<Expression> arguments = lambda.getArguments();
+ Map<String, Expression> nameToArray = Maps.newLinkedHashMap();
+ for (Expression argument : arguments) {
+ if (argument instanceof ArrayItemReference) {
+ nameToArray.put(((ArrayItemReference) argument).getName(), argument.child(0));
+ }
+ }
+
+ List<String> path = context.accessPathBuilder.getPathList();
+ if (!path.isEmpty() && path.get(0).equals("*")) {
+ context.accessPathBuilder.removePrefix();
+ }
+
+ nameToLambdaArguments.push(nameToArray);
+ try {
+ continueCollectAccessPath(arguments.get(0), context);
+ } finally {
+ nameToLambdaArguments.pop();
+ }
+ return null;
+ }
+
+ /** CollectorContext */
+ public static class CollectorContext {
+ private StatementContext statementContext;
+ private AccessPathBuilder accessPathBuilder;
+ private boolean bottomFilter;
+ private TAccessPathType type;
+
+ public CollectorContext(StatementContext statementContext, boolean bottomFilter) {
+ this.statementContext = statementContext;
+ this.accessPathBuilder = new AccessPathBuilder();
+ this.bottomFilter = bottomFilter;
+ this.type = TAccessPathType.DATA;
+ }
+
+ public TAccessPathType getType() {
+ return type;
+ }
+
+ public void setType(TAccessPathType type) {
+ this.type = type;
+ }
+ }
+
+ private static class AccessPathBuilder {
+ private LinkedList<String> accessPath;
+
+ public AccessPathBuilder() {
+ accessPath = new LinkedList<>();
+ }
+
+ public AccessPathBuilder addPrefix(String prefix) {
+ accessPath.addFirst(prefix);
+ return this;
+ }
+
+ public AccessPathBuilder removePrefix() {
+ accessPath.removeFirst();
+ return this;
+ }
+
+ public List<String> getPathList() {
+ return accessPath;
+ }
+
+ public boolean isEmpty() {
+ return accessPath.isEmpty();
+ }
+
+ @Override
+ public String toString() {
+ return String.join(".", accessPath);
+ }
+ }
+
+ /** AccessPathIsPredicate */
+ public static class CollectAccessPathResult {
+ private final List<String> path;
+ private final boolean isPredicate;
+ private final TAccessPathType type;
+
+ public CollectAccessPathResult(List<String> path, boolean isPredicate, TAccessPathType type) {
+ this.path = path;
+ this.isPredicate = isPredicate;
+ this.type = type;
+ }
+
+ public TAccessPathType getType() {
+ return type;
+ }
+
+ public List<String> getPath() {
+ return path;
+ }
+
+ public boolean isPredicate() {
+ return isPredicate;
+ }
+
+ @Override
+ public String toString() {
+ return String.join(".", path) + ", " + isPredicate;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ CollectAccessPathResult that = (CollectAccessPathResult) o;
+ return isPredicate == that.isPredicate && Objects.equals(path, that.path);
+ }
+
+ @Override
+ public int hashCode() {
+ return path.hashCode();
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java
new file mode 100644
index 00000000000..514f7bb1e8c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectAccessPathResult;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
+import org.apache.doris.nereids.types.NestedColumnPrunable;
+
+import com.google.common.collect.LinkedHashMultimap;
+import com.google.common.collect.Multimap;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/** AccessPathPlanCollector */
+public class AccessPathPlanCollector extends DefaultPlanVisitor<Void, StatementContext> {
+ private Multimap<Integer, CollectAccessPathResult> allSlotToAccessPaths = LinkedHashMultimap.create();
+ private Map<Slot, List<CollectAccessPathResult>> scanSlotToAccessPaths = new LinkedHashMap<>();
+
+ public Map<Slot, List<CollectAccessPathResult>> collect(Plan root, StatementContext context) {
+ root.accept(this, context);
+ return scanSlotToAccessPaths;
+ }
+
+ @Override
+ public Void visitLogicalFilter(LogicalFilter<? extends Plan> filter, StatementContext context) {
+ boolean bottomFilter = filter.child().arity() == 0;
+ collectByExpressions(filter, context, bottomFilter);
+ return filter.child().accept(this, context);
+ }
+
+ @Override
+ public Void visitLogicalCTEAnchor(
+ LogicalCTEAnchor<? extends Plan, ? extends Plan> cteAnchor, StatementContext context) {
+ // first, collect access paths in the outer slots, and propagate outer slot's access path to inner slots
+ cteAnchor.right().accept(this, context);
+
+ // second, push down access path in the inner slots
+ cteAnchor.left().accept(this, context);
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalCTEConsumer(LogicalCTEConsumer cteConsumer, StatementContext context) {
+ // propagate outer slot's access path to inner slots
+ for (Entry<Slot, Slot> slots : cteConsumer.getConsumerToProducerOutputMap().entrySet()) {
+ Slot outerSlot = slots.getKey();
+
+ if (outerSlot.getDataType() instanceof NestedColumnPrunable) {
+ int outerSlotId = outerSlot.getExprId().asInt();
+ int innerSlotId = slots.getValue().getExprId().asInt();
+ allSlotToAccessPaths.putAll(innerSlotId, allSlotToAccessPaths.get(outerSlotId));
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalCTEProducer(LogicalCTEProducer<? extends Plan> cteProducer, StatementContext context) {
+ return cteProducer.child().accept(this, context);
+ }
+
+ @Override
+ public Void visitLogicalUnion(LogicalUnion union, StatementContext context) {
+ // now we will not prune complex type through union, because we can not prune the complex type's literal,
+ // for example, we can not prune the literal now: array(map(1, named_struct('a', 100, 'b', 100))),
+ // so we can not prune this sql:
+ // select struct_element(map_values(s[0]), 'a')
+ // from (
+ // select s from tbl
+ // union all
+ // select array(map(1, named_struct('a', 100, 'b', 100)))
+ // ) tbl;
+ //
+ // so we will not propagate access paths through the union
+ for (Plan child : union.children()) {
+ child.accept(this, context);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalOlapScan(LogicalOlapScan olapScan, StatementContext context) {
+ for (Slot slot : olapScan.getOutput()) {
+ if (!(slot.getDataType() instanceof NestedColumnPrunable)) {
+ continue;
+ }
+ Collection<CollectAccessPathResult> accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt());
+ if (!accessPaths.isEmpty()) {
+ scanSlotToAccessPaths.put(slot, new ArrayList<>(accessPaths));
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalFileScan(LogicalFileScan fileScan, StatementContext context) {
+ for (Slot slot : fileScan.getOutput()) {
+ if (!(slot.getDataType() instanceof NestedColumnPrunable)) {
+ continue;
+ }
+ Collection<CollectAccessPathResult> accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt());
+ if (!accessPaths.isEmpty()) {
+ scanSlotToAccessPaths.put(slot, new ArrayList<>(accessPaths));
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, StatementContext context) {
+ for (Slot slot : tvfRelation.getOutput()) {
+ if (!(slot.getDataType() instanceof NestedColumnPrunable)) {
+ continue;
+ }
+ Collection<CollectAccessPathResult> accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt());
+ if (!accessPaths.isEmpty()) {
+ scanSlotToAccessPaths.put(slot, new ArrayList<>(accessPaths));
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(Plan plan, StatementContext context) {
+ collectByExpressions(plan, context);
+
+ for (Plan child : plan.children()) {
+ child.accept(this, context);
+ }
+ return null;
+ }
+
+ private void collectByExpressions(Plan plan, StatementContext context) {
+ collectByExpressions(plan, context, false);
+ }
+
+ private void collectByExpressions(Plan plan, StatementContext context, boolean bottomPredicate) {
+ AccessPathExpressionCollector exprCollector
+ = new AccessPathExpressionCollector(context, allSlotToAccessPaths, bottomPredicate);
+ for (Expression expression : plan.getExpressions()) {
+ exprCollector.collect(expression);
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
new file mode 100644
index 00000000000..de22e9d2101
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
@@ -0,0 +1,487 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.analysis.AccessPathInfo;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.jobs.JobContext;
+import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectAccessPathResult;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
+import org.apache.doris.nereids.types.ArrayType;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.MapType;
+import org.apache.doris.nereids.types.NullType;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.thrift.TAccessPathType;
+import org.apache.doris.thrift.TColumnAccessPath;
+import org.apache.doris.thrift.TDataAccessPath;
+import org.apache.doris.thrift.TMetaAccessPath;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.TreeMultimap;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+
+/**
+ * <li> 1. prune the data type of struct/map
+ *
+ * <p> for example, column s is a struct<id: int, value: double>,
+ * and `select struct(s, 'id') from tbl` will return the data type: `struct<id: int>`
+ * </p>
+ * </li>
+ *
+ * <li> 2. collect the access paths
+ * <p> for example, select struct(s, 'id'), struct(s, 'data') from tbl` will collect the access path:
+ * [s.id, s.data]
+ * </p>
+ * </li>
+ */
+public class NestedColumnPruning implements CustomRewriter {
+ public static final Logger LOG = LogManager.getLogger(NestedColumnPruning.class);
+
+ @Override
+ public Plan rewriteRoot(Plan plan, JobContext jobContext) {
+ try {
+ StatementContext statementContext = jobContext.getCascadesContext().getStatementContext();
+ SessionVariable sessionVariable = statementContext.getConnectContext().getSessionVariable();
+ if (!sessionVariable.enablePruneNestedColumns || !statementContext.hasNestedColumns()) {
+ return plan;
+ }
+
+ AccessPathPlanCollector collector = new AccessPathPlanCollector();
+ Map<Slot, List<CollectAccessPathResult>> slotToAccessPaths = collector.collect(plan, statementContext);
+ Map<Integer, AccessPathInfo> slotToResult = pruneDataType(slotToAccessPaths);
+
+ if (!slotToResult.isEmpty()) {
+ Map<Integer, AccessPathInfo> slotIdToPruneType = Maps.newLinkedHashMap();
+ for (Entry<Integer, AccessPathInfo> kv : slotToResult.entrySet()) {
+ Integer slotId = kv.getKey();
+ AccessPathInfo accessPathInfo = kv.getValue();
+ slotIdToPruneType.put(slotId, accessPathInfo);
+ }
+ SlotTypeReplacer typeReplacer = new SlotTypeReplacer(slotIdToPruneType);
+ return plan.accept(typeReplacer, null);
+ }
+ return plan;
+ } catch (Throwable t) {
+ LOG.warn("NestedColumnPruning failed.", t);
+ return plan;
+ }
+ }
+
+ private static Map<Integer, AccessPathInfo> pruneDataType(
+ Map<Slot, List<CollectAccessPathResult>> slotToAccessPaths) {
+ Map<Integer, AccessPathInfo> result = new LinkedHashMap<>();
+ Map<Slot, DataTypeAccessTree> slotIdToAllAccessTree = new LinkedHashMap<>();
+ Map<Slot, DataTypeAccessTree> slotIdToPredicateAccessTree = new LinkedHashMap<>();
+
+ Comparator<Pair<TAccessPathType, List<String>>> pathComparator = Comparator.comparing(
+ l -> StringUtils.join(l.second, "."));
+
+ Multimap<Integer, Pair<TAccessPathType, List<String>>> allAccessPaths = TreeMultimap.create(
+ Comparator.naturalOrder(), pathComparator);
+ Multimap<Integer, Pair<TAccessPathType, List<String>>> predicateAccessPaths = TreeMultimap.create(
+ Comparator.naturalOrder(), pathComparator);
+
+ // first: build access data type tree
+ for (Entry<Slot, List<CollectAccessPathResult>> kv : slotToAccessPaths.entrySet()) {
+ Slot slot = kv.getKey();
+ List<CollectAccessPathResult> collectAccessPathResults = kv.getValue();
+ for (CollectAccessPathResult collectAccessPathResult : collectAccessPathResults) {
+ List<String> path = collectAccessPathResult.getPath();
+ TAccessPathType pathType = collectAccessPathResult.getType();
+ DataTypeAccessTree allAccessTree = slotIdToAllAccessTree.computeIfAbsent(
+ slot, i -> DataTypeAccessTree.ofRoot(slot, pathType)
+ );
+ allAccessTree.setAccessByPath(path, 0, pathType);
+ allAccessPaths.put(slot.getExprId().asInt(), Pair.of(pathType, path));
+
+ if (collectAccessPathResult.isPredicate()) {
+ DataTypeAccessTree predicateAccessTree = slotIdToPredicateAccessTree.computeIfAbsent(
+ slot, i -> DataTypeAccessTree.ofRoot(slot, pathType)
+ );
+ predicateAccessTree.setAccessByPath(path, 0, pathType);
+ predicateAccessPaths.put(
+ slot.getExprId().asInt(), Pair.of(pathType, path)
+ );
+ }
+ }
+ }
+
+ // second: build non-predicate access paths
+ for (Entry<Slot, DataTypeAccessTree> kv : slotIdToAllAccessTree.entrySet()) {
+ Slot slot = kv.getKey();
+ DataTypeAccessTree accessTree = kv.getValue();
+ DataType prunedDataType = accessTree.pruneDataType().orElse(slot.getDataType());
+
+ List<TColumnAccessPath> allPaths = new ArrayList<>();
+ boolean accessWholeColumn = false;
+ TAccessPathType accessWholeColumnType = TAccessPathType.META;
+ for (Pair<TAccessPathType, List<String>> pathInfo : allAccessPaths.get(slot.getExprId().asInt())) {
+ if (pathInfo.first == TAccessPathType.DATA) {
+ TDataAccessPath dataAccessPath = new TDataAccessPath();
+ dataAccessPath.setPath(new ArrayList<>(pathInfo.second));
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.DATA);
+ accessPath.setDataAccessPath(dataAccessPath);
+ allPaths.add(accessPath);
+ } else {
+ TMetaAccessPath dataAccessPath = new TMetaAccessPath();
+ dataAccessPath.setPath(new ArrayList<>(pathInfo.second));
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.META);
+ accessPath.setMetaAccessPath(dataAccessPath);
+ allPaths.add(accessPath);
+ }
+ // only retain access the whole root
+ if (pathInfo.second.size() == 1) {
+ accessWholeColumn = true;
+ if (pathInfo.first == TAccessPathType.DATA) {
+ accessWholeColumnType = TAccessPathType.DATA;
+ }
+ } else {
+ accessWholeColumnType = TAccessPathType.DATA;
+ }
+ }
+ if (accessWholeColumn) {
+ TColumnAccessPath accessPath = new TColumnAccessPath(accessWholeColumnType);
+ SlotReference slotReference = (SlotReference) kv.getKey();
+ String wholeColumnName = slotReference.getOriginalColumn().get().getName();
+ if (accessWholeColumnType == TAccessPathType.DATA) {
+ accessPath.setDataAccessPath(new TDataAccessPath(ImmutableList.of(wholeColumnName)));
+ } else {
+ accessPath.setMetaAccessPath(new TMetaAccessPath(ImmutableList.of(wholeColumnName)));
+ }
+ allPaths = ImmutableList.of(accessPath);
+ }
+ result.put(slot.getExprId().asInt(), new AccessPathInfo(prunedDataType, allPaths, new ArrayList<>()));
+ }
+
+ // third: build predicate access path
+ for (Entry<Slot, DataTypeAccessTree> kv : slotIdToPredicateAccessTree.entrySet()) {
+ Slot slot = kv.getKey();
+
+ List<TColumnAccessPath> predicatePaths = new ArrayList<>();
+ boolean accessWholeColumn = false;
+ TAccessPathType accessWholeColumnType = TAccessPathType.META;
+ for (Pair<TAccessPathType, List<String>> pathInfo : predicateAccessPaths.get(slot.getExprId().asInt())) {
+ if (pathInfo == null) {
+ throw new AnalysisException("This is a bug, please report this");
+ }
+
+ if (pathInfo.first == TAccessPathType.DATA) {
+ TDataAccessPath dataAccessPath = new TDataAccessPath();
+ dataAccessPath.setPath(new ArrayList<>(pathInfo.second));
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.DATA);
+ accessPath.setDataAccessPath(dataAccessPath);
+ predicatePaths.add(accessPath);
+ } else {
+ TMetaAccessPath dataAccessPath = new TMetaAccessPath();
+ dataAccessPath.setPath(new ArrayList<>(pathInfo.second));
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.META);
+ accessPath.setMetaAccessPath(dataAccessPath);
+ predicatePaths.add(accessPath);
+ }
+ // only retain access the whole root
+ if (pathInfo.second.size() == 1) {
+ accessWholeColumn = true;
+ if (pathInfo.first == TAccessPathType.DATA) {
+ accessWholeColumnType = TAccessPathType.DATA;
+ }
+ } else {
+ accessWholeColumnType = TAccessPathType.DATA;
+ }
+ }
+
+ if (accessWholeColumn) {
+ TColumnAccessPath accessPath = new TColumnAccessPath(accessWholeColumnType);
+ SlotReference slotReference = (SlotReference) kv.getKey();
+ String wholeColumnName = slotReference.getOriginalColumn().get().getName();
+ if (accessWholeColumnType == TAccessPathType.DATA) {
+ accessPath.setDataAccessPath(new TDataAccessPath(ImmutableList.of(wholeColumnName)));
+ } else {
+ accessPath.setMetaAccessPath(new TMetaAccessPath(ImmutableList.of(wholeColumnName)));
+ }
+ predicatePaths = ImmutableList.of(accessPath);
+ }
+
+ AccessPathInfo accessPathInfo = result.get(slot.getExprId().asInt());
+ accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths);
+ }
+
+ return result;
+ }
+
+ /** DataTypeAccessTree */
+ public static class DataTypeAccessTree {
+ private DataType type;
+ private boolean isRoot;
+ private boolean accessPartialChild;
+ private boolean accessAll;
+ private TAccessPathType pathType;
+ private Map<String, DataTypeAccessTree> children = new LinkedHashMap<>();
+
+ public DataTypeAccessTree(DataType type, TAccessPathType pathType) {
+ this(false, type, pathType);
+ }
+
+ public DataTypeAccessTree(boolean isRoot, DataType type, TAccessPathType pathType) {
+ this.isRoot = isRoot;
+ this.type = type;
+ this.pathType = pathType;
+ }
+
+ /** pruneCastType */
+ public DataType pruneCastType(DataTypeAccessTree origin, DataTypeAccessTree cast) {
+ if (type instanceof StructType) {
+ Map<String, String> nameMapping = new LinkedHashMap<>();
+ List<String> castNames = new ArrayList<>(cast.children.keySet());
+ int i = 0;
+ for (String s : origin.children.keySet()) {
+ nameMapping.put(s, castNames.get(i++));
+ }
+ List<StructField> mappingFields = new ArrayList<>();
+ StructType originCastStructType = (StructType) cast.type;
+ for (Entry<String, DataTypeAccessTree> kv : children.entrySet()) {
+ String originName = kv.getKey();
+ String mappingName = nameMapping.getOrDefault(originName, originName);
+ DataTypeAccessTree originPrunedTree = kv.getValue();
+ DataType mappingType = originPrunedTree.pruneCastType(
+ origin.children.get(originName),
+ cast.children.get(mappingName)
+ );
+ StructField originCastField = originCastStructType.getField(mappingName);
+ mappingFields.add(
+ new StructField(mappingName, mappingType,
+ originCastField.isNullable(), originCastField.getComment())
+ );
+ }
+ return new StructType(mappingFields);
+ } else if (type instanceof ArrayType) {
+ return ArrayType.of(
+ children.values().iterator().next().pruneCastType(
+ origin.children.values().iterator().next(),
+ cast.children.values().iterator().next()
+ ),
+ ((ArrayType) cast.type).containsNull()
+ );
+ } else if (type instanceof MapType) {
+ return MapType.of(
+ children.get("KEYS").pruneCastType(origin.children.get("KEYS"), cast.children.get("KEYS")),
+ children.get("VALUES").pruneCastType(origin.children.get("VALUES"), cast.children.get("VALUES"))
+ );
+ } else {
+ return cast.type;
+ }
+ }
+
+ /** replacePathByAnotherTree */
+ public boolean replacePathByAnotherTree(DataTypeAccessTree cast, List<String> path, int index) {
+ if (index >= path.size()) {
+ return true;
+ }
+ if (cast.type instanceof StructType) {
+ List<StructField> fields = ((StructType) cast.type).getFields();
+ for (int i = 0; i < fields.size(); i++) {
+ String castFieldName = path.get(index);
+ if (fields.get(i).getName().equalsIgnoreCase(castFieldName)) {
+ String originFieldName = ((StructType) type).getFields().get(i).getName();
+ path.set(index, originFieldName);
+ return children.get(originFieldName).replacePathByAnotherTree(
+ cast.children.get(castFieldName), path, index + 1
+ );
+ }
+ }
+ } else if (cast.type instanceof ArrayType) {
+ return children.values().iterator().next().replacePathByAnotherTree(
+ cast.children.values().iterator().next(), path, index + 1);
+ } else if (cast.type instanceof MapType) {
+ String fieldName = path.get(index);
+ return children.get("VALUES").replacePathByAnotherTree(
+ cast.children.get(fieldName), path, index + 1
+ );
+ }
+ return false;
+ }
+
+ /** setAccessByPath */
+ public void setAccessByPath(List<String> path, int accessIndex, TAccessPathType pathType) {
+ if (accessIndex >= path.size()) {
+ accessAll = true;
+ return;
+ } else {
+ accessPartialChild = true;
+ }
+
+ if (pathType == TAccessPathType.DATA) {
+ this.pathType = TAccessPathType.DATA;
+ }
+
+ if (this.type.isStructType()) {
+ String fieldName = path.get(accessIndex).toLowerCase();
+ DataTypeAccessTree child = children.get(fieldName);
+ if (child != null) {
+ child.setAccessByPath(path, accessIndex + 1, pathType);
+ } else {
+ // can not find the field
+ accessAll = true;
+ }
+ return;
+ } else if (this.type.isArrayType()) {
+ DataTypeAccessTree child = children.get("*");
+ if (path.get(accessIndex).equals("*")) {
+ // enter this array and skip next *
+ child.setAccessByPath(path, accessIndex + 1, pathType);
+ }
+ return;
+ } else if (this.type.isMapType()) {
+ String fieldName = path.get(accessIndex);
+ if (fieldName.equals("*")) {
+ // access value by the key, so we should access key and access value, then prune the value's type.
+ // e.g. map_column['id'] should access the keys, and access the values
+ DataTypeAccessTree keysChild = children.get("KEYS");
+ DataTypeAccessTree valuesChild = children.get("VALUES");
+ keysChild.accessAll = true;
+ valuesChild.setAccessByPath(path, accessIndex + 1, pathType);
+ return;
+ } else if (fieldName.equals("KEYS")) {
+ // only access the keys and not need enter keys, because it must be primitive type.
+ // e.g. map_keys(map_column)
+ DataTypeAccessTree keysChild = children.get("KEYS");
+ keysChild.accessAll = true;
+ return;
+ } else if (fieldName.equals("VALUES")) {
+ // only access the values without keys, and maybe prune the value's data type.
+ // e.g. map_values(map_columns)[0] will access the array of values first,
+ // and then access the array, so the access path is ['VALUES', '*']
+ DataTypeAccessTree valuesChild = children.get("VALUES");
+ valuesChild.setAccessByPath(path, accessIndex + 1, pathType);
+ return;
+ }
+ } else if (isRoot) {
+ children.get(path.get(accessIndex).toLowerCase()).setAccessByPath(path, accessIndex + 1, pathType);
+ return;
+ }
+ throw new AnalysisException("unsupported data type: " + this.type);
+ }
+
+ public static DataTypeAccessTree ofRoot(Slot slot, TAccessPathType pathType) {
+ DataTypeAccessTree child = of(slot.getDataType(), pathType);
+ DataTypeAccessTree root = new DataTypeAccessTree(true, NullType.INSTANCE, pathType);
+ root.children.put(slot.getName().toLowerCase(), child);
+ return root;
+ }
+
+ /** of */
+ public static DataTypeAccessTree of(DataType type, TAccessPathType pathType) {
+ DataTypeAccessTree root = new DataTypeAccessTree(type, pathType);
+ if (type instanceof StructType) {
+ StructType structType = (StructType) type;
+ for (Entry<String, StructField> kv : structType.getNameToFields().entrySet()) {
+ root.children.put(kv.getKey().toLowerCase(), of(kv.getValue().getDataType(), pathType));
+ }
+ } else if (type instanceof ArrayType) {
+ root.children.put("*", of(((ArrayType) type).getItemType(), pathType));
+ } else if (type instanceof MapType) {
+ root.children.put("KEYS", of(((MapType) type).getKeyType(), pathType));
+ root.children.put("VALUES", of(((MapType) type).getValueType(), pathType));
+ }
+ return root;
+ }
+
+ /** pruneDataType */
+ public Optional<DataType> pruneDataType() {
+ if (isRoot) {
+ return children.values().iterator().next().pruneDataType();
+ } else if (accessAll) {
+ return Optional.of(type);
+ } else if (!accessPartialChild) {
+ return Optional.empty();
+ }
+
+ List<Pair<String, DataType>> accessedChildren = new ArrayList<>();
+
+ if (type instanceof StructType) {
+ for (Entry<String, DataTypeAccessTree> kv : children.entrySet()) {
+ DataTypeAccessTree childTypeTree = kv.getValue();
+ Optional<DataType> childDataType = childTypeTree.pruneDataType();
+ if (childDataType.isPresent()) {
+ accessedChildren.add(Pair.of(kv.getKey(), childDataType.get()));
+ }
+ }
+ } else if (type instanceof ArrayType) {
+ Optional<DataType> childDataType = children.get("*").pruneDataType();
+ if (childDataType.isPresent()) {
+ accessedChildren.add(Pair.of("*", childDataType.get()));
+ }
+ } else if (type instanceof MapType) {
+ DataType prunedValueType = children.get("VALUES")
+ .pruneDataType()
+ .orElse(((MapType) type).getValueType());
+ // can not prune keys but can prune values
+ accessedChildren.add(Pair.of("KEYS", ((MapType) type).getKeyType()));
+ accessedChildren.add(Pair.of("VALUES", prunedValueType));
+ }
+ if (accessedChildren.isEmpty()) {
+ return Optional.of(type);
+ }
+
+ return Optional.of(pruneDataType(type, accessedChildren));
+ }
+
+ private DataType pruneDataType(DataType dataType, List<Pair<String, DataType>> newChildrenTypes) {
+ if (dataType instanceof StructType) {
+ // prune struct fields
+ StructType structType = (StructType) dataType;
+ Map<String, StructField> nameToFields = structType.getNameToFields();
+ List<StructField> newFields = new ArrayList<>();
+ for (Pair<String, DataType> kv : newChildrenTypes) {
+ String fieldName = kv.first;
+ StructField originField = nameToFields.get(fieldName);
+ DataType prunedType = kv.second;
+ newFields.add(new StructField(
+ originField.getName(), prunedType, originField.isNullable(), originField.getComment()
+ ));
+ }
+ return new StructType(newFields);
+ } else if (dataType instanceof ArrayType) {
+ return ArrayType.of(newChildrenTypes.get(0).second, ((ArrayType) dataType).containsNull());
+ } else if (dataType instanceof MapType) {
+ return MapType.of(newChildrenTypes.get(0).second, newChildrenTypes.get(1).second);
+ } else {
+ throw new AnalysisException("unsupported data type: " + dataType);
+ }
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java
new file mode 100644
index 00000000000..6ef42183c7b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java
@@ -0,0 +1,315 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.pattern.MatchingContext;
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+
+/** PushDownNestedColumnThroughJoin */
+public class PushDownNestedColumn implements RewriteRuleFactory {
+ @Override
+ public List<Rule> buildRules() {
+ return ImmutableList.of(
+ RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_JOIN.build(
+ logicalProject(logicalJoin()).thenApply(this::defaultPushDown)
+ ),
+ RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_WINDOW.build(
+ logicalProject(logicalWindow()).thenApply(this::defaultPushDown)
+ ),
+ RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_PARTITION_TOP_N.build(
+ logicalProject(logicalPartitionTopN()).thenApply(this::defaultPushDown)
+ ),
+ // RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_DEFER_MATERIALIZE_TOP_N.build(
+ // logicalProject(logicalDeferMaterializeTopN()).thenApply(this::defaultPushDown)
+ // ),
+ RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_UNION.build(
+ logicalProject(
+ logicalUnion().when(u -> u.getQualifier() == Qualifier.ALL)
+ ).thenApply(this::pushThroughUnion)
+ )
+ );
+ }
+
+ private <C extends LogicalPlan> Plan defaultPushDown(MatchingContext<LogicalProject<C>> ctx) {
+ if (!ctx.connectContext.getSessionVariable().enablePruneNestedColumns) {
+ return ctx.root;
+ }
+
+ LogicalProject<C> project = ctx.root;
+ C child = project.child();
+ PushdownProjectHelper pushdownProjectHelper
+ = new PushdownProjectHelper(ctx.statementContext, child);
+
+ Pair<Boolean, List<NamedExpression>> pushProjects
+ = pushdownProjectHelper.pushDownExpressions(project.getProjects());
+
+ if (pushProjects.first) {
+ List<Plan> newJoinChildren = pushdownProjectHelper.buildNewChildren();
+ return new LogicalProject<>(
+ pushProjects.second,
+ child.withChildren(newJoinChildren)
+ );
+ }
+ return project;
+ }
+
+ private Plan pushThroughUnion(MatchingContext<LogicalProject<LogicalUnion>> ctx) {
+ if (!ctx.connectContext.getSessionVariable().enablePruneNestedColumns) {
+ return ctx.root;
+ }
+ LogicalProject<LogicalUnion> project = ctx.root;
+ LogicalUnion union = project.child();
+ PushdownProjectHelper pushdownProjectHelper
+ = new PushdownProjectHelper(ctx.statementContext, project);
+
+ Pair<Boolean, List<NamedExpression>> pushProjects
+ = pushdownProjectHelper.pushDownExpressions(project.getProjects());
+ if (pushProjects.first) {
+ List<NamedExpression> unionOutputs = union.getOutputs();
+ Map<Slot, Integer> slotToColumnIndex = new LinkedHashMap<>();
+ for (int i = 0; i < unionOutputs.size(); i++) {
+ NamedExpression output = unionOutputs.get(i);
+ slotToColumnIndex.put(output.toSlot(), i);
+ }
+
+ Collection<NamedExpression> pushDownProjections
+ = pushdownProjectHelper.childToPushDownProjects.values();
+ List<Plan> newChildren = new ArrayList<>();
+ List<List<SlotReference>> newChildrenOutputs = new ArrayList<>();
+ for (Plan child : union.children()) {
+ List<NamedExpression> pushedOutput = replaceSlot(
+ ctx.statementContext,
+ pushDownProjections,
+ slot -> {
+ Integer sourceColumnIndex = slotToColumnIndex.get(slot);
+ if (sourceColumnIndex != null) {
+ return child.getOutput().get(sourceColumnIndex).toSlot();
+ }
+ return slot;
+ }
+ );
+
+ LogicalProject<Plan> newChild = new LogicalProject<>(
+ ImmutableList.<NamedExpression>builder()
+ .addAll(child.getOutput())
+ .addAll(pushedOutput)
+ .build(),
+ child
+ );
+
+ newChildrenOutputs.add((List) newChild.getOutput());
+ newChildren.add(newChild);
+ }
+
+ for (List<NamedExpression> originConstantExprs : union.getConstantExprsList()) {
+ List<NamedExpression> pushedOutput = replaceSlot(
+ ctx.statementContext,
+ pushDownProjections,
+ slot -> {
+ Integer sourceColumnIndex = slotToColumnIndex.get(slot);
+ if (sourceColumnIndex != null) {
+ return originConstantExprs.get(sourceColumnIndex).toSlot();
+ }
+ return slot;
+ }
+ );
+
+ LogicalOneRowRelation originOneRowRelation = new LogicalOneRowRelation(
+ ctx.statementContext.getNextRelationId(),
+ originConstantExprs
+ );
+
+ LogicalProject<Plan> newChild = new LogicalProject<>(
+ ImmutableList.<NamedExpression>builder()
+ .addAll(originOneRowRelation.getOutput())
+ .addAll(pushedOutput)
+ .build(),
+ originOneRowRelation
+ );
+
+ newChildrenOutputs.add((List) newChild.getOutput());
+ newChildren.add(newChild);
+ }
+
+ List<NamedExpression> newUnionOutputs = new ArrayList<>(union.getOutputs());
+ for (NamedExpression projection : pushDownProjections) {
+ newUnionOutputs.add(projection.toSlot());
+ }
+
+ return new LogicalProject<>(
+ pushProjects.second,
+ new LogicalUnion(
+ union.getQualifier(),
+ newUnionOutputs,
+ newChildrenOutputs,
+ ImmutableList.of(),
+ union.hasPushedFilter(),
+ newChildren
+ )
+ );
+ }
+ return project;
+ }
+
+ private List<NamedExpression> replaceSlot(
+ StatementContext statementContext,
+ Collection<NamedExpression> pushDownProjections,
+ Function<Slot, Slot> slotReplace) {
+ List<NamedExpression> pushedOutput = new ArrayList<>();
+ for (NamedExpression projection : pushDownProjections) {
+ NamedExpression newOutput = (NamedExpression) projection.rewriteUp(e -> {
+ if (e instanceof Slot) {
+ Slot newSlot = slotReplace.apply((Slot) e);
+ if (newSlot != null) {
+ return newSlot;
+ }
+ }
+ return e;
+ });
+ if (newOutput instanceof Alias) {
+ pushedOutput.add(new Alias(statementContext.getNextExprId(), newOutput.child(0)));
+ } else {
+ pushedOutput.add(new Alias(statementContext.getNextExprId(), newOutput));
+ }
+ }
+ return pushedOutput;
+ }
+
+ private static class PushdownProjectHelper {
+ private final Plan plan;
+ private final StatementContext statementContext;
+ private final Map<Expression, Pair<Slot, Plan>> exprToChildAndSlot;
+ private final Multimap<Plan, NamedExpression> childToPushDownProjects;
+
+ public PushdownProjectHelper(StatementContext statementContext, Plan plan) {
+ this.statementContext = statementContext;
+ this.plan = plan;
+ this.exprToChildAndSlot = new LinkedHashMap<>();
+ this.childToPushDownProjects = ArrayListMultimap.create();
+ }
+
+ public <C extends Collection<E>, E extends Expression> Pair<Boolean, C> pushDownExpressions(C expressions) {
+ ImmutableCollection.Builder<E> builder;
+ if (expressions instanceof List) {
+ builder = ImmutableList.builderWithExpectedSize(expressions.size());
+ } else {
+ builder = ImmutableSet.builderWithExpectedSize(expressions.size());
+ }
+
+ boolean extracted = false;
+ for (E expression : expressions) {
+ Optional<E> result = pushDownExpression(expression);
+ if (!result.isPresent()) {
+ builder.add(expression);
+ } else {
+ extracted = true;
+ builder.add(result.get());
+ }
+ }
+
+ if (extracted) {
+ return Pair.of(true, (C) builder.build());
+ } else {
+ return Pair.of(false, expressions);
+ }
+ }
+
+ public <E extends Expression> Optional<E> pushDownExpression(E expression) {
+ if (!(expression instanceof PreferPushDownProject
+ || (expression instanceof Alias && expression.child(0) instanceof PreferPushDownProject))) {
+ return Optional.empty();
+ }
+ Pair<Slot, Plan> existPushdown = exprToChildAndSlot.get(expression);
+ if (existPushdown != null) {
+ return Optional.of((E) existPushdown.first);
+ }
+
+ Alias pushDownAlias = null;
+ if (expression instanceof Alias) {
+ pushDownAlias = (Alias) expression;
+ } else {
+ pushDownAlias = new Alias(statementContext.getNextExprId(), expression);
+ }
+
+ Set<Slot> inputSlots = expression.getInputSlots();
+ for (Plan child : plan.children()) {
+ if (child.getOutputSet().containsAll(inputSlots)) {
+ Slot remaimSlot = pushDownAlias.toSlot();
+ exprToChildAndSlot.put(expression, Pair.of(remaimSlot, child));
+ childToPushDownProjects.put(child, pushDownAlias);
+ return Optional.of((E) remaimSlot);
+ }
+ }
+ return Optional.empty();
+ }
+
+ public List<Plan> buildNewChildren() {
+ if (childToPushDownProjects.isEmpty()) {
+ return plan.children();
+ }
+ ImmutableList.Builder<Plan> newChildren = ImmutableList.builderWithExpectedSize(plan.arity());
+ for (Plan child : plan.children()) {
+ Collection<NamedExpression> newProject = childToPushDownProjects.get(child);
+ if (newProject.isEmpty()) {
+ newChildren.add(child);
+ } else {
+ newChildren.add(
+ new LogicalProject<>(
+ ImmutableList.<NamedExpression>builder()
+ .addAll(child.getOutput())
+ .addAll(newProject)
+ .build(),
+ child
+ )
+ );
+ }
+ }
+ return newChildren.build();
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
new file mode 100644
index 00000000000..459c7362758
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
@@ -0,0 +1,674 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.analysis.AccessPathInfo;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.common.Pair;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.nereids.properties.OrderKey;
+import org.apache.doris.nereids.rules.rewrite.NestedColumnPruning.DataTypeAccessTree;
+import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
+import org.apache.doris.nereids.trees.expressions.Cast;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.OrderExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.Function;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
+import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN;
+import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
+import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
+import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
+import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
+import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
+import org.apache.doris.nereids.types.ArrayType;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.MapType;
+import org.apache.doris.nereids.types.NestedColumnPrunable;
+import org.apache.doris.nereids.types.StructType;
+import org.apache.doris.nereids.util.MoreFieldsThread;
+import org.apache.doris.thrift.TAccessPathType;
+import org.apache.doris.thrift.TColumnAccessPath;
+import org.apache.doris.thrift.TDataAccessPath;
+import org.apache.doris.thrift.TMetaAccessPath;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableMultimap.Builder;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.Set;
+
+/** SlotTypeReplacer */
+public class SlotTypeReplacer extends DefaultPlanRewriter<Void> {
+ private Map<Integer, AccessPathInfo> replacedDataTypes;
+
+ public SlotTypeReplacer(Map<Integer, AccessPathInfo> bottomReplacedDataTypes) {
+ this.replacedDataTypes = Maps.newLinkedHashMap(bottomReplacedDataTypes);
+ }
+
+ @Override
+ public Plan visitLogicalCTEProducer(LogicalCTEProducer<? extends Plan> cteProducer, Void context) {
+ return super.visitLogicalCTEProducer(cteProducer, context);
+ }
+
+ @Override
+ public Plan visitLogicalWindow(LogicalWindow<? extends Plan> window, Void context) {
+ window = visitChildren(this, window, context);
+ Pair<Boolean, ? extends List<? extends Expression>> replaced = replaceExpressions(
+ window.getExpressions(), false, false);
+ if (replaced.first) {
+ return window.withExpressionsAndChild((List) replaced.second, window.child());
+ }
+ return window;
+ }
+
+ @Override
+ public Plan visitLogicalCTEConsumer(LogicalCTEConsumer cteConsumer, Void context) {
+ Map<Slot, Slot> consumerToProducerOutputMap = cteConsumer.getConsumerToProducerOutputMap();
+ Multimap<Slot, Slot> producerToConsumerOutputMap = cteConsumer.getProducerToConsumerOutputMap();
+
+ Map<Slot, Slot> replacedConsumerToProducerOutputMap = new LinkedHashMap<>();
+ Builder<Slot, Slot> replacedProducerToConsumerOutputMap = ImmutableMultimap.builder();
+
+ boolean changed = false;
+ for (Entry<Slot, Slot> kv : consumerToProducerOutputMap.entrySet()) {
+ Slot consumerSlot = kv.getKey();
+ Slot producerSlot = kv.getValue();
+ AccessPathInfo accessPathInfo = replacedDataTypes.get(producerSlot.getExprId().asInt());
+ if (accessPathInfo != null) {
+ DataType prunedType = accessPathInfo.getPrunedType();
+ if (!prunedType.equals(producerSlot.getDataType())) {
+ replacedDataTypes.put(consumerSlot.getExprId().asInt(), accessPathInfo);
+ changed = true;
+ producerSlot = producerSlot.withNullableAndDataType(producerSlot.nullable(), prunedType);
+ consumerSlot = consumerSlot.withNullableAndDataType(consumerSlot.nullable(), prunedType);
+ }
+ }
+ replacedConsumerToProducerOutputMap.put(consumerSlot, producerSlot);
+ }
+
+ for (Entry<Slot, Collection<Slot>> kv : producerToConsumerOutputMap.asMap().entrySet()) {
+ Slot producerSlot = kv.getKey();
+ Collection<Slot> consumerSlots = kv.getValue();
+ AccessPathInfo accessPathInfo = replacedDataTypes.get(producerSlot.getExprId().asInt());
+ if (accessPathInfo != null && !accessPathInfo.getPrunedType().equals(producerSlot.getDataType())) {
+ DataType replacedDataType = accessPathInfo.getPrunedType();
+ changed = true;
+ producerSlot = producerSlot.withNullableAndDataType(producerSlot.nullable(), replacedDataType);
+ for (Slot consumerSlot : consumerSlots) {
+ consumerSlot = consumerSlot.withNullableAndDataType(consumerSlot.nullable(), replacedDataType);
+ replacedProducerToConsumerOutputMap.put(producerSlot, consumerSlot);
+ }
+ } else {
+ replacedProducerToConsumerOutputMap.putAll(producerSlot, consumerSlots);
+ }
+ }
+
+ if (changed) {
+ return new LogicalCTEConsumer(
+ cteConsumer.getRelationId(), cteConsumer.getCteId(), cteConsumer.getName(),
+ replacedConsumerToProducerOutputMap, replacedProducerToConsumerOutputMap.build()
+ );
+ }
+ return cteConsumer;
+ }
+
+ @Override
+ public Plan visitLogicalJoin(LogicalJoin<? extends Plan, ? extends Plan> join, Void context) {
+ join = visitChildren(this, join, context);
+ Pair<Boolean, List<Expression>> replacedHashJoinConjuncts
+ = replaceExpressions(join.getHashJoinConjuncts(), false, false);
+ Pair<Boolean, List<Expression>> replacedOtherJoinConjuncts
+ = replaceExpressions(join.getOtherJoinConjuncts(), false, false);
+
+ if (replacedHashJoinConjuncts.first || replacedOtherJoinConjuncts.first) {
+ return join.withJoinConjuncts(
+ replacedHashJoinConjuncts.second,
+ replacedOtherJoinConjuncts.second,
+ join.getJoinReorderContext());
+ }
+ return join;
+ }
+
+ @Override
+ public Plan visitLogicalProject(LogicalProject<? extends Plan> project, Void context) {
+ project = visitChildren(this, project, context);
+
+ Pair<Boolean, List<NamedExpression>> projects = replaceExpressions(project.getProjects(), true, false);
+ if (projects.first) {
+ return project.withProjects(projects.second);
+ }
+ return project;
+ }
+
+ @Override
+ public Plan visitLogicalPartitionTopN(LogicalPartitionTopN<? extends Plan> partitionTopN, Void context) {
+ partitionTopN = visitChildren(this, partitionTopN, context);
+
+ Pair<Boolean, List<Expression>> replacedPartitionKeys = replaceExpressions(
+ partitionTopN.getPartitionKeys(), false, false);
+ Pair<Boolean, List<OrderExpression>> replacedOrderExpressions
+ = replaceOrderExpressions(partitionTopN.getOrderKeys());
+ if (replacedPartitionKeys.first || replacedOrderExpressions.first) {
+ return partitionTopN.withPartitionKeysAndOrderKeys(
+ replacedPartitionKeys.second, replacedOrderExpressions.second);
+ }
+ return partitionTopN;
+ }
+
+ @Override
+ public Plan visitLogicalDeferMaterializeTopN(LogicalDeferMaterializeTopN<? extends Plan> topN, Void context) {
+ topN = visitChildren(this, topN, context);
+
+ LogicalTopN logicalTopN = (LogicalTopN) topN.getLogicalTopN().accept(this, context);
+ if (logicalTopN != topN.getLogicalTopN()) {
+ SlotReference replacedColumnIdSlot = replaceExpressions(
+ ImmutableList.of(topN.getColumnIdSlot()), false, false).second.get(0);
+ return new LogicalDeferMaterializeTopN(
+ logicalTopN, topN.getDeferMaterializeSlotIds(), replacedColumnIdSlot);
+ }
+
+ return topN;
+ }
+
+ @Override
+ public Plan visitLogicalExcept(LogicalExcept except, Void context) {
+ except = visitChildren(this, except, context);
+
+ Pair<Boolean, List<List<SlotReference>>> replacedRegularChildrenOutputs = replaceMultiExpressions(
+ except.getRegularChildrenOutputs());
+
+ Pair<Boolean, List<NamedExpression>> replacedOutputs
+ = replaceExpressions(except.getOutputs(), true, false);
+
+ if (replacedRegularChildrenOutputs.first || replacedOutputs.first) {
+ return new LogicalExcept(except.getQualifier(), except.getOutputs(),
+ except.getRegularChildrenOutputs(), except.children());
+ }
+
+ return except;
+ }
+
+ @Override
+ public Plan visitLogicalIntersect(LogicalIntersect intersect, Void context) {
+ intersect = visitChildren(this, intersect, context);
+
+ Pair<Boolean, List<List<SlotReference>>> replacedRegularChildrenOutputs = replaceMultiExpressions(
+ intersect.getRegularChildrenOutputs());
+
+ Pair<Boolean, List<NamedExpression>> replacedOutputs
+ = replaceExpressions(intersect.getOutputs(), true, false);
+
+ if (replacedRegularChildrenOutputs.first || replacedOutputs.first) {
+ return new LogicalIntersect(intersect.getQualifier(), intersect.getOutputs(),
+ intersect.getRegularChildrenOutputs(), intersect.children());
+ }
+ return intersect;
+ }
+
+ @Override
+ public Plan visitLogicalUnion(LogicalUnion union, Void context) {
+ union = visitChildren(this, union, context);
+
+ Pair<Boolean, List<List<SlotReference>>> replacedRegularChildrenOutputs = replaceMultiExpressions(
+ union.getRegularChildrenOutputs());
+
+ Pair<Boolean, List<NamedExpression>> replacedOutputs
+ = replaceExpressions(union.getOutputs(), true, false);
+
+ if (replacedRegularChildrenOutputs.first || replacedOutputs.first) {
+ return new LogicalUnion(
+ union.getQualifier(),
+ replacedOutputs.second,
+ replacedRegularChildrenOutputs.second,
+ union.getConstantExprsList(),
+ union.hasPushedFilter(),
+ union.children()
+ );
+ }
+
+ return union;
+ }
+
+ @Override
+ public Plan visitLogicalRepeat(LogicalRepeat<? extends Plan> repeat, Void context) {
+ repeat = visitChildren(this, repeat, context);
+
+ Pair<Boolean, List<List<Expression>>> replacedGroupingSets
+ = replaceMultiExpressions(repeat.getGroupingSets());
+ Pair<Boolean, List<NamedExpression>> replacedOutputs
+ = replaceExpressions(repeat.getOutputExpressions(), true, false);
+
+ if (replacedGroupingSets.first || replacedOutputs.first) {
+ return repeat.withGroupSetsAndOutput(replacedGroupingSets.second, replacedOutputs.second);
+ }
+ return repeat;
+ }
+
+ @Override
+ public Plan visitLogicalGenerate(LogicalGenerate<? extends Plan> generate, Void context) {
+ generate = visitChildren(this, generate, context);
+
+ Pair<Boolean, List<Function>> replacedGenerators
+ = replaceExpressions(generate.getGenerators(), false, false);
+ Pair<Boolean, List<Slot>> replacedGeneratorOutput
+ = replaceExpressions(generate.getGeneratorOutput(), false, false);
+ if (replacedGenerators.first || replacedGeneratorOutput.first) {
+ return new LogicalGenerate<>(replacedGenerators.second, replacedGeneratorOutput.second,
+ generate.getExpandColumnAlias(), generate.child());
+ }
+ return generate;
+ }
+
+ @Override
+ public Plan visitLogicalAggregate(LogicalAggregate<? extends Plan> aggregate, Void context) {
+ aggregate = visitChildren(this, aggregate, context);
+
+ Pair<Boolean, List<Expression>> replacedGroupBy = replaceExpressions(
+ aggregate.getGroupByExpressions(), false, false);
+ Pair<Boolean, List<NamedExpression>> replacedOutput = replaceExpressions(
+ aggregate.getOutputExpressions(), true, false);
+
+ if (replacedGroupBy.first || replacedOutput.first) {
+ return aggregate.withGroupByAndOutput(replacedGroupBy.second, replacedOutput.second);
+ }
+ return aggregate;
+ }
+
+ @Override
+ public Plan visitLogicalSort(LogicalSort<? extends Plan> sort, Void context) {
+ sort = visitChildren(this, sort, context);
+
+ Pair<Boolean, List<OrderKey>> replaced = replaceOrderKeys(sort.getOrderKeys());
+ if (replaced.first) {
+ return sort.withOrderKeys(replaced.second);
+ }
+ return sort;
+ }
+
+ @Override
+ public Plan visitLogicalTopN(LogicalTopN<? extends Plan> topN, Void context) {
+ topN = visitChildren(this, topN, context);
+
+ Pair<Boolean, List<OrderKey>> replaced = replaceOrderKeys(topN.getOrderKeys());
+ if (replaced.first) {
+ return topN.withOrderKeys(replaced.second);
+ }
+ return topN;
+ }
+
+ @Override
+ public Plan visitLogicalDeferMaterializeOlapScan(
+ LogicalDeferMaterializeOlapScan deferMaterializeOlapScan, Void context) {
+
+ LogicalOlapScan logicalOlapScan
+ = (LogicalOlapScan) deferMaterializeOlapScan.getLogicalOlapScan().accept(this, context);
+
+ if (logicalOlapScan != deferMaterializeOlapScan.getLogicalOlapScan()) {
+ SlotReference replacedColumnIdSlot = replaceExpressions(
+ ImmutableList.of(deferMaterializeOlapScan.getColumnIdSlot()), false, false).second.get(0);
+ return new LogicalDeferMaterializeOlapScan(
+ logicalOlapScan, deferMaterializeOlapScan.getDeferMaterializeSlotIds(), replacedColumnIdSlot
+ );
+ }
+ return deferMaterializeOlapScan;
+ }
+
+ @Override
+ public Plan visitLogicalFilter(LogicalFilter<? extends Plan> filter, Void context) {
+ filter = visitChildren(this, filter, context);
+
+ Pair<Boolean, Set<Expression>> replaced = replaceExpressions(filter.getConjuncts(), false, false);
+ if (replaced.first) {
+ return filter.withConjuncts(replaced.second);
+ }
+ return filter;
+ }
+
+ @Override
+ public Plan visitLogicalFileScan(LogicalFileScan fileScan, Void context) {
+ Pair<Boolean, List<Slot>> replaced = replaceExpressions(fileScan.getOutput(), false, true);
+ if (replaced.first) {
+ List<Slot> replaceSlots = new ArrayList<>(replaced.second);
+ if (fileScan.getTable() instanceof IcebergExternalTable) {
+ for (int i = 0; i < replaceSlots.size(); i++) {
+ Slot slot = replaceSlots.get(i);
+ if (!(slot instanceof SlotReference)) {
+ continue;
+ }
+ SlotReference slotReference = (SlotReference) slot;
+ Optional<List<TColumnAccessPath>> allAccessPaths = slotReference.getAllAccessPaths();
+ if (!allAccessPaths.isPresent() || !slotReference.getOriginalColumn().isPresent()) {
+ continue;
+ }
+ List<TColumnAccessPath> allAccessPathsWithId
+ = replaceIcebergAccessPathToId(allAccessPaths.get(), slotReference);
+ List<TColumnAccessPath> predicateAccessPathsWithId = replaceIcebergAccessPathToId(
+ slotReference.getPredicateAccessPaths().get(), slotReference);
+ replaceSlots.set(i, ((SlotReference) slot).withAccessPaths(
+ allAccessPathsWithId,
+ predicateAccessPathsWithId,
+ allAccessPaths.get(),
+ slotReference.getPredicateAccessPaths().get()
+ ));
+ }
+ }
+ return fileScan.withCachedOutput(replaceSlots);
+ }
+ return fileScan;
+ }
+
+ @Override
+ public Plan visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, Void context) {
+ Pair<Boolean, List<Slot>> replaced
+ = replaceExpressions(tvfRelation.getOutput(), false, true);
+ if (replaced.first) {
+ return tvfRelation.withCachedOutputs(replaced.second);
+ }
+ return tvfRelation;
+ }
+
+ @Override
+ public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Void context) {
+ Pair<Boolean, List<Slot>> replaced = replaceExpressions(olapScan.getOutput(), false, true);
+ if (replaced.first) {
+ return olapScan.withPrunedTypeSlots(replaced.second);
+ }
+ return olapScan;
+ }
+
+ @Override
+ public Plan visitLogicalEmptyRelation(LogicalEmptyRelation emptyRelation, Void context) {
+ Pair<Boolean, List<NamedExpression>> replacedProjects
+ = replaceExpressions(emptyRelation.getProjects(), true, false);
+
+ if (replacedProjects.first) {
+ return emptyRelation.withProjects(replacedProjects.second);
+ }
+ return emptyRelation;
+ }
+
+ @Override
+ public Plan visitLogicalOneRowRelation(LogicalOneRowRelation oneRowRelation, Void context) {
+ Pair<Boolean, List<NamedExpression>> replacedProjects
+ = replaceExpressions(oneRowRelation.getProjects(), true, false);
+
+ if (replacedProjects.first) {
+ return oneRowRelation.withProjects(replacedProjects.second);
+ }
+ return oneRowRelation;
+ }
+
+ @Override
+ public Plan visitLogicalResultSink(LogicalResultSink<? extends Plan> logicalResultSink, Void context) {
+ logicalResultSink = visitChildren(this, logicalResultSink, context);
+
+ Pair<Boolean, List<NamedExpression>> replacedOutput = replaceExpressions(logicalResultSink.getOutputExprs(),
+ false, false);
+ if (replacedOutput.first) {
+ return logicalResultSink.withOutputExprs(replacedOutput.second);
+ }
+ return logicalResultSink;
+ }
+
+ private Pair<Boolean, List<OrderExpression>> replaceOrderExpressions(List<OrderExpression> orderExpressions) {
+ ImmutableList.Builder<OrderExpression> newOrderKeys
+ = ImmutableList.builderWithExpectedSize(orderExpressions.size());
+ boolean changed = false;
+ for (OrderExpression orderExpression : orderExpressions) {
+ Expression newOrderKeyExpr = replaceSlot(orderExpression.getOrderKey().getExpr(), false);
+ if (newOrderKeyExpr != orderExpression.getOrderKey().getExpr()) {
+ newOrderKeys.add(new OrderExpression(orderExpression.getOrderKey().withExpression(newOrderKeyExpr)));
+ changed = true;
+ } else {
+ newOrderKeys.add(orderExpression);
+ }
+ }
+ return Pair.of(changed, newOrderKeys.build());
+ }
+
+ private Pair<Boolean, List<OrderKey>> replaceOrderKeys(List<OrderKey> orderKeys) {
+ ImmutableList.Builder<OrderKey> newOrderKeys = ImmutableList.builderWithExpectedSize(orderKeys.size());
+ boolean changed = false;
+ for (OrderKey orderKey : orderKeys) {
+ Expression newOrderKeyExpr = replaceSlot(orderKey.getExpr(), false);
+ if (newOrderKeyExpr != orderKey.getExpr()) {
+ newOrderKeys.add(orderKey.withExpression(newOrderKeyExpr));
+ changed = true;
+ } else {
+ newOrderKeys.add(orderKey);
+ }
+ }
+ return Pair.of(changed, newOrderKeys.build());
+ }
+
+ private <C extends Collection<E>, E extends Expression>
+ Pair<Boolean, List<C>> replaceMultiExpressions(List<C> expressionsList) {
+ ImmutableList.Builder<C> result = ImmutableList.builderWithExpectedSize(expressionsList.size());
+ boolean changed = false;
+ for (C expressions : expressionsList) {
+ Pair<Boolean, C> replaced = replaceExpressions(expressions, false, false);
+ changed |= replaced.first;
+ result.add(replaced.second);
+ }
+ return Pair.of(changed, result.build());
+ }
+
+ private <C extends Collection<E>, E extends Expression> Pair<Boolean, C> replaceExpressions(
+ C expressions, boolean propagateType, boolean fillAccessPaths) {
+ ImmutableCollection.Builder<E> newExprs;
+ if (expressions instanceof List) {
+ newExprs = ImmutableList.builder();
+ } else {
+ newExprs = ImmutableSet.builder();
+ }
+
+ boolean changed = false;
+ for (Expression oldExpr : expressions) {
+ Expression newExpr = replaceSlot(oldExpr, fillAccessPaths);
+ if (newExpr != oldExpr) {
+ newExprs.add((E) newExpr);
+ changed = true;
+
+ if (propagateType && oldExpr instanceof NamedExpression
+ && !oldExpr.getDataType().equals(newExpr.getDataType())) {
+ replacedDataTypes.put(
+ ((NamedExpression) oldExpr).getExprId().asInt(),
+ // not need access path in the upper slots
+ new AccessPathInfo(newExpr.getDataType(), null, null)
+ );
+ }
+ } else {
+ newExprs.add((E) oldExpr);
+ }
+ }
+ return Pair.of(changed, (C) newExprs.build());
+ }
+
+ private Expression replaceSlot(Expression e, boolean fillAccessPath) {
+ return MoreFieldsThread.keepFunctionSignature(false,
+ () -> doRewriteExpression(e, fillAccessPath)
+ );
+ }
+
+ private Expression doRewriteExpression(Expression e, boolean fillAccessPath) {
+ if (e instanceof Lambda) {
+ return rewriteLambda((Lambda) e, fillAccessPath);
+ } else if (e instanceof Cast) {
+ return rewriteCast((Cast) e, fillAccessPath);
+ } else if (e instanceof SlotReference) {
+ AccessPathInfo accessPathInfo = replacedDataTypes.get(((SlotReference) e).getExprId().asInt());
+ if (accessPathInfo != null) {
+ SlotReference newSlot = (SlotReference) ((SlotReference) e).withNullableAndDataType(
+ e.nullable(), accessPathInfo.getPrunedType());
+ if (fillAccessPath) {
+ newSlot = newSlot.withAccessPaths(
+ accessPathInfo.getAllAccessPaths(), accessPathInfo.getPredicateAccessPaths()
+ );
+ }
+ return newSlot;
+ }
+ }
+
+ ImmutableList.Builder<Expression> newChildren = ImmutableList.builderWithExpectedSize(e.arity());
+ boolean changed = false;
+ for (Expression child : e.children()) {
+ Expression newChild = doRewriteExpression(child, fillAccessPath);
+ changed |= child != newChild;
+ newChildren.add(newChild);
+ }
+ return changed ? e.withChildren(newChildren.build()) : e;
+ }
+
+ private Expression rewriteLambda(Lambda e, boolean fillAccessPath) {
+ // we should rewrite ArrayItemReference first, then we can replace the ArrayItemSlot int the lambda
+ Expression[] newChildren = new Expression[e.arity()];
+ for (int i = 0; i < e.arity(); i++) {
+ Expression child = e.child(i);
+ if (child instanceof ArrayItemReference) {
+ Expression newRef = child.withChildren(doRewriteExpression(child.child(0), fillAccessPath));
+ replacedDataTypes.put(((ArrayItemReference) child).getExprId().asInt(),
+ new AccessPathInfo(newRef.getDataType(), null, null));
+ newChildren[i] = newRef;
+ } else {
+ newChildren[i] = child;
+ }
+ }
+
+ for (int i = 0; i < newChildren.length; i++) {
+ Expression child = newChildren[i];
+ if (!(child instanceof ArrayItemReference)) {
+ newChildren[i] = doRewriteExpression(child, fillAccessPath);
+ }
+ }
+
+ return e.withChildren(newChildren);
+ }
+
+ private Expression rewriteCast(Cast cast, boolean fillAccessPath) {
+ Expression newChild = doRewriteExpression(cast.child(0), fillAccessPath);
+ if (newChild == cast.child(0)) {
+ return cast;
+ }
+
+ DataType newType = cast.getDataType();
+ if (cast.getDataType() instanceof NestedColumnPrunable
+ && newChild.getDataType() instanceof NestedColumnPrunable) {
+ DataTypeAccessTree originTree = DataTypeAccessTree.of(cast.child().getDataType(), TAccessPathType.DATA);
+ DataTypeAccessTree prunedTree = DataTypeAccessTree.of(newChild.getDataType(), TAccessPathType.DATA);
+ DataTypeAccessTree castTree = DataTypeAccessTree.of(cast.getDataType(), TAccessPathType.DATA);
+ newType = prunedTree.pruneCastType(originTree, castTree);
+ }
+
+ return new Cast(newChild, newType);
+ }
+
+ private List<TColumnAccessPath> replaceIcebergAccessPathToId(
+ List<TColumnAccessPath> originAccessPaths, SlotReference slotReference) {
+ Column column = slotReference.getOriginalColumn().get();
+ List<TColumnAccessPath> replacedAccessPaths = new ArrayList<>();
+ for (TColumnAccessPath accessPath : originAccessPaths) {
+ List<String> icebergColumnAccessPath = new ArrayList<>();
+ if (accessPath.type == TAccessPathType.DATA) {
+ icebergColumnAccessPath.addAll(accessPath.data_access_path.path);
+ replaceIcebergAccessPathToId(
+ icebergColumnAccessPath, 0, slotReference.getDataType(), column
+ );
+ TColumnAccessPath newAccessPath = new TColumnAccessPath(TAccessPathType.DATA);
+ newAccessPath.data_access_path = new TDataAccessPath(icebergColumnAccessPath);
+ replacedAccessPaths.add(newAccessPath);
+ } else {
+ icebergColumnAccessPath.addAll(accessPath.meta_access_path.path);
+ replaceIcebergAccessPathToId(
+ icebergColumnAccessPath, 0, slotReference.getDataType(), column
+ );
+ TColumnAccessPath newAccessPath = new TColumnAccessPath(TAccessPathType.META);
+ newAccessPath.meta_access_path = new TMetaAccessPath(icebergColumnAccessPath);
+ replacedAccessPaths.add(newAccessPath);
+ }
+ }
+ return replacedAccessPaths;
+ }
+
+ private void replaceIcebergAccessPathToId(List<String> originPath, int index, DataType type, Column column) {
+ if (index >= originPath.size()) {
+ return;
+ }
+ if (index == 0) {
+ originPath.set(index, String.valueOf(column.getUniqueId()));
+ replaceIcebergAccessPathToId(originPath, index + 1, type, column);
+ } else {
+ String fieldName = originPath.get(index);
+ if (type instanceof ArrayType) {
+ // skip replace *
+ replaceIcebergAccessPathToId(
+ originPath, index + 1, ((ArrayType) type).getItemType(), column.getChildren().get(0)
+ );
+ } else if (type instanceof MapType) {
+ if (fieldName.equals("*") || fieldName.equals("VALUES")) {
+ replaceIcebergAccessPathToId(
+ originPath, index + 1, ((MapType) type).getValueType(), column.getChildren().get(1)
+ );
+ }
+ } else if (type instanceof StructType) {
+ for (Column child : column.getChildren()) {
+ if (child.getName().equals(fieldName)) {
+ originPath.set(index, String.valueOf(child.getUniqueId()));
+ DataType childType = ((StructType) type).getNameToFields().get(fieldName).getDataType();
+ replaceIcebergAccessPathToId(originPath, index + 1, childType, child);
+ break;
+ }
+ }
+ } else {
+ originPath.set(index, String.valueOf(column.getUniqueId()));
+ }
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
index 1ed8d151270..abb65a283ed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
@@ -22,7 +22,6 @@ import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.properties.OrderKey;
-import org.apache.doris.nereids.rules.rewrite.ColumnPruning.PruneContext;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -87,7 +86,7 @@ import java.util.Set;
* generating the slots for the required sub path on scan, union, and cte consumer.
* Then, it replaces the element_at with the corresponding slot.
*/
-public class VariantSubPathPruning extends DefaultPlanRewriter<PruneContext> implements CustomRewriter {
+public class VariantSubPathPruning implements CustomRewriter {
public static final Logger LOG = LogManager.getLogger(VariantSubPathPruning.class);
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PreferPushDownProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PreferPushDownProject.java
new file mode 100644
index 00000000000..4c9c3bebe5a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PreferPushDownProject.java
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions;
+
+import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
+
+/**
+ * some expressions prefer push down project under the plan,
+ *
+ * e.g. the element_at function prefer push down to a project in under the plan:
+ * Project(element_at(left.column, 'id') as a)
+ * |
+ * Join
+ * / |
+ * left right
+ *
+ * we will push down project through the Join, so we can prune the column which has complex type:
+ *
+ * Join
+ * / \
+ * Project(element_at(left.column, 'id') as a, ...) right
+ * |
+ * left
+ */
+public interface PreferPushDownProject extends ExpressionTrait {
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
index ce2d3dbb3e5..1c77dc669ac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
@@ -24,6 +24,7 @@ import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.thrift.TColumnAccessPath;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -56,6 +57,10 @@ public class SlotReference extends Slot {
// that need return original table and name for view not its original table if u query a view
private final TableIf oneLevelTable;
private final Column oneLevelColumn;
+ private final Optional<List<TColumnAccessPath>> allAccessPaths;
+ private final Optional<List<TColumnAccessPath>> predicateAccessPaths;
+ private final Optional<List<TColumnAccessPath>> displayAllAccessPaths;
+ private final Optional<List<TColumnAccessPath>> displayPredicateAccessPaths;
public SlotReference(String name, DataType dataType) {
this(StatementScopeIdGenerator.newExprId(), name, dataType, true, ImmutableList.of(),
@@ -92,6 +97,15 @@ public class SlotReference extends Slot {
subPath, Optional.empty());
}
+ public SlotReference(ExprId exprId, Supplier<String> name, DataType dataType, boolean nullable,
+ List<String> qualifier, @Nullable TableIf originalTable, @Nullable Column originalColumn,
+ @Nullable TableIf oneLevelTable, Column oneLevelColumn,
+ List<String> subPath, Optional<Pair<Integer, Integer>> indexInSql) {
+ this(exprId, name, dataType, nullable, qualifier, originalTable, originalColumn, oneLevelTable,
+ oneLevelColumn, subPath, indexInSql, Optional.empty(), Optional.empty(),
+ Optional.empty(), Optional.empty());
+ }
+
/**
* Constructor for SlotReference.
*
@@ -106,7 +120,10 @@ public class SlotReference extends Slot {
public SlotReference(ExprId exprId, Supplier<String> name, DataType dataType, boolean nullable,
List<String> qualifier, @Nullable TableIf originalTable, @Nullable Column originalColumn,
@Nullable TableIf oneLevelTable, Column oneLevelColumn,
- List<String> subPath, Optional<Pair<Integer, Integer>> indexInSql) {
+ List<String> subPath, Optional<Pair<Integer, Integer>> indexInSql,
+ Optional<List<TColumnAccessPath>> allAccessPaths, Optional<List<TColumnAccessPath>> predicateAccessPaths,
+ Optional<List<TColumnAccessPath>> displayAllAccessPaths,
+ Optional<List<TColumnAccessPath>> displayPredicateAccessPaths) {
super(indexInSql);
this.exprId = exprId;
this.name = name;
@@ -119,6 +136,10 @@ public class SlotReference extends Slot {
this.oneLevelTable = oneLevelTable;
this.oneLevelColumn = oneLevelColumn;
this.subPath = Objects.requireNonNull(subPath, "subPath can not be null");
+ this.allAccessPaths = allAccessPaths;
+ this.predicateAccessPaths = predicateAccessPaths;
+ this.displayAllAccessPaths = displayAllAccessPaths;
+ this.displayPredicateAccessPaths = displayPredicateAccessPaths;
}
public static SlotReference of(String name, DataType type) {
@@ -342,4 +363,37 @@ public class SlotReference extends Slot {
public boolean hasAutoInc() {
return originalColumn != null ? originalColumn.isAutoInc() : false;
}
+
+ public SlotReference withAccessPaths(
+ List<TColumnAccessPath> allAccessPaths, List<TColumnAccessPath> predicateAccessPaths) {
+ return new SlotReference(exprId, name, dataType, nullable, qualifier,
+ originalTable, originalColumn, oneLevelTable, oneLevelColumn,
+ subPath, indexInSqlString, Optional.of(allAccessPaths), Optional.of(predicateAccessPaths),
+ Optional.of(allAccessPaths), Optional.of(predicateAccessPaths));
+ }
+
+ public SlotReference withAccessPaths(
+ List<TColumnAccessPath> allAccessPaths, List<TColumnAccessPath> predicateAccessPaths,
+ List<TColumnAccessPath> displayAllAccessPaths, List<TColumnAccessPath> displayPredicateAccessPaths) {
+ return new SlotReference(exprId, name, dataType, nullable, qualifier,
+ originalTable, originalColumn, oneLevelTable, oneLevelColumn,
+ subPath, indexInSqlString, Optional.of(allAccessPaths), Optional.of(predicateAccessPaths),
+ Optional.of(displayAllAccessPaths), Optional.of(displayPredicateAccessPaths));
+ }
+
+ public Optional<List<TColumnAccessPath>> getAllAccessPaths() {
+ return allAccessPaths;
+ }
+
+ public Optional<List<TColumnAccessPath>> getPredicateAccessPaths() {
+ return predicateAccessPaths;
+ }
+
+ public Optional<List<TColumnAccessPath>> getDisplayAllAccessPaths() {
+ return displayAllAccessPaths;
+ }
+
+ public Optional<List<TColumnAccessPath>> getDisplayPredicateAccessPaths() {
+ return displayPredicateAccessPaths;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirst.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirst.java
index 8c0babc39ca..5410de371a7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirst.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirst.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import java.util.List;
@@ -51,4 +52,9 @@ public class ArrayFirst extends ElementAt
public List<FunctionSignature> getImplSignature() {
return SIGNATURES;
}
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitArrayFirst(this, context);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLast.java
index e1ed4f4d27b..b9f5650156f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLast.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLast.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import java.util.List;
@@ -51,4 +52,9 @@ public class ArrayLast extends ElementAt
public ElementAt withChildren(List<Expression> children) {
return new ArrayLast(getFunctionParams(children));
}
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitArrayLast(this, context);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
index 5d74f5e1457..1a1a246cea6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ImplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -37,7 +38,7 @@ import java.util.List;
* ScalarFunction 'array_map'.
*/
public class ArrayMap extends ScalarFunction
- implements ImplicitlyCastableSignature, PropagateNullable {
+ implements ImplicitlyCastableSignature, PropagateNullable, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(new FollowToAnyDataType(0)).args(LambdaType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java
index b24d46c98a4..cd2cb974364 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.RewriteWhenAnalyze;
@@ -42,7 +43,8 @@ import java.util.List;
* ScalarFunction 'element_at'. This class is generated by GenerateFunction.
*/
public class ElementAt extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, RewriteWhenAnalyze {
+ implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable,
+ RewriteWhenAnalyze, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(new FollowToAnyDataType(0))
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java
index a929f2dd5d5..e7a054fc864 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.types.LambdaType;
import org.apache.doris.nereids.types.coercion.AnyDataType;
@@ -29,7 +30,7 @@ import java.util.List;
/**
* Interface of HighOrderFunction, provide default FunctionSignature of LambdaType argument
*/
-public interface HighOrderFunction extends ExplicitlyCastableSignature {
+public interface HighOrderFunction extends ExplicitlyCastableSignature, PreferPushDownProject {
@Override
default List<FunctionSignature> getSignatures() {
return new ImmutableList.Builder<FunctionSignature>()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java
index b8ecc8667c7..517bfaa7748 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -36,7 +37,7 @@ import java.util.List;
* ScalarFunction 'map_contains_entry'.
*/
public class MapContainsEntry extends ScalarFunction
- implements TernaryExpression, ExplicitlyCastableSignature {
+ implements TernaryExpression, ExplicitlyCastableSignature, PreferPushDownProject {
/**
* Basic signature, forcing search key/value type to convert to the same type as map's key/value type.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
index 4fc4c32f233..845e84f71b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -36,7 +37,7 @@ import java.util.List;
* ScalarFunction 'map_contains_key'. This class is generated by GenerateFunction.
*/
public class MapContainsKey extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature {
+ implements BinaryExpression, ExplicitlyCastableSignature, PreferPushDownProject {
public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
FunctionSignature.ret(BooleanType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
index a60f6653cde..fd42c42fb65 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -36,7 +37,7 @@ import java.util.List;
* ScalarFunction 'map_contains_value'. This class is generated by GenerateFunction.
*/
public class MapContainsValue extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature {
+ implements BinaryExpression, ExplicitlyCastableSignature, PreferPushDownProject {
public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
FunctionSignature.ret(BooleanType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java
index 4be48638e5e..22a14c31df3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.functions.SearchSignature;
@@ -41,7 +42,7 @@ import java.util.List;
* fields 'key' and 'value'.
*/
public class MapEntries extends ScalarFunction
- implements UnaryExpression, CustomSignature, PropagateNullable {
+ implements UnaryExpression, CustomSignature, PropagateNullable, PreferPushDownProject {
/**
* constructor with 1 argument.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java
index 957dae2844a..a3123d29cb0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -37,7 +38,7 @@ import java.util.List;
* ScalarFunction 'map_keys'.
*/
public class MapKeys extends ScalarFunction
- implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable {
+ implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(ArrayType.of(new FollowToAnyDataType(0))).args(MapType.of(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java
index e7ca7f48dd3..1622b277156 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -37,7 +38,7 @@ import java.util.List;
* ScalarFunction 'map_values'.
*/
public class MapValues extends ScalarFunction
- implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable {
+ implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(ArrayType.of(new FollowToAnyDataType(0))).args(MapType.of(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java
index 85bb424de19..ed29818b688 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral;
@@ -29,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.NullType;
+import org.apache.doris.nereids.types.StructField;
import org.apache.doris.nereids.types.StructType;
import com.google.common.base.Preconditions;
@@ -40,7 +42,7 @@ import java.util.List;
* ScalarFunction 'struct_element'.
*/
public class StructElement extends ScalarFunction
- implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral {
+ implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral, PreferPushDownProject {
/**
* constructor with 0 or more arguments.
@@ -98,10 +100,11 @@ public class StructElement extends ScalarFunction
}
} else if (child(1) instanceof StringLikeLiteral) {
String name = ((StringLikeLiteral) child(1)).getStringValue();
- if (!structArgType.getNameToFields().containsKey(name.toLowerCase())) {
+ StructField field = structArgType.getField(name);
+ if (field == null) {
throw new AnalysisException("the specified field name " + name + " was not found: " + this.toSql());
} else {
- retType = structArgType.getNameToFields().get(name).getDataType();
+ retType = field.getDataType();
}
} else {
throw new AnalysisException("struct_element only allows"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
index d347352c5ef..d48ba889cd0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
@@ -54,10 +54,12 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayEnumerat
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExcept;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExists;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFilter;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirst;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirstIndex;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFlatten;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayIntersect;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayJoin;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLast;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLastIndex;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMap;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMatchAll;
@@ -641,6 +643,10 @@ public interface ScalarFunctionVisitor<R, C> {
return visitScalarFunction(arrayFilter, context);
}
+ default R visitArrayFirst(ArrayFirst arrayFirst, C context) {
+ return visitElementAt(arrayFirst, context);
+ }
+
default R visitArrayFirstIndex(ArrayFirstIndex arrayFirstIndex, C context) {
return visitScalarFunction(arrayFirstIndex, context);
}
@@ -653,6 +659,10 @@ public interface ScalarFunctionVisitor<R, C> {
return visitScalarFunction(arrayJoin, context);
}
+ default R visitArrayLast(ArrayLast arrayLast, C context) {
+ return visitElementAt(arrayLast, context);
+ }
+
default R visitArrayLastIndex(ArrayLastIndex arrayLastIndex, C context) {
return visitScalarFunction(arrayLastIndex, context);
}
@@ -757,8 +767,8 @@ public interface ScalarFunctionVisitor<R, C> {
return visitScalarFunction(arrayFlatten, context);
}
- default R visitArrayMap(ArrayMap arraySort, C context) {
- return visitScalarFunction(arraySort, context);
+ default R visitArrayMap(ArrayMap arrayMap, C context) {
+ return visitScalarFunction(arrayMap, context);
}
default R visitArrayMatchAll(ArrayMatchAll arrayMatchAll, C context) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
index 6ae9e5dd32a..860fbee192e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
@@ -51,16 +51,17 @@ public class LogicalFileScan extends LogicalCatalogRelation {
protected final Optional<TableSample> tableSample;
protected final Optional<TableSnapshot> tableSnapshot;
protected final Optional<TableScanParams> scanParams;
+ protected final Optional<List<Slot>> cachedOutputs;
public LogicalFileScan(RelationId id, ExternalTable table, List<String> qualifier,
Collection<Slot> operativeSlots,
Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot,
- Optional<TableScanParams> scanParams) {
+ Optional<TableScanParams> scanParams, Optional<List<Slot>> cachedOutputs) {
this(id, table, qualifier,
table.initSelectedPartitions(MvccUtil.getSnapshotFromContext(table)),
operativeSlots, ImmutableList.of(),
tableSample, tableSnapshot,
- scanParams, Optional.empty(), Optional.empty());
+ scanParams, Optional.empty(), Optional.empty(), cachedOutputs);
}
/**
@@ -71,22 +72,24 @@ public class LogicalFileScan extends LogicalCatalogRelation {
List<NamedExpression> virtualColumns, Optional<TableSample> tableSample,
Optional<TableSnapshot> tableSnapshot, Optional<TableScanParams> scanParams,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
- String tableAlias) {
+ String tableAlias, Optional<List<Slot>> cachedSlots) {
super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, operativeSlots, virtualColumns,
groupExpression, logicalProperties, tableAlias);
this.selectedPartitions = selectedPartitions;
this.tableSample = tableSample;
this.tableSnapshot = tableSnapshot;
this.scanParams = scanParams;
+ this.cachedOutputs = cachedSlots;
}
protected LogicalFileScan(RelationId id, ExternalTable table, List<String> qualifier,
SelectedPartitions selectedPartitions, Collection<Slot> operativeSlots,
List<NamedExpression> virtualColumns, Optional<TableSample> tableSample,
Optional<TableSnapshot> tableSnapshot, Optional<TableScanParams> scanParams,
- Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties) {
+ Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
+ Optional<List<Slot>> cachedOutputs) {
this(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, groupExpression, logicalProperties, "");
+ scanParams, groupExpression, logicalProperties, "", cachedOutputs);
}
public SelectedPartitions getSelectedPartitions() {
@@ -126,7 +129,7 @@ public class LogicalFileScan extends LogicalCatalogRelation {
public LogicalFileScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, groupExpression, Optional.of(getLogicalProperties()), tableAlias);
+ scanParams, groupExpression, Optional.of(getLogicalProperties()), tableAlias, cachedOutputs);
}
@Override
@@ -134,26 +137,26 @@ public class LogicalFileScan extends LogicalCatalogRelation {
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, groupExpression, logicalProperties, tableAlias);
+ scanParams, groupExpression, logicalProperties, tableAlias, cachedOutputs);
}
public LogicalFileScan withSelectedPartitions(SelectedPartitions selectedPartitions) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, Optional.empty(), Optional.of(getLogicalProperties()), tableAlias);
+ scanParams, Optional.empty(), Optional.of(getLogicalProperties()), tableAlias, cachedOutputs);
}
@Override
public LogicalFileScan withRelationId(RelationId relationId) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, Optional.empty(), Optional.empty(), tableAlias);
+ scanParams, Optional.empty(), Optional.empty(), tableAlias, cachedOutputs);
}
public LogicalFileScan withTableAlias(String tableAlias) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, Optional.empty(), Optional.of(getLogicalProperties()), tableAlias);
+ scanParams, Optional.empty(), Optional.of(getLogicalProperties()), tableAlias, cachedOutputs);
}
@Override
@@ -166,6 +169,19 @@ public class LogicalFileScan extends LogicalCatalogRelation {
return super.equals(o) && Objects.equals(selectedPartitions, ((LogicalFileScan) o).selectedPartitions);
}
+ @Override
+ public List<Slot> computeOutput() {
+ if (cachedOutputs.isPresent()) {
+ return cachedOutputs.get();
+ }
+ return super.computeOutput();
+ }
+
+ @Override
+ public List<Slot> computeAsteriskOutput() {
+ return super.computeAsteriskOutput();
+ }
+
/**
* SelectedPartitions contains the selected partitions and the total partition number.
* Mainly for hive table partition pruning.
@@ -223,7 +239,13 @@ public class LogicalFileScan extends LogicalCatalogRelation {
public LogicalFileScan withOperativeSlots(Collection<Slot> operativeSlots) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, groupExpression, Optional.of(getLogicalProperties()));
+ scanParams, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
+ }
+
+ public LogicalFileScan withCachedOutput(List<Slot> cachedOutputs) {
+ return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
+ selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
+ scanParams, groupExpression, Optional.empty(), Optional.of(cachedOutputs));
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
index 9a123a04d2b..134f48309d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
@@ -77,9 +77,10 @@ public class LogicalHudiScan extends LogicalFileScan {
Collection<Slot> operativeSlots,
List<NamedExpression> virtualColumns,
Optional<GroupExpression> groupExpression,
- Optional<LogicalProperties> logicalProperties) {
+ Optional<LogicalProperties> logicalProperties,
+ Optional<List<Slot>> cachedOutputs) {
super(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns,
- tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties);
+ tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties, cachedOutputs);
Objects.requireNonNull(scanParams, "scanParams should not null");
Objects.requireNonNull(incrementalRelation, "incrementalRelation should not null");
this.incrementalRelation = incrementalRelation;
@@ -87,10 +88,11 @@ public class LogicalHudiScan extends LogicalFileScan {
public LogicalHudiScan(RelationId id, ExternalTable table, List<String> qualifier,
Collection<Slot> operativeSlots, Optional<TableScanParams> scanParams,
- Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot) {
+ Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot,
+ Optional<List<Slot>> cachedOutputs) {
this(id, table, qualifier, ((HMSExternalTable) table).initHudiSelectedPartitions(tableSnapshot),
tableSample, tableSnapshot, scanParams, Optional.empty(), operativeSlots, ImmutableList.of(),
- Optional.empty(), Optional.empty());
+ Optional.empty(), Optional.empty(), cachedOutputs);
}
public Optional<TableScanParams> getScanParams() {
@@ -140,7 +142,7 @@ public class LogicalHudiScan extends LogicalFileScan {
public LogicalHudiScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
@Override
@@ -148,20 +150,20 @@ public class LogicalHudiScan extends LogicalFileScan {
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, logicalProperties);
+ operativeSlots, virtualColumns, groupExpression, logicalProperties, cachedOutputs);
}
public LogicalHudiScan withSelectedPartitions(SelectedPartitions selectedPartitions) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
@Override
public LogicalHudiScan withRelationId(RelationId relationId) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
@Override
@@ -173,7 +175,7 @@ public class LogicalHudiScan extends LogicalFileScan {
public LogicalFileScan withOperativeSlots(Collection<Slot> operativeSlots) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
/**
@@ -226,6 +228,6 @@ public class LogicalHudiScan extends LogicalFileScan {
}
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, newIncrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
index 4278c22f123..394e414cc2e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
@@ -851,4 +851,21 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
}
return replaceMap;
}
+
+ /** withPrunedTypeSlots */
+ public LogicalOlapScan withPrunedTypeSlots(List<Slot> outputSlots) {
+ Map<Pair<Long, String>, Slot> replaceSlotMap = new HashMap<>();
+ for (Slot outputSlot : outputSlots) {
+ Pair<Long, String> key = Pair.of(selectedIndexId, outputSlot.getName());
+ replaceSlotMap.put(key, outputSlot);
+ }
+
+ return new LogicalOlapScan(relationId, (Table) table, qualifier,
+ groupExpression, Optional.empty(),
+ selectedPartitionIds, partitionPruned, selectedTabletIds,
+ selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
+ hints, replaceSlotMap, tableSample, directMvScan, colToSubPathsMap,
+ manuallySpecifiedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit,
+ annOrderKeys, annLimit, tableAlias);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java
index a434f81b231..c6d0241f393 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java
@@ -47,20 +47,31 @@ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation,
private final TableValuedFunction function;
private final ImmutableList<String> qualifier;
private final ImmutableList<Slot> operativeSlots;
+ private final Optional<List<Slot>> cachedOutputs;
public LogicalTVFRelation(RelationId id, TableValuedFunction function, ImmutableList<Slot> operativeSlots) {
super(id, PlanType.LOGICAL_TVF_RELATION);
this.operativeSlots = operativeSlots;
this.function = function;
- qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName());
+ this.qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName());
+ this.cachedOutputs = Optional.empty();
}
public LogicalTVFRelation(RelationId id, TableValuedFunction function, ImmutableList<Slot> operativeSlots,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties) {
+ this(id, function, operativeSlots, Optional.empty(), groupExpression, logicalProperties);
+ }
+
+ public LogicalTVFRelation(RelationId id, TableValuedFunction function,
+ ImmutableList<Slot> operativeSlots,
+ Optional<List<Slot>> cachedOutputs,
+ Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties) {
super(id, PlanType.LOGICAL_TVF_RELATION, groupExpression, logicalProperties);
this.operativeSlots = operativeSlots;
this.function = function;
- qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName());
+ this.cachedOutputs = Objects.requireNonNull(cachedOutputs, "cachedOutputs can not be null");
+ this.qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName());
}
@Override
@@ -120,6 +131,9 @@ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation,
@Override
public List<Slot> computeOutput() {
+ if (cachedOutputs.isPresent()) {
+ return cachedOutputs.get();
+ }
IdGenerator<ExprId> exprIdGenerator = StatementScopeIdGenerator.getExprIdGenerator();
return function.getTable().getBaseSchema()
.stream()
@@ -138,4 +152,9 @@ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation,
public TableValuedFunction getFunction() {
return function;
}
+
+ public LogicalTVFRelation withCachedOutputs(List<Slot> replaceSlots) {
+ return new LogicalTVFRelation(relationId, function, Utils.fastToImmutableList(operativeSlots),
+ Optional.of(replaceSlots), Optional.empty(), Optional.empty());
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java
index 9a34ab740d3..6d4ec539ff3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java
@@ -25,7 +25,7 @@ import java.util.Objects;
/**
* Array type in Nereids.
*/
-public class ArrayType extends DataType implements ComplexDataType {
+public class ArrayType extends DataType implements ComplexDataType, NestedColumnPrunable {
public static final ArrayType SYSTEM_DEFAULT = new ArrayType(NullType.INSTANCE, true);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java
index fb8346987f7..176c1db1d0d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java
@@ -27,7 +27,7 @@ import java.util.Objects;
* Struct type in Nereids.
*/
@Developing
-public class MapType extends DataType implements ComplexDataType {
+public class MapType extends DataType implements ComplexDataType, NestedColumnPrunable {
public static final MapType SYSTEM_DEFAULT = new MapType();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/NestedColumnPrunable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/NestedColumnPrunable.java
new file mode 100644
index 00000000000..2d5e464f042
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/NestedColumnPrunable.java
@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.types;
+
+/** NestedColumnPrunable */
+public interface NestedColumnPrunable {
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java
index 88317682c4c..63e94bc369b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java
@@ -19,7 +19,6 @@ package org.apache.doris.nereids.types;
import org.apache.doris.nereids.util.Utils;
-import java.util.Locale;
import java.util.Objects;
/**
@@ -41,7 +40,7 @@ public class StructField {
* @param nullable Indicates if values of this field can be `null` values
*/
public StructField(String name, DataType dataType, boolean nullable, String comment) {
- this.name = Objects.requireNonNull(name, "name should not be null").toLowerCase(Locale.ROOT);
+ this.name = Objects.requireNonNull(name, "name should not be null").toLowerCase();
this.dataType = Objects.requireNonNull(dataType, "dataType should not be null");
this.nullable = nullable;
this.comment = Objects.requireNonNull(comment, "comment should not be null");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java
index ffbff7c61e1..8296ac28281 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java
@@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -35,7 +36,7 @@ import java.util.stream.Collectors;
* Struct type in Nereids.
*/
@Developing
-public class StructType extends DataType implements ComplexDataType {
+public class StructType extends DataType implements ComplexDataType, NestedColumnPrunable {
public static final StructType SYSTEM_DEFAULT = new StructType();
@@ -55,7 +56,8 @@ public class StructType extends DataType implements ComplexDataType {
public StructType(List<StructField> fields) {
this.fields = ImmutableList.copyOf(Objects.requireNonNull(fields, "fields should not be null"));
// field name should be lowercase and check the same or not
- this.nameToFields = new HashMap<>();
+ // ATTN: should use LinkedHashMap to keep order
+ this.nameToFields = new LinkedHashMap<>();
for (StructField field : this.fields) {
String fieldName = field.getName().toLowerCase();
StructField existingField = this.nameToFields.put(fieldName, field);
@@ -73,6 +75,10 @@ public class StructType extends DataType implements ComplexDataType {
return nameToFields;
}
+ public StructField getField(String name) {
+ return nameToFields.get(name.toLowerCase());
+ }
+
@Override
public DataType conversion() {
return new StructType(fields.stream().map(StructField::conversion).collect(Collectors.toList()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index 98399c93d31..f9c6ed0b237 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -1088,6 +1088,9 @@ public class OlapScanNode extends ScanNode {
output.append(prefix).append("rewrittenProjectList: ").append(
getExplainString(rewrittenProjectList)).append("\n");
}
+
+ printNestedColumns(output, prefix);
+
return output.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index 27369e40e18..382782c393a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -518,7 +518,7 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
return expBuilder.toString();
}
- private String getplanNodeExplainString(String prefix, TExplainLevel detailLevel) {
+ private String getPlanNodeExplainString(String prefix, TExplainLevel detailLevel) {
StringBuilder expBuilder = new StringBuilder();
expBuilder.append(getNodeExplainString(prefix, detailLevel));
if (limit != -1) {
@@ -533,7 +533,7 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
}
public void getExplainStringMap(TExplainLevel detailLevel, Map<Integer, String> planNodeMap) {
- planNodeMap.put(id.asInt(), getplanNodeExplainString("", detailLevel));
+ planNodeMap.put(id.asInt(), getPlanNodeExplainString("", detailLevel));
for (int i = 0; i < children.size(); ++i) {
children.get(i).getExplainStringMap(detailLevel, planNodeMap);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
index fee9b0f8afc..ee8270fbd79 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
@@ -47,10 +47,13 @@ import org.apache.doris.datasource.FederationBackendPolicy;
import org.apache.doris.datasource.SplitAssignment;
import org.apache.doris.datasource.SplitGenerator;
import org.apache.doris.datasource.SplitSource;
+import org.apache.doris.datasource.iceberg.source.IcebergScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rpc.RpcException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.system.Backend;
+import org.apache.doris.thrift.TAccessPathType;
+import org.apache.doris.thrift.TColumnAccessPath;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TScanRange;
@@ -65,6 +68,7 @@ import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeRangeSet;
import org.apache.commons.collections.map.CaseInsensitiveMap;
+import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -539,6 +543,111 @@ public abstract class ScanNode extends PlanNode implements SplitGenerator {
.addValue(super.debugString()).toString();
}
+ protected void printNestedColumns(StringBuilder output, String prefix) {
+ boolean printNestedColumnsHeader = true;
+ for (SlotDescriptor slot : getTupleDesc().getSlots()) {
+ String prunedType = null;
+ if (!slot.getType().equals(slot.getColumn().getType())) {
+ prunedType = slot.getType().toString();
+ }
+ String displayAllAccessPathsString = null;
+ if (slot.getDisplayAllAccessPaths() != null
+ && slot.getDisplayAllAccessPaths() != null
+ && !slot.getDisplayAllAccessPaths().isEmpty()) {
+ if (this instanceof IcebergScanNode) {
+ displayAllAccessPathsString = mergeIcebergAccessPathsWithId(
+ slot.getAllAccessPaths(),
+ slot.getDisplayAllAccessPaths()
+ );
+ } else {
+ displayAllAccessPathsString = slot.getDisplayAllAccessPaths()
+ .stream()
+ .map(a -> {
+ if (a.type == TAccessPathType.DATA) {
+ return StringUtils.join(a.data_access_path.path, ".");
+ } else {
+ return StringUtils.join(a.meta_access_path.path, ".");
+ }
+ })
+ .collect(Collectors.joining(", "));
+ }
+ }
+ String displayPredicateAccessPathsString = null;
+ if (slot.getDisplayPredicateAccessPaths() != null
+ && slot.getDisplayPredicateAccessPaths() != null
+ && !slot.getDisplayPredicateAccessPaths().isEmpty()) {
+ if (this instanceof IcebergScanNode) {
+ displayPredicateAccessPathsString = mergeIcebergAccessPathsWithId(
+ slot.getPredicateAccessPaths(),
+ slot.getDisplayPredicateAccessPaths()
+ );
+ } else {
+ displayPredicateAccessPathsString = slot.getPredicateAccessPaths()
+ .stream()
+ .map(a -> {
+ if (a.type == TAccessPathType.DATA) {
+ return StringUtils.join(a.data_access_path.path, ".");
+ } else {
+ return StringUtils.join(a.meta_access_path.path, ".");
+ }
+ })
+ .collect(Collectors.joining(", "));
+ }
+ }
+
+
+ if (prunedType == null
+ && displayAllAccessPathsString == null
+ && displayPredicateAccessPathsString == null) {
+ continue;
+ }
+
+ if (printNestedColumnsHeader) {
+ output.append(prefix).append("nested columns:\n");
+ printNestedColumnsHeader = false;
+ }
+ output.append(prefix).append(" ").append(slot.getColumn().getName()).append(":\n");
+ output.append(prefix).append(" origin type: ").append(slot.getColumn().getType()).append("\n");
+ if (prunedType != null) {
+ output.append(prefix).append(" pruned type: ").append(prunedType).append("\n");
+ }
+ if (displayAllAccessPathsString != null) {
+ output.append(prefix).append(" all access paths: [")
+ .append(displayAllAccessPathsString).append("]\n");
+ }
+ if (displayPredicateAccessPathsString != null) {
+ output.append(prefix).append(" predicate access paths: [")
+ .append(displayPredicateAccessPathsString).append("]\n");
+ }
+ }
+ }
+
+ private String mergeIcebergAccessPathsWithId(
+ List<TColumnAccessPath> accessPaths, List<TColumnAccessPath> displayAccessPaths) {
+ List<String> mergeDisplayAccessPaths = Lists.newArrayList();
+ for (int i = 0; i < displayAccessPaths.size(); i++) {
+ TColumnAccessPath displayAccessPath = displayAccessPaths.get(i);
+ TColumnAccessPath idAccessPath = accessPaths.get(i);
+ List<String> nameAccessPathStrings = displayAccessPath.type == TAccessPathType.DATA
+ ? displayAccessPath.data_access_path.path : displayAccessPath.meta_access_path.path;
+ List<String> idAccessPathStrings = idAccessPath.type == TAccessPathType.DATA
+ ? idAccessPath.data_access_path.path : idAccessPath.meta_access_path.path;
+
+ List<String> mergedPath = new ArrayList<>();
+ for (int j = 0; j < idAccessPathStrings.size(); j++) {
+ String name = nameAccessPathStrings.get(j);
+ String id = idAccessPathStrings.get(j);
+ if (name.equals(id)) {
+ mergedPath.add(name);
+ } else {
+ mergedPath.add(name + "(" + id + ")");
+ }
+ }
+ mergeDisplayAccessPaths.add(StringUtils.join(mergedPath, "."));
+ }
+ return StringUtils.join(mergeDisplayAccessPaths, ", ");
+ }
+
public List<TupleId> getOutputTupleIds() {
if (outputTupleDesc != null) {
return Lists.newArrayList(outputTupleDesc.getId());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 52b1d90ac26..7a06cbc7823 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -409,6 +409,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_RUNTIME_FILTER_PARTITION_PRUNE =
"enable_runtime_filter_partition_prune";
+ public static final String ENABLE_PRUNE_NESTED_COLUMN = "enable_prune_nested_column";
+
static final String SESSION_CONTEXT = "session_context";
public static final String DEFAULT_ORDER_BY_LIMIT = "default_order_by_limit";
@@ -1557,6 +1559,13 @@ public class SessionVariable implements Serializable, Writable {
varType = VariableAnnotation.EXPERIMENTAL)
public int topNLazyMaterializationThreshold = 1024;
+ @VariableMgr.VarAttr(name = ENABLE_PRUNE_NESTED_COLUMN, needForward = true,
+ fuzzy = false,
+ varType = VariableAnnotation.EXPERIMENTAL,
+ description = {"是否裁剪map/struct类型", "Whether to prune the type of map/struct"}
+ )
+ public boolean enablePruneNestedColumns = true;
+
public boolean enableTopnLazyMaterialization() {
return ConnectContext.get() != null
&& ConnectContext.get().getSessionVariable().topNLazyMaterializationThreshold > 0;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
new file mode 100644
index 00000000000..8e8d7c44d16
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -0,0 +1,625 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.analysis.SlotDescriptor;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.Pair;
+import org.apache.doris.common.Triple;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.util.MemoPatternMatchSupported;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.planner.OlapScanNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.thrift.TAccessPathType;
+import org.apache.doris.thrift.TColumnAccessPath;
+import org.apache.doris.thrift.TDataAccessPath;
+import org.apache.doris.thrift.TMetaAccessPath;
+import org.apache.doris.utframe.TestWithFeService;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeSet;
+import java.util.function.Consumer;
+
+public class PruneNestedColumnTest extends TestWithFeService implements MemoPatternMatchSupported {
+ @BeforeAll
+ public void createTable() throws Exception {
+ createDatabase("test");
+ useDatabase("test");
+
+ createTable("create table tbl(\n"
+ + " id int,\n"
+ + " s struct<\n"
+ + " city: string,\n"
+ + " data: array<map<\n"
+ + " int,\n"
+ + " struct<a: int, b: double>\n"
+ + " >>\n"
+ + ">)\n"
+ + "properties ('replication_num'='1')");
+
+ createTable("create table tbl2(\n"
+ + " id2 int,\n"
+ + " s2 struct<\n"
+ + " city2: string,\n"
+ + " data2: array<map<\n"
+ + " int,\n"
+ + " struct<a2: int, b2: double>\n"
+ + " >>\n"
+ + ">)\n"
+ + "properties ('replication_num'='1')");
+
+ connectContext.getSessionVariable().setDisableNereidsRules(RuleType.PRUNE_EMPTY_PARTITION.name());
+ connectContext.getSessionVariable().enableNereidsTimeout = false;
+ }
+
+ @Test
+ public void testCaseInsensitive() throws Exception {
+ assertColumn("select struct_element(MAP_VALUES(struct_element(S, 'DATA')[1])[1], 'B') from tbl",
+ "struct<data:array<map<int,struct<b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "b")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testMap() throws Exception {
+ assertColumn("select MAP_KEYS(struct_element(s, 'data')[0])[1] from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "KEYS")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select MAP_VALUES(struct_element(s, 'data')[0])[1] from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testStruct() throws Throwable {
+ assertColumn("select struct_element(s, 1) from tbl",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select struct_element(map_values(struct_element(s, 'data')[0])[0], 1) from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testPruneCast() throws Exception {
+ assertColumn("select struct_element(cast(s as struct<k:text,l:array<map<int,struct<x:int,y:double>>>>), 'k') from tbl",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select struct_element(map_values(struct_element(cast(s as struct<k:text,l:array<map<int,struct<x:double,y:double>>>>), 'l')[0])[0], 'x') from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of()
+ );
+
+ assertColumns("select struct_element(s, 'city') from (select * from tbl union all select * from tbl2)t",
+ ImmutableList.of(
+ Triple.of(
+ "struct<city2:text>",
+ ImmutableList.of(path("s2", "city2")),
+ ImmutableList.of()
+ ),
+ Triple.of(
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ )
+ )
+ );
+
+ assertColumns("select struct_element(s, 'city'), struct_element(map_values(struct_element(s, 'data')[0])[0], 'b') from (select * from tbl union all select * from tbl2)t",
+ ImmutableList.of(
+ Triple.of(
+ "struct<city2:text,data2:array<map<int,struct<b2:double>>>>",
+ ImmutableList.of(path("s2", "city2"), path("s2", "data2", "*", "VALUES", "b2")),
+ ImmutableList.of()
+ ),
+ Triple.of(
+ "struct<city:text,data:array<map<int,struct<b:double>>>>",
+ ImmutableList.of(path("s", "city"), path("s", "data", "*", "VALUES", "b")),
+ ImmutableList.of()
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testPruneArrayLambda() throws Exception {
+ // map_values(struct_element(s, 'data').*)[0].a
+ assertColumn("select struct_element(array_map(x -> map_values(x)[0], struct_element(s, 'data'))[0], 'a') from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select array_map((x, y) -> struct_element(map_values(x)[0], 'a') + struct_element(map_values(y)[0], 'b'), struct_element(s, 'data'), struct_element(s, 'data')) from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a"), path("s", "data", "*", "VALUES", "b")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testProject() throws Exception {
+ assertColumn("select 100 from tbl", null, null, null);
+ assertColumn("select * from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select tbl.* from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select test.tbl.* from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select internal.test.tbl.* from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select s from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'city'), s from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'city') from tbl",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'data') from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'data')[1] from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*")),
+ ImmutableList.of()
+ );
+ assertColumn("select map_keys(struct_element(s, 'data')[1]) from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "KEYS")),
+ ImmutableList.of()
+ );
+ assertColumn("select map_values(struct_element(s, 'data')[1]) from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(map_values(struct_element(s, 'data')[1])[1], 'a') from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'data')[1][1] from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "*")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(struct_element(s, 'data')[1][1], 'a') from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "*", "a")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(struct_element(s, 'data')[1][1], 'b') from tbl",
+ "struct<data:array<map<int,struct<b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "*", "b")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testFilter() throws Throwable {
+ assertColumn("select 100 from tbl where s is not null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of(path("s"))
+ );
+
+ assertColumn("select 100 from tbl where if(id = 1, null, s) is not null and struct_element(s, 'city') = 'beijing'",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of(path("s"))
+ );
+
+ assertColumn("select 100 from tbl where struct_element(s, 'city') is not null",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of(path("s", "city"))
+ );
+
+ assertColumn("select 100 from tbl where struct_element(s, 'data') is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data")),
+ ImmutableList.of(path("s", "data"))
+ );
+ assertColumn("select 100 from tbl where struct_element(s, 'data')[1] is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*")),
+ ImmutableList.of(path("s", "data", "*"))
+ );
+ assertColumn("select 100 from tbl where map_keys(struct_element(s, 'data')[1]) is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "KEYS")),
+ ImmutableList.of(path("s", "data", "*", "KEYS"))
+ );
+ assertColumn("select 100 from tbl where map_values(struct_element(s, 'data')[1]) is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES")),
+ ImmutableList.of(path("s", "data", "*", "VALUES"))
+ );
+ assertColumn("select 100 from tbl where struct_element(map_values(struct_element(s, 'data')[1])[1], 'a') is not null",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a"))
+ );
+ assertColumn("select 100 from tbl where struct_element(s, 'data')[1][1] is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "*")),
+ ImmutableList.of(path("s", "data", "*", "*"))
+ );
+ assertColumn("select 100 from tbl where struct_element(struct_element(s, 'data')[1][1], 'a') is not null",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "*", "a")),
+ ImmutableList.of(path("s", "data", "*", "*", "a"))
+ );
+ assertColumn("select 100 from tbl where struct_element(struct_element(s, 'data')[1][1], 'b') is not null",
+ "struct<data:array<map<int,struct<b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "*", "b")),
+ ImmutableList.of(path("s", "data", "*", "*", "b"))
+ );
+ }
+
+ @Test
+ public void testProjectFilter() throws Throwable {
+ assertColumn("select s from tbl where struct_element(s, 'city') is not null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of(path("s", "city"))
+ );
+
+ assertColumn("select struct_element(s, 'data') from tbl where struct_element(s, 'city') is not null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data"), path("s", "city")),
+ ImmutableList.of(path("s", "city"))
+ );
+
+ assertColumn("select struct_element(s, 'data') from tbl where struct_element(s, 'city') is not null and struct_element(s, 'data') is not null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data"), path("s", "city")),
+ ImmutableList.of(path("s", "data"), path("s", "city"))
+ );
+ }
+
+ @Test
+ public void testCte() throws Throwable {
+ assertColumn("with t as (select id, s from tbl) select struct_element(t1.s, 'city') from t t1 join t t2 on t1.id = t2.id",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("with t as (select id, struct_element(s, 'city') as c from tbl) select t1.c from t t1 join t t2 on t1.id = t2.id",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testUnion() throws Throwable {
+ assertColumn("select struct_element(s, 'city') from (select s from tbl union all select null)a",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select * from (select struct_element(s, 'city') from tbl union all select null)a",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testCteAndUnion() throws Throwable {
+ assertColumn("with t as (select id, s from tbl) select struct_element(s, 'city') from (select * from t union all select 1, null) tmp",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("with t as (select id, s from tbl) select * from (select struct_element(s, 'city') from t union all select null) tmp",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testPushDownThroughJoin() {
+ PlanChecker.from(connectContext)
+ .analyze("select struct_element(s, 'city') from (select * from tbl)a join (select 100 id, 'f1' name)b on a.id=b.id")
+ .rewrite()
+ .matches(
+ logicalResultSink(
+ logicalProject(
+ logicalJoin(
+ logicalProject(
+ logicalFilter(
+ logicalOlapScan()
+ )
+ ).when(p -> {
+ Assertions.assertEquals(2, p.getProjects().size());
+ Assertions.assertTrue(p.getProjects().stream()
+ .anyMatch(o -> o instanceof Alias && o.child(0) instanceof StructElement));
+ return true;
+ }),
+ logicalOneRowRelation()
+ )
+ ).when(p -> {
+ Assertions.assertTrue(p.getProjects().size() == 1 && p.getProjects().get(0) instanceof SlotReference);
+ return true;
+ })
+ )
+ );
+ }
+
+ @Test
+ public void testPushDownThroughWindow() {
+ PlanChecker.from(connectContext)
+ .analyze("select struct_element(s, 'city'), r from (select s, rank() over(partition by id) r from tbl t)a")
+ .rewrite()
+ .matches(
+ logicalResultSink(
+ logicalProject(
+ logicalWindow(
+ logicalProject(
+ logicalOlapScan()
+ ).when(p -> {
+ Assertions.assertEquals(2, p.getProjects().size());
+ Assertions.assertTrue(p.getProjects().stream()
+ .anyMatch(o -> o instanceof Alias && o.child(0) instanceof StructElement));
+ return true;
+ })
+ )
+ ).when(p -> {
+ Assertions.assertTrue(p.getProjects().size() == 2 && p.getProjects().get(0) instanceof SlotReference);
+ return true;
+ })
+ )
+ );
+ }
+
+ @Test
+ public void testPushDownThroughPartitionTopN() {
+ PlanChecker.from(connectContext)
+ .analyze("select struct_element(s, 'city'), r from (select s, rank() over(partition by id) r from tbl t limit 10)a")
+ .rewrite()
+ .matches(
+ logicalResultSink(
+ logicalLimit(
+ logicalLimit(
+ logicalProject(
+ logicalWindow(
+ logicalPartitionTopN(
+ logicalProject(
+ logicalOlapScan()
+ ).when(p -> {
+ Assertions.assertEquals(2, p.getProjects().size());
+ Assertions.assertTrue(p.getProjects().stream()
+ .anyMatch(o -> o instanceof Alias && o.child(0) instanceof StructElement));
+ return true;
+ })
+ )
+ )
+ ).when(p -> {
+ Assertions.assertTrue(p.getProjects().size() == 2 && p.getProjects().get(0) instanceof SlotReference);
+ return true;
+ })
+ )
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testPushDownThroughUnion() {
+ PlanChecker.from(connectContext)
+ .analyze("select struct_element(s, 'city') from (select id, s from tbl union all select 1, null) tmp")
+ .rewrite()
+ .matches(
+ logicalResultSink(
+ logicalUnion(
+ logicalProject(
+ logicalOlapScan()
+ ).when(p -> {
+ Assertions.assertEquals(1, p.getProjects().size());
+ Assertions.assertInstanceOf(StructElement.class, p.getProjects().get(0).child(0));
+ return true;
+ })
+ ).when(u -> {
+ Assertions.assertEquals(1, u.getConstantExprsList().size());
+ Assertions.assertInstanceOf(NullLiteral.class, u.getConstantExprsList().get(0).get(0).child(0));
+ return true;
+ })
+ )
+ );
+ }
+
+ private void assertColumn(String sql, String expectType,
+ List<TColumnAccessPath> expectAllAccessPaths,
+ List<TColumnAccessPath> expectPredicateAccessPaths) throws Exception {
+ assertColumns(sql, expectType == null ? null : ImmutableList.of(Triple.of(expectType, expectAllAccessPaths, expectPredicateAccessPaths)));
+ }
+
+ private void assertColumns(String sql,
+ List<Triple<String, List<TColumnAccessPath>, List<TColumnAccessPath>>> expectResults) throws Exception {
+ Pair<PhysicalPlan, List<SlotDescriptor>> result = collectComplexSlots(sql);
+ PhysicalPlan physicalPlan = result.first;
+ List<SlotDescriptor> slotDescriptors = result.second;
+ if (expectResults == null) {
+ Assertions.assertEquals(0, slotDescriptors.size());
+ return;
+ }
+
+ Assertions.assertEquals(expectResults.size(), slotDescriptors.size());
+ int slotIndex = 0;
+ for (Triple<String, List<TColumnAccessPath>, List<TColumnAccessPath>> expectResult : expectResults) {
+ String expectType = expectResult.left;
+ List<TColumnAccessPath> expectAllAccessPaths = expectResult.middle;
+ List<TColumnAccessPath> expectPredicateAccessPaths = expectResult.right;
+ SlotDescriptor slotDescriptor = slotDescriptors.get(slotIndex++);
+ Assertions.assertEquals(expectType, slotDescriptor.getType().toString());
+
+ TreeSet<TColumnAccessPath> expectAllAccessPathSet = new TreeSet<>(expectAllAccessPaths);
+ TreeSet<TColumnAccessPath> actualAllAccessPaths
+ = new TreeSet<>(slotDescriptor.getAllAccessPaths());
+ Assertions.assertEquals(expectAllAccessPathSet, actualAllAccessPaths);
+
+ TreeSet<TColumnAccessPath> expectPredicateAccessPathSet = new TreeSet<>(expectPredicateAccessPaths);
+ TreeSet<TColumnAccessPath> actualPredicateAccessPaths
+ = new TreeSet<>(slotDescriptor.getPredicateAccessPaths());
+ Assertions.assertEquals(expectPredicateAccessPathSet, actualPredicateAccessPaths);
+
+ Map<Integer, DataType> slotIdToDataTypes = new LinkedHashMap<>();
+ Consumer<Expression> assertHasSameType = e -> {
+ if (e instanceof NamedExpression) {
+ DataType dataType = slotIdToDataTypes.get(((NamedExpression) e).getExprId().asInt());
+ if (dataType != null) {
+ Assertions.assertEquals(dataType, e.getDataType());
+ } else {
+ slotIdToDataTypes.put(((NamedExpression) e).getExprId().asInt(), e.getDataType());
+ }
+ }
+ };
+
+ // assert same slot id has same type
+ physicalPlan.foreachUp(plan -> {
+ List<? extends Expression> expressions = ((PhysicalPlan) plan).getExpressions();
+ for (Expression expression : expressions) {
+ expression.foreach(e -> {
+ assertHasSameType.accept((Expression) e);
+ if (e instanceof Alias && e.child(0) instanceof Slot) {
+ assertHasSameType.accept((Alias) e);
+ } else if (e instanceof ArrayItemReference) {
+ assertHasSameType.accept((ArrayItemReference) e);
+ }
+ });
+ }
+
+ if (plan instanceof PhysicalCTEConsumer) {
+ for (Entry<Slot, Collection<Slot>> kv : ((PhysicalCTEConsumer) plan).getProducerToConsumerSlotMap()
+ .asMap().entrySet()) {
+ Slot producerSlot = kv.getKey();
+ for (Slot consumerSlot : kv.getValue()) {
+ Assertions.assertEquals(producerSlot.getDataType(), consumerSlot.getDataType());
+ }
+ }
+ } else if (plan instanceof PhysicalUnion) {
+ List<Slot> output = ((PhysicalUnion) plan).getOutput();
+ for (List<SlotReference> regularChildrenOutput : ((PhysicalUnion) plan).getRegularChildrenOutputs()) {
+ Assertions.assertEquals(output.size(), regularChildrenOutput.size());
+ for (int i = 0; i < output.size(); i++) {
+ Assertions.assertEquals(output.get(i).getDataType(), regularChildrenOutput.get(i).getDataType());
+ }
+ }
+ }
+ });
+ }
+ }
+
+ private Pair<PhysicalPlan, List<SlotDescriptor>> collectComplexSlots(String sql) throws Exception {
+ NereidsPlanner planner = (NereidsPlanner) executeNereidsSql(sql).planner();
+ List<SlotDescriptor> complexSlots = new ArrayList<>();
+ PhysicalPlan physicalPlan = planner.getPhysicalPlan();
+ for (PlanFragment fragment : planner.getFragments()) {
+ List<OlapScanNode> olapScanNodes = fragment.getPlanRoot().collectInCurrentFragment(OlapScanNode.class::isInstance);
+ for (OlapScanNode olapScanNode : olapScanNodes) {
+ List<SlotDescriptor> slots = olapScanNode.getTupleDesc().getSlots();
+ for (SlotDescriptor slot : slots) {
+ Type type = slot.getType();
+ if (type.isComplexType() || type.isVariantType()) {
+ complexSlots.add(slot);
+ }
+ }
+ }
+ }
+ return Pair.of(physicalPlan, complexSlots);
+ }
+
+ private TColumnAccessPath path(String... path) {
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.DATA);
+ accessPath.data_access_path = new TDataAccessPath(ImmutableList.copyOf(path));
+ return accessPath;
+ }
+
+ private TColumnAccessPath metaPath(String... path) {
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.META);
+ accessPath.meta_access_path = new TMetaAccessPath(ImmutableList.copyOf(path));
+ return accessPath;
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
index 21c45c7135c..55d8e70d50d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
@@ -559,8 +559,8 @@ public abstract class TestWithFeService {
&& connectContext.getState().getErrorCode() == null) {
return stmtExecutor;
} else {
- // throw new IllegalStateException(connectContext.getState().getErrorMessage());
- return null;
+ throw new IllegalStateException(connectContext.getState().getErrorMessage());
+ // return null;
}
}
@@ -574,7 +574,7 @@ public abstract class TestWithFeService {
}
}
- public void executeNereidsSql(String queryStr) throws Exception {
+ public StmtExecutor executeNereidsSql(String queryStr) throws Exception {
connectContext.getState().reset();
StatementContext statementContext = new StatementContext(connectContext, new OriginStatement(queryStr, 0));
@@ -590,6 +590,7 @@ public abstract class TestWithFeService {
|| connectContext.getState().getErrorCode() != null) {
throw new IllegalStateException(connectContext.getState().getErrorMessage());
}
+ return stmtExecutor;
}
public void createDatabase(String db) throws Exception {
diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift
index a265db13e5d..23f72d4bc68 100644
--- a/gensrc/thrift/Descriptors.thrift
+++ b/gensrc/thrift/Descriptors.thrift
@@ -27,6 +27,49 @@ enum TPatternType {
MATCH_NAME_GLOB = 2
}
+enum TAccessPathType {
+ DATA = 1,
+ META = 2 // use to prune `where s.data is not null` by only scan the meta of s.data
+}
+
+struct TDataAccessPath {
+ // the specification of special path:
+ // <empty>: access the whole complex column
+ // *:
+ // 1. access every items when the type is array
+ // 2. access key and value when the type is map
+ // KEYS: only access the keys of map
+ // VALUES: only access the keys of map
+ //
+ // example:
+ // s: struct<
+ // data: array<
+ // map<
+ // int,
+ // struct<
+ // a: id
+ // b: double
+ // >
+ // >
+ // >
+ // >
+ // if we want to access `map_keys(s.data[0])`, the path will be: ['s', 'data', '*', 'KEYS'],
+ // if we want to access `map_values(s.data[0])[0].b`, the path will be: ['s', 'data', '*', 'VALUES', 'b'],
+ // if we want to access `s.data[0]['k'].b`, the path will be ['s', 'data', '*', '*', 'b']
+ // if we want to access the whole struct of s, the path will be: ['s'],
+ 1: required list<string> path
+}
+
+struct TMetaAccessPath {
+ 1: required list<string> path
+}
+
+struct TColumnAccessPath {
+ 1: required TAccessPathType type
+ 2: optional TDataAccessPath data_access_path
+ 3: optional TMetaAccessPath meta_access_path
+}
+
struct TColumn {
1: required string column_name
2: required Types.TColumnType column_type
@@ -69,14 +112,17 @@ struct TSlotDescriptor {
11: optional i32 col_unique_id = -1
12: optional bool is_key = false
// If set to false, then such slots will be ignored during
- // materialize them.Used to optmize to read less data and less memory usage
+ // materialize them.Used to optimize to read less data and less memory usage
13: optional bool need_materialize = true
14: optional bool is_auto_increment = false;
// subcolumn path info list for semi structure column(variant)
+ // deprecated: will be replaced to column_access_paths
15: optional list<string> column_paths
16: optional string col_default_value
17: optional Types.TPrimitiveType primitive_type = Types.TPrimitiveType.INVALID_TYPE
18: optional Exprs.TExpr virtual_column_expr
+ 19: optional list<TColumnAccessPath> all_access_paths
+ 20: optional list<TColumnAccessPath> predicate_access_paths
}
struct TTupleDescriptor {
diff --git a/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out b/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out
new file mode 100644
index 00000000000..74f336d27e5
--- /dev/null
+++ b/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out
@@ -0,0 +1,24 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+1 {"city":"beijing", "data":[{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}]}
+2 {"city":"shanghai", "data":[{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}]}
+
+-- !sql1 --
+1 [10]
+
+-- !sql2 --
+1 beijing
+2 shanghai
+
+-- !sql3 --
+1 [{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}]
+2 [{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}]
+
+-- !sql4 --
+1 [{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}]
+2 [{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}]
+
+-- !sql5 --
+1 beijing
+2 shanghai
+
diff --git a/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy b/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy
new file mode 100644
index 00000000000..ad032fab8f2
--- /dev/null
+++ b/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_pruned_columns") {
+ sql """DROP TABLE IF EXISTS `tbl_test_pruned_columns`"""
+ sql """
+ CREATE TABLE `tbl_test_pruned_columns` (
+ `id` int NULL,
+ `s` struct<city:text,data:array<map<int,struct<a:int,b:double>>>> NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY RANDOM BUCKETS AUTO
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """
+ insert into `tbl_test_pruned_columns` values
+ (1, named_struct('city', 'beijing', 'data', array(map(1, named_struct('a', 10, 'b', 20.0), 2, named_struct('a', 30, 'b', 40))))),
+ (2, named_struct('city', 'shanghai', 'data', array(map(2, named_struct('a', 50, 'b', 40.0), 1, named_struct('a', 70, 'b', 80)))));
+ """
+
+ qt_sql """
+ select * from `tbl_test_pruned_columns` order by 1;
+ """
+
+ qt_sql1 """
+ select b.id, array_map(x -> struct_element(map_values(x)[1], 'a'), struct_element(s, 'data')) from `tbl_test_pruned_columns` t join (select 1 id) b on t.id = b.id order by 1;
+ """
+
+ qt_sql2 """
+ select id, struct_element(s, 'city') from `tbl_test_pruned_columns` order by 1;
+ """
+
+ qt_sql3 """
+ select id, struct_element(s, 'data') from `tbl_test_pruned_columns` order by 1;
+ """
+
+ qt_sql4 """
+ select id, struct_element(s, 'data') from `tbl_test_pruned_columns` where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 order by 1;
+ """
+
+ qt_sql5 """
+ select id, struct_element(s, 'city') from `tbl_test_pruned_columns` where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 order by 1;
+ """
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 07/18: fix topN late materialzation access_path.
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4000802e5fb7ae4f89f77548f4557c9db4f1302f
Author: kakachen <ch...@selectdb.com>
AuthorDate: Wed Oct 29 22:08:27 2025 +0800
fix topN late materialzation access_path.
---
be/src/runtime/descriptors.cpp | 53 ++++++++++++++++++++++-
be/src/runtime/descriptors.h | 4 +-
be/src/vec/exec/format/table/hive_reader.cpp | 56 +++++++------------------
be/src/vec/exec/format/table/iceberg_reader.cpp | 28 ++++---------
gensrc/proto/descriptors.proto | 45 ++++++++++++++++++++
5 files changed, 123 insertions(+), 63 deletions(-)
diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp
index a17f0926752..bc27c6a41a0 100644
--- a/be/src/runtime/descriptors.cpp
+++ b/be/src/runtime/descriptors.cpp
@@ -103,7 +103,31 @@ SlotDescriptor::SlotDescriptor(const PSlotDescriptor& pdesc)
_is_materialized(pdesc.is_materialized()),
_is_key(pdesc.is_key()),
_column_paths(pdesc.column_paths().begin(), pdesc.column_paths().end()),
- _is_auto_increment(pdesc.is_auto_increment()) {}
+ _is_auto_increment(pdesc.is_auto_increment()) {
+ auto convert_to_thrift_column_access_path = [](const PColumnAccessPath& pb_path) {
+ TColumnAccessPath thrift_path;
+ thrift_path.type = (TAccessPathType::type)pb_path.type();
+ if (pb_path.has_data_access_path()) {
+ thrift_path.__isset.data_access_path = true;
+ for (int i = 0; i < pb_path.data_access_path().path_size(); ++i) {
+ thrift_path.data_access_path.path.push_back(pb_path.data_access_path().path(i));
+ }
+ }
+ if (pb_path.has_meta_access_path()) {
+ thrift_path.__isset.meta_access_path = true;
+ for (int i = 0; i < pb_path.meta_access_path().path_size(); ++i) {
+ thrift_path.meta_access_path.path.push_back(pb_path.meta_access_path().path(i));
+ }
+ }
+ return thrift_path;
+ };
+ for (const auto& pb_path : pdesc.all_access_paths()) {
+ _all_access_paths.push_back(convert_to_thrift_column_access_path(pb_path));
+ }
+ for (const auto& pb_path : pdesc.predicate_access_paths()) {
+ _predicate_access_paths.push_back(convert_to_thrift_column_access_path(pb_path));
+ }
+}
#ifdef BE_TEST
SlotDescriptor::SlotDescriptor()
@@ -137,6 +161,33 @@ void SlotDescriptor::to_protobuf(PSlotDescriptor* pslot) const {
for (const std::string& path : _column_paths) {
pslot->add_column_paths(path);
}
+ auto convert_to_protobuf_column_access_path = [](const TColumnAccessPath& thrift_path,
+ doris::PColumnAccessPath* pb_path) {
+ pb_path->Clear();
+ pb_path->set_type((PAccessPathType)thrift_path.type); // 使用 reinterpret_cast 进行类型转换
+ if (thrift_path.__isset.data_access_path) {
+ auto* pb_data = pb_path->mutable_data_access_path();
+ pb_data->Clear();
+ for (const auto& s : thrift_path.data_access_path.path) {
+ pb_data->add_path(s);
+ }
+ }
+ if (thrift_path.__isset.meta_access_path) {
+ auto* pb_meta = pb_path->mutable_meta_access_path();
+ pb_meta->Clear();
+ for (const auto& s : thrift_path.meta_access_path.path) {
+ pb_meta->add_path(s);
+ }
+ }
+ };
+ for (const auto& path : _all_access_paths) {
+ auto* pb_path = pslot->add_all_access_paths();
+ convert_to_protobuf_column_access_path(path, pb_path);
+ }
+ for (const auto& path : _predicate_access_paths) {
+ auto* pb_path = pslot->add_predicate_access_paths();
+ convert_to_protobuf_column_access_path(path, pb_path);
+ }
}
vectorized::DataTypePtr SlotDescriptor::get_data_type_ptr() const {
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index b34e54622b5..1446ce4c3d9 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -137,8 +137,8 @@ private:
const bool _is_key;
const std::vector<std::string> _column_paths;
- const TColumnAccessPaths _all_access_paths;
- const TColumnAccessPaths _predicate_access_paths;
+ TColumnAccessPaths _all_access_paths;
+ TColumnAccessPaths _predicate_access_paths;
const bool _is_auto_increment;
const std::string _col_default_value;
diff --git a/be/src/vec/exec/format/table/hive_reader.cpp b/be/src/vec/exec/format/table/hive_reader.cpp
index eea7a126517..81c8cf47f42 100644
--- a/be/src/vec/exec/format/table/hive_reader.cpp
+++ b/be/src/vec/exec/format/table/hive_reader.cpp
@@ -120,8 +120,7 @@ ColumnIdResult HiveOrcReader::_create_column_ids(const orc::Type* orc_type,
auto process_access_paths = [](const orc::Type* orc_field,
const std::vector<TColumnAccessPath>& access_paths,
std::set<uint64_t>& out_ids) {
- if (!orc_field) return;
- if (access_paths.empty()) return;
+ bool access_paths_empty = access_paths.empty();
std::vector<std::vector<std::string>> paths;
bool has_top_level_only = false;
@@ -134,7 +133,6 @@ ColumnIdResult HiveOrcReader::_create_column_ids(const orc::Type* orc_type,
} else {
continue;
}
- DCHECK(path.size() >= 1);
std::vector<std::string> remaining_path;
if (path.size() > 1) {
remaining_path.assign(path.begin() + 1, path.end());
@@ -148,7 +146,7 @@ ColumnIdResult HiveOrcReader::_create_column_ids(const orc::Type* orc_type,
paths.push_back(std::move(remaining_path));
}
- if (has_top_level_only) {
+ if (has_top_level_only || access_paths_empty) {
uint64_t start_id = orc_field->getColumnId();
uint64_t max_column_id = orc_field->getMaximumColumnId();
for (uint64_t id = start_id; id <= max_column_id; ++id) {
@@ -183,15 +181,11 @@ ColumnIdResult HiveOrcReader::_create_column_ids(const orc::Type* orc_type,
// complex types:
// collect and process all_access_paths -> column_ids
- if (!all_access_paths.empty()) {
- process_access_paths(orc_field, all_access_paths, column_ids);
- }
+ process_access_paths(orc_field, all_access_paths, column_ids);
// collect and process predicate_access_paths -> filter_column_ids
const auto& predicate_access_paths = slot->predicate_access_paths();
- if (!predicate_access_paths.empty()) {
- process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
- }
+ process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
}
return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
@@ -221,8 +215,7 @@ ColumnIdResult HiveOrcReader::_create_column_ids_by_top_level_col_index(
auto process_access_paths = [](const orc::Type* orc_field,
const std::vector<TColumnAccessPath>& access_paths,
std::set<uint64_t>& out_ids) {
- if (!orc_field) return;
- if (access_paths.empty()) return;
+ bool access_paths_empty = access_paths.empty();
std::vector<std::vector<std::string>> paths;
bool has_top_level_only = false;
@@ -235,7 +228,6 @@ ColumnIdResult HiveOrcReader::_create_column_ids_by_top_level_col_index(
} else {
continue;
}
- DCHECK(path.size() >= 1);
std::vector<std::string> remaining_path;
if (path.size() > 1) {
remaining_path.assign(path.begin() + 1, path.end());
@@ -249,7 +241,7 @@ ColumnIdResult HiveOrcReader::_create_column_ids_by_top_level_col_index(
paths.push_back(std::move(remaining_path));
}
- if (has_top_level_only) {
+ if (has_top_level_only || access_paths_empty) {
uint64_t start_id = orc_field->getColumnId();
uint64_t max_column_id = orc_field->getMaximumColumnId();
for (uint64_t id = start_id; id <= max_column_id; ++id) {
@@ -284,15 +276,11 @@ ColumnIdResult HiveOrcReader::_create_column_ids_by_top_level_col_index(
// complex types
// collect and process all_access_paths -> column_ids
- if (!all_access_paths.empty()) {
- process_access_paths(orc_field, all_access_paths, column_ids);
- }
+ process_access_paths(orc_field, all_access_paths, column_ids);
// collect and process predicate_access_paths -> filter_column_ids
const auto& predicate_access_paths = slot->predicate_access_paths();
- if (!predicate_access_paths.empty()) {
- process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
- }
+ process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
}
return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
@@ -403,8 +391,7 @@ ColumnIdResult HiveParquetReader::_create_column_ids(const FieldDescriptor* fiel
auto process_access_paths = [](const FieldSchema* parquet_field,
const std::vector<TColumnAccessPath>& access_paths,
std::set<uint64_t>& out_ids) {
- if (!parquet_field) return;
- if (access_paths.empty()) return;
+ bool access_paths_empty = access_paths.empty();
std::vector<std::vector<std::string>> paths;
bool has_top_level_only = false;
@@ -417,7 +404,6 @@ ColumnIdResult HiveParquetReader::_create_column_ids(const FieldDescriptor* fiel
} else {
continue;
}
- DCHECK(path.size() >= 1);
std::vector<std::string> remaining_path;
if (path.size() > 1) {
remaining_path.assign(path.begin() + 1, path.end());
@@ -431,7 +417,7 @@ ColumnIdResult HiveParquetReader::_create_column_ids(const FieldDescriptor* fiel
paths.push_back(std::move(remaining_path));
}
- if (has_top_level_only) {
+ if (has_top_level_only || access_paths_empty) {
uint64_t start_id = parquet_field->get_column_id();
uint64_t max_column_id = parquet_field->get_max_column_id();
for (uint64_t id = start_id; id <= max_column_id; ++id) {
@@ -468,15 +454,11 @@ ColumnIdResult HiveParquetReader::_create_column_ids(const FieldDescriptor* fiel
// complex types:
// collect and process all_access_paths -> column_ids
- if (!all_access_paths.empty()) {
- process_access_paths(field_schema, all_access_paths, column_ids);
- }
+ process_access_paths(field_schema, all_access_paths, column_ids);
// collect and process predicate_access_paths -> filter_column_ids
const auto& predicate_access_paths = slot->predicate_access_paths();
- if (!predicate_access_paths.empty()) {
- process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
- }
+ process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
}
return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
@@ -509,8 +491,7 @@ ColumnIdResult HiveParquetReader::_create_column_ids_by_top_level_col_index(
auto process_access_paths = [](const FieldSchema* parquet_field,
const std::vector<TColumnAccessPath>& access_paths,
std::set<uint64_t>& out_ids) {
- if (!parquet_field) return;
- if (access_paths.empty()) return;
+ bool access_paths_empty = access_paths.empty();
std::vector<std::vector<std::string>> paths;
bool has_top_level_only = false;
@@ -523,7 +504,6 @@ ColumnIdResult HiveParquetReader::_create_column_ids_by_top_level_col_index(
} else {
continue;
}
- DCHECK(path.size() >= 1);
std::vector<std::string> remaining_path;
if (path.size() > 1) {
remaining_path.assign(path.begin() + 1, path.end());
@@ -537,7 +517,7 @@ ColumnIdResult HiveParquetReader::_create_column_ids_by_top_level_col_index(
paths.push_back(std::move(remaining_path));
}
- if (has_top_level_only) {
+ if (has_top_level_only || access_paths_empty) {
uint64_t start_id = parquet_field->get_column_id();
uint64_t max_column_id = parquet_field->get_max_column_id();
for (uint64_t id = start_id; id <= max_column_id; ++id) {
@@ -572,15 +552,11 @@ ColumnIdResult HiveParquetReader::_create_column_ids_by_top_level_col_index(
}
// collect and process all_access_paths -> column_ids
- if (!all_access_paths.empty()) {
- process_access_paths(field_schema, all_access_paths, column_ids);
- }
+ process_access_paths(field_schema, all_access_paths, column_ids);
// collect and process predicate_access_paths -> filter_column_ids
const auto& predicate_access_paths = slot->predicate_access_paths();
- if (!predicate_access_paths.empty()) {
- process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
- }
+ process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
}
return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
diff --git a/be/src/vec/exec/format/table/iceberg_reader.cpp b/be/src/vec/exec/format/table/iceberg_reader.cpp
index 9a9e984a9d9..b08229165ea 100644
--- a/be/src/vec/exec/format/table/iceberg_reader.cpp
+++ b/be/src/vec/exec/format/table/iceberg_reader.cpp
@@ -497,8 +497,7 @@ ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* f
auto process_access_paths = [](const FieldSchema* parquet_field,
const std::vector<TColumnAccessPath>& access_paths,
std::set<uint64_t>& out_ids) {
- if (!parquet_field) return;
- if (access_paths.empty()) return;
+ bool access_paths_empty = access_paths.empty();
std::vector<std::vector<std::string>> paths;
bool has_top_level_only = false;
@@ -511,7 +510,6 @@ ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* f
} else {
continue;
}
- DCHECK(path.size() >= 1);
std::vector<std::string> remaining_path;
if (path.size() > 1) {
remaining_path.assign(path.begin() + 1, path.end());
@@ -525,7 +523,7 @@ ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* f
paths.push_back(std::move(remaining_path));
}
- if (has_top_level_only) {
+ if (has_top_level_only || access_paths_empty) {
uint64_t start_id = parquet_field->get_column_id();
uint64_t max_column_id = parquet_field->get_max_column_id();
for (uint64_t id = start_id; id <= max_column_id; ++id) {
@@ -562,15 +560,11 @@ ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* f
// complex types:
// collect and process all_access_paths -> column_ids
- if (!all_access_paths.empty()) {
- process_access_paths(field_schema, all_access_paths, column_ids);
- }
+ process_access_paths(field_schema, all_access_paths, column_ids);
// collect and process predicate_access_paths -> filter_column_ids
const auto& predicate_access_paths = slot->predicate_access_paths();
- if (!predicate_access_paths.empty()) {
- process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
- }
+ process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
}
return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
}
@@ -697,8 +691,7 @@ ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type,
auto process_access_paths = [](const orc::Type* orc_field,
const std::vector<TColumnAccessPath>& access_paths,
std::set<uint64_t>& out_ids) {
- if (!orc_field) return;
- if (access_paths.empty()) return;
+ bool access_paths_empty = access_paths.empty();
std::vector<std::vector<std::string>> paths;
bool has_top_level_only = false;
@@ -711,7 +704,6 @@ ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type,
} else {
continue;
}
- DCHECK(path.size() >= 1);
std::vector<std::string> remaining_path;
if (path.size() > 1) {
remaining_path.assign(path.begin() + 1, path.end());
@@ -725,7 +717,7 @@ ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type,
paths.push_back(std::move(remaining_path));
}
- if (has_top_level_only) {
+ if (has_top_level_only || access_paths_empty) {
uint64_t start_id = orc_field->getColumnId();
uint64_t max_column_id = orc_field->getMaximumColumnId();
for (uint64_t id = start_id; id <= max_column_id; ++id) {
@@ -760,15 +752,11 @@ ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type,
// nested types:
// collect and process all_access_paths -> column_ids
- if (!all_access_paths.empty()) {
- process_access_paths(orc_field, all_access_paths, column_ids);
- }
+ process_access_paths(orc_field, all_access_paths, column_ids);
// collect and process predicate_access_paths -> filter_column_ids
const auto& predicate_access_paths = slot->predicate_access_paths();
- if (!predicate_access_paths.empty()) {
- process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
- }
+ process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
}
return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
diff --git a/gensrc/proto/descriptors.proto b/gensrc/proto/descriptors.proto
index 31008c90725..f764e90c771 100644
--- a/gensrc/proto/descriptors.proto
+++ b/gensrc/proto/descriptors.proto
@@ -23,6 +23,49 @@ option java_package = "org.apache.doris.proto";
import "types.proto";
import "olap_file.proto";
+message PDataAccessPath {
+ // the specification of special path:
+ // <empty>: access the whole complex column
+ // *:
+ // 1. access every items when the type is array
+ // 2. access key and value when the type is map
+ // KEYS: only access the keys of map
+ // VALUES: only access the keys of map
+ //
+ // example:
+ // s: struct<
+ // data: array<
+ // map<
+ // int,
+ // struct<
+ // a: id
+ // b: double
+ // >
+ // >
+ // >
+ // >
+ // if we want to access `map_keys(s.data[0])`, the path will be: ['s', 'data', '*', 'KEYS'],
+ // if we want to access `map_values(s.data[0])[0].b`, the path will be: ['s', 'data', '*', 'VALUES', 'b'],
+ // if we want to access `s.data[0]['k'].b`, the path will be ['s', 'data', '*', '*', 'b']
+ // if we want to access the whole struct of s, the path will be: ['s'],
+ repeated string path = 1;
+}
+
+message PMetaAccessPath {
+ repeated string path = 1;
+}
+
+enum PAccessPathType {
+ DATA = 1;
+ META = 2; // use to prune `where s.data is not null` by only scan the meta of s.data
+}
+
+message PColumnAccessPath {
+ required PAccessPathType type = 1;
+ optional PDataAccessPath data_access_path = 2;
+ optional PMetaAccessPath meta_access_path = 3;
+}
+
message PSlotDescriptor {
required int32 id = 1;
required int32 parent = 2; // tuple id which this slot is belong to
@@ -39,6 +82,8 @@ message PSlotDescriptor {
optional bool is_auto_increment = 13;
optional int32 col_type = 14 [default = 0];
repeated string column_paths = 15;
+ repeated PColumnAccessPath all_access_paths = 16;
+ repeated PColumnAccessPath predicate_access_paths = 17;
};
message PTupleDescriptor {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 11/18: Fix some tests's bugs of external table.
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0279d9c575853e06e59d84eb57a7b317414e12f2
Author: kakachen <ch...@selectdb.com>
AuthorDate: Thu Oct 30 19:15:40 2025 +0800
Fix some tests's bugs of external table.
---
be/src/vec/exec/format/orc/vorc_reader.cpp | 17 +-
.../table/hive/hive_orc_nested_column_utils.cpp | 21 +++
be/src/vec/exec/format/table/hive_reader.cpp | 40 ++---
.../iceberg/iceberg_orc_nested_column_utils.cpp | 20 +++
.../iceberg_parquet_nested_column_utils.cpp | 16 +-
be/src/vec/exec/format/table/iceberg_reader.cpp | 22 +--
.../hive/hive_reader_create_column_ids_test.cpp | 184 ++++++++++++---------
.../iceberg_reader_create_column_ids_test.cpp | 122 +++++++++-----
8 files changed, 274 insertions(+), 168 deletions(-)
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp
index cfbad191772..11b75b820ca 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -1984,12 +1984,9 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
}
} else {
// Normal processing: convert ORC column to Doris column
- auto status = _orc_column_to_doris_column<false>(
+ RETURN_IF_ERROR(_orc_column_to_doris_column<false>(
key_col_name, doris_key_column, doris_key_type, root_node->get_key_node(),
- orc_key_type, orc_map->keys.get(), element_size);
- if (!status.ok()) {
- return status;
- }
+ orc_key_type, orc_map->keys.get(), element_size));
}
// Handle value column: if still missing, fill with default values
@@ -2005,14 +2002,14 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
} else {
mutable_value_column->insert_many_defaults(element_size);
}
- return Status::OK();
} else {
// Normal processing: convert ORC column to Doris column
- return _orc_column_to_doris_column<false>(value_col_name, doris_value_column,
- doris_value_type, root_node->get_value_node(),
- orc_value_type, orc_map->elements.get(),
- element_size);
+ RETURN_IF_ERROR(_orc_column_to_doris_column<false>(
+ value_col_name, doris_value_column, doris_value_type,
+ root_node->get_value_node(), orc_value_type, orc_map->elements.get(),
+ element_size));
}
+ return doris_map.deduplicate_keys();
}
case PrimitiveType::TYPE_STRUCT: {
if (orc_column_type->getKind() != orc::TypeKind::STRUCT) {
diff --git a/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.cpp b/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.cpp
index 6e9fa381fc7..1cd02967f0d 100644
--- a/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.cpp
+++ b/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.cpp
@@ -95,6 +95,27 @@ void HiveOrcNestedColumnUtils::extract_nested_column_ids(
} else if (i == 1) {
child_field_name = "VALUES";
}
+
+ // Special handling for Orc MAP structure:
+ // When accessing only VALUES, we still need KEY structure for levels
+ // Check if we're at key child (i==0) and only VALUES is requested (no KEYS)
+ if (i == 0) {
+ bool has_keys_access = child_paths_by_table_col_name.find("KEYS") !=
+ child_paths_by_table_col_name.end();
+ bool has_values_access = child_paths_by_table_col_name.find("VALUES") !=
+ child_paths_by_table_col_name.end();
+
+ // If only VALUES is accessed (not KEYS), still include key structure for deduplicate_keys
+ if (!has_keys_access && has_values_access) {
+ uint64_t key_start_id = child->getColumnId();
+ uint64_t key_max_id = child->getMaximumColumnId();
+ for (uint64_t id = key_start_id; id <= key_max_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ continue; // Skip further processing of key child
+ }
+ }
break;
default:
child_field_name = "";
diff --git a/be/src/vec/exec/format/table/hive_reader.cpp b/be/src/vec/exec/format/table/hive_reader.cpp
index 81c8cf47f42..b9bc01fe7b6 100644
--- a/be/src/vec/exec/format/table/hive_reader.cpp
+++ b/be/src/vec/exec/format/table/hive_reader.cpp
@@ -166,8 +166,6 @@ ColumnIdResult HiveOrcReader::_create_column_ids(const orc::Type* orc_type,
}
const orc::Type* orc_field = it->second;
- const auto& all_access_paths = slot->all_access_paths();
-
// primitive (non-nested) types: direct mapping by name
if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
slot->col_type() != TYPE_MAP)) {
@@ -179,13 +177,13 @@ ColumnIdResult HiveOrcReader::_create_column_ids(const orc::Type* orc_type,
}
// complex types:
-
- // collect and process all_access_paths -> column_ids
+ const auto& all_access_paths = slot->all_access_paths();
process_access_paths(orc_field, all_access_paths, column_ids);
- // collect and process predicate_access_paths -> filter_column_ids
const auto& predicate_access_paths = slot->predicate_access_paths();
- process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
+ }
}
return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
@@ -261,8 +259,6 @@ ColumnIdResult HiveOrcReader::_create_column_ids_by_top_level_col_index(
}
const orc::Type* orc_field = it->second;
- const auto& all_access_paths = slot->all_access_paths();
-
// primitive (non-nested) types: direct mapping by pos
if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
slot->col_type() != TYPE_MAP)) {
@@ -273,14 +269,14 @@ ColumnIdResult HiveOrcReader::_create_column_ids_by_top_level_col_index(
continue;
}
+ const auto& all_access_paths = slot->all_access_paths();
// complex types
-
- // collect and process all_access_paths -> column_ids
process_access_paths(orc_field, all_access_paths, column_ids);
- // collect and process predicate_access_paths -> filter_column_ids
const auto& predicate_access_paths = slot->predicate_access_paths();
- process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
+ }
}
return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
@@ -438,8 +434,6 @@ ColumnIdResult HiveParquetReader::_create_column_ids(const FieldDescriptor* fiel
}
auto field_schema = it->second;
- const auto& all_access_paths = slot->all_access_paths();
-
// primitive (non-nested) types: direct mapping by name
if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
slot->col_type() != TYPE_MAP)) {
@@ -452,13 +446,13 @@ ColumnIdResult HiveParquetReader::_create_column_ids(const FieldDescriptor* fiel
}
// complex types:
-
- // collect and process all_access_paths -> column_ids
+ const auto& all_access_paths = slot->all_access_paths();
process_access_paths(field_schema, all_access_paths, column_ids);
- // collect and process predicate_access_paths -> filter_column_ids
const auto& predicate_access_paths = slot->predicate_access_paths();
- process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
+ }
}
return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
@@ -538,8 +532,6 @@ ColumnIdResult HiveParquetReader::_create_column_ids_by_top_level_col_index(
}
auto field_schema = it->second;
- const auto& all_access_paths = slot->all_access_paths();
-
// primitive (non-nested) types: direct mapping by position
if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
slot->col_type() != TYPE_MAP)) {
@@ -551,12 +543,14 @@ ColumnIdResult HiveParquetReader::_create_column_ids_by_top_level_col_index(
continue;
}
- // collect and process all_access_paths -> column_ids
+ // complex types:
+ const auto& all_access_paths = slot->all_access_paths();
process_access_paths(field_schema, all_access_paths, column_ids);
- // collect and process predicate_access_paths -> filter_column_ids
const auto& predicate_access_paths = slot->predicate_access_paths();
- process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
+ }
}
return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
diff --git a/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.cpp b/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.cpp
index 5dbbbfb2af2..e3aacded188 100644
--- a/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.cpp
+++ b/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.cpp
@@ -97,6 +97,26 @@ void IcebergOrcNestedColumnUtils::extract_nested_column_ids(
} else if (i == 1) {
child_field_id = "VALUES";
}
+ // Special handling for Orc MAP structure:
+ // When accessing only VALUES, we still need KEY structure for levels
+ // Check if we're at key child (i==0) and only VALUES is requested (no KEYS)
+ if (i == 0) {
+ bool has_keys_access =
+ child_paths_by_field_id.find("KEYS") != child_paths_by_field_id.end();
+ bool has_values_access =
+ child_paths_by_field_id.find("VALUES") != child_paths_by_field_id.end();
+
+ // If only VALUES is accessed (not KEYS), still include key structure for deduplicate_keys
+ if (!has_keys_access && has_values_access) {
+ uint64_t key_start_id = child->getColumnId();
+ uint64_t key_max_id = child->getMaximumColumnId();
+ for (uint64_t id = key_start_id; id <= key_max_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ continue; // Skip further processing of key child
+ }
+ }
break;
default:
child_field_id = "";
diff --git a/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp b/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp
index cb2b2e6b8c7..7741d498f99 100644
--- a/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp
+++ b/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp
@@ -114,7 +114,10 @@ void IcebergParquetNestedColumnUtils::extract_nested_column_ids(
uint64_t key_start_id = child.get_column_id();
uint64_t key_max_id = child.get_max_column_id();
for (uint64_t id = key_start_id; id <= key_max_id; ++id) {
- column_ids.insert(id);
+ auto inserted = column_ids.insert(id);
+ if (inserted.second) {
+ std::cout << "[IcebergNested] added column id: " << id << std::endl;
+ }
}
has_child_columns = true;
continue; // Skip further processing of key child
@@ -144,7 +147,10 @@ void IcebergParquetNestedColumnUtils::extract_nested_column_ids(
uint64_t start_id = child.get_column_id();
uint64_t max_column_id = child.get_max_column_id();
for (uint64_t id = start_id; id <= max_column_id; ++id) {
- column_ids.insert(id);
+ auto inserted = column_ids.insert(id);
+ if (inserted.second) {
+ std::cout << "[IcebergNested] added column id: " << id << std::endl;
+ }
}
has_child_columns = true;
} else {
@@ -166,7 +172,11 @@ void IcebergParquetNestedColumnUtils::extract_nested_column_ids(
// This ensures parent struct/container nodes are included when their children are needed
if (has_child_columns) {
// Set automatically handles deduplication, so no need to check if it already exists
- column_ids.insert(field_schema.get_column_id());
+ auto inserted = column_ids.insert(field_schema.get_column_id());
+ if (inserted.second) {
+ std::cout << "[IcebergNested] added parent column id: " << field_schema.get_column_id()
+ << std::endl;
+ }
}
}
diff --git a/be/src/vec/exec/format/table/iceberg_reader.cpp b/be/src/vec/exec/format/table/iceberg_reader.cpp
index b08229165ea..1a010dc4454 100644
--- a/be/src/vec/exec/format/table/iceberg_reader.cpp
+++ b/be/src/vec/exec/format/table/iceberg_reader.cpp
@@ -544,8 +544,6 @@ ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* f
}
auto field_schema = it->second;
- const auto& all_access_paths = slot->all_access_paths();
-
// primitive (non-nested) types: direct mapping by name
if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
slot->col_type() != TYPE_MAP)) {
@@ -558,13 +556,13 @@ ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* f
}
// complex types:
-
- // collect and process all_access_paths -> column_ids
+ const auto& all_access_paths = slot->all_access_paths();
process_access_paths(field_schema, all_access_paths, column_ids);
- // collect and process predicate_access_paths -> filter_column_ids
const auto& predicate_access_paths = slot->predicate_access_paths();
- process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
+ }
}
return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
}
@@ -737,8 +735,6 @@ ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type,
}
const orc::Type* orc_field = it->second;
- const auto& all_access_paths = slot->all_access_paths();
-
// primitive (non-nested) types: direct mapping by name
if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
slot->col_type() != TYPE_MAP)) {
@@ -749,14 +745,14 @@ ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type,
continue;
}
- // nested types:
-
- // collect and process all_access_paths -> column_ids
+ // complex types:
+ const auto& all_access_paths = slot->all_access_paths();
process_access_paths(orc_field, all_access_paths, column_ids);
- // collect and process predicate_access_paths -> filter_column_ids
const auto& predicate_access_paths = slot->predicate_access_paths();
- process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
+ }
}
return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
diff --git a/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp b/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp
index 7efca721fcd..d46acbdc134 100644
--- a/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp
+++ b/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp
@@ -738,7 +738,8 @@ protected:
}
// Helper function: Run Parquet test with different column ID extraction methods
- void run_parquet_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ void run_parquet_test_with_method(const std::vector<std::string>& table_column_names,
+ const std::vector<ColumnAccessPathConfig>& access_configs,
const std::set<uint64_t>& expected_column_ids,
const std::set<uint64_t>& expected_filter_column_ids,
bool use_top_level_method = false,
@@ -760,12 +761,13 @@ protected:
TTableDescriptor t_table_desc;
// Define all columns according to the schema
- std::vector<std::string> table_column_names = {
- "name", "profile", "tags", "friends", "recent_activity",
- "attributes", "complex_attributes"};
- std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
- std::vector<TPrimitiveType::type> table_column_types = {
- // TPrimitiveType::BIGINT, // id
+ std::vector<std::string> all_table_column_names = {"id", "name",
+ "profile", "tags",
+ "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> all_table_column_positions = {0, 1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> all_table_column_types = {
+ TPrimitiveType::BIGINT, // id
TPrimitiveType::STRING, // name
TPrimitiveType::STRUCT, // profile
TPrimitiveType::ARRAY, // tags
@@ -775,6 +777,18 @@ protected:
TPrimitiveType::MAP // complex_attributes
};
+ std::vector<int> table_column_positions;
+ std::vector<TPrimitiveType::type> table_column_types;
+ for (const auto& col_name : table_column_names) {
+ auto it = std::find(all_table_column_names.begin(), all_table_column_names.end(),
+ col_name);
+ if (it != all_table_column_names.end()) {
+ int idx = std::distance(all_table_column_names.begin(), it);
+ table_column_positions.push_back(idx);
+ table_column_types.push_back(all_table_column_types[idx]);
+ }
+ }
+
const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
&desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
table_column_positions, table_column_types, access_configs);
@@ -795,7 +809,8 @@ protected:
}
// Helper function: Run ORC test with different column ID extraction methods
- void run_orc_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ void run_orc_test_with_method(const std::vector<std::string>& table_column_names,
+ const std::vector<ColumnAccessPathConfig>& access_configs,
const std::set<uint64_t>& expected_column_ids,
const std::set<uint64_t>& expected_filter_column_ids,
bool use_top_level_method = false,
@@ -817,12 +832,13 @@ protected:
TTableDescriptor t_table_desc;
// Define all columns according to the schema
- std::vector<std::string> table_column_names = {
- "name", "profile", "tags", "friends", "recent_activity",
- "attributes", "complex_attributes"};
- std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
- std::vector<TPrimitiveType::type> table_column_types = {
- // TPrimitiveType::BIGINT, // id
+ std::vector<std::string> all_table_column_names = {"id", "name",
+ "profile", "tags",
+ "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> all_table_column_positions = {0, 1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> all_table_column_types = {
+ TPrimitiveType::BIGINT, // id
TPrimitiveType::STRING, // name
TPrimitiveType::STRUCT, // profile
TPrimitiveType::ARRAY, // tags
@@ -832,6 +848,18 @@ protected:
TPrimitiveType::MAP // complex_attributes
};
+ std::vector<int> table_column_positions;
+ std::vector<TPrimitiveType::type> table_column_types;
+ for (const auto& col_name : table_column_names) {
+ auto it = std::find(all_table_column_names.begin(), all_table_column_names.end(),
+ col_name);
+ if (it != all_table_column_names.end()) {
+ int idx = std::distance(all_table_column_names.begin(), it);
+ table_column_positions.push_back(idx);
+ table_column_types.push_back(all_table_column_types[idx]);
+ }
+ }
+
const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
&desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
table_column_positions, table_column_types, access_configs);
@@ -868,18 +896,19 @@ TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_1) {
{"profile", "contact", "email"}};
// column_ids should contain all necessary column IDs (set automatically deduplicates)
- // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::vector<std::string> table_column_names = {"name", "profile"};
std::set<uint64_t> expected_column_ids = {2, 3, 4, 7, 8, 9, 10, 11, 15, 16, 18};
- // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
- run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
- run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
- true);
+ run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
- run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
- run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
- true);
+ run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
}
TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_2) {
@@ -915,20 +944,21 @@ TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_2) {
access_config.predicate_paths = {{"profile", "address", "coordinates", "lat"},
{"profile", "contact", "email"}};
+ std::vector<std::string> table_column_names = {"name", "profile"};
// column_ids should contain all necessary column IDs (set automatically deduplicates)
- // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18};
- // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
- run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
- run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
- true);
+ run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
- run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
- run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
- true);
+ run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
}
TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_3) {
@@ -964,19 +994,20 @@ TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_3) {
access_config.predicate_paths = {{"profile", "address", "coordinates"},
{"profile", "contact", "email"}};
+ std::vector<std::string> table_column_names = {"name", "profile"};
// column_ids should contain all necessary column IDs (set automatically deduplicates)
- // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14};
- // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 9, 10, 11};
- run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
- run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
- true);
+ run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
- run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
- run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
- true);
+ run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
}
TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_4) {
@@ -988,19 +1019,21 @@ TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_4) {
access_config.all_column_paths = {};
access_config.predicate_paths = {};
+ std::vector<std::string> table_column_names = {"name", "profile"};
// column_ids should contain all necessary column IDs (set automatically deduplicates)
- // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
- std::set<uint64_t> expected_column_ids = {2};
- // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10,
+ 11, 12, 13, 14, 15, 16, 17, 18};
std::set<uint64_t> expected_filter_column_ids = {};
- run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
- run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
- true);
+ run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
- run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
- run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
- true);
+ run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
}
TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_5) {
@@ -1049,18 +1082,20 @@ TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_5) {
access_configs.push_back(access_config);
}
+ std::vector<std::string> table_column_names = {"name", "friends", "recent_activity"};
// column_ids should contain all necessary column IDs (set automatically deduplicates)
- // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
std::set<uint64_t> expected_column_ids = {2, 21, 22, 24, 25, 26, 27, 28, 29, 30, 32};
- // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
std::set<uint64_t> expected_filter_column_ids = {21, 22, 24, 26, 27, 28};
- run_parquet_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
- run_parquet_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids,
- true);
+ run_parquet_test_with_method(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
+ run_parquet_test_with_method(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids, true);
- run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
- run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids, true);
+ run_orc_test_with_method(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test_with_method(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids, true);
}
TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_6) {
@@ -1146,33 +1181,32 @@ TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_6) {
}
{
+ std::vector<std::string> table_column_names = {"name", "complex_attributes"};
// parquet values should access keys
// column_ids should contain all necessary column IDs (set automatically deduplicates)
- // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
- std::set<uint64_t> parquet_expected_column_ids = {
- 2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53, 54, 61, 62, 63, 64,
- 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
- // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
- std::set<uint64_t> parquet_expected_filter_column_ids = {36, 37, 38, 39, 40};
-
- run_parquet_test_with_method(access_configs, parquet_expected_column_ids,
- parquet_expected_filter_column_ids);
- run_parquet_test_with_method(access_configs, parquet_expected_column_ids,
- parquet_expected_filter_column_ids, true);
+ std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53,
+ 54, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
+ 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
+ std::set<uint64_t> expected_filter_column_ids = {36, 37, 38, 39, 40};
+
+ run_parquet_test_with_method(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
+ run_parquet_test_with_method(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids, true);
}
{
+ std::vector<std::string> table_column_names = {"name", "complex_attributes"};
+ // orc values should access keys because need to deduplicate by keys
// column_ids should contain all necessary column IDs (set automatically deduplicates)
- // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
- std::set<uint64_t> orc_expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52,
- 53, 54, 61, 63, 64, 65, 66, 67, 68, 69, 70,
- 71, 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
- // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
- std::set<uint64_t> orc_expected_filter_column_ids = {36, 37, 38, 39, 40};
- run_orc_test_with_method(access_configs, orc_expected_column_ids,
- orc_expected_filter_column_ids);
- run_orc_test_with_method(access_configs, orc_expected_column_ids,
- orc_expected_filter_column_ids, true);
+ std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53,
+ 54, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
+ 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
+ std::set<uint64_t> expected_filter_column_ids = {36, 37, 38, 39, 40};
+ run_orc_test_with_method(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test_with_method(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids, true);
}
}
diff --git a/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp b/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp
index 47e0ad87a4a..b1152f159bd 100644
--- a/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp
+++ b/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp
@@ -770,7 +770,8 @@ protected:
}
// Helper function: run Parquet test with different column ID extraction methods
- void run_parquet_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ void run_parquet_test_with_method(const std::vector<std::string>& table_column_names,
+ const std::vector<ColumnAccessPathConfig>& access_configs,
const std::set<uint64_t>& expected_column_ids,
const std::set<uint64_t>& expected_filter_column_ids,
bool use_top_level_method = false,
@@ -792,12 +793,13 @@ protected:
TTableDescriptor t_table_desc;
// Define all columns according to the schema
- std::vector<std::string> table_column_names = {
- "name", "profile", "tags", "friends", "recent_activity",
- "attributes", "complex_attributes"};
- std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
- std::vector<TPrimitiveType::type> table_column_types = {
- // TPrimitiveType::BIGINT, // id
+ std::vector<std::string> all_table_column_names = {"id", "name",
+ "profile", "tags",
+ "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> all_table_column_positions = {0, 1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> all_table_column_types = {
+ TPrimitiveType::BIGINT, // id
TPrimitiveType::STRING, // name
TPrimitiveType::STRUCT, // profile
TPrimitiveType::ARRAY, // tags
@@ -807,6 +809,18 @@ protected:
TPrimitiveType::MAP // complex_attributes
};
+ std::vector<int> table_column_positions;
+ std::vector<TPrimitiveType::type> table_column_types;
+ for (const auto& col_name : table_column_names) {
+ auto it = std::find(all_table_column_names.begin(), all_table_column_names.end(),
+ col_name);
+ if (it != all_table_column_names.end()) {
+ int idx = std::distance(all_table_column_names.begin(), it);
+ table_column_positions.push_back(idx);
+ table_column_types.push_back(all_table_column_types[idx]);
+ }
+ }
+
const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
&desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
table_column_positions, table_column_types, access_configs);
@@ -827,7 +841,8 @@ protected:
}
// Helper function: run Orc test with different column ID extraction methods
- void run_orc_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ void run_orc_test_with_method(const std::vector<std::string>& table_column_names,
+ const std::vector<ColumnAccessPathConfig>& access_configs,
const std::set<uint64_t>& expected_column_ids,
const std::set<uint64_t>& expected_filter_column_ids,
bool use_top_level_method = false,
@@ -849,12 +864,13 @@ protected:
TTableDescriptor t_table_desc;
// Define all columns according to the schema
- std::vector<std::string> table_column_names = {
- "name", "profile", "tags", "friends", "recent_activity",
- "attributes", "complex_attributes"};
- std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
- std::vector<TPrimitiveType::type> table_column_types = {
- // TPrimitiveType::BIGINT, // id
+ std::vector<std::string> all_table_column_names = {"id", "name",
+ "profile", "tags",
+ "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> all_table_column_positions = {0, 1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> all_table_column_types = {
+ TPrimitiveType::BIGINT, // id
TPrimitiveType::STRING, // name
TPrimitiveType::STRUCT, // profile
TPrimitiveType::ARRAY, // tags
@@ -864,6 +880,18 @@ protected:
TPrimitiveType::MAP // complex_attributes
};
+ std::vector<int> table_column_positions;
+ std::vector<TPrimitiveType::type> table_column_types;
+ for (const auto& col_name : table_column_names) {
+ auto it = std::find(all_table_column_names.begin(), all_table_column_names.end(),
+ col_name);
+ if (it != all_table_column_names.end()) {
+ int idx = std::distance(all_table_column_names.begin(), it);
+ table_column_positions.push_back(idx);
+ table_column_types.push_back(all_table_column_types[idx]);
+ }
+ }
+
const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
&desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
table_column_positions, table_column_types, access_configs);
@@ -898,14 +926,15 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_1) {
{"3", "11", "*", "23"}};
access_config.predicate_paths = {{"3", "9", "14", "15"}, {"3", "10", "17"}};
+ std::vector<std::string> table_column_names = {"name", "profile"};
// column_ids should contain all necessary column IDs (set automatically deduplicates)
- // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
std::set<uint64_t> expected_column_ids = {2, 3, 4, 7, 8, 9, 10, 11, 15, 16, 18};
- // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
- run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
- run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
}
TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_2) {
@@ -940,15 +969,16 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_2) {
access_config.all_column_paths = {{"3"}};
access_config.predicate_paths = {{"3", "9", "14", "15"}, {"3", "10", "17"}};
+ std::vector<std::string> table_column_names = {"name", "profile"};
// column_ids should contain all necessary column IDs (set automatically deduplicates)
- // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18};
- // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
- run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
- run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
}
TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_3) {
@@ -987,14 +1017,15 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_3) {
// access_config.predicate_paths = {{"profile", "address", "coordinates"},
// {"profile", "contact", "email"}};
+ std::vector<std::string> table_column_names = {"name", "profile"};
// column_ids should contain all necessary column IDs (set automatically deduplicates)
- // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14};
- // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 9, 10, 11};
- run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
- run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
}
TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_4) {
@@ -1006,14 +1037,16 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_4) {
access_config.all_column_paths = {};
access_config.predicate_paths = {};
+ std::vector<std::string> table_column_names = {"name", "profile"};
// column_ids should contain all necessary column IDs (set automatically deduplicates)
- // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
- std::set<uint64_t> expected_column_ids = {2};
- // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10,
+ 11, 12, 13, 14, 15, 16, 17, 18};
std::set<uint64_t> expected_filter_column_ids = {};
- run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
- run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
}
TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_5) {
@@ -1060,14 +1093,15 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_5) {
access_configs.push_back(access_config);
}
+ std::vector<std::string> table_column_names = {"name", "friends", "recent_activity"};
// column_ids should contain all necessary column IDs (set automatically deduplicates)
- // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
std::set<uint64_t> expected_column_ids = {2, 21, 22, 24, 25, 26, 27, 28, 29, 30, 32};
- // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
std::set<uint64_t> expected_filter_column_ids = {21, 22, 24, 26, 27, 28};
- run_parquet_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
- run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test_with_method(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
}
TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_6) {
@@ -1148,29 +1182,29 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_6) {
}
{
+ std::vector<std::string> table_column_names = {"name", "complex_attributes"};
// parquet values should access keys
// column_ids should contain all necessary column IDs (set automatically deduplicates)
- // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53,
54, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
- // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
std::set<uint64_t> expected_filter_column_ids = {36, 37, 38, 39, 40};
- run_parquet_test_with_method(access_configs, expected_column_ids,
+ run_parquet_test_with_method(table_column_names, access_configs, expected_column_ids,
expected_filter_column_ids);
}
{
+ std::vector<std::string> table_column_names = {"name", "complex_attributes"};
+ // orc values should access keys because need to deduplicate by keys
// column_ids should contain all necessary column IDs (set automatically deduplicates)
- // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
- std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52,
- 53, 54, 61, 63, 64, 65, 66, 67, 68, 69, 70,
- 71, 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
- // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53,
+ 54, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
+ 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
std::set<uint64_t> expected_filter_column_ids = {36, 37, 38, 39, 40};
- run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/18: Fix parquet_thrift_test unit test.
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 28ba969f5b53d2520dd179f63a34857f5a5e24ef
Author: kakachen <ch...@selectdb.com>
AuthorDate: Tue Oct 28 20:58:48 2025 +0800
Fix parquet_thrift_test unit test.
---
be/test/vec/exec/format/parquet/parquet_thrift_test.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp b/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
index edd131a2410..4f4202e1143 100644
--- a/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
+++ b/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
@@ -150,8 +150,8 @@ TEST_F(ParquetThriftReaderTest, complex_nested_file) {
// R=0,D=1 R=1,D=2 R=1,D=3 R=2,D=4
// \------ R=2,D=4
// \------ R=2,D=5
- auto h_key = hobby->children[0];
- auto h_value = hobby->children[1];
+ auto h_key = hobby->children[0].children[0];
+ auto h_value = hobby->children[0].children[1];
ASSERT_TRUE(h_key.repetition_level == 2 && h_key.definition_level == 4);
ASSERT_TRUE(h_value.repetition_level == 2 && h_value.definition_level == 5);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 16/18: Clean external table implementation's code.
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6572e52de834374948f104d1f8501a8f0803a02e
Author: kakachen <ch...@selectdb.com>
AuthorDate: Mon Nov 3 11:28:25 2025 +0800
Clean external table implementation's code.
---
be/src/vec/exec/format/orc/vorc_reader.cpp | 134 ++++++------------
be/src/vec/exec/format/orc/vorc_reader.h | 6 +-
.../exec/format/parquet/vparquet_column_reader.cpp | 67 +--------
.../exec/format/parquet/vparquet_column_reader.h | 4 -
.../exec/format/parquet/vparquet_group_reader.cpp | 24 +---
.../hive/hive_reader_create_column_ids_test.cpp | 152 +++++++++------------
.../exec/format/table/hive/hive_reader_test.cpp | 4 +-
.../iceberg_reader_create_column_ids_test.cpp | 113 ++++++---------
.../format/table/iceberg/iceberg_reader_test.cpp | 8 +-
9 files changed, 168 insertions(+), 344 deletions(-)
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp
index 11b75b820ca..62d56c0c8cd 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -406,9 +406,6 @@ Status OrcReader::_init_read_columns() {
SCOPED_RAW_TIMER(&_statistics.init_column_time);
const auto& root_type = _reader->getType();
- // Save file root type for later use in type conversion
- _file_root_type = &root_type;
-
// Build column ID to file type mapping for all columns in file
_column_id_to_file_type.clear();
std::function<void(const orc::Type*, int)> build_column_id_map = [&](const orc::Type* type,
@@ -1191,7 +1188,7 @@ Status OrcReader::set_fill_columns(
if (!_column_ids.empty()) {
std::list<uint64_t> column_ids_list(_column_ids.begin(), _column_ids.end());
_row_reader_options.includeTypes(column_ids_list);
- } else {
+ } else { // If column_ids is empty, include all top-level columns to be read.
_row_reader_options.include(_read_file_cols);
}
_row_reader_options.setEnableLazyDecoding(true);
@@ -1281,35 +1278,6 @@ Status OrcReader::set_fill_columns(
}
}
- // Helper function to recursively print ORC type structure
- std::function<void(const orc::Type*, int, const std::string&)> print_orc_type_tree =
- [&](const orc::Type* type, int indent, const std::string& name) {
- if (type) {
- std::string indent_str(indent * 2, ' ');
- VLOG_DEBUG << indent_str << "- " << name
- << " (type_id=" << type->getColumnId()
- << ", kind=" << static_cast<int>(type->getKind()) << ")";
- }
-
- if (type->getKind() == orc::TypeKind::STRUCT) {
- for (uint64_t i = 0; i < type->getSubtypeCount(); ++i) {
- std::string field_name = type->getFieldName(i);
- print_orc_type_tree(type->getSubtype(i), indent + 1, field_name);
- }
- } else if (type->getKind() == orc::TypeKind::LIST) {
- print_orc_type_tree(type->getSubtype(0), indent + 1, "element");
- } else if (type->getKind() == orc::TypeKind::MAP) {
- print_orc_type_tree(type->getSubtype(0), indent + 1, "key");
- print_orc_type_tree(type->getSubtype(1), indent + 1, "value");
- } else {
- // Primitive type, no children
- }
- };
-
- // VLOG_DEBUG << "\n[OrcReader] ========== ORC Type Structure ==========";
- // print_orc_type_tree(&selected_type, 0, "root");
- // VLOG_DEBUG << "[OrcReader] ==========================================\n";
-
_type_map.clear();
if (_is_acid) {
for (uint64_t i = 0; i < selected_type.getSubtypeCount(); ++i) {
@@ -1429,11 +1397,6 @@ Status OrcReader::_fill_missing_columns(
block->erase(result_column_id);
}
}
-
- // // no default column, fill with null
- // auto mutable_column = block->get_by_name(kv.first).column->assume_mutable();
- // auto* nullable_column = static_cast<vectorized::ColumnNullable*>(mutable_column.get());
- // nullable_column->insert_many_defaults(rows);
}
return Status::OK();
}
@@ -1542,22 +1505,22 @@ DataTypePtr OrcReader::convert_to_doris_type(const orc::Type* orc_type) {
orc_type->getSubtype(1) == nullptr) {
// Try to find the complete type from _column_id_to_file_type
uint64_t column_id = orc_type->getColumnId();
- VLOG(1) << "[OrcReader] Detected incomplete MAP type: column_id=" << column_id
- << ", subtype_count=" << orc_type->getSubtypeCount() << ", subtype(0)="
- << (orc_type->getSubtypeCount() > 0 && orc_type->getSubtype(0) != nullptr
- ? "not null"
- : "null")
- << ", subtype(1)="
- << (orc_type->getSubtypeCount() > 1 && orc_type->getSubtype(1) != nullptr
- ? "not null"
- : "null");
+ VLOG_DEBUG << "[OrcReader] Detected incomplete MAP type: column_id=" << column_id
+ << ", subtype_count=" << orc_type->getSubtypeCount() << ", subtype(0)="
+ << (orc_type->getSubtypeCount() > 0 && orc_type->getSubtype(0) != nullptr
+ ? "not null"
+ : "null")
+ << ", subtype(1)="
+ << (orc_type->getSubtypeCount() > 1 && orc_type->getSubtype(1) != nullptr
+ ? "not null"
+ : "null");
auto it = _column_id_to_file_type.find(column_id);
if (it != _column_id_to_file_type.end() && it->second != nullptr) {
const orc::Type* complete_type = it->second;
- VLOG(1) << "[OrcReader] Found complete type in mapping: column_id=" << column_id
- << ", complete_type_kind=" << static_cast<int>(complete_type->getKind())
- << ", complete_subtype_count=" << complete_type->getSubtypeCount();
+ VLOG_DEBUG << "[OrcReader] Found complete type in mapping: column_id=" << column_id
+ << ", complete_type_kind=" << static_cast<int>(complete_type->getKind())
+ << ", complete_subtype_count=" << complete_type->getSubtypeCount();
// Print subtypes information
for (uint64_t i = 0; i < complete_type->getSubtypeCount(); ++i) {
@@ -1573,7 +1536,8 @@ DataTypePtr OrcReader::convert_to_doris_type(const orc::Type* orc_type) {
if (complete_type->getKind() == orc::TypeKind::MAP &&
complete_type->getSubtypeCount() == 2) {
- VLOG(1) << "[OrcReader] Using complete MAP type from file schema for column_id="
+ VLOG_DEBUG
+ << "[OrcReader] Using complete MAP type from file schema for column_id="
<< column_id;
// Get subtypes with extra validation
@@ -1925,18 +1889,18 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
const orc::Type* orc_key_type = orc_column_type->getSubtype(0);
const orc::Type* orc_value_type = orc_column_type->getSubtype(1);
- VLOG(1) << "[OrcReader] MAP column '" << col_name
- << "': orc_key_type=" << (orc_key_type != nullptr ? "not null" : "NULL")
- << ", orc_value_type=" << (orc_value_type != nullptr ? "not null" : "NULL")
- << ", element_size=" << element_size;
+ VLOG_DEBUG << "[OrcReader] MAP column '" << col_name
+ << "': orc_key_type=" << (orc_key_type != nullptr ? "not null" : "NULL")
+ << ", orc_value_type=" << (orc_value_type != nullptr ? "not null" : "NULL")
+ << ", element_size=" << element_size;
// Handle incomplete MAP type - if key or value type is nullptr, try to recover from mapping
bool key_is_missing = (orc_key_type == nullptr);
bool value_is_missing = (orc_value_type == nullptr);
if (key_is_missing || value_is_missing) {
- VLOG(1) << "[OrcReader] Detected incomplete MAP subtypes for column '" << col_name
- << "', attempting to recover from mapping...";
+ VLOG_DEBUG << "[OrcReader] Detected incomplete MAP subtypes for column '" << col_name
+ << "', attempting to recover from mapping...";
uint64_t column_id = orc_column_type->getColumnId();
auto it = _column_id_to_file_type.find(column_id);
@@ -1948,15 +1912,16 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
orc_key_type = complete_map_type->getSubtype(0);
if (orc_key_type != nullptr) {
// key_is_missing = false;
- VLOG(1) << "[OrcReader] Recovered key type from mapping for column '"
- << col_name << "'";
+ VLOG_DEBUG << "[OrcReader] Recovered key type from mapping for column '"
+ << col_name << "'";
}
}
if (value_is_missing) {
orc_value_type = complete_map_type->getSubtype(1);
if (orc_value_type != nullptr) {
// value_is_missing = false;
- VLOG(1) << "[OrcReader] Recovered value type from mapping for column '"
+ VLOG_DEBUG
+ << "[OrcReader] Recovered value type from mapping for column '"
<< col_name << "'";
}
}
@@ -1971,9 +1936,6 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
// Handle key column: if still missing, fill with default values
if (key_is_missing) {
- LOG(INFO) << "[OrcReader] Key type is missing for MAP column '" << col_name
- << "', filling with default values (element_size=" << element_size << ")";
-
// Fill key column with default values (nulls or empty values)
auto mutable_key_column = doris_key_column->assume_mutable();
if (mutable_key_column->is_nullable()) {
@@ -1991,9 +1953,6 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
// Handle value column: if still missing, fill with default values
if (value_is_missing) {
- LOG(INFO) << "[OrcReader] Value type is missing for MAP column '" << col_name
- << "', filling with default values (element_size=" << element_size << ")";
-
// Fill value column with default values (nulls or empty values)
auto mutable_value_column = doris_value_column->assume_mutable();
if (mutable_value_column->is_nullable()) {
@@ -2072,7 +2031,6 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
for (auto read_field : read_fields) {
orc::ColumnVectorBatch* orc_field = orc_struct->fields[read_field.second];
-
const orc::Type* orc_type = orc_column_type->getSubtype(read_field.second);
std::string field_name =
col_name + "." + orc_column_type->getFieldName(read_field.second);
@@ -2099,7 +2057,6 @@ Status OrcReader::_orc_column_to_doris_column(
const orc::ColumnVectorBatch* cvb, size_t num_values) {
DataTypePtr resolved_type;
ColumnPtr resolved_column;
- converter::ColumnTypeConverter* converter_ptr = nullptr;
MutableColumnPtr data_column;
if (orc_column_type != nullptr) {
auto src_type = convert_to_doris_type(orc_column_type);
@@ -2129,9 +2086,9 @@ Status OrcReader::_orc_column_to_doris_column(
// reuse the cached converter
_converters[converter_key] = std::move(converter);
}
- converter_ptr = _converters[converter_key].get();
- resolved_column = converter_ptr->get_column(src_type, doris_column, data_type);
- resolved_type = converter_ptr->get_type();
+ converter::ColumnTypeConverter* converter = _converters[converter_key].get();
+ resolved_column = converter->get_column(src_type, doris_column, data_type);
+ resolved_type = converter->get_type();
if (resolved_column->is_nullable()) {
SCOPED_RAW_TIMER(&_statistics.decode_null_map_time);
@@ -2158,17 +2115,20 @@ Status OrcReader::_orc_column_to_doris_column(
data_column = resolved_column->assume_mutable();
}
- RETURN_IF_ERROR((_fill_doris_data_column<is_filter>(
+ RETURN_IF_ERROR(_fill_doris_data_column<is_filter>(
col_name, data_column, remove_nullable(resolved_type), root_node, orc_column_type,
- cvb, num_values)));
- // if (converter_ptr != nullptr) {
+ cvb, num_values));
// resolve schema change
auto converted_column = doris_column->assume_mutable();
- return converter_ptr->convert(resolved_column, converted_column);
- // }
+ return converter->convert(resolved_column, converted_column);
} else {
- reinterpret_cast<ColumnNullable*>(doris_column->assume_mutable().get())
- ->insert_many_defaults(num_values);
+ auto mutable_column = doris_column->assume_mutable();
+ if (mutable_column->is_nullable()) {
+ auto* nullable_column = static_cast<ColumnNullable*>(mutable_column.get());
+ nullable_column->insert_many_defaults(num_values);
+ } else {
+ mutable_column->insert_many_defaults(num_values);
+ }
}
return Status::OK();
@@ -2272,10 +2232,10 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
if (orc_col_idx == _colname_to_idx.end()) {
return Status::InternalError("Wrong read column '{}' in orc file", col_name);
}
- RETURN_IF_ERROR((_orc_column_to_doris_column<true>(
+ RETURN_IF_ERROR(_orc_column_to_doris_column<true>(
col_name, column_ptr, column_type,
_table_info_node_ptr->get_children_node(col_name), _type_map[file_column_name],
- batch_vec[orc_col_idx->second], _batch->numElements)));
+ batch_vec[orc_col_idx->second], _batch->numElements));
}
RETURN_IF_ERROR(_fill_partition_columns(block, _batch->numElements,
@@ -2330,12 +2290,6 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
}
}
- //std::cout << "[OrcReader] _column_ids: ";
- //for (const auto& cid : _column_ids) {
- // std::cout << cid << " ";
- //}
- //std::cout << std::endl;
-
if (!_dict_cols_has_converted && !_dict_filter_cols.empty()) {
for (auto& dict_filter_cols : _dict_filter_cols) {
MutableColumnPtr dict_col_ptr = ColumnInt32::create();
@@ -2369,10 +2323,10 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
if (orc_col_idx == _colname_to_idx.end()) {
return Status::InternalError("Wrong read column '{}' in orc file", col_name);
}
- RETURN_IF_ERROR((_orc_column_to_doris_column<false>(
+ RETURN_IF_ERROR(_orc_column_to_doris_column<false>(
col_name, column_ptr, column_type,
_table_info_node_ptr->get_children_node(col_name), _type_map[file_column_name],
- batch_vec[orc_col_idx->second], _batch->numElements)));
+ batch_vec[orc_col_idx->second], _batch->numElements));
}
RETURN_IF_ERROR(_fill_partition_columns(block, _batch->numElements,
@@ -2535,10 +2489,10 @@ Status OrcReader::filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t s
if (orc_col_idx == _colname_to_idx.end()) {
return Status::InternalError("Wrong read column '{}' in orc file", table_col_name);
}
- RETURN_IF_ERROR((_orc_column_to_doris_column<false>(
+ RETURN_IF_ERROR(_orc_column_to_doris_column<false>(
table_col_name, column_ptr, column_type,
_table_info_node_ptr->get_children_node(table_col_name),
- _type_map[file_column_name], batch_vec[orc_col_idx->second], data.numElements)));
+ _type_map[file_column_name], batch_vec[orc_col_idx->second], data.numElements));
}
RETURN_IF_ERROR(
_fill_partition_columns(block, size, _lazy_read_ctx.predicate_partition_columns));
diff --git a/be/src/vec/exec/format/orc/vorc_reader.h b/be/src/vec/exec/format/orc/vorc_reader.h
index 341e40a9ce2..d1313bc4639 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.h
+++ b/be/src/vec/exec/format/orc/vorc_reader.h
@@ -639,7 +639,6 @@ private:
size_t _batch_size;
int64_t _range_start_offset;
int64_t _range_size;
- // const std::string& _ctz;
std::string _ctz;
int32_t _offset_days = 0;
@@ -663,11 +662,8 @@ private:
// file column name to orc type
std::unordered_map<std::string, const orc::Type*> _type_map;
- // Optimized type conversion support
- // column ID to file original type mapping for handling partial column selection
+ // Column ID to file original type mapping for handling incomplete MAP type due to column pruning.
std::unordered_map<uint64_t, const orc::Type*> _column_id_to_file_type;
- // Keep reference to file root type for complete type information
- const orc::Type* _file_root_type = nullptr;
std::unique_ptr<ORCFileInputStream> _file_input_stream;
Statistics _statistics;
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
index a7455455198..589570c7ab0 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
@@ -111,23 +111,6 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
size_t max_buf_size, const tparquet::OffsetIndex* offset_index,
const std::set<uint64_t>& column_ids,
const std::set<uint64_t>& filter_column_ids) {
- // if (!column_ids.empty()) {
- // uint64_t field_column_id = field->get_column_id();
- // VLOG_DEBUG << "[ParquetReader] Checking column_id: " << field_column_id
- // << ", column_ids.size(): " << column_ids.size();
- // if (column_ids.find(field_column_id) == column_ids.end()) {
- // VLOG_DEBUG << "[ParquetReader] Setting skip_reading=true for column_id: "
- // << field_column_id;
- // auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx, field);
- // skip_reader->_filter_column_ids = filter_column_ids;
- // reader = std::move(skip_reader);
- // return Status::OK();
- // } else {
- // VLOG_DEBUG << "[ParquetReader] Column_id " << field_column_id
- // << " found in column_ids, skip_reading=false";
- // }
- // }
-
if (field->data_type->get_primitive_type() == TYPE_ARRAY) {
std::unique_ptr<ParquetColumnReader> element_reader;
RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
@@ -177,7 +160,7 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
child_readers.reserve(field->children.size());
int non_skip_reader_idx = -1;
for (int i = 0; i < field->children.size(); ++i) {
- auto& child = field->children[i]; // 去掉 const，保证 &child 类型为 FieldSchema*
+ auto& child = field->children[i];
std::unique_ptr<ParquetColumnReader> child_reader;
if (column_ids.empty() || column_ids.find(child.get_column_id()) != column_ids.end()) {
RETURN_IF_ERROR(create(file, &child, row_group, row_ranges, ctz, io_ctx,
@@ -185,32 +168,25 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
filter_column_ids));
child_reader->set_nested_column();
child_readers[child.name] = std::move(child_reader);
- // 记录第一个非 SkippingReader
+ // Record the first non-SkippingReader
if (non_skip_reader_idx == -1) {
non_skip_reader_idx = i;
}
} else {
- // 不在 column_ids 的 child，创建 SkippingReader
auto skip_reader =
std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx, &child);
skip_reader->_filter_column_ids = filter_column_ids;
child_readers[child.name] = std::move(skip_reader);
}
}
- // 如果所有都是 SkipReadingReader，则强制第一个 child 调用 create
+ // If all children are SkipReadingReader, force the first child to call create
if (non_skip_reader_idx == -1) {
- // for (int i = 0; i < field->children.size(); ++i) {
- // 只处理 SkipReadingReader（即刚才创建的）
- // if (dynamic_cast<SkipReadingReader*>(child_readers[field->children[i].name].get()) != nullptr) {
std::unique_ptr<ParquetColumnReader> child_reader;
RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
child_reader, max_buf_size, nullptr, column_ids,
filter_column_ids));
child_reader->set_nested_column();
child_readers[field->children[0].name] = std::move(child_reader);
- // break;
- // }
- // }
}
auto struct_reader = StructColumnReader::create_unique(row_ranges, ctz, io_ctx);
RETURN_IF_ERROR(struct_reader->init(std::move(child_readers), field));
@@ -878,7 +854,6 @@ Status MapColumnReader::read_column_data(
value_rows += loop_rows;
}
DCHECK_EQ(key_rows, value_rows);
- // DCHECK_EQ(key_eof, value_eof);
*read_rows = key_rows;
*eof = key_eof;
@@ -948,10 +923,6 @@ Status StructColumnReader::read_column_data(
continue;
}
auto file_name = root_node->children_file_column_name(doris_name);
- // if (_child_readers.find(file_name) == _child_readers.end()) {
- // missing_column_idxs.push_back(i);
- // continue;
- // }
// Check if this is a SkipReadingReader - we should skip it when choosing reference column
// because SkipReadingReader doesn't know the actual data size in nested context
@@ -978,13 +949,6 @@ Status StructColumnReader::read_column_data(
batch_size, &field_rows, &field_eof, is_dict_filter));
*read_rows = field_rows;
*eof = field_eof;
- // Debug: print the first non-missing child column read info
- //std::cout << "[ParquetReader] struct '" << _field_schema->name << "' first non-missing child='"
- // << file_name << "' doris_name='" << doris_name << "' field_rows=" << field_rows
- // << " field_eof=" << field_eof << " doris_field_size=" << doris_field->size()
- // << " child_rep_levels=" << _child_readers[file_name]->get_rep_level().size()
- // << " child_def_levels=" << _child_readers[file_name]->get_def_level().size()
- // << std::endl;
/*
* Considering the issue in the `_read_nested_column` function where data may span across pages, leading
* to missing definition and repetition levels, when filling the null_map of the struct later, it is
@@ -1001,12 +965,6 @@ Status StructColumnReader::read_column_data(
doris_field, doris_type, root_node->get_children_node(doris_name),
filter_map, *read_rows - field_rows, &loop_rows, &field_eof,
is_dict_filter));
- // Debug: print each loop iteration for non-first child columns
- //std::cout << "[ParquetReader] struct '" << _field_schema->name << "' non-first child='"
- // << file_name << "' doris_name='" << doris_name << "' loop_rows=" << loop_rows
- // << " field_rows=" << field_rows << " target_rows=" << *read_rows
- // << " field_eof=" << field_eof << " doris_field_size=" << doris_field->size()
- // << std::endl;
field_rows += loop_rows;
}
DCHECK_EQ(*read_rows, field_rows);
@@ -1014,20 +972,11 @@ Status StructColumnReader::read_column_data(
}
}
- // After scanning children, print summary of missing/skip info
- //std::cout << "[ParquetReader] struct '" << _field_schema->name << "' summary: not_missing_column_id="
- // << not_missing_column_id << " missing_columns_count=" << missing_column_idxs.size()
- // << " skip_reading_count=" << skip_reading_column_idxs.size() << std::endl;
-
int64_t missing_column_sz = -1;
if (not_missing_column_id == -1) {
// All queried columns are missing in the file (e.g., all added after schema change)
- // We need to pick a column from _field_schema that exists in the file to get RL/DL
- //std::cout << "[ParquetReader] All queried columns missing for struct '" << _field_schema->name
- // << "', searching for reference column in _field_schema" << std::endl;
-
- // Find a column from _field_schema children that exists in the file for RL/DL reference
+ // We need to pick a column from _field_schema children that exists in the file for RL/DL reference
std::string reference_file_column_name;
std::unique_ptr<ParquetColumnReader>* reference_reader = nullptr;
@@ -1039,8 +988,6 @@ Status StructColumnReader::read_column_data(
if (!is_skip_reader) {
reference_file_column_name = child.name;
reference_reader = &(it->second);
- //std::cout << "[ParquetReader] Found reference column: " << reference_file_column_name
- // << " for struct '" << _field_schema->name << "'" << std::endl;
break;
}
}
@@ -1087,10 +1034,6 @@ Status StructColumnReader::read_column_data(
// Store this reference column name for get_rep_level/get_def_level to use
_read_column_names.emplace_back(reference_file_column_name);
- //std::cout << "[ParquetReader] Read " << field_rows << " rows from reference column '"
- // << reference_file_column_name << "', temp_column.size=" << temp_column->size()
- // << " for struct '" << _field_schema->name << "'" << std::endl;
-
missing_column_sz = temp_column->size() - not_missing_orig_column_size;
} else {
return Status::Corruption(
@@ -1139,8 +1082,6 @@ Status StructColumnReader::read_column_data(
}
if (null_map_ptr != nullptr) {
- //std::cout << "[ParquetReader] struct '" << _field_schema->name << "' before fill_struct_null_map: rep_levels="
- // << this->get_rep_level().size() << " def_levels=" << this->get_def_level().size() << std::endl;
fill_struct_null_map(_field_schema, *null_map_ptr, this->get_rep_level(),
this->get_def_level());
}
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
index 7b37f2e0e6c..893b3bc784a 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
@@ -412,8 +412,6 @@ public:
<< _field_schema->name
<< ". This indicates the SkipReadingReader was incorrectly used as a reference "
"column.";
- // static std::vector<level_t> empty_levels;
- // return empty_levels;
__builtin_unreachable();
}
@@ -422,8 +420,6 @@ public:
<< _field_schema->name
<< ". This indicates the SkipReadingReader was incorrectly used as a reference "
"column.";
- // static std::vector<level_t> empty_levels;
- // return empty_levels;
__builtin_unreachable();
}
diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
index 19ddb8d1b99..947dcee3408 100644
--- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
@@ -98,13 +98,7 @@ RowGroupReader::RowGroupReader(io::FileReaderSPtr file_reader,
_state(state),
_obj_pool(new ObjectPool()),
_column_ids(column_ids),
- _filter_column_ids(filter_column_ids) {
- //std::cout << "[RowGroupReader] _column_ids: ";
- //for (const auto& col_id : _column_ids) {
- // std::cout << col_id << " ";
- //}
- //std::cout << std::endl;
-}
+ _filter_column_ids(filter_column_ids) {}
RowGroupReader::~RowGroupReader() {
_column_readers.clear();
@@ -438,14 +432,13 @@ Status RowGroupReader::_read_column_data(Block* block,
RETURN_IF_ERROR(_column_readers[read_col_name]->read_column_data(
column_ptr, column_type, _table_info_node_ptr->get_children_node(read_col_name),
filter_map, batch_size - col_read_rows, &loop_rows, &col_eof, is_dict_filter));
- // Debug: print loop_rows for this iteration
- //std::cout << "[RowGroupReader] column '" << read_col_name << "' loop_rows=" << loop_rows
- // << " col_read_rows_so_far=" << col_read_rows << std::endl;
+ VLOG_DEBUG << "[RowGroupReader] column '" << read_col_name
+ << "' loop_rows=" << loop_rows << " col_read_rows_so_far=" << col_read_rows
+ << std::endl;
col_read_rows += loop_rows;
}
- // Debug: print rows read for this column
- //std::cout << "[RowGroupReader] column '" << read_col_name << "' read_rows=" << col_read_rows
- // << std::endl;
+ VLOG_DEBUG << "[RowGroupReader] column '" << read_col_name
+ << "' read_rows=" << col_read_rows << std::endl;
if (batch_read_rows > 0 && batch_read_rows != col_read_rows) {
LOG(WARNING) << "[RowGroupReader] Mismatched read rows among parquet columns. "
"previous_batch_read_rows="
@@ -741,10 +734,6 @@ Status RowGroupReader::_fill_missing_columns(
block->erase(result_column_id);
}
}
-
- // auto mutable_column = block->get_by_name(kv.first).column->assume_mutable();
- // auto* nullable_column = assert_cast<vectorized::ColumnNullable*>(mutable_column.get());
- // nullable_column->insert_many_defaults(rows);
}
return Status::OK();
}
@@ -1135,7 +1124,6 @@ ParquetColumnReader::Statistics RowGroupReader::statistics() {
}
return st;
}
-
#include "common/compile_check_end.h"
} // namespace doris::vectorized
diff --git a/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp b/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp
index d46acbdc134..eea63192857 100644
--- a/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp
+++ b/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp
@@ -60,7 +60,6 @@ struct ColumnAccessPathConfig {
std::string column_name;
std::vector<std::vector<std::string>> all_column_paths; // For all_column_access_paths
std::vector<std::vector<std::string>> predicate_paths; // For predicate_column_access_paths
- bool use_name_paths = true; // true=use field names, false=use indices
};
// ORC column IDs are assigned in a tree-increment order, root is 0 and children increment sequentially.
@@ -738,12 +737,11 @@ protected:
}
// Helper function: Run Parquet test with different column ID extraction methods
- void run_parquet_test_with_method(const std::vector<std::string>& table_column_names,
- const std::vector<ColumnAccessPathConfig>& access_configs,
- const std::set<uint64_t>& expected_column_ids,
- const std::set<uint64_t>& expected_filter_column_ids,
- bool use_top_level_method = false,
- bool should_skip_assertion = false) {
+ void run_parquet_test(const std::vector<std::string>& table_column_names,
+ const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false, bool should_skip_assertion = false) {
std::string test_file =
"./be/test/exec/test_data/nested_user_profiles_parquet/"
"part-00000-64a7a390-1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet";
@@ -809,12 +807,11 @@ protected:
}
// Helper function: Run ORC test with different column ID extraction methods
- void run_orc_test_with_method(const std::vector<std::string>& table_column_names,
- const std::vector<ColumnAccessPathConfig>& access_configs,
- const std::set<uint64_t>& expected_column_ids,
- const std::set<uint64_t>& expected_filter_column_ids,
- bool use_top_level_method = false,
- bool should_skip_assertion = false) {
+ void run_orc_test(const std::vector<std::string>& table_column_names,
+ const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false, bool should_skip_assertion = false) {
std::string test_file =
"./be/test/exec/test_data/nested_user_profiles_orc/"
"part-00000-62614f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc";
@@ -881,12 +878,8 @@ protected:
};
TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_1) {
- // Properties:
-
- // Configure access paths for profile column
ColumnAccessPathConfig access_config;
access_config.column_name = "profile";
- access_config.use_name_paths = true;
access_config.all_column_paths = {{"profile", "address", "coordinates", "lat"},
{"profile", "address", "coordinates", "lng"},
@@ -900,21 +893,20 @@ TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_1) {
std::set<uint64_t> expected_column_ids = {2, 3, 4, 7, 8, 9, 10, 11, 15, 16, 18};
std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
- run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
- run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids, true);
+ run_parquet_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_parquet_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
- run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
- run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids, true);
+ run_orc_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
}
TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_2) {
- // Properties:
- // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
- // ORC column IDs are assigned in a tree-increment order, root is 0 and children increment sequentially.
+ // ORC column IDs are assigned in a tree-like incremental manner: the root node is 0, and child nodes increase sequentially.
+ // Currently, Parquet uses a similar design.
// 0: struct (table/root)
// 1: id (int64)
// 2: name (string)
@@ -935,36 +927,32 @@ TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_2) {
// 17: name (string)
// 18: level (int32)
- // 配置profile列的访问路径
ColumnAccessPathConfig access_config;
access_config.column_name = "profile";
- access_config.use_name_paths = true;
access_config.all_column_paths = {{"profile"}};
access_config.predicate_paths = {{"profile", "address", "coordinates", "lat"},
{"profile", "contact", "email"}};
std::vector<std::string> table_column_names = {"name", "profile"};
- // column_ids should contain all necessary column IDs (set automatically deduplicates)
std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18};
std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
- run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
- run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids, true);
+ run_parquet_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_parquet_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
- run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
- run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids, true);
+ run_orc_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
}
TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_3) {
- // Properties:
- // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
- // ORC column IDs are assigned in a tree-increment order, root is 0 and children increment sequentially.
+ // ORC column IDs are assigned in a tree-like incremental manner: the root node is 0, and child nodes increase sequentially.
+ // Currently, Parquet uses a similar design.
// 0: struct (table/root)
// 1: id (int64)
// 2: name (string)
@@ -985,58 +973,54 @@ TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_3) {
// 17: name (string)
// 18: level (int32)
- // 配置profile列的访问路径
ColumnAccessPathConfig access_config;
access_config.column_name = "profile";
- access_config.use_name_paths = true;
access_config.all_column_paths = {{"profile", "contact"}, {"profile", "address"}};
access_config.predicate_paths = {{"profile", "address", "coordinates"},
{"profile", "contact", "email"}};
std::vector<std::string> table_column_names = {"name", "profile"};
- // column_ids should contain all necessary column IDs (set automatically deduplicates)
std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14};
std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 9, 10, 11};
- run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
- run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids, true);
+ run_parquet_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_parquet_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
- run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
- run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids, true);
+ run_orc_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
}
TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_4) {
- // 配置profile列的访问路径
ColumnAccessPathConfig access_config;
access_config.column_name = "profile";
- access_config.use_name_paths = true;
access_config.all_column_paths = {};
access_config.predicate_paths = {};
std::vector<std::string> table_column_names = {"name", "profile"};
- // column_ids should contain all necessary column IDs (set automatically deduplicates)
std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18};
std::set<uint64_t> expected_filter_column_ids = {};
- run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
- run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids, true);
+ run_parquet_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_parquet_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
- run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
- run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids, true);
+ run_orc_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids, true);
}
TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_5) {
+ // ORC column IDs are assigned in a tree-like incremental manner: the root node is 0, and child nodes increase sequentially.
+ // Currently, Parquet uses a similar design.
// 19: tags (array)
// 20: element (string)
// 21: friends (array)
@@ -1058,7 +1042,6 @@ TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_5) {
// Configure access paths for friends column
ColumnAccessPathConfig access_config;
access_config.column_name = "friends";
- access_config.use_name_paths = true;
access_config.all_column_paths = {{"friends", "*", "nickname"},
{"friends", "*", "friendship_level"}};
@@ -1072,7 +1055,6 @@ TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_5) {
// Configure access paths for recent_activity column
ColumnAccessPathConfig access_config;
access_config.column_name = "recent_activity";
- access_config.use_name_paths = true;
access_config.all_column_paths = {{"recent_activity", "*", "action"},
{"recent_activity", "*", "details", "*", "value"}};
@@ -1083,22 +1065,23 @@ TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_5) {
}
std::vector<std::string> table_column_names = {"name", "friends", "recent_activity"};
- // column_ids should contain all necessary column IDs (set automatically deduplicates)
std::set<uint64_t> expected_column_ids = {2, 21, 22, 24, 25, 26, 27, 28, 29, 30, 32};
std::set<uint64_t> expected_filter_column_ids = {21, 22, 24, 26, 27, 28};
- run_parquet_test_with_method(table_column_names, access_configs, expected_column_ids,
- expected_filter_column_ids);
- run_parquet_test_with_method(table_column_names, access_configs, expected_column_ids,
- expected_filter_column_ids, true);
+ run_parquet_test(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
+ run_parquet_test(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids, true);
- run_orc_test_with_method(table_column_names, access_configs, expected_column_ids,
- expected_filter_column_ids);
- run_orc_test_with_method(table_column_names, access_configs, expected_column_ids,
- expected_filter_column_ids, true);
+ run_orc_test(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids, true);
}
TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_6) {
+ // ORC column IDs are assigned in a tree-like incremental manner: the root node is 0, and child nodes increase sequentially.
+ // Currently, Parquet uses a similar design.
// 33: attributes (map)
// 34: key (string)
// 35: value (string)
@@ -1165,7 +1148,6 @@ TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_6) {
// Configure access paths for complex_attributes column
ColumnAccessPathConfig access_config;
access_config.column_name = "complex_attributes";
- access_config.use_name_paths = true;
access_config.all_column_paths = {
{"complex_attributes", "*", "metadata", "version"},
@@ -1183,30 +1165,28 @@ TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_6) {
{
std::vector<std::string> table_column_names = {"name", "complex_attributes"};
// parquet values should access keys
- // column_ids should contain all necessary column IDs (set automatically deduplicates)
std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53,
54, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
std::set<uint64_t> expected_filter_column_ids = {36, 37, 38, 39, 40};
- run_parquet_test_with_method(table_column_names, access_configs, expected_column_ids,
- expected_filter_column_ids);
- run_parquet_test_with_method(table_column_names, access_configs, expected_column_ids,
- expected_filter_column_ids, true);
+ run_parquet_test(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
+ run_parquet_test(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids, true);
}
{
std::vector<std::string> table_column_names = {"name", "complex_attributes"};
// orc values should access keys because need to deduplicate by keys
- // column_ids should contain all necessary column IDs (set automatically deduplicates)
std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53,
54, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
std::set<uint64_t> expected_filter_column_ids = {36, 37, 38, 39, 40};
- run_orc_test_with_method(table_column_names, access_configs, expected_column_ids,
- expected_filter_column_ids);
- run_orc_test_with_method(table_column_names, access_configs, expected_column_ids,
- expected_filter_column_ids, true);
+ run_orc_test(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids, true);
}
}
diff --git a/be/test/vec/exec/format/table/hive/hive_reader_test.cpp b/be/test/vec/exec/format/table/hive/hive_reader_test.cpp
index 16bde5c7bae..ca2cada932f 100644
--- a/be/test/vec/exec/format/table/hive/hive_reader_test.cpp
+++ b/be/test/vec/exec/format/table/hive/hive_reader_test.cpp
@@ -462,7 +462,7 @@ protected:
};
// Test reading real Hive Parquet file using HiveTableReader
-TEST_F(HiveReaderTest, ReadHiveParquetFile) {
+TEST_F(HiveReaderTest, read_hive_parquet_file) {
// Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email
// Setup table descriptor for test columns with new schema:
/**
@@ -604,7 +604,7 @@ TEST_F(HiveReaderTest, ReadHiveParquetFile) {
}
// Test reading real Hive Orc file using HiveTableReader
-TEST_F(HiveReaderTest, ReadHiveOrcFile) {
+TEST_F(HiveReaderTest, read_hive_rrc_file) {
// Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email
// Setup table descriptor for test columns with new schema:
/**
diff --git a/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp b/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp
index b1152f159bd..d897f0e4762 100644
--- a/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp
+++ b/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp
@@ -60,10 +60,9 @@ struct ColumnAccessPathConfig {
std::string column_name;
std::vector<std::vector<std::string>> all_column_paths; // For all_column_access_paths
std::vector<std::vector<std::string>> predicate_paths; // For predicate_column_access_paths
- bool use_name_paths = true; // true=use field names, false=use indices
};
-// Iceberg 列ID分配 (基于 Protobuf Schema 定义)
+// Iceberg column ID assignment (based on Protobuf schema definition)
// 1: id (int64)
// 2: name (string)
// 3: profile (struct)
@@ -770,12 +769,11 @@ protected:
}
// Helper function: run Parquet test with different column ID extraction methods
- void run_parquet_test_with_method(const std::vector<std::string>& table_column_names,
- const std::vector<ColumnAccessPathConfig>& access_configs,
- const std::set<uint64_t>& expected_column_ids,
- const std::set<uint64_t>& expected_filter_column_ids,
- bool use_top_level_method = false,
- bool should_skip_assertion = false) {
+ void run_parquet_test(const std::vector<std::string>& table_column_names,
+ const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false, bool should_skip_assertion = false) {
std::string test_file =
"./be/test/exec/test_data/nested_user_profiles_iceberg_parquet/data/"
"00000-9-a7e0135f-d581-40e4-8d56-a929aded99e4-0-00001.parquet";
@@ -841,12 +839,11 @@ protected:
}
// Helper function: run Orc test with different column ID extraction methods
- void run_orc_test_with_method(const std::vector<std::string>& table_column_names,
- const std::vector<ColumnAccessPathConfig>& access_configs,
- const std::set<uint64_t>& expected_column_ids,
- const std::set<uint64_t>& expected_filter_column_ids,
- bool use_top_level_method = false,
- bool should_skip_assertion = false) {
+ void run_orc_test(const std::vector<std::string>& table_column_names,
+ const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false, bool should_skip_assertion = false) {
std::string test_file =
"./be/test/exec/test_data/nested_user_profiles_iceberg_orc/data/"
"00000-8-5a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc";
@@ -913,12 +910,8 @@ protected:
};
TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_1) {
- // Properties:
-
- // Configure access paths for the profile column
ColumnAccessPathConfig access_config;
access_config.column_name = "profile";
- access_config.use_name_paths = true;
access_config.all_column_paths = {{"3", "9", "14", "15"},
{"3", "9", "14", "16"},
@@ -931,16 +924,15 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_1) {
std::set<uint64_t> expected_column_ids = {2, 3, 4, 7, 8, 9, 10, 11, 15, 16, 18};
std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
- run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
- run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
+ run_parquet_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
}
TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_2) {
- // Properties:
- // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
- // ORC 列ID分配为树型递增，根节点为0，子节点依次递增。
+ // ORC column IDs are assigned in a tree-like incremental manner: the root node is 0, and child nodes increase sequentially.
+ // Currently, Parquet uses a similar design.
// 0: struct (table/root)
// 1: id (int64)
// 2: name (string)
@@ -961,30 +953,26 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_2) {
// 17: name (string)
// 18: level (int32)
- // 配置profile列的访问路径
ColumnAccessPathConfig access_config;
access_config.column_name = "profile";
- access_config.use_name_paths = true;
access_config.all_column_paths = {{"3"}};
access_config.predicate_paths = {{"3", "9", "14", "15"}, {"3", "10", "17"}};
std::vector<std::string> table_column_names = {"name", "profile"};
- // column_ids should contain all necessary column IDs (set automatically deduplicates)
std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18};
std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
- run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
- run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
+ run_parquet_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
}
TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_3) {
- // Properties:
- // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
- // ORC 列ID分配为树型递增，根节点为0，子节点依次递增。
+ // ORC column IDs are assigned in a tree-like incremental manner: the root node is 0, and child nodes increase sequentially.
+ // Currently, Parquet uses a similar design.
// 0: struct (table/root)
// 1: id (int64)
// 2: name (string)
@@ -1005,51 +993,43 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_3) {
// 17: name (string)
// 18: level (int32)
- // 配置profile列的访问路径
ColumnAccessPathConfig access_config;
access_config.column_name = "profile";
- access_config.use_name_paths = true;
access_config.all_column_paths = {{"3", "10"}, {"3", "9"}};
access_config.predicate_paths = {{"3", "9", "14"}, {"3", "10", "17"}};
- // access_config.all_column_paths = {{"profile", "contact"}, {"profile", "address"}};
- // access_config.predicate_paths = {{"profile", "address", "coordinates"},
- // {"profile", "contact", "email"}};
-
std::vector<std::string> table_column_names = {"name", "profile"};
- // column_ids should contain all necessary column IDs (set automatically deduplicates)
std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14};
std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 9, 10, 11};
- run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
- run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
+ run_parquet_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
}
TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_4) {
- // 配置profile列的访问路径
ColumnAccessPathConfig access_config;
access_config.column_name = "profile";
- access_config.use_name_paths = true;
access_config.all_column_paths = {};
access_config.predicate_paths = {};
std::vector<std::string> table_column_names = {"name", "profile"};
- // column_ids should contain all necessary column IDs (set automatically deduplicates)
std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18};
std::set<uint64_t> expected_filter_column_ids = {};
- run_parquet_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
- run_orc_test_with_method(table_column_names, {access_config}, expected_column_ids,
- expected_filter_column_ids);
+ run_parquet_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test(table_column_names, {access_config}, expected_column_ids,
+ expected_filter_column_ids);
}
TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_5) {
+ // ORC column IDs are assigned in a tree-like incremental manner: the root node is 0, and child nodes increase sequentially.
+ // Currently, Parquet uses a similar design.
// 19: tags (array)
// 20: element (string)
// 21: friends (array)
@@ -1068,10 +1048,8 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_5) {
std::vector<ColumnAccessPathConfig> access_configs;
{
- // 配置friends列的访问路径
ColumnAccessPathConfig access_config;
access_config.column_name = "friends";
- access_config.use_name_paths = true;
access_config.all_column_paths = {{"5", "*", "27"}, {"5", "*", "28"}};
access_config.predicate_paths = {
@@ -1081,10 +1059,8 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_5) {
}
{
- // 配置recent_activity列的访问路径
ColumnAccessPathConfig access_config;
access_config.column_name = "recent_activity";
- access_config.use_name_paths = true;
access_config.all_column_paths = {{"6", "*", "30"}, {"6", "*", "31", "*", "34"}};
access_config.predicate_paths = {
@@ -1094,17 +1070,18 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_5) {
}
std::vector<std::string> table_column_names = {"name", "friends", "recent_activity"};
- // column_ids should contain all necessary column IDs (set automatically deduplicates)
std::set<uint64_t> expected_column_ids = {2, 21, 22, 24, 25, 26, 27, 28, 29, 30, 32};
std::set<uint64_t> expected_filter_column_ids = {21, 22, 24, 26, 27, 28};
- run_parquet_test_with_method(table_column_names, access_configs, expected_column_ids,
- expected_filter_column_ids);
- run_orc_test_with_method(table_column_names, access_configs, expected_column_ids,
- expected_filter_column_ids);
+ run_parquet_test(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
+ run_orc_test(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
}
TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_6) {
+ // ORC column IDs are assigned in a tree-like incremental manner: the root node is 0, and child nodes increase sequentially.
+ // Currently, Parquet uses a similar design.
// 33: attributes (map)
// 34: key (string)
// 35: value (string)
@@ -1168,10 +1145,8 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_6) {
std::vector<ColumnAccessPathConfig> access_configs;
{
- // 配置complex_attributes列的访问路径
ColumnAccessPathConfig access_config;
access_config.column_name = "complex_attributes";
- access_config.use_name_paths = true;
access_config.all_column_paths = {{"8", "*", "39", "43"},
{"8", "*", "40", "*", "50", "*", "55"},
@@ -1184,27 +1159,25 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_6) {
{
std::vector<std::string> table_column_names = {"name", "complex_attributes"};
// parquet values should access keys
- // column_ids should contain all necessary column IDs (set automatically deduplicates)
std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53,
54, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
std::set<uint64_t> expected_filter_column_ids = {36, 37, 38, 39, 40};
- run_parquet_test_with_method(table_column_names, access_configs, expected_column_ids,
- expected_filter_column_ids);
+ run_parquet_test(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
}
{
std::vector<std::string> table_column_names = {"name", "complex_attributes"};
// orc values should access keys because need to deduplicate by keys
- // column_ids should contain all necessary column IDs (set automatically deduplicates)
std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53,
54, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
std::set<uint64_t> expected_filter_column_ids = {36, 37, 38, 39, 40};
- run_orc_test_with_method(table_column_names, access_configs, expected_column_ids,
- expected_filter_column_ids);
+ run_orc_test(table_column_names, access_configs, expected_column_ids,
+ expected_filter_column_ids);
}
}
diff --git a/be/test/vec/exec/format/table/iceberg/iceberg_reader_test.cpp b/be/test/vec/exec/format/table/iceberg/iceberg_reader_test.cpp
index 12d6bb0299f..b7dedcf834e 100644
--- a/be/test/vec/exec/format/table/iceberg/iceberg_reader_test.cpp
+++ b/be/test/vec/exec/format/table/iceberg/iceberg_reader_test.cpp
@@ -465,12 +465,10 @@ protected:
};
// Test reading real Iceberg Parquet file using IcebergTableReader
-TEST_F(IcebergReaderTest, ReadIcebergParquetFile) {
+TEST_F(IcebergReaderTest, read_iceberg_parquet_file) {
// Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email
// Setup table descriptor for test columns with new schema:
/**
- Properties:
- iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type":{"type" [...]
Schema:
message table {
required int64 id = 1;
@@ -603,12 +601,10 @@ TEST_F(IcebergReaderTest, ReadIcebergParquetFile) {
}
// Test reading real Iceberg Orc file using IcebergTableReader
-TEST_F(IcebergReaderTest, ReadIcebergOrcFile) {
+TEST_F(IcebergReaderTest, read_iceberg_orc_file) {
// Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email
// Setup table descriptor for test columns with new schema:
/**
- Properties:
- iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type":{"type" [...]
Schema:
message table {
required int64 id = 1;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/18: use slot->col_name_lower_case().
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3dfbdea9e40d4077de976b6e1633b6de98ee1d5f
Author: kakachen <ch...@selectdb.com>
AuthorDate: Tue Oct 28 21:20:36 2025 +0800
use slot->col_name_lower_case().
---
be/src/vec/exec/format/table/hive_reader.cpp | 16 ++++------------
be/src/vec/exec/format/table/iceberg_reader.cpp | 3 ---
2 files changed, 4 insertions(+), 15 deletions(-)
diff --git a/be/src/vec/exec/format/table/hive_reader.cpp b/be/src/vec/exec/format/table/hive_reader.cpp
index 96ca2dff2d0..eea7a126517 100644
--- a/be/src/vec/exec/format/table/hive_reader.cpp
+++ b/be/src/vec/exec/format/table/hive_reader.cpp
@@ -53,7 +53,7 @@ Status HiveOrcReader::init_reader(
// hive1 / use index
std::map<std::string, const SlotDescriptor*> slot_map; // table_name to slot
for (const auto& slot : tuple_descriptor->slots()) {
- slot_map.emplace(slot->col_name(), slot);
+ slot_map.emplace(slot->col_name_lower_case(), slot);
}
// For top-level columns, use indexes to match, and for sub-columns, still use name to match columns.
@@ -161,7 +161,7 @@ ColumnIdResult HiveOrcReader::_create_column_ids(const orc::Type* orc_type,
};
for (const auto* slot : tuple_descriptor->slots()) {
- auto it = table_col_name_to_orc_type_map.find(slot->col_name());
+ auto it = table_col_name_to_orc_type_map.find(slot->col_name_lower_case());
if (it == table_col_name_to_orc_type_map.end()) {
// Column not found in file (e.g., partition column, added column)
continue;
@@ -318,7 +318,7 @@ Status HiveParquetReader::init_reader(
} else { // use idx
std::map<std::string, const SlotDescriptor*> slot_map; //table_name to slot
for (const auto& slot : tuple_descriptor->slots()) {
- slot_map.emplace(slot->col_name(), slot);
+ slot_map.emplace(slot->col_name_lower_case(), slot);
}
// For top-level columns, use indexes to match, and for sub-columns, still use name to match columns.
@@ -444,12 +444,8 @@ ColumnIdResult HiveParquetReader::_create_column_ids(const FieldDescriptor* fiel
};
for (const auto* slot : tuple_descriptor->slots()) {
- // if (slot->col_name().starts_with(BeConsts::GLOBAL_ROWID_COL)) {
- // continue;
- // }
-
// Find the field schema for this slot (may not exist for partition columns, etc.)
- auto it = table_col_name_to_field_schema_map.find(slot->col_name());
+ auto it = table_col_name_to_field_schema_map.find(slot->col_name_lower_case());
if (it == table_col_name_to_field_schema_map.end()) {
// Column not found in file (e.g., partition column, added column)
continue;
@@ -554,10 +550,6 @@ ColumnIdResult HiveParquetReader::_create_column_ids_by_top_level_col_index(
};
for (const auto* slot : tuple_descriptor->slots()) {
- // if (slot->col_name().starts_with(BeConsts::GLOBAL_ROWID_COL)) {
- // continue;
- // }
-
// Find the field schema for this slot (may not exist for partition columns, etc.)
auto it = table_col_pos_to_field_schema_map.find(slot->col_pos());
if (it == table_col_pos_to_field_schema_map.end()) {
diff --git a/be/src/vec/exec/format/table/iceberg_reader.cpp b/be/src/vec/exec/format/table/iceberg_reader.cpp
index 95aa41a43e6..9a9e984a9d9 100644
--- a/be/src/vec/exec/format/table/iceberg_reader.cpp
+++ b/be/src/vec/exec/format/table/iceberg_reader.cpp
@@ -539,9 +539,6 @@ ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* f
};
for (const auto* slot : tuple_descriptor->slots()) {
- // if (slot->col_name().starts_with(BeConsts::GLOBAL_ROWID_COL)) {
- // continue;
- // }
auto it = iceberg_id_to_field_schema_map.find(slot->col_unique_id());
if (it == iceberg_id_to_field_schema_map.end()) {
// Column not found in file (e.g., partition column, added column)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 06/18: fix case error in BE
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9b00e6fa10770bc2b9bd089460a6171610ed0619
Author: Hu Shenggang <hu...@selectdb.com>
AuthorDate: Wed Oct 29 16:23:58 2025 +0800
fix case error in BE
---
be/src/olap/rowset/segment_v2/column_reader.cpp | 4 ++--
be/src/vec/functions/function.cpp | 3 +++
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 9c68eee2fea..d825423c1ab 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -935,7 +935,7 @@ Result<TColumnAccessPaths> ColumnIterator::_get_sub_access_paths(
Status::InternalError("Invalid access path for struct column: path is empty"));
}
- if (name_path.data_access_path.path[0] != _column_name) {
+ if (!StringCaseEqual()(name_path.data_access_path.path[0], _column_name)) {
return ResultError(Status::InternalError(
R"(Invalid access path for column: expected name "{}", got "{}")", _column_name,
name_path.data_access_path.path[0]));
@@ -1329,7 +1329,7 @@ Status StructFileColumnIterator::set_access_paths(
if (!no_predicate_sub_column) {
for (const auto& paths : sub_predicate_access_paths) {
- if (paths.data_access_path.path[0] == name) {
+ if (!StringCaseEqual()(paths.data_access_path.path[0], name)) {
sub_predicate_access_paths_of_this.emplace_back(paths);
}
}
diff --git a/be/src/vec/functions/function.cpp b/be/src/vec/functions/function.cpp
index 17ce7b846ee..77b2b05612d 100644
--- a/be/src/vec/functions/function.cpp
+++ b/be/src/vec/functions/function.cpp
@@ -279,6 +279,9 @@ DataTypePtr FunctionBuilderImpl::get_return_type(const ColumnsWithTypeAndName& a
create_block_with_nested_columns(Block(arguments), numbers, false);
auto return_type = get_return_type_impl(
ColumnsWithTypeAndName(nested_block.begin(), nested_block.end()));
+ if (!return_type) {
+ return nullptr;
+ }
return make_nullable(return_type);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 14/18: update orc thirdparty lib commit.
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6fdbdbd020d4431998c2253fa4bd67f7556bed05
Author: kakachen <ch...@selectdb.com>
AuthorDate: Fri Oct 31 13:29:02 2025 +0800
update orc thirdparty lib commit.
---
contrib/apache-orc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/contrib/apache-orc b/contrib/apache-orc
index 70b673c7d29..8c0930fd239 160000
--- a/contrib/apache-orc
+++ b/contrib/apache-orc
@@ -1 +1 @@
-Subproject commit 70b673c7d299690ced7c4c600fabaee9e1601198
+Subproject commit 8c0930fd239b2434b53bb1979075ce637e75a467
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 15/18: use selected column ids for parquet merge io ranges.
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4f4b423b6ad3d3c2990a070206254e0fb299f057
Author: kakachen <ch...@selectdb.com>
AuthorDate: Fri Oct 31 16:52:18 2025 +0800
use selected column ids for parquet merge io ranges.
---
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 43 ++++++++++++----------
1 file changed, 23 insertions(+), 20 deletions(-)
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
index e47a0eddaac..9a9b6c302e1 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
@@ -796,27 +796,30 @@ std::vector<io::PrefetchRange> ParquetReader::_generate_random_access_ranges(
size_t total_io_size = 0;
std::function<void(const FieldSchema*, const tparquet::RowGroup&)> scalar_range =
[&](const FieldSchema* field, const tparquet::RowGroup& row_group) {
- if (field->data_type->get_primitive_type() == TYPE_ARRAY) {
- scalar_range(&field->children[0], row_group);
- } else if (field->data_type->get_primitive_type() == TYPE_MAP) {
- scalar_range(&field->children[0], row_group);
- scalar_range(&field->children[1], row_group);
- } else if (field->data_type->get_primitive_type() == TYPE_STRUCT) {
- for (int i = 0; i < field->children.size(); ++i) {
- scalar_range(&field->children[i], row_group);
+ if (_column_ids.empty() ||
+ _column_ids.find(field->get_column_id()) != _column_ids.end()) {
+ if (field->data_type->get_primitive_type() == TYPE_ARRAY) {
+ scalar_range(&field->children[0], row_group);
+ } else if (field->data_type->get_primitive_type() == TYPE_MAP) {
+ scalar_range(&field->children[0], row_group);
+ scalar_range(&field->children[1], row_group);
+ } else if (field->data_type->get_primitive_type() == TYPE_STRUCT) {
+ for (int i = 0; i < field->children.size(); ++i) {
+ scalar_range(&field->children[i], row_group);
+ }
+ } else {
+ const tparquet::ColumnChunk& chunk =
+ row_group.columns[field->physical_column_index];
+ auto& chunk_meta = chunk.meta_data;
+ int64_t chunk_start = has_dict_page(chunk_meta)
+ ? chunk_meta.dictionary_page_offset
+ : chunk_meta.data_page_offset;
+ int64_t chunk_end = chunk_start + chunk_meta.total_compressed_size;
+ DCHECK_GE(chunk_start, last_chunk_end);
+ result.emplace_back(chunk_start, chunk_end);
+ total_io_size += chunk_meta.total_compressed_size;
+ last_chunk_end = chunk_end;
}
- } else {
- const tparquet::ColumnChunk& chunk =
- row_group.columns[field->physical_column_index];
- auto& chunk_meta = chunk.meta_data;
- int64_t chunk_start = has_dict_page(chunk_meta)
- ? chunk_meta.dictionary_page_offset
- : chunk_meta.data_page_offset;
- int64_t chunk_end = chunk_start + chunk_meta.total_compressed_size;
- DCHECK_GE(chunk_start, last_chunk_end);
- result.emplace_back(chunk_start, chunk_end);
- total_io_size += chunk_meta.total_compressed_size;
- last_chunk_end = chunk_end;
}
};
const tparquet::RowGroup& row_group = _t_metadata->row_groups[group.row_group_id];
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 09/18: fix be ut
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit cd95b8eed1d04bdd41f5f98023f560744a9c06e7
Author: Hu Shenggang <hu...@selectdb.com>
AuthorDate: Thu Oct 30 15:52:24 2025 +0800
fix be ut
---
be/src/olap/iterators.h | 3 ---
be/src/olap/rowset/segment_v2/column_reader.cpp | 2 +-
2 files changed, 1 insertion(+), 4 deletions(-)
diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h
index 1b63aad4909..783e057d753 100644
--- a/be/src/olap/iterators.h
+++ b/be/src/olap/iterators.h
@@ -46,9 +46,6 @@ struct IteratorRowRef;
namespace segment_v2 {
struct SubstreamIterator;
}
-
-using TColumnAccessPaths = std::vector<TColumnAccessPath>;
-
class StorageReadOptions {
public:
struct KeyRange {
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index d825423c1ab..0cecd06e60e 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -1329,7 +1329,7 @@ Status StructFileColumnIterator::set_access_paths(
if (!no_predicate_sub_column) {
for (const auto& paths : sub_predicate_access_paths) {
- if (!StringCaseEqual()(paths.data_access_path.path[0], name)) {
+ if (StringCaseEqual()(paths.data_access_path.path[0], name)) {
sub_predicate_access_paths_of_this.emplace_back(paths);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/18: [Feature](catalog) Prune nested columns for hive/iceberg parquet and orc reader.
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1e5c96140550164cfd257053d276f66265a53035
Author: kakachen <ch...@selectdb.com>
AuthorDate: Thu Oct 23 09:54:32 2025 +0800
[Feature](catalog) Prune nested columns for hive/iceberg parquet and orc reader.
---
.gitmodules | 2 +-
be/src/runtime/descriptors.h | 6 +
be/src/vec/exec/format/orc/vorc_reader.cpp | 436 ++++++--
be/src/vec/exec/format/orc/vorc_reader.h | 24 +-
be/src/vec/exec/format/parquet/schema_desc.cpp | 87 +-
be/src/vec/exec/format/parquet/schema_desc.h | 30 +-
.../exec/format/parquet/vparquet_column_reader.cpp | 264 ++++-
.../exec/format/parquet/vparquet_column_reader.h | 105 +-
.../exec/format/parquet/vparquet_group_reader.cpp | 40 +-
.../exec/format/parquet/vparquet_group_reader.h | 8 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 21 +-
be/src/vec/exec/format/parquet/vparquet_reader.h | 5 +-
.../table/hive/hive_orc_nested_column_utils.cpp | 148 +++
.../table/hive/hive_orc_nested_column_utils.h | 43 +
.../hive/hive_parquet_nested_column_utils.cpp | 172 +++
.../table/hive/hive_parquet_nested_column_utils.h | 41 +
be/src/vec/exec/format/table/hive_reader.cpp | 453 +++++++-
be/src/vec/exec/format/table/hive_reader.h | 14 +
.../iceberg/iceberg_orc_nested_column_utils.cpp | 149 +++
.../iceberg/iceberg_orc_nested_column_utils.h | 46 +
.../iceberg_parquet_nested_column_utils.cpp | 173 +++
.../iceberg/iceberg_parquet_nested_column_utils.h | 42 +
be/src/vec/exec/format/table/iceberg_reader.cpp | 253 ++++-
be/src/vec/exec/format/table/iceberg_reader.h | 7 +
.../vec/exec/format/table/table_format_reader.cpp | 21 +-
be/src/vec/exec/format/table/table_format_reader.h | 36 +
be/src/vec/exec/scan/file_scanner.cpp | 6 +-
be/src/vec/exec/scan/file_scanner.h | 2 +-
...4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc | Bin 0 -> 2277 bytes
...4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc | Bin 0 -> 2334 bytes
...2c4c8-e0bd-4db6-b550-6323977446a4.metadata.json | 145 +++
...66011-3129-4bfb-9b4b-c1510c566d73.metadata.json | 176 +++
.../6a876742-85c2-4f84-ae24-292482fa9288-m0.avro | Bin 0 -> 8175 bytes
...212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro | Bin 0 -> 4264 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4068 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4020 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4043 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4082 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4067 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4047 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4077 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4147 bytes
...38ed9-31af-46c8-9274-506c7f2709c9.metadata.json | 145 +++
...4732d-f823-4900-8185-3eb3cb23e4a9.metadata.json | 176 +++
.../51fb6058-7fc2-4791-96a9-f3111933728a-m0.avro | Bin 0 -> 8783 bytes
...552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro | Bin 0 -> 4264 bytes
...a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc | Bin 0 -> 7181 bytes
...20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json | 548 +++++++++
...ddc20-6aa8-4864-9d15-546dd5a6bae4.metadata.json | 579 ++++++++++
.../d9aed717-61ef-422b-b044-a16ca31f9e86-m0.avro | 1 +
...228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro | Bin 0 -> 4261 bytes
...35f-d581-40e4-8d56-a929aded99e4-0-00001.parquet | Bin 0 -> 21586 bytes
...ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json | 548 +++++++++
...56265-e89e-4c9f-a0fb-7f3075d4e8f9.metadata.json | 579 ++++++++++
.../bee22dff-a1d3-420d-bfba-c1b4193ab0df-m0.avro | 5 +
...002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro | Bin 0 -> 4262 bytes
.../test_data/nested_user_profiles_orc/_SUCCESS | 0
...f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc | Bin 0 -> 8396 bytes
.../nested_user_profiles_parquet/_SUCCESS | 0
...1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet | Bin 0 -> 21157 bytes
.../exec/format/parquet/parquet_thrift_test.cpp | 12 +-
.../hive/hive_reader_create_column_ids_test.cpp | 1179 ++++++++++++++++++++
.../exec/format/table/hive/hive_reader_test.cpp | 743 ++++++++++++
.../iceberg_reader_create_column_ids_test.cpp | 1177 +++++++++++++++++++
.../format/table/iceberg/iceberg_reader_test.cpp | 739 ++++++++++++
.../org/apache/doris/datasource/FileScanNode.java | 1 -
.../trees/plans/logical/LogicalFileScan.java | 19 +-
.../trees/plans/logical/LogicalHudiScan.java | 2 +-
68 files changed, 9229 insertions(+), 179 deletions(-)
diff --git a/.gitmodules b/.gitmodules
index 54c1a8a3636..63e4b14f70d 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -20,7 +20,7 @@
[submodule "contrib/apache-orc"]
path = contrib/apache-orc
url = https://github.com/apache/doris-thirdparty.git
- branch = orc
+ branch = cq_nested_column_prune_external_table
[submodule "doris-faiss"]
path = contrib/faiss
url = https://github.com/apache/doris-thirdparty.git
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index c66ccd80b1d..b34e54622b5 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -100,6 +100,10 @@ public:
return virtual_column_expr;
}
+ void set_is_predicate(bool is_predicate) { _is_predicate = is_predicate; }
+
+ bool is_predicate() const { return _is_predicate; }
+
private:
friend class DescriptorTbl;
friend class TupleDescriptor;
@@ -141,6 +145,8 @@ private:
std::shared_ptr<doris::TExpr> virtual_column_expr = nullptr;
+ bool _is_predicate = false;
+
SlotDescriptor(const TSlotDescriptor& tdesc);
SlotDescriptor(const PSlotDescriptor& pdesc);
MOCK_DEFINE(SlotDescriptor();)
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp
index bd996f2189b..cfbad191772 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -26,6 +26,7 @@
#include <algorithm>
#include <cctype>
+#include <list>
#include "vec/exprs/vdirect_in_predicate.h"
#include "vec/exprs/vexpr.h"
@@ -355,8 +356,11 @@ Status OrcReader::init_reader(
const RowDescriptor* row_descriptor,
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts,
- std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr) {
+ std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr,
+ const std::set<uint64_t>& column_ids, const std::set<uint64_t>& filter_column_ids) {
_table_column_names = column_names;
+ _column_ids = column_ids;
+ _filter_column_ids = filter_column_ids;
_colname_to_value_range = colname_to_value_range;
_lazy_read_ctx.conjuncts = conjuncts;
_is_acid = is_acid;
@@ -401,6 +405,32 @@ Status OrcReader::get_parsed_schema(std::vector<std::string>* col_names,
Status OrcReader::_init_read_columns() {
SCOPED_RAW_TIMER(&_statistics.init_column_time);
const auto& root_type = _reader->getType();
+
+ // Save file root type for later use in type conversion
+ _file_root_type = &root_type;
+
+ // Build column ID to file type mapping for all columns in file
+ _column_id_to_file_type.clear();
+ std::function<void(const orc::Type*, int)> build_column_id_map = [&](const orc::Type* type,
+ int depth) {
+ if (type == nullptr) return;
+ uint64_t col_id = type->getColumnId();
+ _column_id_to_file_type[col_id] = type;
+
+ std::string indent(depth * 2, ' ');
+ VLOG_DEBUG << indent << "[OrcReader] Mapping column_id=" << col_id
+ << ", kind=" << static_cast<int>(type->getKind())
+ << ", subtype_count=" << type->getSubtypeCount();
+
+ for (uint64_t i = 0; i < type->getSubtypeCount(); ++i) {
+ build_column_id_map(type->getSubtype(i), depth + 1);
+ }
+ };
+
+ VLOG_DEBUG << "[OrcReader] Building column ID to file type mapping...";
+ build_column_id_map(&root_type, 0);
+ VLOG_DEBUG << "[OrcReader] Total mapped columns: " << _column_id_to_file_type.size();
+
if (_is_acid) {
for (uint64_t i = 0; i < root_type.getSubtypeCount(); ++i) {
if (root_type.getSubtype(i)->getKind() == orc::TypeKind::STRUCT) {
@@ -650,7 +680,7 @@ std::pair<bool, orc::Literal> OrcReader::_make_orc_literal(const VSlotRef* slot_
auto* slot = _tuple_descriptor->slots()[slot_ref->column_id()];
auto slot_type = slot->type();
auto primitive_type = slot_type->get_primitive_type();
- auto src_type = OrcReader::convert_to_doris_type(orc_type)->get_primitive_type();
+ auto src_type = convert_to_doris_type(orc_type)->get_primitive_type();
// should not down predicate for string type change from other type
if (src_type != primitive_type && !is_string_type(src_type) && is_string_type(primitive_type)) {
LOG(WARNING) << "Unsupported Push Down Schema Changed Column " << primitive_type << " to "
@@ -1158,7 +1188,12 @@ Status OrcReader::set_fill_columns(
try {
_row_reader_options.range(_range_start_offset, _range_size);
_row_reader_options.setTimezoneName(_ctz == "CST" ? "Asia/Shanghai" : _ctz);
- _row_reader_options.include(_read_file_cols);
+ if (!_column_ids.empty()) {
+ std::list<uint64_t> column_ids_list(_column_ids.begin(), _column_ids.end());
+ _row_reader_options.includeTypes(column_ids_list);
+ } else {
+ _row_reader_options.include(_read_file_cols);
+ }
_row_reader_options.setEnableLazyDecoding(true);
//orc reader should not use the tiny stripe optimization when reading by row id.
@@ -1246,6 +1281,57 @@ Status OrcReader::set_fill_columns(
}
}
+ // Helper function to recursively print ORC type structure
+ std::function<void(const orc::Type*, int, const std::string&)> print_orc_type_tree =
+ [&](const orc::Type* type, int indent, const std::string& name) {
+ if (type) {
+ std::string indent_str(indent * 2, ' ');
+ VLOG_DEBUG << indent_str << "- " << name
+ << " (type_id=" << type->getColumnId()
+ << ", kind=" << static_cast<int>(type->getKind()) << ")";
+ }
+
+ if (type->getKind() == orc::TypeKind::STRUCT) {
+ for (uint64_t i = 0; i < type->getSubtypeCount(); ++i) {
+ std::string field_name = type->getFieldName(i);
+ print_orc_type_tree(type->getSubtype(i), indent + 1, field_name);
+ }
+ } else if (type->getKind() == orc::TypeKind::LIST) {
+ print_orc_type_tree(type->getSubtype(0), indent + 1, "element");
+ } else if (type->getKind() == orc::TypeKind::MAP) {
+ print_orc_type_tree(type->getSubtype(0), indent + 1, "key");
+ print_orc_type_tree(type->getSubtype(1), indent + 1, "value");
+ } else {
+ // Primitive type, no children
+ }
+ };
+
+ // VLOG_DEBUG << "\n[OrcReader] ========== ORC Type Structure ==========";
+ // print_orc_type_tree(&selected_type, 0, "root");
+ // VLOG_DEBUG << "[OrcReader] ==========================================\n";
+
+ _type_map.clear();
+ if (_is_acid) {
+ for (uint64_t i = 0; i < selected_type.getSubtypeCount(); ++i) {
+ if (selected_type.getSubtype(i)->getKind() == orc::TypeKind::STRUCT) {
+ auto row_orc_type = selected_type.getSubtype(i);
+ for (uint64_t j = 0; j < row_orc_type->getSubtypeCount(); j++) {
+ std::string field_name =
+ TransactionalHive::ROW + "." + row_orc_type->getFieldName(j);
+ _type_map.emplace(field_name, row_orc_type->getSubtype(j));
+ }
+ } else {
+ std::string field_name = selected_type.getFieldName(i);
+ _type_map.emplace(field_name, selected_type.getSubtype(i));
+ }
+ }
+ } else {
+ for (int i = 0; i < selected_type.getSubtypeCount(); ++i) {
+ std::string field_name = selected_type.getFieldName(i);
+ _type_map.emplace(field_name, selected_type.getSubtype(i));
+ }
+ }
+
_remaining_rows = _row_reader->getNumberOfRows();
} catch (std::exception& e) {
@@ -1343,6 +1429,11 @@ Status OrcReader::_fill_missing_columns(
block->erase(result_column_id);
}
}
+
+ // // no default column, fill with null
+ // auto mutable_column = block->get_by_name(kv.first).column->assume_mutable();
+ // auto* nullable_column = static_cast<vectorized::ColumnNullable*>(mutable_column.get());
+ // nullable_column->insert_many_defaults(rows);
}
return Status::OK();
}
@@ -1391,6 +1482,14 @@ void OrcReader::_init_file_description() {
}
DataTypePtr OrcReader::convert_to_doris_type(const orc::Type* orc_type) {
+ // Critical: check for nullptr BEFORE accessing any methods
+ if (orc_type == nullptr) {
+ LOG(WARNING) << "[OrcReader] ERROR: convert_to_doris_type called with nullptr orc_type! "
+ "Falling back to STRING";
+ // Return a safe default type instead of crashing
+ return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_STRING, true);
+ }
+
switch (orc_type->getKind()) {
case orc::TypeKind::BOOLEAN:
return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_BOOLEAN, true);
@@ -1437,6 +1536,74 @@ DataTypePtr OrcReader::convert_to_doris_type(const orc::Type* orc_type) {
std::make_shared<DataTypeArray>(convert_to_doris_type(orc_type->getSubtype(0))));
}
case orc::TypeKind::MAP: {
+ // Handle incomplete MAP type due to column pruning
+ // If MAP doesn't have both key and value subtypes, try to find the complete type from file
+ if (orc_type->getSubtypeCount() < 2 || orc_type->getSubtype(0) == nullptr ||
+ orc_type->getSubtype(1) == nullptr) {
+ // Try to find the complete type from _column_id_to_file_type
+ uint64_t column_id = orc_type->getColumnId();
+ VLOG(1) << "[OrcReader] Detected incomplete MAP type: column_id=" << column_id
+ << ", subtype_count=" << orc_type->getSubtypeCount() << ", subtype(0)="
+ << (orc_type->getSubtypeCount() > 0 && orc_type->getSubtype(0) != nullptr
+ ? "not null"
+ : "null")
+ << ", subtype(1)="
+ << (orc_type->getSubtypeCount() > 1 && orc_type->getSubtype(1) != nullptr
+ ? "not null"
+ : "null");
+
+ auto it = _column_id_to_file_type.find(column_id);
+ if (it != _column_id_to_file_type.end() && it->second != nullptr) {
+ const orc::Type* complete_type = it->second;
+ VLOG(1) << "[OrcReader] Found complete type in mapping: column_id=" << column_id
+ << ", complete_type_kind=" << static_cast<int>(complete_type->getKind())
+ << ", complete_subtype_count=" << complete_type->getSubtypeCount();
+
+ // Print subtypes information
+ for (uint64_t i = 0; i < complete_type->getSubtypeCount(); ++i) {
+ const orc::Type* subtype = complete_type->getSubtype(i);
+ VLOG_DEBUG << "[OrcReader] complete_type->subtype[" << i << "]: "
+ << (subtype != nullptr
+ ? ("kind=" +
+ std::to_string(static_cast<int>(subtype->getKind())) +
+ ", column_id=" +
+ std::to_string(subtype->getColumnId()))
+ : "nullptr");
+ }
+
+ if (complete_type->getKind() == orc::TypeKind::MAP &&
+ complete_type->getSubtypeCount() == 2) {
+ VLOG(1) << "[OrcReader] Using complete MAP type from file schema for column_id="
+ << column_id;
+
+ // Get subtypes with extra validation
+ const orc::Type* key_type = complete_type->getSubtype(0);
+ const orc::Type* value_type = complete_type->getSubtype(1);
+
+ VLOG_DEBUG << "[OrcReader] About to convert key_type: "
+ << (key_type != nullptr ? "not null" : "NULL");
+ VLOG_DEBUG << "[OrcReader] About to convert value_type: "
+ << (value_type != nullptr ? "not null" : "NULL");
+
+ // Use the complete type from file - with null checks
+ DataTypePtr key_doris_type = convert_to_doris_type(key_type);
+ VLOG_DEBUG << "[OrcReader] Successfully converted key_type";
+
+ DataTypePtr value_doris_type = convert_to_doris_type(value_type);
+ VLOG_DEBUG << "[OrcReader] Successfully converted value_type";
+
+ return make_nullable(
+ std::make_shared<DataTypeMap>(key_doris_type, value_doris_type));
+ } else {
+ LOG(WARNING) << "[OrcReader] Warning: Complete type is not a valid MAP or has "
+ "wrong subtype count";
+ }
+ } else {
+ LOG(WARNING) << "[OrcReader] Warning: Could not find complete type in mapping for "
+ "column_id="
+ << column_id << ", mapping_size=" << _column_id_to_file_type.size();
+ }
+ }
return make_nullable(
std::make_shared<DataTypeMap>(convert_to_doris_type(orc_type->getSubtype(0)),
convert_to_doris_type(orc_type->getSubtype(1))));
@@ -1682,6 +1849,7 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
const orc::Type* orc_column_type,
const orc::ColumnVectorBatch* cvb, size_t num_values) {
auto logical_type = data_type->get_primitive_type();
+
switch (logical_type) {
#define DISPATCH(FlatType, CppType, OrcColumnType) \
case FlatType: \
@@ -1752,21 +1920,99 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
const DataTypePtr& doris_value_type =
reinterpret_cast<const DataTypeMap*>(remove_nullable(data_type).get())
->get_value_type();
+
+ // Get ORC key and value types with null checks
const orc::Type* orc_key_type = orc_column_type->getSubtype(0);
const orc::Type* orc_value_type = orc_column_type->getSubtype(1);
+
+ VLOG(1) << "[OrcReader] MAP column '" << col_name
+ << "': orc_key_type=" << (orc_key_type != nullptr ? "not null" : "NULL")
+ << ", orc_value_type=" << (orc_value_type != nullptr ? "not null" : "NULL")
+ << ", element_size=" << element_size;
+
+ // Handle incomplete MAP type - if key or value type is nullptr, try to recover from mapping
+ bool key_is_missing = (orc_key_type == nullptr);
+ bool value_is_missing = (orc_value_type == nullptr);
+
+ if (key_is_missing || value_is_missing) {
+ VLOG(1) << "[OrcReader] Detected incomplete MAP subtypes for column '" << col_name
+ << "', attempting to recover from mapping...";
+
+ uint64_t column_id = orc_column_type->getColumnId();
+ auto it = _column_id_to_file_type.find(column_id);
+ if (it != _column_id_to_file_type.end() && it->second != nullptr) {
+ const orc::Type* complete_map_type = it->second;
+ if (complete_map_type->getKind() == orc::TypeKind::MAP &&
+ complete_map_type->getSubtypeCount() == 2) {
+ if (key_is_missing) {
+ orc_key_type = complete_map_type->getSubtype(0);
+ if (orc_key_type != nullptr) {
+ // key_is_missing = false;
+ VLOG(1) << "[OrcReader] Recovered key type from mapping for column '"
+ << col_name << "'";
+ }
+ }
+ if (value_is_missing) {
+ orc_value_type = complete_map_type->getSubtype(1);
+ if (orc_value_type != nullptr) {
+ // value_is_missing = false;
+ VLOG(1) << "[OrcReader] Recovered value type from mapping for column '"
+ << col_name << "'";
+ }
+ }
+ }
+ }
+ }
+
ColumnPtr& doris_key_column = doris_map.get_keys_ptr();
ColumnPtr& doris_value_column = doris_map.get_values_ptr();
std::string key_col_name = col_name + ".key";
std::string value_col_name = col_name + ".value";
- RETURN_IF_ERROR(_orc_column_to_doris_column<false>(
- key_col_name, doris_key_column, doris_key_type, root_node->get_key_node(),
+ // Handle key column: if still missing, fill with default values
+ if (key_is_missing) {
+ LOG(INFO) << "[OrcReader] Key type is missing for MAP column '" << col_name
+ << "', filling with default values (element_size=" << element_size << ")";
+
+ // Fill key column with default values (nulls or empty values)
+ auto mutable_key_column = doris_key_column->assume_mutable();
+ if (mutable_key_column->is_nullable()) {
+ auto* nullable_column = static_cast<ColumnNullable*>(mutable_key_column.get());
+ nullable_column->insert_many_defaults(element_size);
+ } else {
+ mutable_key_column->insert_many_defaults(element_size);
+ }
+ } else {
+ // Normal processing: convert ORC column to Doris column
+ auto status = _orc_column_to_doris_column<false>(
+ key_col_name, doris_key_column, doris_key_type, root_node->get_key_node(),
+ orc_key_type, orc_map->keys.get(), element_size);
+ if (!status.ok()) {
+ return status;
+ }
+ }
+
+ // Handle value column: if still missing, fill with default values
+ if (value_is_missing) {
+ LOG(INFO) << "[OrcReader] Value type is missing for MAP column '" << col_name
+ << "', filling with default values (element_size=" << element_size << ")";
- orc_key_type, orc_map->keys.get(), element_size));
- RETURN_IF_ERROR(_orc_column_to_doris_column<false>(
- value_col_name, doris_value_column, doris_value_type, root_node->get_value_node(),
- orc_value_type, orc_map->elements.get(), element_size));
- return doris_map.deduplicate_keys();
+ // Fill value column with default values (nulls or empty values)
+ auto mutable_value_column = doris_value_column->assume_mutable();
+ if (mutable_value_column->is_nullable()) {
+ auto* nullable_column = static_cast<ColumnNullable*>(mutable_value_column.get());
+ nullable_column->insert_many_defaults(element_size);
+ } else {
+ mutable_value_column->insert_many_defaults(element_size);
+ }
+ return Status::OK();
+ } else {
+ // Normal processing: convert ORC column to Doris column
+ return _orc_column_to_doris_column<false>(value_col_name, doris_value_column,
+ doris_value_type, root_node->get_value_node(),
+ orc_value_type, orc_map->elements.get(),
+ element_size);
+ }
}
case PrimitiveType::TYPE_STRUCT: {
if (orc_column_type->getKind() != orc::TypeKind::STRUCT) {
@@ -1781,6 +2027,14 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
const auto* doris_struct_type =
assert_cast<const DataTypeStruct*>(remove_nullable(data_type).get());
+ // Build ORC field name to index map for faster lookup
+ std::unordered_map<std::string, int> orc_field_name_to_idx;
+ for (int j = 0; j < orc_column_type->getSubtypeCount(); ++j) {
+ std::string field_name = orc_column_type->getFieldName(j);
+ std::transform(field_name.begin(), field_name.end(), field_name.begin(), ::tolower);
+ orc_field_name_to_idx[field_name] = j;
+ }
+
for (int i = 0; i < doris_struct.tuple_size(); ++i) {
const auto& table_column_name = doris_struct_type->get_name_by_position(i);
if (!root_node->children_column_exists(table_column_name)) {
@@ -1788,10 +2042,23 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
continue;
}
const auto& file_column_name = root_node->children_file_column_name(table_column_name);
- for (int j = 0; j < orc_column_type->getSubtypeCount(); ++j) {
- if (boost::iequals(orc_column_type->getFieldName(j), file_column_name)) {
- read_fields[i] = j;
- }
+ std::string file_column_name_lower = file_column_name;
+ std::transform(file_column_name_lower.begin(), file_column_name_lower.end(),
+ file_column_name_lower.begin(), ::tolower);
+
+ auto it = orc_field_name_to_idx.find(file_column_name_lower);
+ if (it != orc_field_name_to_idx.end()) {
+ read_fields[i] = it->second;
+ VLOG_DEBUG << "[OrcReader] Found field mapping: doris_field[" << i
+ << "] -> orc_field[" << it->second
+ << "], table_column: " << table_column_name
+ << ", file_column: " << file_column_name_lower;
+ } else {
+ missing_fields.insert(i);
+ VLOG_DEBUG << "[OrcReader] Missing field: doris_field[" << i
+ << "], table_column: " << table_column_name
+ << ", file_column: " << file_column_name_lower
+ << " (not found in ORC file)";
}
}
@@ -1808,6 +2075,7 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
for (auto read_field : read_fields) {
orc::ColumnVectorBatch* orc_field = orc_struct->fields[read_field.second];
+
const orc::Type* orc_type = orc_column_type->getSubtype(read_field.second);
std::string field_name =
col_name + "." + orc_column_type->getFieldName(read_field.second);
@@ -1832,67 +2100,81 @@ Status OrcReader::_orc_column_to_doris_column(
const std::string& col_name, ColumnPtr& doris_column, const DataTypePtr& data_type,
std::shared_ptr<TableSchemaChangeHelper::Node> root_node, const orc::Type* orc_column_type,
const orc::ColumnVectorBatch* cvb, size_t num_values) {
- auto src_type = convert_to_doris_type(orc_column_type);
- bool is_dict_filter_col = false;
- for (const std::pair<std::string, int>& dict_col : _dict_filter_cols) {
- if (col_name == dict_col.first) {
- src_type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true);
- is_dict_filter_col = true;
- break;
- }
- }
- // If the column can be dictionary filtered, there will be two types.
- // It may be plain or a dictionary, because the same field in different stripes may have different types.
- // Here we use the $dict_ prefix to represent the dictionary type converter.
- auto converter_key = !is_dict_filter_col ? col_name : fmt::format("$dict_{}", col_name);
-
- if (!_converters.contains(converter_key)) {
- std::unique_ptr<converter::ColumnTypeConverter> converter =
- converter::ColumnTypeConverter::get_converter(src_type, data_type,
- converter::FileFormat::ORC);
- if (!converter->support()) {
- return Status::InternalError(
- "The column type of '{}' has changed and is not supported: ", col_name,
- converter->get_error_msg());
- }
- // reuse the cached converter
- _converters[converter_key] = std::move(converter);
- }
- converter::ColumnTypeConverter* converter = _converters[converter_key].get();
- ColumnPtr resolved_column = converter->get_column(src_type, doris_column, data_type);
- const DataTypePtr& resolved_type = converter->get_type();
-
+ DataTypePtr resolved_type;
+ ColumnPtr resolved_column;
+ converter::ColumnTypeConverter* converter_ptr = nullptr;
MutableColumnPtr data_column;
- if (resolved_column->is_nullable()) {
- SCOPED_RAW_TIMER(&_statistics.decode_null_map_time);
- auto* nullable_column =
- reinterpret_cast<ColumnNullable*>(resolved_column->assume_mutable().get());
- data_column = nullable_column->get_nested_column_ptr();
- NullMap& map_data_column = nullable_column->get_null_map_data();
- auto origin_size = map_data_column.size();
- map_data_column.resize(origin_size + num_values);
- if (cvb->hasNulls) {
- const auto* cvb_nulls = cvb->notNull.data();
- for (int i = 0; i < num_values; ++i) {
- map_data_column[origin_size + i] = !cvb_nulls[i];
+ if (orc_column_type != nullptr) {
+ auto src_type = convert_to_doris_type(orc_column_type);
+ bool is_dict_filter_col = false;
+ for (const std::pair<std::string, int>& dict_col : _dict_filter_cols) {
+ if (col_name == dict_col.first) {
+ src_type =
+ DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true);
+ is_dict_filter_col = true;
+ break;
+ }
+ }
+ // If the column can be dictionary filtered, there will be two types.
+ // It may be plain or a dictionary, because the same field in different stripes may have different types.
+ // Here we use the $dict_ prefix to represent the dictionary type converter.
+ auto converter_key = !is_dict_filter_col ? col_name : fmt::format("$dict_{}", col_name);
+
+ if (!_converters.contains(converter_key)) {
+ std::unique_ptr<converter::ColumnTypeConverter> converter =
+ converter::ColumnTypeConverter::get_converter(src_type, data_type,
+ converter::FileFormat::ORC);
+ if (!converter->support()) {
+ return Status::InternalError(
+ "The column type of '{}' has changed and is not supported: ", col_name,
+ converter->get_error_msg());
+ }
+ // reuse the cached converter
+ _converters[converter_key] = std::move(converter);
+ }
+ converter_ptr = _converters[converter_key].get();
+ resolved_column = converter_ptr->get_column(src_type, doris_column, data_type);
+ resolved_type = converter_ptr->get_type();
+
+ if (resolved_column->is_nullable()) {
+ SCOPED_RAW_TIMER(&_statistics.decode_null_map_time);
+ auto* nullable_column =
+ reinterpret_cast<ColumnNullable*>(resolved_column->assume_mutable().get());
+ data_column = nullable_column->get_nested_column_ptr();
+
+ NullMap& map_data_column = nullable_column->get_null_map_data();
+ auto origin_size = map_data_column.size();
+ map_data_column.resize(origin_size + num_values);
+ if (cvb->hasNulls) {
+ const auto* cvb_nulls = cvb->notNull.data();
+ for (int i = 0; i < num_values; ++i) {
+ map_data_column[origin_size + i] = !cvb_nulls[i];
+ }
+ } else {
+ memset(map_data_column.data() + origin_size, 0, num_values);
}
} else {
- memset(map_data_column.data() + origin_size, 0, num_values);
+ if (cvb->hasNulls) {
+ return Status::InternalError("Not nullable column {} has null values in orc file",
+ col_name);
+ }
+ data_column = resolved_column->assume_mutable();
}
+
+ RETURN_IF_ERROR((_fill_doris_data_column<is_filter>(
+ col_name, data_column, remove_nullable(resolved_type), root_node, orc_column_type,
+ cvb, num_values)));
+ // if (converter_ptr != nullptr) {
+ // resolve schema change
+ auto converted_column = doris_column->assume_mutable();
+ return converter_ptr->convert(resolved_column, converted_column);
+ // }
} else {
- if (cvb->hasNulls) {
- return Status::InternalError("Not nullable column {} has null values in orc file",
- col_name);
- }
- data_column = resolved_column->assume_mutable();
+ reinterpret_cast<ColumnNullable*>(doris_column->assume_mutable().get())
+ ->insert_many_defaults(num_values);
}
- RETURN_IF_ERROR(_fill_doris_data_column<is_filter>(col_name, data_column,
- remove_nullable(resolved_type), root_node,
- orc_column_type, cvb, num_values));
- // resolve schema change
- auto converted_column = doris_column->assume_mutable();
- return converter->convert(resolved_column, converted_column);
+ return Status::OK();
}
std::string OrcReader::get_field_name_lower_case(const orc::Type* orc_type, int pos) {
@@ -1993,10 +2275,10 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
if (orc_col_idx == _colname_to_idx.end()) {
return Status::InternalError("Wrong read column '{}' in orc file", col_name);
}
- RETURN_IF_ERROR(_orc_column_to_doris_column<true>(
+ RETURN_IF_ERROR((_orc_column_to_doris_column<true>(
col_name, column_ptr, column_type,
_table_info_node_ptr->get_children_node(col_name), _type_map[file_column_name],
- batch_vec[orc_col_idx->second], _batch->numElements));
+ batch_vec[orc_col_idx->second], _batch->numElements)));
}
RETURN_IF_ERROR(_fill_partition_columns(block, _batch->numElements,
@@ -2051,6 +2333,12 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
}
}
+ //std::cout << "[OrcReader] _column_ids: ";
+ //for (const auto& cid : _column_ids) {
+ // std::cout << cid << " ";
+ //}
+ //std::cout << std::endl;
+
if (!_dict_cols_has_converted && !_dict_filter_cols.empty()) {
for (auto& dict_filter_cols : _dict_filter_cols) {
MutableColumnPtr dict_col_ptr = ColumnInt32::create();
@@ -2084,10 +2372,10 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
if (orc_col_idx == _colname_to_idx.end()) {
return Status::InternalError("Wrong read column '{}' in orc file", col_name);
}
- RETURN_IF_ERROR(_orc_column_to_doris_column(
+ RETURN_IF_ERROR((_orc_column_to_doris_column<false>(
col_name, column_ptr, column_type,
_table_info_node_ptr->get_children_node(col_name), _type_map[file_column_name],
- batch_vec[orc_col_idx->second], _batch->numElements));
+ batch_vec[orc_col_idx->second], _batch->numElements)));
}
RETURN_IF_ERROR(_fill_partition_columns(block, _batch->numElements,
@@ -2250,10 +2538,10 @@ Status OrcReader::filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t s
if (orc_col_idx == _colname_to_idx.end()) {
return Status::InternalError("Wrong read column '{}' in orc file", table_col_name);
}
- RETURN_IF_ERROR(_orc_column_to_doris_column(
+ RETURN_IF_ERROR((_orc_column_to_doris_column<false>(
table_col_name, column_ptr, column_type,
_table_info_node_ptr->get_children_node(table_col_name),
- _type_map[file_column_name], batch_vec[orc_col_idx->second], data.numElements));
+ _type_map[file_column_name], batch_vec[orc_col_idx->second], data.numElements)));
}
RETURN_IF_ERROR(
_fill_partition_columns(block, size, _lazy_read_ctx.predicate_partition_columns));
diff --git a/be/src/vec/exec/format/orc/vorc_reader.h b/be/src/vec/exec/format/orc/vorc_reader.h
index 358ba1d2468..341e40a9ce2 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.h
+++ b/be/src/vec/exec/format/orc/vorc_reader.h
@@ -112,6 +112,12 @@ struct LazyReadContext {
std::unordered_map<std::string, VExprContextSPtr> predicate_missing_columns;
// lazy read missing columns or all missing columns
std::unordered_map<std::string, VExprContextSPtr> missing_columns;
+
+ std::vector<std::string> partial_predicate_columns;
+
+ // Record the number of rows filled in filter phase for lazy materialization
+ // This is used to check if a column was already processed in filter phase
+ size_t filter_phase_rows = 0;
};
class OrcReader : public GenericReader {
@@ -158,7 +164,9 @@ public:
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts,
std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr =
- TableSchemaChangeHelper::ConstNode::get_instance());
+ TableSchemaChangeHelper::ConstNode::get_instance(),
+ const std::set<uint64_t>& column_ids = {},
+ const std::set<uint64_t>& filter_column_ids = {});
Status set_fill_columns(
const std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>&
@@ -195,7 +203,8 @@ public:
std::unordered_map<std::string, orc::StringDictionary*>& column_name_to_dict_map,
bool* is_stripe_filtered);
- static DataTypePtr convert_to_doris_type(const orc::Type* orc_type);
+ DataTypePtr convert_to_doris_type(const orc::Type* orc_type);
+
static std::string get_field_name_lower_case(const orc::Type* orc_type, int pos);
void set_row_id_column_iterator(
@@ -630,7 +639,8 @@ private:
size_t _batch_size;
int64_t _range_start_offset;
int64_t _range_size;
- const std::string& _ctz;
+ // const std::string& _ctz;
+ std::string _ctz;
int32_t _offset_days = 0;
cctz::time_zone _time_zone;
@@ -653,6 +663,12 @@ private:
// file column name to orc type
std::unordered_map<std::string, const orc::Type*> _type_map;
+ // Optimized type conversion support
+ // column ID to file original type mapping for handling partial column selection
+ std::unordered_map<uint64_t, const orc::Type*> _column_id_to_file_type;
+ // Keep reference to file root type for complete type information
+ const orc::Type* _file_root_type = nullptr;
+
std::unique_ptr<ORCFileInputStream> _file_input_stream;
Statistics _statistics;
OrcProfile _orc_profile;
@@ -673,6 +689,8 @@ private:
std::vector<DecimalScaleParams> _decimal_scale_params;
size_t _decimal_scale_params_index;
+ std::set<uint64_t> _column_ids;
+ std::set<uint64_t> _filter_column_ids;
const std::unordered_map<std::string, ColumnValueRangeType>* _colname_to_value_range = nullptr;
bool _is_acid = false;
std::unique_ptr<IColumn::Filter> _filter;
diff --git a/be/src/vec/exec/format/parquet/schema_desc.cpp b/be/src/vec/exec/format/parquet/schema_desc.cpp
index 82e6e0c9d61..c022f1e3cfe 100644
--- a/be/src/vec/exec/format/parquet/schema_desc.cpp
+++ b/be/src/vec/exec/format/parquet/schema_desc.cpp
@@ -176,6 +176,7 @@ Status FieldDescriptor::parse_node_field(const std::vector<tparquet::SchemaEleme
parse_physical_field(t_schema, false, child);
node_field->name = t_schema.name;
+ node_field->lower_case_name = to_lower(t_schema.name);
node_field->data_type = std::make_shared<DataTypeArray>(make_nullable(child->data_type));
_next_schema_pos = curr_pos + 1;
node_field->field_id = t_schema.__isset.field_id ? t_schema.field_id : -1;
@@ -193,8 +194,10 @@ Status FieldDescriptor::parse_node_field(const std::vector<tparquet::SchemaEleme
void FieldDescriptor::parse_physical_field(const tparquet::SchemaElement& physical_schema,
bool is_nullable, FieldSchema* physical_field) {
physical_field->name = physical_schema.name;
+ physical_field->lower_case_name = to_lower(physical_field->name);
physical_field->parquet_schema = physical_schema;
physical_field->physical_type = physical_schema.type;
+ physical_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
_physical_fields.push_back(physical_field);
physical_field->physical_column_index = cast_set<int>(_physical_fields.size() - 1);
auto type = get_doris_type(physical_schema, is_nullable);
@@ -393,6 +396,8 @@ Status FieldDescriptor::parse_group_field(const std::vector<tparquet::SchemaElem
RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, struct_field));
group_field->name = group_schema.name;
+ group_field->lower_case_name = to_lower(group_field->name);
+ group_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
group_field->data_type =
std::make_shared<DataTypeArray>(make_nullable(struct_field->data_type));
group_field->field_id = group_schema.__isset.field_id ? group_schema.field_id : -1;
@@ -461,6 +466,8 @@ Status FieldDescriptor::parse_list_field(const std::vector<tparquet::SchemaEleme
}
list_field->name = first_level.name;
+ list_field->lower_case_name = to_lower(first_level.name);
+ list_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
list_field->data_type =
std::make_shared<DataTypeArray>(make_nullable(list_field->children[0].data_type));
if (is_optional) {
@@ -519,21 +526,24 @@ Status FieldDescriptor::parse_map_field(const std::vector<tparquet::SchemaElemen
map_field->repetition_level++;
map_field->definition_level++;
- map_field->children.resize(1);
+ // Directly create key and value children instead of intermediate key_value node
+ map_field->children.resize(2);
// map is a repeated node, we should set the `repeated_parent_def_level` of its children as `definition_level`
set_child_node_level(map_field, map_field->definition_level);
- auto map_kv_field = &map_field->children[0];
- // produce MAP<STRUCT<KEY, VALUE>>
- RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, map_kv_field));
+
+ auto key_field = &map_field->children[0];
+ auto value_field = &map_field->children[1];
+
+ // Parse key and value fields directly from the key_value group's children
+ _next_schema_pos = curr_pos + 2; // Skip key_value group, go directly to key
+ RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, key_field));
+ RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, value_field));
map_field->name = map_schema.name;
- map_field->data_type = std::make_shared<DataTypeMap>(
- make_nullable(assert_cast<const DataTypeStruct*>(
- remove_nullable(map_kv_field->data_type).get())
- ->get_element(0)),
- make_nullable(assert_cast<const DataTypeStruct*>(
- remove_nullable(map_kv_field->data_type).get())
- ->get_element(1)));
+ map_field->lower_case_name = to_lower(map_field->name);
+ map_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
+ map_field->data_type = std::make_shared<DataTypeMap>(make_nullable(key_field->data_type),
+ make_nullable(value_field->data_type));
if (is_optional) {
map_field->data_type = make_nullable(map_field->data_type);
}
@@ -558,6 +568,8 @@ Status FieldDescriptor::parse_struct_field(const std::vector<tparquet::SchemaEle
RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, &struct_field->children[i]));
}
struct_field->name = struct_schema.name;
+ struct_field->lower_case_name = to_lower(struct_field->name);
+ struct_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
struct_field->field_id = struct_schema.__isset.field_id ? struct_schema.field_id : -1;
DataTypes res_data_types;
@@ -610,6 +622,59 @@ std::string FieldDescriptor::debug_string() const {
ss << "]";
return ss.str();
}
+
+void FieldDescriptor::assign_ids() {
+ uint64_t next_id = 1;
+ for (auto& field : _fields) {
+ field.assign_ids(next_id);
+ }
+}
+
+const FieldSchema* FieldDescriptor::find_column_by_id(uint64_t column_id) const {
+ for (const auto& field : _fields) {
+ if (auto result = field.find_column_by_id(column_id)) {
+ return result;
+ }
+ }
+ return nullptr;
+}
+
+void FieldSchema::assign_ids(uint64_t& next_id) {
+ column_id = next_id++;
+
+ for (auto& child : children) {
+ child.assign_ids(next_id);
+ }
+
+ max_column_id = next_id - 1;
+}
+
+const FieldSchema* FieldSchema::find_column_by_id(uint64_t target_id) const {
+ if (column_id == target_id) {
+ return this;
+ }
+
+ for (const auto& child : children) {
+ if (auto result = child.find_column_by_id(target_id)) {
+ return result;
+ }
+ }
+
+ return nullptr;
+}
+
+uint64_t FieldSchema::get_column_id() const {
+ return column_id;
+}
+
+void FieldSchema::set_column_id(uint64_t id) {
+ column_id = id;
+}
+
+uint64_t FieldSchema::get_max_column_id() const {
+ return max_column_id;
+}
+
#include "common/compile_check_end.h"
} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/schema_desc.h b/be/src/vec/exec/format/parquet/schema_desc.h
index 09a52a4251a..3ba38071406 100644
--- a/be/src/vec/exec/format/parquet/schema_desc.h
+++ b/be/src/vec/exec/format/parquet/schema_desc.h
@@ -36,8 +36,13 @@
namespace doris::vectorized {
#include "common/compile_check_begin.h"
+
+// Constant for unassigned column IDs
+constexpr uint64_t UNASSIGNED_COLUMN_ID = UINT64_MAX;
+
struct FieldSchema {
std::string name;
+ std::string lower_case_name; // for hms column name case insensitive match
// the referenced parquet schema element
tparquet::SchemaElement parquet_schema;
@@ -56,12 +61,22 @@ struct FieldSchema {
//For UInt8 -> Int16,UInt16 -> Int32,UInt32 -> Int64,UInt64 -> Int128.
bool is_type_compatibility = false;
- FieldSchema() : data_type(std::make_shared<DataTypeNothing>()) {}
+ FieldSchema()
+ : data_type(std::make_shared<DataTypeNothing>()), column_id(UNASSIGNED_COLUMN_ID) {}
~FieldSchema() = default;
FieldSchema(const FieldSchema& fieldSchema) = default;
std::string debug_string() const;
int32_t field_id = -1;
+ uint64_t column_id = UNASSIGNED_COLUMN_ID;
+ uint64_t max_column_id = 0; // Maximum column ID for this field and its children
+
+ // Column ID assignment and lookup methods
+ void assign_ids(uint64_t& next_id);
+ const FieldSchema* find_column_by_id(uint64_t target_id) const;
+ uint64_t get_column_id() const;
+ void set_column_id(uint64_t id);
+ uint64_t get_max_column_id() const;
};
class FieldDescriptor {
@@ -133,6 +148,19 @@ public:
int32_t size() const { return cast_set<int32_t>(_fields.size()); }
const std::vector<FieldSchema>& get_fields_schema() const { return _fields; }
+
+ /**
+ * Assign stable column IDs to schema fields.
+ *
+ * This uses an ORC-compatible encoding so that the results of
+ * create_column_ids() are consistent across formats. IDs start from 1
+ * and are assigned in a pre-order traversal (parent before children).
+ * After calling this, each FieldSchema will have column_id and
+ * max_column_id populated.
+ */
+ void assign_ids();
+
+ const FieldSchema* find_column_by_id(uint64_t column_id) const;
};
#include "common/compile_check_end.h"
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
index 74fe4c3601b..a7455455198 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
@@ -108,45 +108,120 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
const std::vector<RowRange>& row_ranges,
const cctz::time_zone* ctz, io::IOContext* io_ctx,
std::unique_ptr<ParquetColumnReader>& reader,
- size_t max_buf_size, const tparquet::OffsetIndex* offset_index) {
+ size_t max_buf_size, const tparquet::OffsetIndex* offset_index,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids) {
+ // if (!column_ids.empty()) {
+ // uint64_t field_column_id = field->get_column_id();
+ // VLOG_DEBUG << "[ParquetReader] Checking column_id: " << field_column_id
+ // << ", column_ids.size(): " << column_ids.size();
+ // if (column_ids.find(field_column_id) == column_ids.end()) {
+ // VLOG_DEBUG << "[ParquetReader] Setting skip_reading=true for column_id: "
+ // << field_column_id;
+ // auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx, field);
+ // skip_reader->_filter_column_ids = filter_column_ids;
+ // reader = std::move(skip_reader);
+ // return Status::OK();
+ // } else {
+ // VLOG_DEBUG << "[ParquetReader] Column_id " << field_column_id
+ // << " found in column_ids, skip_reading=false";
+ // }
+ // }
+
if (field->data_type->get_primitive_type() == TYPE_ARRAY) {
std::unique_ptr<ParquetColumnReader> element_reader;
RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
- element_reader, max_buf_size));
+ element_reader, max_buf_size, nullptr, column_ids,
+ filter_column_ids));
element_reader->set_nested_column();
auto array_reader = ArrayColumnReader::create_unique(row_ranges, ctz, io_ctx);
RETURN_IF_ERROR(array_reader->init(std::move(element_reader), field));
+ array_reader->_filter_column_ids = filter_column_ids;
reader.reset(array_reader.release());
} else if (field->data_type->get_primitive_type() == TYPE_MAP) {
std::unique_ptr<ParquetColumnReader> key_reader;
std::unique_ptr<ParquetColumnReader> value_reader;
- RETURN_IF_ERROR(create(file, &field->children[0].children[0], row_group, row_ranges, ctz,
- io_ctx, key_reader, max_buf_size));
- RETURN_IF_ERROR(create(file, &field->children[0].children[1], row_group, row_ranges, ctz,
- io_ctx, value_reader, max_buf_size));
- key_reader->set_nested_column();
- value_reader->set_nested_column();
+
+ if (column_ids.empty() ||
+ column_ids.find(field->children[0].get_column_id()) != column_ids.end()) {
+ // Create key reader
+ RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
+ key_reader, max_buf_size, nullptr, column_ids,
+ filter_column_ids));
+ key_reader->set_nested_column();
+ } else {
+ auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx,
+ &field->children[0]);
+ key_reader = std::move(skip_reader);
+ }
+
+ if (column_ids.empty() ||
+ column_ids.find(field->children[1].get_column_id()) != column_ids.end()) {
+ // Create value reader
+ RETURN_IF_ERROR(create(file, &field->children[1], row_group, row_ranges, ctz, io_ctx,
+ value_reader, max_buf_size, nullptr, column_ids,
+ filter_column_ids));
+ value_reader->set_nested_column();
+ } else {
+ auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx,
+ &field->children[0]);
+ value_reader = std::move(skip_reader);
+ }
+
auto map_reader = MapColumnReader::create_unique(row_ranges, ctz, io_ctx);
RETURN_IF_ERROR(map_reader->init(std::move(key_reader), std::move(value_reader), field));
+ map_reader->_filter_column_ids = filter_column_ids;
reader.reset(map_reader.release());
} else if (field->data_type->get_primitive_type() == TYPE_STRUCT) {
std::unordered_map<std::string, std::unique_ptr<ParquetColumnReader>> child_readers;
child_readers.reserve(field->children.size());
+ int non_skip_reader_idx = -1;
for (int i = 0; i < field->children.size(); ++i) {
+ auto& child = field->children[i]; // 去掉 const，保证 &child 类型为 FieldSchema*
+ std::unique_ptr<ParquetColumnReader> child_reader;
+ if (column_ids.empty() || column_ids.find(child.get_column_id()) != column_ids.end()) {
+ RETURN_IF_ERROR(create(file, &child, row_group, row_ranges, ctz, io_ctx,
+ child_reader, max_buf_size, nullptr, column_ids,
+ filter_column_ids));
+ child_reader->set_nested_column();
+ child_readers[child.name] = std::move(child_reader);
+ // 记录第一个非 SkippingReader
+ if (non_skip_reader_idx == -1) {
+ non_skip_reader_idx = i;
+ }
+ } else {
+ // 不在 column_ids 的 child，创建 SkippingReader
+ auto skip_reader =
+ std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx, &child);
+ skip_reader->_filter_column_ids = filter_column_ids;
+ child_readers[child.name] = std::move(skip_reader);
+ }
+ }
+ // 如果所有都是 SkipReadingReader，则强制第一个 child 调用 create
+ if (non_skip_reader_idx == -1) {
+ // for (int i = 0; i < field->children.size(); ++i) {
+ // 只处理 SkipReadingReader（即刚才创建的）
+ // if (dynamic_cast<SkipReadingReader*>(child_readers[field->children[i].name].get()) != nullptr) {
std::unique_ptr<ParquetColumnReader> child_reader;
- RETURN_IF_ERROR(create(file, &field->children[i], row_group, row_ranges, ctz, io_ctx,
- child_reader, max_buf_size));
+ RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
+ child_reader, max_buf_size, nullptr, column_ids,
+ filter_column_ids));
child_reader->set_nested_column();
- child_readers[field->children[i].name] = std::move(child_reader);
+ child_readers[field->children[0].name] = std::move(child_reader);
+ // break;
+ // }
+ // }
}
auto struct_reader = StructColumnReader::create_unique(row_ranges, ctz, io_ctx);
RETURN_IF_ERROR(struct_reader->init(std::move(child_readers), field));
+ struct_reader->_filter_column_ids = filter_column_ids;
reader.reset(struct_reader.release());
} else {
const tparquet::ColumnChunk& chunk = row_group.columns[field->physical_column_index];
auto scalar_reader =
ScalarColumnReader::create_unique(row_ranges, chunk, offset_index, ctz, io_ctx);
RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size));
+ scalar_reader->_filter_column_ids = filter_column_ids;
reader.reset(scalar_reader.release());
}
return Status::OK();
@@ -584,7 +659,8 @@ Status ScalarColumnReader::_try_load_dict_page(bool* loaded, bool* has_dict) {
Status ScalarColumnReader::read_column_data(
ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node, FilterMap& filter_map,
- size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter) {
+ size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter,
+ int64_t real_column_size) {
if (_converter == nullptr) {
_converter = parquet::PhysicalToLogicalConverter::get_converter(
_field_schema, _field_schema->data_type, type, _ctz, is_dict_filter);
@@ -640,7 +716,7 @@ Status ScalarColumnReader::read_column_data(
}
if (batch_size >= remaining_num_values &&
filter_map.can_filter_all(remaining_num_values, _filter_map_index)) {
- // We can skip the whole page if the remaining values is filtered by predicate columns
+ // We can skip the whole page if the remaining values are filtered by predicate columns
_filter_map_index += remaining_num_values;
_current_row_index += _chunk_reader->remaining_num_values();
RETURN_IF_ERROR(_chunk_reader->skip_page());
@@ -701,7 +777,8 @@ Status ArrayColumnReader::init(std::unique_ptr<ParquetColumnReader> element_read
Status ArrayColumnReader::read_column_data(
ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node, FilterMap& filter_map,
- size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter) {
+ size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter,
+ int64_t real_column_size) {
MutableColumnPtr data_column;
NullMap* null_map_ptr = nullptr;
if (doris_column->is_nullable()) {
@@ -753,7 +830,8 @@ Status MapColumnReader::init(std::unique_ptr<ParquetColumnReader> key_reader,
Status MapColumnReader::read_column_data(
ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node, FilterMap& filter_map,
- size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter) {
+ size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter,
+ int64_t real_column_size) {
MutableColumnPtr data_column;
NullMap* null_map_ptr = nullptr;
if (doris_column->is_nullable()) {
@@ -785,6 +863,7 @@ Status MapColumnReader::read_column_data(
size_t value_rows = 0;
bool key_eof = false;
bool value_eof = false;
+ int64_t orig_col_column_size = key_column->size();
RETURN_IF_ERROR(_key_reader->read_column_data(key_column, key_type, root_node->get_key_node(),
filter_map, batch_size, &key_rows, &key_eof,
@@ -794,11 +873,12 @@ Status MapColumnReader::read_column_data(
size_t loop_rows = 0;
RETURN_IF_ERROR(_value_reader->read_column_data(
value_column, value_type, root_node->get_value_node(), filter_map,
- key_rows - value_rows, &loop_rows, &value_eof, is_dict_filter));
+ key_rows - value_rows, &loop_rows, &value_eof, is_dict_filter,
+ key_column->size() - orig_col_column_size));
value_rows += loop_rows;
}
DCHECK_EQ(key_rows, value_rows);
- DCHECK_EQ(key_eof, value_eof);
+ // DCHECK_EQ(key_eof, value_eof);
*read_rows = key_rows;
*eof = key_eof;
@@ -826,7 +906,8 @@ Status StructColumnReader::init(
Status StructColumnReader::read_column_data(
ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node, FilterMap& filter_map,
- size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter) {
+ size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter,
+ int64_t real_column_size) {
MutableColumnPtr data_column;
NullMap* null_map_ptr = nullptr;
if (doris_column->is_nullable()) {
@@ -850,7 +931,9 @@ Status StructColumnReader::read_column_data(
const auto* doris_struct_type = assert_cast<const DataTypeStruct*>(remove_nullable(type).get());
int64_t not_missing_column_id = -1;
+ size_t not_missing_orig_column_size = 0;
std::vector<size_t> missing_column_idxs {};
+ std::vector<size_t> skip_reading_column_idxs {};
_read_column_names.clear();
@@ -860,21 +943,48 @@ Status StructColumnReader::read_column_data(
auto& doris_name = doris_struct_type->get_element_name(i);
if (!root_node->children_column_exists(doris_name)) {
missing_column_idxs.push_back(i);
+ VLOG_DEBUG << "[ParquetReader] Missing column in schema: column_idx[" << i
+ << "], doris_name: " << doris_name << " (column not exists in root node)";
continue;
}
auto file_name = root_node->children_file_column_name(doris_name);
+ // if (_child_readers.find(file_name) == _child_readers.end()) {
+ // missing_column_idxs.push_back(i);
+ // continue;
+ // }
+
+ // Check if this is a SkipReadingReader - we should skip it when choosing reference column
+ // because SkipReadingReader doesn't know the actual data size in nested context
+ bool is_skip_reader =
+ dynamic_cast<SkipReadingReader*>(_child_readers[file_name].get()) != nullptr;
+
+ if (is_skip_reader) {
+ // Store SkipReadingReader columns to fill them later based on reference column size
+ skip_reading_column_idxs.push_back(i);
+ continue;
+ }
+ // Only add non-SkipReadingReader columns to _read_column_names
+ // This ensures get_rep_level() and get_def_level() return valid levels
_read_column_names.emplace_back(file_name);
size_t field_rows = 0;
bool field_eof = false;
if (not_missing_column_id == -1) {
not_missing_column_id = i;
+ not_missing_orig_column_size = doris_field->size();
RETURN_IF_ERROR(_child_readers[file_name]->read_column_data(
doris_field, doris_type, root_node->get_children_node(doris_name), filter_map,
batch_size, &field_rows, &field_eof, is_dict_filter));
*read_rows = field_rows;
*eof = field_eof;
+ // Debug: print the first non-missing child column read info
+ //std::cout << "[ParquetReader] struct '" << _field_schema->name << "' first non-missing child='"
+ // << file_name << "' doris_name='" << doris_name << "' field_rows=" << field_rows
+ // << " field_eof=" << field_eof << " doris_field_size=" << doris_field->size()
+ // << " child_rep_levels=" << _child_readers[file_name]->get_rep_level().size()
+ // << " child_def_levels=" << _child_readers[file_name]->get_def_level().size()
+ // << std::endl;
/*
* Considering the issue in the `_read_nested_column` function where data may span across pages, leading
* to missing definition and repetition levels, when filling the null_map of the struct later, it is
@@ -891,6 +1001,12 @@ Status StructColumnReader::read_column_data(
doris_field, doris_type, root_node->get_children_node(doris_name),
filter_map, *read_rows - field_rows, &loop_rows, &field_eof,
is_dict_filter));
+ // Debug: print each loop iteration for non-first child columns
+ //std::cout << "[ParquetReader] struct '" << _field_schema->name << "' non-first child='"
+ // << file_name << "' doris_name='" << doris_name << "' loop_rows=" << loop_rows
+ // << " field_rows=" << field_rows << " target_rows=" << *read_rows
+ // << " field_eof=" << field_eof << " doris_field_size=" << doris_field->size()
+ // << std::endl;
field_rows += loop_rows;
}
DCHECK_EQ(*read_rows, field_rows);
@@ -898,10 +1014,90 @@ Status StructColumnReader::read_column_data(
}
}
+ // After scanning children, print summary of missing/skip info
+ //std::cout << "[ParquetReader] struct '" << _field_schema->name << "' summary: not_missing_column_id="
+ // << not_missing_column_id << " missing_columns_count=" << missing_column_idxs.size()
+ // << " skip_reading_count=" << skip_reading_column_idxs.size() << std::endl;
+
+ int64_t missing_column_sz = -1;
+
if (not_missing_column_id == -1) {
- // TODO: support read struct which columns are all missing
- return Status::Corruption("Not support read struct '{}' which columns are all missing",
- _field_schema->name);
+ // All queried columns are missing in the file (e.g., all added after schema change)
+ // We need to pick a column from _field_schema that exists in the file to get RL/DL
+ //std::cout << "[ParquetReader] All queried columns missing for struct '" << _field_schema->name
+ // << "', searching for reference column in _field_schema" << std::endl;
+
+ // Find a column from _field_schema children that exists in the file for RL/DL reference
+ std::string reference_file_column_name;
+ std::unique_ptr<ParquetColumnReader>* reference_reader = nullptr;
+
+ for (const auto& child : _field_schema->children) {
+ auto it = _child_readers.find(child.name);
+ if (it != _child_readers.end()) {
+ // Skip SkipReadingReader as they don't have valid RL/DL
+ bool is_skip_reader = dynamic_cast<SkipReadingReader*>(it->second.get()) != nullptr;
+ if (!is_skip_reader) {
+ reference_file_column_name = child.name;
+ reference_reader = &(it->second);
+ //std::cout << "[ParquetReader] Found reference column: " << reference_file_column_name
+ // << " for struct '" << _field_schema->name << "'" << std::endl;
+ break;
+ }
+ }
+ }
+
+ if (reference_reader != nullptr) {
+ // Read the reference column to get correct RL/DL information
+ // We need to find the FieldSchema for the reference column from _field_schema children
+ FieldSchema* ref_field_schema = nullptr;
+ for (auto& child : _field_schema->children) {
+ if (child.name == reference_file_column_name) {
+ ref_field_schema = &child;
+ break;
+ }
+ }
+
+ if (ref_field_schema == nullptr) {
+ return Status::InternalError(
+ "Cannot find field schema for reference column '{}' in struct '{}'",
+ reference_file_column_name, _field_schema->name);
+ }
+
+ // Create a temporary column to hold the data (we'll use its size for missing_column_sz)
+ ColumnPtr temp_column = ref_field_schema->data_type->create_column();
+ auto temp_type = ref_field_schema->data_type;
+
+ size_t field_rows = 0;
+ bool field_eof = false;
+
+ // Use root_node to get the correct child node for the reference column
+ // reference_file_column_name is the file column name, use get_children_node_by_file_column_name
+ auto ref_child_node =
+ root_node->get_children_node_by_file_column_name(reference_file_column_name);
+ not_missing_orig_column_size = temp_column->size();
+
+ RETURN_IF_ERROR((*reference_reader)
+ ->read_column_data(temp_column, temp_type, ref_child_node,
+ filter_map, batch_size, &field_rows,
+ &field_eof, is_dict_filter));
+
+ *read_rows = field_rows;
+ *eof = field_eof;
+
+ // Store this reference column name for get_rep_level/get_def_level to use
+ _read_column_names.emplace_back(reference_file_column_name);
+
+ //std::cout << "[ParquetReader] Read " << field_rows << " rows from reference column '"
+ // << reference_file_column_name << "', temp_column.size=" << temp_column->size()
+ // << " for struct '" << _field_schema->name << "'" << std::endl;
+
+ missing_column_sz = temp_column->size() - not_missing_orig_column_size;
+ } else {
+ return Status::Corruption(
+ "Cannot read struct '{}': all queried columns are missing and no reference "
+ "column found in file",
+ _field_schema->name);
+ }
}
// This missing_column_sz is not *read_rows. Because read_rows returns the number of rows.
@@ -912,8 +1108,27 @@ Status StructColumnReader::read_column_data(
// [{4,null},{5,null}]
// When you first read subcolumn a, you read 5 data items and the value of *read_rows is 2.
// You should insert 5 records into subcolumn b instead of 2.
- auto missing_column_sz = doris_struct.get_column(not_missing_column_id).size();
- // fill missing column with null or default value
+ if (missing_column_sz == -1) {
+ missing_column_sz = doris_struct.get_column(not_missing_column_id).size() -
+ not_missing_orig_column_size;
+ }
+
+ // Fill SkipReadingReader columns with the correct amount of data based on reference column
+ // Let SkipReadingReader handle the data filling through its read_column_data method
+ for (auto idx : skip_reading_column_idxs) {
+ auto& doris_field = doris_struct.get_column_ptr(idx);
+ auto& doris_type = const_cast<DataTypePtr&>(doris_struct_type->get_element(idx));
+ auto& doris_name = const_cast<String&>(doris_struct_type->get_element_name(idx));
+ auto file_name = root_node->children_file_column_name(doris_name);
+
+ size_t field_rows = 0;
+ bool field_eof = false;
+ RETURN_IF_ERROR(_child_readers[file_name]->read_column_data(
+ doris_field, doris_type, root_node->get_children_node(doris_name), filter_map,
+ missing_column_sz, &field_rows, &field_eof, is_dict_filter, missing_column_sz));
+ }
+
+ // Fill truly missing columns (not in root_node) with null or default value
for (auto idx : missing_column_idxs) {
auto& doris_field = doris_struct.get_column_ptr(idx);
auto& doris_type = doris_struct_type->get_element(idx);
@@ -924,10 +1139,11 @@ Status StructColumnReader::read_column_data(
}
if (null_map_ptr != nullptr) {
+ //std::cout << "[ParquetReader] struct '" << _field_schema->name << "' before fill_struct_null_map: rep_levels="
+ // << this->get_rep_level().size() << " def_levels=" << this->get_def_level().size() << std::endl;
fill_struct_null_map(_field_schema, *null_map_ptr, this->get_rep_level(),
this->get_def_level());
}
-
return Status::OK();
}
#include "common/compile_check_end.h"
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
index f0d02b0d3e4..7b37f2e0e6c 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
@@ -123,7 +123,8 @@ public:
virtual Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
FilterMap& filter_map, size_t batch_size, size_t* read_rows,
- bool* eof, bool is_dict_filter) = 0;
+ bool* eof, bool is_dict_filter,
+ int64_t real_column_size = -1) = 0;
virtual Status read_dict_values_to_column(MutableColumnPtr& doris_column, bool* has_dict) {
return Status::NotSupported("read_dict_values_to_column is not supported");
@@ -138,7 +139,9 @@ public:
const tparquet::RowGroup& row_group,
const std::vector<RowRange>& row_ranges, const cctz::time_zone* ctz,
io::IOContext* io_ctx, std::unique_ptr<ParquetColumnReader>& reader,
- size_t max_buf_size, const tparquet::OffsetIndex* offset_index = nullptr);
+ size_t max_buf_size, const tparquet::OffsetIndex* offset_index = nullptr,
+ const std::set<uint64_t>& column_ids = {},
+ const std::set<uint64_t>& filter_column_ids = {});
void set_nested_column() { _nested_column = true; }
virtual const std::vector<level_t>& get_rep_level() const = 0;
virtual const std::vector<level_t>& get_def_level() const = 0;
@@ -147,6 +150,8 @@ public:
virtual void reset_filter_map_index() = 0;
+ FieldSchema* get_field_schema() const { return _field_schema; }
+
protected:
void _generate_read_ranges(int64_t start_index, int64_t end_index,
std::list<RowRange>& read_ranges);
@@ -162,6 +167,7 @@ protected:
int64_t _decode_null_map_time = 0;
size_t _filter_map_index = 0;
+ std::set<uint64_t> _filter_column_ids;
};
class ScalarColumnReader : public ParquetColumnReader {
@@ -179,7 +185,7 @@ public:
Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof,
- bool is_dict_filter) override;
+ bool is_dict_filter, int64_t real_column_size = -1) override;
Status read_dict_values_to_column(MutableColumnPtr& doris_column, bool* has_dict) override;
MutableColumnPtr convert_dict_column_to_string_column(const ColumnInt32* dict_column) override;
const std::vector<level_t>& get_rep_level() const override { return _rep_levels; }
@@ -226,7 +232,7 @@ public:
Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof,
- bool is_dict_filter) override;
+ bool is_dict_filter, int64_t real_column_size = -1) override;
const std::vector<level_t>& get_rep_level() const override {
return _element_reader->get_rep_level();
}
@@ -255,7 +261,7 @@ public:
Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof,
- bool is_dict_filter) override;
+ bool is_dict_filter, int64_t real_column_size = -1) override;
const std::vector<level_t>& get_rep_level() const override {
return _key_reader->get_rep_level();
@@ -297,7 +303,7 @@ public:
Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof,
- bool is_dict_filter) override;
+ bool is_dict_filter, int64_t real_column_size = -1) override;
const std::vector<level_t>& get_rep_level() const override {
if (!_read_column_names.empty()) {
@@ -346,6 +352,93 @@ private:
std::vector<std::string> _read_column_names;
//Need to use vector instead of set,see `get_rep_level()` for the reason.
};
+
+// A special reader that skips actual reading but provides empty data with correct structure
+// This is used when a column is not needed but its structure is required (e.g., for map keys)
+class SkipReadingReader : public ParquetColumnReader {
+public:
+ SkipReadingReader(const std::vector<RowRange>& row_ranges, const cctz::time_zone* ctz,
+ io::IOContext* io_ctx, FieldSchema* field_schema)
+ : ParquetColumnReader(row_ranges, ctz, io_ctx) {
+ _field_schema = field_schema; // Use inherited member from base class
+ VLOG_DEBUG << "[ParquetReader] Created SkipReadingReader for field: "
+ << _field_schema->name;
+ }
+
+ Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
+ const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
+ FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof,
+ bool is_dict_filter, int64_t real_column_size = -1) override {
+ VLOG_DEBUG << "[ParquetReader] SkipReadingReader::read_column_data for field: "
+ << _field_schema->name << ", batch_size: " << batch_size;
+ DCHECK(real_column_size >= 0); // real_column_size for filtered column size.
+
+ // Simulate reading without actually reading data
+ // Fill with default/null values based on column type
+ MutableColumnPtr data_column = doris_column->assume_mutable();
+
+ if (real_column_size > 0) {
+ if (doris_column->is_nullable()) {
+ auto* nullable_column = static_cast<vectorized::ColumnNullable*>(data_column.get());
+ nullable_column->insert_many_defaults(real_column_size);
+ } else {
+ // For non-nullable columns, insert appropriate default values
+ for (size_t i = 0; i < real_column_size; ++i) {
+ data_column->insert_default();
+ }
+ }
+ }
+
+ *read_rows = batch_size; // Indicate we "read" batch_size rows
+ *eof = false; // We can always provide more empty data
+
+ VLOG_DEBUG << "[ParquetReader] SkipReadingReader generated " << batch_size
+ << " default values for field: " << _field_schema->name;
+
+ return Status::OK();
+ }
+
+ static std::unique_ptr<SkipReadingReader> create_unique(const std::vector<RowRange>& row_ranges,
+ cctz::time_zone* ctz,
+ io::IOContext* io_ctx,
+ FieldSchema* field_schema) {
+ return std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx, field_schema);
+ }
+
+ // These methods should not be called for SkipReadingReader
+ // If they are called, it indicates a logic error in the code
+ const std::vector<level_t>& get_rep_level() const override {
+ LOG(FATAL) << "get_rep_level() should not be called on SkipReadingReader for field: "
+ << _field_schema->name
+ << ". This indicates the SkipReadingReader was incorrectly used as a reference "
+ "column.";
+ // static std::vector<level_t> empty_levels;
+ // return empty_levels;
+ __builtin_unreachable();
+ }
+
+ const std::vector<level_t>& get_def_level() const override {
+ LOG(FATAL) << "get_def_level() should not be called on SkipReadingReader for field: "
+ << _field_schema->name
+ << ". This indicates the SkipReadingReader was incorrectly used as a reference "
+ "column.";
+ // static std::vector<level_t> empty_levels;
+ // return empty_levels;
+ __builtin_unreachable();
+ }
+
+ // Implement required pure virtual methods from base class
+ Statistics statistics() override {
+ return Statistics(); // Return empty statistics
+ }
+
+ void close() override {
+ // Nothing to close for skip reading
+ }
+
+ void reset_filter_map_index() override { _filter_map_index = 0; }
+};
+
#include "common/compile_check_end.h"
}; // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
index 7d1f9d27430..19ddb8d1b99 100644
--- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
@@ -25,6 +25,7 @@
#include <algorithm>
#include <boost/iterator/iterator_facade.hpp>
+#include <iostream>
#include <ostream>
#include "common/config.h"
@@ -82,7 +83,9 @@ RowGroupReader::RowGroupReader(io::FileReaderSPtr file_reader,
const int32_t row_group_id, const tparquet::RowGroup& row_group,
const cctz::time_zone* ctz, io::IOContext* io_ctx,
const PositionDeleteContext& position_delete_ctx,
- const LazyReadContext& lazy_read_ctx, RuntimeState* state)
+ const LazyReadContext& lazy_read_ctx, RuntimeState* state,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids)
: _file_reader(file_reader),
_read_table_columns(read_columns),
_row_group_id(row_group_id),
@@ -93,7 +96,15 @@ RowGroupReader::RowGroupReader(io::FileReaderSPtr file_reader,
_position_delete_ctx(position_delete_ctx),
_lazy_read_ctx(lazy_read_ctx),
_state(state),
- _obj_pool(new ObjectPool()) {}
+ _obj_pool(new ObjectPool()),
+ _column_ids(column_ids),
+ _filter_column_ids(filter_column_ids) {
+ //std::cout << "[RowGroupReader] _column_ids: ";
+ //for (const auto& col_id : _column_ids) {
+ // std::cout << col_id << " ";
+ //}
+ //std::cout << std::endl;
+}
RowGroupReader::~RowGroupReader() {
_column_readers.clear();
@@ -130,9 +141,9 @@ Status RowGroupReader::init(
const tparquet::OffsetIndex* offset_index =
col_offsets.find(physical_index) != col_offsets.end() ? &col_offsets[physical_index]
: nullptr;
- RETURN_IF_ERROR(ParquetColumnReader::create(_file_reader, field, _row_group_meta,
- _read_ranges, _ctz, _io_ctx, reader,
- max_buf_size, offset_index));
+ RETURN_IF_ERROR(ParquetColumnReader::create(
+ _file_reader, field, _row_group_meta, _read_ranges, _ctz, _io_ctx, reader,
+ max_buf_size, offset_index, _column_ids, _filter_column_ids));
if (reader == nullptr) {
VLOG_DEBUG << "Init row group(" << _row_group_id << ") reader failed";
return Status::Corruption("Init row group reader failed");
@@ -320,8 +331,8 @@ Status RowGroupReader::next_batch(Block* block, size_t batch_size, size_t* read_
return _do_lazy_read(block, batch_size, read_rows, batch_eof);
} else {
FilterMap filter_map;
- RETURN_IF_ERROR(_read_column_data(block, _lazy_read_ctx.all_read_columns, batch_size,
- read_rows, batch_eof, filter_map));
+ RETURN_IF_ERROR((_read_column_data(block, _lazy_read_ctx.all_read_columns, batch_size,
+ read_rows, batch_eof, filter_map)));
RETURN_IF_ERROR(
_fill_partition_columns(block, *read_rows, _lazy_read_ctx.partition_columns));
RETURN_IF_ERROR(_fill_missing_columns(block, *read_rows, _lazy_read_ctx.missing_columns));
@@ -427,9 +438,19 @@ Status RowGroupReader::_read_column_data(Block* block,
RETURN_IF_ERROR(_column_readers[read_col_name]->read_column_data(
column_ptr, column_type, _table_info_node_ptr->get_children_node(read_col_name),
filter_map, batch_size - col_read_rows, &loop_rows, &col_eof, is_dict_filter));
+ // Debug: print loop_rows for this iteration
+ //std::cout << "[RowGroupReader] column '" << read_col_name << "' loop_rows=" << loop_rows
+ // << " col_read_rows_so_far=" << col_read_rows << std::endl;
col_read_rows += loop_rows;
}
+ // Debug: print rows read for this column
+ //std::cout << "[RowGroupReader] column '" << read_col_name << "' read_rows=" << col_read_rows
+ // << std::endl;
if (batch_read_rows > 0 && batch_read_rows != col_read_rows) {
+ LOG(WARNING) << "[RowGroupReader] Mismatched read rows among parquet columns. "
+ "previous_batch_read_rows="
+ << batch_read_rows << ", current_column='" << read_col_name
+ << "', current_col_read_rows=" << col_read_rows;
return Status::Corruption("Can't read the same number of rows among parquet columns");
}
batch_read_rows = col_read_rows;
@@ -720,6 +741,10 @@ Status RowGroupReader::_fill_missing_columns(
block->erase(result_column_id);
}
}
+
+ // auto mutable_column = block->get_by_name(kv.first).column->assume_mutable();
+ // auto* nullable_column = assert_cast<vectorized::ColumnNullable*>(mutable_column.get());
+ // nullable_column->insert_many_defaults(rows);
}
return Status::OK();
}
@@ -1110,6 +1135,7 @@ ParquetColumnReader::Statistics RowGroupReader::statistics() {
}
return st;
}
+
#include "common/compile_check_end.h"
} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.h b/be/src/vec/exec/format/parquet/vparquet_group_reader.h
index 86b7cdf3f0e..5853cd6827d 100644
--- a/be/src/vec/exec/format/parquet/vparquet_group_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.h
@@ -150,7 +150,9 @@ public:
const int32_t row_group_id, const tparquet::RowGroup& row_group,
const cctz::time_zone* ctz, io::IOContext* io_ctx,
const PositionDeleteContext& position_delete_ctx,
- const LazyReadContext& lazy_read_ctx, RuntimeState* state);
+ const LazyReadContext& lazy_read_ctx, RuntimeState* state,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids);
~RowGroupReader();
Status init(const FieldDescriptor& schema, std::vector<RowRange>& row_ranges,
@@ -188,9 +190,11 @@ private:
void _merge_read_ranges(std::vector<RowRange>& row_ranges);
Status _read_empty_batch(size_t batch_size, size_t* read_rows, bool* batch_eof,
bool* modify_row_ids);
+
Status _read_column_data(Block* block, const std::vector<std::string>& columns,
size_t batch_size, size_t* read_rows, bool* batch_eof,
FilterMap& filter_map);
+
Status _do_lazy_read(Block* block, size_t batch_size, size_t* read_rows, bool* batch_eof);
Status _rebuild_filter_map(FilterMap& filter_map,
DorisUniqueBufferPtr<uint8_t>& filter_map_data,
@@ -249,6 +253,8 @@ private:
std::vector<std::pair<std::string, int>> _dict_filter_cols;
RuntimeState* _state = nullptr;
std::shared_ptr<ObjectPool> _obj_pool;
+ const std::set<uint64_t>& _column_ids;
+ const std::set<uint64_t>& _filter_column_ids;
bool _is_row_group_filtered = false;
RowGroupIndex _current_row_group_idx {0, 0, 0};
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
index 5accadf77f7..e47a0eddaac 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
@@ -319,7 +319,8 @@ Status ParquetReader::init_reader(
const std::unordered_map<std::string, int>* colname_to_slot_id,
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts,
- std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr, bool filter_groups) {
+ std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr, bool filter_groups,
+ const std::set<uint64_t>& column_ids, const std::set<uint64_t>& filter_column_ids) {
_tuple_descriptor = tuple_descriptor;
_row_descriptor = row_descriptor;
_colname_to_slot_id = colname_to_slot_id;
@@ -328,6 +329,8 @@ Status ParquetReader::init_reader(
_colname_to_value_range = colname_to_value_range;
_table_info_node_ptr = table_info_node_ptr;
_filter_groups = filter_groups;
+ _column_ids = column_ids;
+ _filter_column_ids = filter_column_ids;
RETURN_IF_ERROR(_open_file());
_t_metadata = &(_file_metadata->to_thrift());
@@ -706,12 +709,12 @@ Status ParquetReader::_next_row_group_reader() {
_profile, _file_reader, io_ranges, merged_read_slice_size)
: _file_reader;
}
- _current_group_reader.reset(
- new RowGroupReader(_io_ctx ? std::make_shared<io::TracingFileReader>(
- group_file_reader, _io_ctx->file_reader_stats)
- : group_file_reader,
- _read_table_columns, row_group_index.row_group_id, row_group, _ctz,
- _io_ctx, position_delete_ctx, _lazy_read_ctx, _state));
+ _current_group_reader.reset(new RowGroupReader(
+ _io_ctx ? std::make_shared<io::TracingFileReader>(group_file_reader,
+ _io_ctx->file_reader_stats)
+ : group_file_reader,
+ _read_table_columns, row_group_index.row_group_id, row_group, _ctz, _io_ctx,
+ position_delete_ctx, _lazy_read_ctx, _state, _column_ids, _filter_column_ids));
_row_group_eof = false;
_current_group_reader->set_current_row_group_idx(row_group_index);
@@ -796,8 +799,8 @@ std::vector<io::PrefetchRange> ParquetReader::_generate_random_access_ranges(
if (field->data_type->get_primitive_type() == TYPE_ARRAY) {
scalar_range(&field->children[0], row_group);
} else if (field->data_type->get_primitive_type() == TYPE_MAP) {
- scalar_range(&field->children[0].children[0], row_group);
- scalar_range(&field->children[0].children[1], row_group);
+ scalar_range(&field->children[0], row_group);
+ scalar_range(&field->children[1], row_group);
} else if (field->data_type->get_primitive_type() == TYPE_STRUCT) {
for (int i = 0; i < field->children.size(); ++i) {
scalar_range(&field->children[i], row_group);
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.h b/be/src/vec/exec/format/parquet/vparquet_reader.h
index fecc433dba6..689c45a37d8 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.h
@@ -123,7 +123,8 @@ public:
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts,
std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr =
TableSchemaChangeHelper::ConstNode::get_instance(),
- bool filter_groups = true);
+ bool filter_groups = true, const std::set<uint64_t>& column_ids = {},
+ const std::set<uint64_t>& filter_column_ids = {});
Status get_next_block(Block* block, size_t* read_rows, bool* eof) override;
@@ -316,6 +317,8 @@ private:
-1};
bool _filter_groups;
+ std::set<uint64_t> _column_ids;
+ std::set<uint64_t> _filter_column_ids;
// for page index filter. slot id => expr
std::map<int, std::vector<std::unique_ptr<ColumnPredicate>>> _push_down_simple_predicates;
diff --git a/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.cpp b/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.cpp
new file mode 100644
index 00000000000..6e9fa381fc7
--- /dev/null
+++ b/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.cpp
@@ -0,0 +1,148 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/hive/hive_orc_nested_column_utils.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/logging.h"
+#include "orc/Type.hh"
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+void HiveOrcNestedColumnUtils::extract_nested_column_ids(
+ const orc::Type& type, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ std::unordered_map<std::string, std::vector<std::vector<std::string>>>
+ child_paths_by_table_col_name;
+ for (const auto& path : paths) {
+ if (!path.empty()) {
+ std::string first_table_col_name = path[0];
+ std::vector<std::string> remaining;
+ if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ child_paths_by_table_col_name[first_table_col_name].push_back(std::move(remaining));
+ }
+ }
+
+ // Track whether any child column was added to determine if parent should be included
+ bool has_child_columns = false;
+
+ // For MAP type, normalize wildcard "*" to explicit KEYS/VALUES access
+ // Wildcard in MAP context means accessing both map keys and values
+ // Normalization logic:
+ // path: ["map_col", "*"] → ["map_col", "VALUES"] + ["map_col", "KEYS"]
+ // path: ["map_col", "*", "field"] → ["map_col", "VALUES", "field"] + ["map_col", "KEYS"]
+ // KEYS are always needed for correct RL/DL computation when accessing MAP via wildcard
+ if (type.getKind() == orc::TypeKind::MAP) {
+ auto wildcard_it = child_paths_by_table_col_name.find("*");
+ if (wildcard_it != child_paths_by_table_col_name.end()) {
+ auto& wildcard_paths = wildcard_it->second;
+
+ // All wildcard paths go to VALUES
+ auto& values_paths = child_paths_by_table_col_name["VALUES"];
+ values_paths.insert(values_paths.end(), wildcard_paths.begin(), wildcard_paths.end());
+
+ // Always add KEYS for wildcard access (needed for RL/DL computation)
+ auto& keys_paths = child_paths_by_table_col_name["KEYS"];
+ // Add an empty path to request full KEYS
+ std::vector<std::string> empty_path;
+ keys_paths.push_back(empty_path);
+
+ // Remove wildcard entry as it's been expanded
+ child_paths_by_table_col_name.erase(wildcard_it);
+ }
+ }
+
+ for (uint64_t i = 0; i < type.getSubtypeCount(); ++i) {
+ const orc::Type* child = type.getSubtype(i);
+
+ std::string child_field_name;
+ switch (type.getKind()) {
+ case orc::TypeKind::STRUCT:
+ child_field_name = to_lower(type.getFieldName(i));
+ break;
+ case orc::TypeKind::LIST:
+ child_field_name = "*";
+ break;
+ case orc::TypeKind::MAP:
+ // After wildcard normalization above, all MAP accesses are explicit KEYS/VALUES
+ // Simply assign the appropriate field name based on which child we're processing
+ if (i == 0) {
+ child_field_name = "KEYS";
+ } else if (i == 1) {
+ child_field_name = "VALUES";
+ }
+ break;
+ default:
+ child_field_name = "";
+ break;
+ }
+
+ if (child_field_name.empty()) {
+ continue;
+ }
+
+ const auto child_paths_it = child_paths_by_table_col_name.find(child_field_name);
+
+ if (child_paths_it != child_paths_by_table_col_name.end()) {
+ const auto& child_paths = child_paths_it->second;
+
+ // Check if any child path is empty (meaning full child needed)
+ bool needs_full_child =
+ std::any_of(child_paths.begin(), child_paths.end(),
+ [](const std::vector<std::string>& path) { return path.empty(); });
+
+ if (needs_full_child) {
+ uint64_t start_id = child->getColumnId();
+ uint64_t max_column_id = child->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ } else {
+ // Store current size to check if recursive call added any columns
+ size_t before_size = column_ids.size();
+
+ // Recursively extract from child
+ extract_nested_column_ids(*child, child_paths, column_ids);
+
+ // Check if recursive call added any columns
+ if (column_ids.size() > before_size) {
+ has_child_columns = true;
+ };
+ }
+ }
+ }
+
+ // If any child columns were added, also add the parent column ID
+ // This ensures parent struct/container nodes are included when their children are needed
+ if (has_child_columns) {
+ // Set automatically handles deduplication, so no need to check if it already exists
+ column_ids.insert(type.getColumnId());
+ }
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.h b/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.h
new file mode 100644
index 00000000000..1e1968c09b9
--- /dev/null
+++ b/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.h
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace orc {
+class Type;
+} // namespace orc
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+class HiveOrcNestedColumnUtils {
+public:
+ static void extract_nested_column_ids(const orc::Type& type,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.cpp b/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.cpp
new file mode 100644
index 00000000000..a0052ecfbfc
--- /dev/null
+++ b/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.cpp
@@ -0,0 +1,172 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/hive/hive_parquet_nested_column_utils.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/parquet/schema_desc.h"
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+void HiveParquetNestedColumnUtils::extract_nested_column_ids(
+ const FieldSchema& field_schema, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ // Group paths by first field_id
+ std::unordered_map<std::string, std::vector<std::vector<std::string>>>
+ child_paths_by_table_col_name;
+
+ for (const auto& path : paths) {
+ if (!path.empty()) {
+ std::string first_table_col_name = path[0];
+ std::vector<std::string> remaining;
+ if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ child_paths_by_table_col_name[first_table_col_name].push_back(std::move(remaining));
+ }
+ }
+
+ // Track whether any child column was added to determine if parent should be included
+ bool has_child_columns = false;
+
+ // For MAP type, normalize wildcard "*" to explicit KEYS/VALUES access
+ // Wildcard in MAP context means accessing both map keys and values
+ // Normalization logic:
+ // path: ["map_col", "*"] → ["map_col", "VALUES"] + ["map_col", "KEYS"]
+ // path: ["map_col", "*", "field"] → ["map_col", "VALUES", "field"] + ["map_col", "KEYS"]
+ // KEYS are always needed for correct RL/DL computation when accessing MAP via wildcard
+ if (field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_MAP) {
+ auto wildcard_it = child_paths_by_table_col_name.find("*");
+ if (wildcard_it != child_paths_by_table_col_name.end()) {
+ auto& wildcard_paths = wildcard_it->second;
+
+ // All wildcard paths go to VALUES
+ auto& values_paths = child_paths_by_table_col_name["VALUES"];
+ values_paths.insert(values_paths.end(), wildcard_paths.begin(), wildcard_paths.end());
+
+ // Always add KEYS for wildcard access (needed for RL/DL computation)
+ auto& keys_paths = child_paths_by_table_col_name["KEYS"];
+ // Add an empty path to request full KEYS
+ std::vector<std::string> empty_path;
+ keys_paths.push_back(empty_path);
+
+ // Remove wildcard entry as it's been expanded
+ child_paths_by_table_col_name.erase(wildcard_it);
+ }
+ }
+
+ // Efficiently traverse children
+ for (uint64_t i = 0; i < field_schema.children.size(); ++i) {
+ const auto& child = field_schema.children[i];
+ std::string child_field_name;
+
+ bool is_list = field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_ARRAY;
+ bool is_map = field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_MAP;
+
+ if (is_list) {
+ child_field_name = "*";
+ } else if (is_map) {
+ // After wildcard normalization above, all MAP accesses are explicit KEYS/VALUES
+ // Simply assign the appropriate field name based on which child we're processing
+ if (i == 0) {
+ child_field_name = "KEYS";
+ } else if (i == 1) {
+ child_field_name = "VALUES";
+ }
+
+ // Special handling for Parquet MAP structure:
+ // When accessing only VALUES, we still need KEY structure for levels
+ // Check if we're at key child (i==0) and only VALUES is requested (no KEYS)
+ if (i == 0) {
+ bool has_keys_access = child_paths_by_table_col_name.find("KEYS") !=
+ child_paths_by_table_col_name.end();
+ bool has_values_access = child_paths_by_table_col_name.find("VALUES") !=
+ child_paths_by_table_col_name.end();
+
+ // If only VALUES is accessed (not KEYS), still include key structure for RL/DL
+ if (!has_keys_access && has_values_access) {
+ // For map_values() queries, we need key's structure for correct RL/DL parsing.
+ // If key is a nested type (e.g., STRUCT), RL/DL info is stored at leaf columns.
+ // Add all column IDs from key's start to max (all leaves + intermediate nodes).
+ uint64_t key_start_id = child.get_column_id();
+ uint64_t key_max_id = child.get_max_column_id();
+ for (uint64_t id = key_start_id; id <= key_max_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ continue; // Skip further processing of key child
+ }
+ }
+
+ } else {
+ child_field_name = child.lower_case_name;
+ }
+
+ if (child_field_name.empty()) {
+ continue;
+ }
+
+ auto child_paths_it = child_paths_by_table_col_name.find(child_field_name);
+ if (child_paths_it != child_paths_by_table_col_name.end()) {
+ const auto& child_paths = child_paths_it->second;
+
+ // Check if any child path is empty (meaning full child needed)
+ bool needs_full_child =
+ std::any_of(child_paths.begin(), child_paths.end(),
+ [](const std::vector<std::string>& path) { return path.empty(); });
+
+ if (needs_full_child) {
+ // Add all column IDs from current child node to max_column_id
+ // This efficiently handles all nested/complex cases in one loop
+ uint64_t start_id = child.get_column_id();
+ uint64_t max_column_id = child.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ } else {
+ // Store current size to check if recursive call added any columns
+ size_t before_size = column_ids.size();
+
+ // Recursively extract from child
+ extract_nested_column_ids(child, child_paths, column_ids);
+
+ // Check if recursive call added any columns
+ if (column_ids.size() > before_size) {
+ has_child_columns = true;
+ }
+ }
+ }
+ }
+
+ // If any child columns were added, also add the parent column ID
+ // This ensures parent struct/container nodes are included when their children are needed
+ if (has_child_columns) {
+ // Set automatically handles deduplication, so no need to check if it already exists
+ column_ids.insert(field_schema.get_column_id());
+ }
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.h b/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.h
new file mode 100644
index 00000000000..6b57a0a3a38
--- /dev/null
+++ b/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.h
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+struct FieldSchema;
+
+class HiveParquetNestedColumnUtils {
+public:
+ static void extract_nested_column_ids(const FieldSchema& field_schema,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/hive_reader.cpp b/be/src/vec/exec/format/table/hive_reader.cpp
index a454118b244..96ca2dff2d0 100644
--- a/be/src/vec/exec/format/table/hive_reader.cpp
+++ b/be/src/vec/exec/format/table/hive_reader.cpp
@@ -21,6 +21,8 @@
#include "common/status.h"
#include "runtime/runtime_state.h"
+#include "vec/exec/format/table/hive/hive_orc_nested_column_utils.h"
+#include "vec/exec/format/table/hive/hive_parquet_nested_column_utils.h"
namespace doris::vectorized {
#include "common/compile_check_begin.h"
@@ -77,10 +79,223 @@ Status HiveOrcReader::init_reader(
}
}
+ auto column_id_result = ColumnIdResult();
+ if (_state->query_options().hive_orc_use_column_names && !is_hive_col_name) {
+ column_id_result = _create_column_ids(orc_type_ptr, tuple_descriptor);
+ } else {
+ column_id_result =
+ _create_column_ids_by_top_level_col_index(orc_type_ptr, tuple_descriptor);
+ }
+
+ // const auto& file_col_names = column_id_result.column_names;
+ const auto& column_ids = column_id_result.column_ids;
+ const auto& filter_column_ids = column_id_result.filter_column_ids;
+
return orc_reader->init_reader(&read_table_col_names, table_col_name_to_value_range, conjuncts,
false, tuple_descriptor, row_descriptor,
not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts,
- table_info_node_ptr);
+ table_info_node_ptr, column_ids, filter_column_ids);
+}
+
+ColumnIdResult HiveOrcReader::_create_column_ids(const orc::Type* orc_type,
+ const TupleDescriptor* tuple_descriptor) {
+ if (!orc_type) {
+ return ColumnIdResult();
+ }
+
+ // map top-level table column name (lower-cased) -> orc::Type*
+ std::unordered_map<std::string, const orc::Type*> table_col_name_to_orc_type_map;
+ for (uint64_t i = 0; i < orc_type->getSubtypeCount(); ++i) {
+ auto orc_sub_type = orc_type->getSubtype(i);
+ if (!orc_sub_type) continue;
+
+ std::string table_col_name = to_lower(orc_type->getFieldName(i));
+ table_col_name_to_orc_type_map[table_col_name] = orc_sub_type;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level orc field
+ auto process_access_paths = [](const orc::Type* orc_field,
+ const std::vector<TColumnAccessPath>& access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!orc_field) return;
+ if (access_paths.empty()) return;
+
+ std::vector<std::vector<std::string>> paths;
+ bool has_top_level_only = false;
+ for (const auto& access_path : access_paths) {
+ std::vector<std::string> path;
+ if (access_path.type == TAccessPathType::DATA) {
+ path = access_path.data_access_path.path;
+ } else if (access_path.type == TAccessPathType::META) {
+ path = access_path.meta_access_path.path;
+ } else {
+ continue;
+ }
+ DCHECK(path.size() >= 1);
+ std::vector<std::string> remaining_path;
+ if (path.size() > 1) {
+ remaining_path.assign(path.begin() + 1, path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path = std::vector<std::string>();
+ }
+ if (remaining_path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = orc_field->getColumnId();
+ uint64_t max_column_id = orc_field->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ out_ids.insert(orc_field->getColumnId());
+ HiveOrcNestedColumnUtils::extract_nested_column_ids(*orc_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ auto it = table_col_name_to_orc_type_map.find(slot->col_name());
+ if (it == table_col_name_to_orc_type_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ const orc::Type* orc_field = it->second;
+
+ const auto& all_access_paths = slot->all_access_paths();
+
+ // primitive (non-nested) types: direct mapping by name
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(orc_field->getColumnId());
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(orc_field->getColumnId());
+ }
+ continue;
+ }
+
+ // complex types:
+
+ // collect and process all_access_paths -> column_ids
+ if (!all_access_paths.empty()) {
+ process_access_paths(orc_field, all_access_paths, column_ids);
+ }
+
+ // collect and process predicate_access_paths -> filter_column_ids
+ const auto& predicate_access_paths = slot->predicate_access_paths();
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
+}
+
+ColumnIdResult HiveOrcReader::_create_column_ids_by_top_level_col_index(
+ const orc::Type* orc_type, const TupleDescriptor* tuple_descriptor) {
+ std::shared_ptr<TableSchemaChangeHelper::Node> schema_node = nullptr;
+
+ if (!orc_type) {
+ return ColumnIdResult();
+ }
+
+ // map top-level table column index -> orc::Type*
+ std::unordered_map<uint64_t, const orc::Type*> table_col_pos_to_orc_type_map;
+ for (uint64_t i = 0; i < orc_type->getSubtypeCount(); ++i) {
+ auto orc_sub_type = orc_type->getSubtype(i);
+ if (!orc_sub_type) continue;
+
+ table_col_pos_to_orc_type_map[i] = orc_sub_type;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level orc field
+ auto process_access_paths = [](const orc::Type* orc_field,
+ const std::vector<TColumnAccessPath>& access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!orc_field) return;
+ if (access_paths.empty()) return;
+
+ std::vector<std::vector<std::string>> paths;
+ bool has_top_level_only = false;
+ for (const auto& access_path : access_paths) {
+ std::vector<std::string> path;
+ if (access_path.type == TAccessPathType::DATA) {
+ path = access_path.data_access_path.path;
+ } else if (access_path.type == TAccessPathType::META) {
+ path = access_path.meta_access_path.path;
+ } else {
+ continue;
+ }
+ DCHECK(path.size() >= 1);
+ std::vector<std::string> remaining_path;
+ if (path.size() > 1) {
+ remaining_path.assign(path.begin() + 1, path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path = std::vector<std::string>();
+ }
+ if (remaining_path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = orc_field->getColumnId();
+ uint64_t max_column_id = orc_field->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ out_ids.insert(orc_field->getColumnId());
+ HiveOrcNestedColumnUtils::extract_nested_column_ids(*orc_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ auto it = table_col_pos_to_orc_type_map.find(slot->col_pos());
+ if (it == table_col_pos_to_orc_type_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ const orc::Type* orc_field = it->second;
+
+ const auto& all_access_paths = slot->all_access_paths();
+
+ // primitive (non-nested) types: direct mapping by pos
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(orc_field->getColumnId());
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(orc_field->getColumnId());
+ }
+ continue;
+ }
+
+ // complex types
+
+ // collect and process all_access_paths -> column_ids
+ if (!all_access_paths.empty()) {
+ process_access_paths(orc_field, all_access_paths, column_ids);
+ }
+
+ // collect and process predicate_access_paths -> filter_column_ids
+ const auto& predicate_access_paths = slot->predicate_access_paths();
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
}
Status HiveParquetReader::init_reader(
@@ -145,10 +360,238 @@ Status HiveParquetReader::init_reader(
}
}
- return parquet_reader->init_reader(read_table_col_names, table_col_name_to_value_range,
- conjuncts, tuple_descriptor, row_descriptor,
- colname_to_slot_id, not_single_slot_filter_conjuncts,
- slot_id_to_filter_conjuncts, table_info_node_ptr);
+ auto column_id_result = ColumnIdResult();
+ if (_state->query_options().hive_parquet_use_column_names) {
+ column_id_result = _create_column_ids(field_desc, tuple_descriptor);
+ } else {
+ column_id_result = _create_column_ids_by_top_level_col_index(field_desc, tuple_descriptor);
+ }
+
+ const auto& column_ids = column_id_result.column_ids;
+ const auto& filter_column_ids = column_id_result.filter_column_ids;
+
+ RETURN_IF_ERROR(init_row_filters());
+
+ return parquet_reader->init_reader(
+ read_table_col_names, table_col_name_to_value_range, conjuncts, tuple_descriptor,
+ row_descriptor, colname_to_slot_id, not_single_slot_filter_conjuncts,
+ slot_id_to_filter_conjuncts, table_info_node_ptr, true, column_ids, filter_column_ids);
+}
+
+ColumnIdResult HiveParquetReader::_create_column_ids(const FieldDescriptor* field_desc,
+ const TupleDescriptor* tuple_descriptor) {
+ if (!field_desc) {
+ return ColumnIdResult();
+ }
+
+ // First, assign column IDs to the field descriptor
+ auto* mutable_field_desc = const_cast<FieldDescriptor*>(field_desc);
+ mutable_field_desc->assign_ids();
+
+ std::unordered_map<std::string, const FieldSchema*> table_col_name_to_field_schema_map;
+ for (int i = 0; i < field_desc->size(); ++i) {
+ auto field_schema = field_desc->get_column(i);
+ if (!field_schema) continue;
+
+ table_col_name_to_field_schema_map[field_schema->lower_case_name] = field_schema;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level parquet field
+ auto process_access_paths = [](const FieldSchema* parquet_field,
+ const std::vector<TColumnAccessPath>& access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!parquet_field) return;
+ if (access_paths.empty()) return;
+
+ std::vector<std::vector<std::string>> paths;
+ bool has_top_level_only = false;
+ for (const auto& access_path : access_paths) {
+ std::vector<std::string> path;
+ if (access_path.type == TAccessPathType::DATA) {
+ path = access_path.data_access_path.path;
+ } else if (access_path.type == TAccessPathType::META) {
+ path = access_path.meta_access_path.path;
+ } else {
+ continue;
+ }
+ DCHECK(path.size() >= 1);
+ std::vector<std::string> remaining_path;
+ if (path.size() > 1) {
+ remaining_path.assign(path.begin() + 1, path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path = std::vector<std::string>();
+ }
+ if (remaining_path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = parquet_field->get_column_id();
+ uint64_t max_column_id = parquet_field->get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ out_ids.insert(parquet_field->get_column_id());
+ HiveParquetNestedColumnUtils::extract_nested_column_ids(*parquet_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ // if (slot->col_name().starts_with(BeConsts::GLOBAL_ROWID_COL)) {
+ // continue;
+ // }
+
+ // Find the field schema for this slot (may not exist for partition columns, etc.)
+ auto it = table_col_name_to_field_schema_map.find(slot->col_name());
+ if (it == table_col_name_to_field_schema_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ auto field_schema = it->second;
+
+ const auto& all_access_paths = slot->all_access_paths();
+
+ // primitive (non-nested) types: direct mapping by name
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(field_schema->column_id);
+
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(field_schema->column_id);
+ }
+ continue;
+ }
+
+ // complex types:
+
+ // collect and process all_access_paths -> column_ids
+ if (!all_access_paths.empty()) {
+ process_access_paths(field_schema, all_access_paths, column_ids);
+ }
+
+ // collect and process predicate_access_paths -> filter_column_ids
+ const auto& predicate_access_paths = slot->predicate_access_paths();
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
+}
+
+ColumnIdResult HiveParquetReader::_create_column_ids_by_top_level_col_index(
+ const FieldDescriptor* field_desc, const TupleDescriptor* tuple_descriptor) {
+ std::shared_ptr<TableSchemaChangeHelper::Node> schema_node = nullptr;
+
+ if (!field_desc) {
+ return ColumnIdResult();
+ }
+
+ // First, assign column IDs to the field descriptor
+ auto* mutable_field_desc = const_cast<FieldDescriptor*>(field_desc);
+ mutable_field_desc->assign_ids();
+
+ std::unordered_map<uint64_t, const FieldSchema*> table_col_pos_to_field_schema_map;
+ for (int i = 0; i < field_desc->size(); ++i) {
+ auto field_schema = field_desc->get_column(i);
+ if (!field_schema) continue;
+
+ table_col_pos_to_field_schema_map[i] = field_schema;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level parquet field
+ auto process_access_paths = [](const FieldSchema* parquet_field,
+ const std::vector<TColumnAccessPath>& access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!parquet_field) return;
+ if (access_paths.empty()) return;
+
+ std::vector<std::vector<std::string>> paths;
+ bool has_top_level_only = false;
+ for (const auto& access_path : access_paths) {
+ std::vector<std::string> path;
+ if (access_path.type == TAccessPathType::DATA) {
+ path = access_path.data_access_path.path;
+ } else if (access_path.type == TAccessPathType::META) {
+ path = access_path.meta_access_path.path;
+ } else {
+ continue;
+ }
+ DCHECK(path.size() >= 1);
+ std::vector<std::string> remaining_path;
+ if (path.size() > 1) {
+ remaining_path.assign(path.begin() + 1, path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path = std::vector<std::string>();
+ }
+ if (remaining_path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = parquet_field->get_column_id();
+ uint64_t max_column_id = parquet_field->get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ out_ids.insert(parquet_field->get_column_id());
+ HiveParquetNestedColumnUtils::extract_nested_column_ids(*parquet_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ // if (slot->col_name().starts_with(BeConsts::GLOBAL_ROWID_COL)) {
+ // continue;
+ // }
+
+ // Find the field schema for this slot (may not exist for partition columns, etc.)
+ auto it = table_col_pos_to_field_schema_map.find(slot->col_pos());
+ if (it == table_col_pos_to_field_schema_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ auto field_schema = it->second;
+
+ const auto& all_access_paths = slot->all_access_paths();
+
+ // primitive (non-nested) types: direct mapping by position
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(field_schema->column_id);
+
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(field_schema->column_id);
+ }
+ continue;
+ }
+
+ // collect and process all_access_paths -> column_ids
+ if (!all_access_paths.empty()) {
+ process_access_paths(field_schema, all_access_paths, column_ids);
+ }
+
+ // collect and process predicate_access_paths -> filter_column_ids
+ const auto& predicate_access_paths = slot->predicate_access_paths();
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
}
#include "common/compile_check_end.h"
diff --git a/be/src/vec/exec/format/table/hive_reader.h b/be/src/vec/exec/format/table/hive_reader.h
index 6acd344e417..b9592c43243 100644
--- a/be/src/vec/exec/format/table/hive_reader.h
+++ b/be/src/vec/exec/format/table/hive_reader.h
@@ -66,6 +66,13 @@ public:
const RowDescriptor* row_descriptor,
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts);
+
+private:
+ static ColumnIdResult _create_column_ids(const orc::Type* orc_type,
+ const TupleDescriptor* tuple_descriptor);
+
+ static ColumnIdResult _create_column_ids_by_top_level_col_index(
+ const orc::Type* orc_type, const TupleDescriptor* tuple_descriptor);
};
class HiveParquetReader final : public HiveReader {
@@ -88,6 +95,13 @@ public:
const std::unordered_map<std::string, int>* colname_to_slot_id,
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts);
+
+private:
+ static ColumnIdResult _create_column_ids(const FieldDescriptor* field_desc,
+ const TupleDescriptor* tuple_descriptor);
+
+ static ColumnIdResult _create_column_ids_by_top_level_col_index(
+ const FieldDescriptor* field_desc, const TupleDescriptor* tuple_descriptor);
};
#include "common/compile_check_end.h"
} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.cpp b/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.cpp
new file mode 100644
index 00000000000..5dbbbfb2af2
--- /dev/null
+++ b/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.cpp
@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "orc/Type.hh"
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+void IcebergOrcNestedColumnUtils::extract_nested_column_ids(
+ const orc::Type& type, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ // Group paths by first field_id
+ std::unordered_map<std::string, std::vector<std::vector<std::string>>> child_paths_by_field_id;
+
+ for (const auto& path : paths) {
+ if (!path.empty()) {
+ std::string first_field_id = path[0];
+ std::vector<std::string> remaining;
+ if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ child_paths_by_field_id[first_field_id].push_back(std::move(remaining));
+ }
+ }
+
+ // For MAP type, normalize wildcard "*" to explicit KEYS/VALUES access
+ // Wildcard in MAP context means accessing both map keys and values
+ // Normalization logic:
+ // path: ["map_col", "*"] → ["map_col", "VALUES"] + ["map_col", "KEYS"]
+ // path: ["map_col", "*", "field"] → ["map_col", "VALUES", "field"] + ["map_col", "KEYS"]
+ // KEYS are always needed for correct RL/DL computation when accessing MAP via wildcard
+ if (type.getKind() == orc::TypeKind::MAP) {
+ auto wildcard_it = child_paths_by_field_id.find("*");
+ if (wildcard_it != child_paths_by_field_id.end()) {
+ auto& wildcard_paths = wildcard_it->second;
+
+ // All wildcard paths go to VALUES
+ auto& values_paths = child_paths_by_field_id["VALUES"];
+ values_paths.insert(values_paths.end(), wildcard_paths.begin(), wildcard_paths.end());
+
+ // Always add KEYS for wildcard access (needed for RL/DL computation)
+ auto& keys_paths = child_paths_by_field_id["KEYS"];
+ // Add an empty path to request full KEYS
+ std::vector<std::string> empty_path;
+ keys_paths.push_back(empty_path);
+
+ // Remove wildcard entry as it's been expanded
+ child_paths_by_field_id.erase(wildcard_it);
+ }
+ }
+
+ bool has_child_columns = false;
+ // Efficiently traverse children
+ for (uint64_t i = 0; i < type.getSubtypeCount(); ++i) {
+ const orc::Type* child = type.getSubtype(i);
+
+ std::string child_field_id;
+ switch (type.getKind()) {
+ case orc::TypeKind::STRUCT:
+ if (!child->hasAttributeKey(ICEBERG_ORC_ATTRIBUTE)) {
+ continue;
+ }
+ child_field_id = child->getAttributeValue(ICEBERG_ORC_ATTRIBUTE);
+ break;
+ case orc::TypeKind::LIST:
+ child_field_id = "*";
+ break;
+ case orc::TypeKind::MAP:
+ // After wildcard normalization above, all MAP accesses are explicit KEYS/VALUES
+ // Simply assign the appropriate field name based on which child we're processing
+ if (i == 0) {
+ child_field_id = "KEYS";
+ } else if (i == 1) {
+ child_field_id = "VALUES";
+ }
+ break;
+ default:
+ child_field_id = "";
+ break;
+ }
+
+ if (child_field_id.empty() || child_field_id == "-1") {
+ continue;
+ }
+
+ auto child_paths_it = child_paths_by_field_id.find(child_field_id);
+ if (child_paths_it != child_paths_by_field_id.end()) {
+ const auto& child_paths = child_paths_it->second;
+
+ // Check if any child path is empty (meaning full child needed)
+ bool needs_full_child =
+ std::any_of(child_paths.begin(), child_paths.end(),
+ [](const std::vector<std::string>& path) { return path.empty(); });
+
+ if (needs_full_child) {
+ uint64_t start_id = child->getColumnId();
+ uint64_t max_column_id = child->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ } else {
+ // Store current size to check if recursive call added any columns
+ size_t before_size = column_ids.size();
+
+ // Recursively extract from child
+ extract_nested_column_ids(*child, child_paths, column_ids);
+
+ // Check if recursive call added any columns
+ if (column_ids.size() > before_size) {
+ has_child_columns = true;
+ }
+ }
+ }
+ }
+
+ // If any child columns were added, also add the parent column ID
+ // This ensures parent struct/container nodes are included when their children are needed
+ if (has_child_columns) {
+ // Set automatically handles deduplication, so no need to check if it already exists
+ column_ids.insert(type.getColumnId());
+ }
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h b/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h
new file mode 100644
index 00000000000..576ec598970
--- /dev/null
+++ b/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace orc {
+class Type;
+} // namespace orc
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+class IcebergOrcNestedColumnUtils {
+public:
+ static void extract_nested_column_ids(const orc::Type& type,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+
+private:
+ static constexpr const char* ICEBERG_ORC_ATTRIBUTE = "iceberg.id";
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp b/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp
new file mode 100644
index 00000000000..cb2b2e6b8c7
--- /dev/null
+++ b/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp
@@ -0,0 +1,173 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h"
+
+#include <algorithm>
+#include <iostream>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/parquet/schema_desc.h"
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+void IcebergParquetNestedColumnUtils::extract_nested_column_ids(
+ const FieldSchema& field_schema, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ // Group paths by first field_id
+ std::unordered_map<std::string, std::vector<std::vector<std::string>>> child_paths_by_field_id;
+
+ for (const auto& path : paths) {
+ if (!path.empty()) {
+ std::string first_field_id = path[0];
+ std::vector<std::string> remaining;
+ if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ child_paths_by_field_id[first_field_id].push_back(std::move(remaining));
+ }
+ }
+
+ // Track whether any child column was added to determine if parent should be included
+ bool has_child_columns = false;
+
+ // For MAP type, normalize wildcard "*" to explicit KEYS/VALUES access
+ // Wildcard in MAP context means accessing both map keys and values
+ // Normalization logic:
+ // path: ["map_col", "*"] → ["map_col", "VALUES"] + ["map_col", "KEYS"]
+ // path: ["map_col", "*", "field"] → ["map_col", "VALUES", "field"] + ["map_col", "KEYS"]
+ // KEYS are always needed for correct RL/DL computation when accessing MAP via wildcard
+ if (field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_MAP) {
+ auto wildcard_it = child_paths_by_field_id.find("*");
+ if (wildcard_it != child_paths_by_field_id.end()) {
+ auto& wildcard_paths = wildcard_it->second;
+
+ // All wildcard paths go to VALUES
+ auto& values_paths = child_paths_by_field_id["VALUES"];
+ values_paths.insert(values_paths.end(), wildcard_paths.begin(), wildcard_paths.end());
+
+ // Always add KEYS for wildcard access (needed for RL/DL computation)
+ auto& keys_paths = child_paths_by_field_id["KEYS"];
+ // Add an empty path to request full KEYS
+ std::vector<std::string> empty_path;
+ keys_paths.push_back(empty_path);
+
+ // Remove wildcard entry as it's been expanded
+ child_paths_by_field_id.erase(wildcard_it);
+ }
+ }
+
+ // Efficiently traverse children
+ for (uint64_t i = 0; i < field_schema.children.size(); ++i) {
+ const auto& child = field_schema.children[i];
+
+ std::string child_field_id;
+
+ bool is_list = field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_ARRAY;
+ bool is_map = field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_MAP;
+
+ if (is_list) {
+ child_field_id = "*";
+ } else if (is_map) {
+ // After wildcard normalization above, all MAP accesses are explicit KEYS/VALUES
+ // Simply assign the appropriate field name based on which child we're processing
+ if (i == 0) {
+ child_field_id = "KEYS";
+ } else if (i == 1) {
+ child_field_id = "VALUES";
+ }
+
+ // Special handling for Parquet MAP structure:
+ // When accessing only VALUES, we still need KEY structure for levels
+ // Check if we're at key child (i==0) and only VALUES is requested (no KEYS)
+ if (i == 0) {
+ bool has_keys_access =
+ child_paths_by_field_id.find("KEYS") != child_paths_by_field_id.end();
+ bool has_values_access =
+ child_paths_by_field_id.find("VALUES") != child_paths_by_field_id.end();
+
+ // If only VALUES is accessed (not KEYS), still include key structure for RL/DL
+ if (!has_keys_access && has_values_access) {
+ // For map_values() queries, we need key's structure for correct RL/DL parsing.
+ // If key is a nested type (e.g., STRUCT), RL/DL info is stored at leaf columns.
+ // Add all column IDs from key's start to max (all leaves + intermediate nodes).
+ uint64_t key_start_id = child.get_column_id();
+ uint64_t key_max_id = child.get_max_column_id();
+ for (uint64_t id = key_start_id; id <= key_max_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ continue; // Skip further processing of key child
+ }
+ }
+
+ } else {
+ child_field_id = std::to_string(child.field_id);
+ }
+
+ if (child_field_id.empty() || child_field_id == "-1") {
+ continue;
+ }
+
+ auto child_paths_it = child_paths_by_field_id.find(child_field_id);
+ if (child_paths_it != child_paths_by_field_id.end()) {
+ const auto& child_paths = child_paths_it->second;
+
+ // Check if any child path is empty (meaning full child needed)
+ bool needs_full_child =
+ std::any_of(child_paths.begin(), child_paths.end(),
+ [](const std::vector<std::string>& path) { return path.empty(); });
+
+ if (needs_full_child) {
+ // Add all column IDs from current child node to max_column_id
+ // This efficiently handles all nested/complex cases in one loop
+ uint64_t start_id = child.get_column_id();
+ uint64_t max_column_id = child.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ } else {
+ // Store current size to check if recursive call added any columns
+ size_t before_size = column_ids.size();
+
+ // Recursively extract from child
+ extract_nested_column_ids(child, child_paths, column_ids);
+
+ // Check if recursive call added any columns
+ if (column_ids.size() > before_size) {
+ has_child_columns = true;
+ }
+ }
+ }
+ }
+
+ // If any child columns were added, also add the parent column ID
+ // This ensures parent struct/container nodes are included when their children are needed
+ if (has_child_columns) {
+ // Set automatically handles deduplication, so no need to check if it already exists
+ column_ids.insert(field_schema.get_column_id());
+ }
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h b/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h
new file mode 100644
index 00000000000..58237d8182b
--- /dev/null
+++ b/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+class FieldDescriptor;
+struct FieldSchema;
+
+class IcebergParquetNestedColumnUtils {
+public:
+ static void extract_nested_column_ids(const FieldSchema& field_schema,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/iceberg_reader.cpp b/be/src/vec/exec/format/table/iceberg_reader.cpp
index db6e44c3d4b..95aa41a43e6 100644
--- a/be/src/vec/exec/format/table/iceberg_reader.cpp
+++ b/be/src/vec/exec/format/table/iceberg_reader.cpp
@@ -17,6 +17,7 @@
#include "iceberg_reader.h"
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/Metrics_types.h>
#include <gen_cpp/PlanNodes_types.h>
#include <gen_cpp/parquet_types.h>
@@ -52,6 +53,8 @@
#include "vec/exec/format/orc/vorc_reader.h"
#include "vec/exec/format/parquet/schema_desc.h"
#include "vec/exec/format/parquet/vparquet_column_chunk_reader.h"
+#include "vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h"
+#include "vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h"
#include "vec/exec/format/table/table_format_reader.h"
namespace cctz {
@@ -445,11 +448,134 @@ Status IcebergParquetReader::init_reader(
}
_all_required_col_names = file_col_names;
+
+ auto column_id_result = _create_column_ids(field_desc, tuple_descriptor);
+ // const auto& file_col_names = column_id_result.column_names;
+ auto& column_ids = column_id_result.column_ids;
+ const auto& filter_column_ids = column_id_result.filter_column_ids;
+
RETURN_IF_ERROR(init_row_filters());
- return parquet_reader->init_reader(_all_required_col_names, colname_to_value_range, conjuncts,
- tuple_descriptor, row_descriptor, colname_to_slot_id,
- not_single_slot_filter_conjuncts,
- slot_id_to_filter_conjuncts, table_info_node_ptr);
+ for (int i = 0; i < field_desc->size(); ++i) {
+ auto field_schema = field_desc->get_column(i);
+ std::string col_name = field_schema->name;
+ if (std::find(_expand_col_names.begin(), _expand_col_names.end(), col_name) !=
+ _expand_col_names.end()) {
+ column_ids.insert(field_schema->get_column_id());
+ }
+ }
+ return parquet_reader->init_reader(
+ _all_required_col_names, colname_to_value_range, conjuncts, tuple_descriptor,
+ row_descriptor, colname_to_slot_id, not_single_slot_filter_conjuncts,
+ slot_id_to_filter_conjuncts, table_info_node_ptr, true, column_ids, filter_column_ids);
+}
+
+ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* field_desc,
+ const TupleDescriptor* tuple_descriptor) {
+ if (!field_desc) {
+ return ColumnIdResult();
+ }
+
+ // First, assign column IDs to the field descriptor
+ auto* mutable_field_desc = const_cast<FieldDescriptor*>(field_desc);
+ mutable_field_desc->assign_ids();
+
+ // Create a mapping from iceberg field_id to FieldSchema for quick lookup
+ std::unordered_map<int, const FieldSchema*> iceberg_id_to_field_schema_map;
+
+ for (int i = 0; i < field_desc->size(); ++i) {
+ auto field_schema = field_desc->get_column(i);
+ if (!field_schema) continue;
+
+ int iceberg_id = field_schema->field_id;
+ iceberg_id_to_field_schema_map[iceberg_id] = field_schema;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level parquet field
+ auto process_access_paths = [](const FieldSchema* parquet_field,
+ const std::vector<TColumnAccessPath>& access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!parquet_field) return;
+ if (access_paths.empty()) return;
+
+ std::vector<std::vector<std::string>> paths;
+ bool has_top_level_only = false;
+ for (const auto& access_path : access_paths) {
+ std::vector<std::string> path;
+ if (access_path.type == TAccessPathType::DATA) {
+ path = access_path.data_access_path.path;
+ } else if (access_path.type == TAccessPathType::META) {
+ path = access_path.meta_access_path.path;
+ } else {
+ continue;
+ }
+ DCHECK(path.size() >= 1);
+ std::vector<std::string> remaining_path;
+ if (path.size() > 1) {
+ remaining_path.assign(path.begin() + 1, path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path = std::vector<std::string>();
+ }
+ if (remaining_path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = parquet_field->get_column_id();
+ uint64_t max_column_id = parquet_field->get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ out_ids.insert(parquet_field->get_column_id());
+ IcebergParquetNestedColumnUtils::extract_nested_column_ids(*parquet_field, paths,
+ out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ // if (slot->col_name().starts_with(BeConsts::GLOBAL_ROWID_COL)) {
+ // continue;
+ // }
+ auto it = iceberg_id_to_field_schema_map.find(slot->col_unique_id());
+ if (it == iceberg_id_to_field_schema_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ auto field_schema = it->second;
+
+ const auto& all_access_paths = slot->all_access_paths();
+
+ // primitive (non-nested) types: direct mapping by name
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(field_schema->column_id);
+
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(field_schema->column_id);
+ }
+ continue;
+ }
+
+ // complex types:
+
+ // collect and process all_access_paths -> column_ids
+ if (!all_access_paths.empty()) {
+ process_access_paths(field_schema, all_access_paths, column_ids);
+ }
+
+ // collect and process predicate_access_paths -> filter_column_ids
+ const auto& predicate_access_paths = slot->predicate_access_paths();
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
+ }
+ }
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
}
Status IcebergParquetReader ::_read_position_delete_file(const TFileRangeDesc* delete_range,
@@ -527,11 +653,128 @@ Status IcebergOrcReader::init_reader(
}
}
+ auto column_id_result = _create_column_ids(orc_type_ptr, tuple_descriptor);
+ auto& column_ids = column_id_result.column_ids;
+ const auto& filter_column_ids = column_id_result.filter_column_ids;
+
RETURN_IF_ERROR(init_row_filters());
+ for (uint64_t i = 0; i < orc_type_ptr->getSubtypeCount(); ++i) {
+ const orc::Type* sub_type = orc_type_ptr->getSubtype(i);
+ std::string col_name = orc_type_ptr->getFieldName(i);
+ if (std::find(_expand_col_names.begin(), _expand_col_names.end(), col_name) !=
+ _expand_col_names.end()) {
+ column_ids.insert(sub_type->getColumnId());
+ }
+ }
return orc_reader->init_reader(&_all_required_col_names, colname_to_value_range, conjuncts,
false, tuple_descriptor, row_descriptor,
not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts,
- table_info_node_ptr);
+ table_info_node_ptr, column_ids, filter_column_ids);
+}
+
+ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type,
+ const TupleDescriptor* tuple_descriptor) {
+ std::shared_ptr<TableSchemaChangeHelper::Node> schema_node = nullptr;
+
+ if (!orc_type) {
+ return ColumnIdResult();
+ }
+
+ // map top-level table column name (lower-cased) -> orc::Type*
+ std::unordered_map<int, const orc::Type*> iceberg_id_to_orc_type_map;
+ for (uint64_t i = 0; i < orc_type->getSubtypeCount(); ++i) {
+ auto orc_sub_type = orc_type->getSubtype(i);
+ if (!orc_sub_type) continue;
+
+ if (!orc_sub_type->hasAttributeKey(ICEBERG_ORC_ATTRIBUTE)) {
+ continue;
+ }
+ int iceberg_id = std::stoi(orc_sub_type->getAttributeValue(ICEBERG_ORC_ATTRIBUTE));
+ iceberg_id_to_orc_type_map[iceberg_id] = orc_sub_type;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level orc field
+ auto process_access_paths = [](const orc::Type* orc_field,
+ const std::vector<TColumnAccessPath>& access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!orc_field) return;
+ if (access_paths.empty()) return;
+
+ std::vector<std::vector<std::string>> paths;
+ bool has_top_level_only = false;
+ for (const auto& access_path : access_paths) {
+ std::vector<std::string> path;
+ if (access_path.type == TAccessPathType::DATA) {
+ path = access_path.data_access_path.path;
+ } else if (access_path.type == TAccessPathType::META) {
+ path = access_path.meta_access_path.path;
+ } else {
+ continue;
+ }
+ DCHECK(path.size() >= 1);
+ std::vector<std::string> remaining_path;
+ if (path.size() > 1) {
+ remaining_path.assign(path.begin() + 1, path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path = std::vector<std::string>();
+ }
+ if (remaining_path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = orc_field->getColumnId();
+ uint64_t max_column_id = orc_field->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ out_ids.insert(orc_field->getColumnId());
+ IcebergOrcNestedColumnUtils::extract_nested_column_ids(*orc_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ auto it = iceberg_id_to_orc_type_map.find(slot->col_unique_id());
+ if (it == iceberg_id_to_orc_type_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ const orc::Type* orc_field = it->second;
+
+ const auto& all_access_paths = slot->all_access_paths();
+
+ // primitive (non-nested) types: direct mapping by name
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(orc_field->getColumnId());
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(orc_field->getColumnId());
+ }
+ continue;
+ }
+
+ // nested types:
+
+ // collect and process all_access_paths -> column_ids
+ if (!all_access_paths.empty()) {
+ process_access_paths(orc_field, all_access_paths, column_ids);
+ }
+
+ // collect and process predicate_access_paths -> filter_column_ids
+ const auto& predicate_access_paths = slot->predicate_access_paths();
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
}
Status IcebergOrcReader::_read_position_delete_file(const TFileRangeDesc* delete_range,
diff --git a/be/src/vec/exec/format/table/iceberg_reader.h b/be/src/vec/exec/format/table/iceberg_reader.h
index 6b0e3fe4b3e..d123261714e 100644
--- a/be/src/vec/exec/format/table/iceberg_reader.h
+++ b/be/src/vec/exec/format/table/iceberg_reader.h
@@ -187,6 +187,9 @@ protected:
}
private:
+ static ColumnIdResult _create_column_ids(const FieldDescriptor* field_desc,
+ const TupleDescriptor* tuple_descriptor);
+
Status _read_position_delete_file(const TFileRangeDesc* delete_range,
DeleteFile* position_delete) final;
};
@@ -226,6 +229,10 @@ protected:
_meta_cache);
}
+private:
+ static ColumnIdResult _create_column_ids(const orc::Type* orc_type,
+ const TupleDescriptor* tuple_descriptor);
+
private:
static const std::string ICEBERG_ORC_ATTRIBUTE;
};
diff --git a/be/src/vec/exec/format/table/table_format_reader.cpp b/be/src/vec/exec/format/table/table_format_reader.cpp
index b3b7e1df336..2c614777a89 100644
--- a/be/src/vec/exec/format/table/table_format_reader.cpp
+++ b/be/src/vec/exec/format/table/table_format_reader.cpp
@@ -76,8 +76,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(
if (file_field.data_type->get_primitive_type() != TYPE_MAP) [[unlikely]] {
return SCHEMA_ERROR;
}
- MOCK_REMOVE(DCHECK(file_field.children.size() == 1));
- MOCK_REMOVE(DCHECK(file_field.children[0].children.size() == 2));
+ MOCK_REMOVE(DCHECK(file_field.children.size() == 2));
std::shared_ptr<TableSchemaChangeHelper::Node> key_node = nullptr;
{
@@ -85,8 +84,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(
assert_cast<const DataTypeMap*>(remove_nullable(table_data_type).get())
->get_key_type());
- RETURN_IF_ERROR(
- by_parquet_name(key_type, file_field.children[0].children[0], key_node));
+ RETURN_IF_ERROR(by_parquet_name(key_type, file_field.children[0], key_node));
}
std::shared_ptr<TableSchemaChangeHelper::Node> value_node = nullptr;
@@ -95,8 +93,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(
assert_cast<const DataTypeMap*>(remove_nullable(table_data_type).get())
->get_value_type());
- RETURN_IF_ERROR(
- by_parquet_name(value_type, file_field.children[0].children[1], value_node));
+ RETURN_IF_ERROR(by_parquet_name(value_type, file_field.children[1], value_node));
}
node = std::make_shared<TableSchemaChangeHelper::MapNode>(key_node, value_node);
break;
@@ -430,19 +427,17 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id(
MOCK_REMOVE(DCHECK(table_schema.nestedField.map_field.key_field.field_ptr != nullptr));
MOCK_REMOVE(DCHECK(table_schema.nestedField.map_field.value_field.field_ptr != nullptr));
- MOCK_REMOVE(DCHECK(parquet_field.children.size() == 1));
- MOCK_REMOVE(DCHECK(parquet_field.children[0].children.size() == 2));
+ MOCK_REMOVE(DCHECK(parquet_field.children.size() == 2));
std::shared_ptr<TableSchemaChangeHelper::Node> key_node = nullptr;
std::shared_ptr<TableSchemaChangeHelper::Node> value_node = nullptr;
RETURN_IF_ERROR(by_parquet_field_id(*table_schema.nestedField.map_field.key_field.field_ptr,
- parquet_field.children[0].children[0], key_node,
- exist_field_id));
+ parquet_field.children[0], key_node, exist_field_id));
- RETURN_IF_ERROR(by_parquet_field_id(
- *table_schema.nestedField.map_field.value_field.field_ptr,
- parquet_field.children[0].children[1], value_node, exist_field_id));
+ RETURN_IF_ERROR(
+ by_parquet_field_id(*table_schema.nestedField.map_field.value_field.field_ptr,
+ parquet_field.children[1], value_node, exist_field_id));
node = std::make_shared<TableSchemaChangeHelper::MapNode>(key_node, value_node);
break;
diff --git a/be/src/vec/exec/format/table/table_format_reader.h b/be/src/vec/exec/format/table/table_format_reader.h
index 1f2a41af930..00442dc45f9 100644
--- a/be/src/vec/exec/format/table/table_format_reader.h
+++ b/be/src/vec/exec/format/table/table_format_reader.h
@@ -133,6 +133,13 @@ public:
throw std::logic_error("get_children_node should not be called on base TableInfoNode");
};
+ virtual std::shared_ptr<Node> get_children_node_by_file_column_name(
+ std::string file_column_name) const {
+ throw std::logic_error(
+ "get_children_node_by_file_column_name should not be called on base "
+ "TableInfoNode");
+ };
+
virtual std::string children_file_column_name(std::string table_column_name) const {
throw std::logic_error(
"children_file_column_name should not be called on base TableInfoNode");
@@ -184,6 +191,19 @@ public:
return children.at(table_column_name).node;
}
+ std::shared_ptr<Node> get_children_node_by_file_column_name(
+ std::string file_column_name) const override {
+ // Search for the child by file column name
+ for (const auto& [table_name, child] : children) {
+ if (child.exists && child.column_name == file_column_name) {
+ return child.node;
+ }
+ }
+ // Not found - throw or return nullptr
+ throw std::runtime_error("File column name '" + file_column_name +
+ "' not found in struct children");
+ }
+
std::string children_file_column_name(std::string table_column_name) const override {
DCHECK(children.contains(table_column_name));
DCHECK(children_column_exists(table_column_name));
@@ -238,6 +258,11 @@ public:
return get_instance();
};
+ std::shared_ptr<Node> get_children_node_by_file_column_name(
+ std::string file_column_name) const override {
+ return get_instance();
+ };
+
std::string children_file_column_name(std::string table_column_name) const override {
return table_column_name;
}
@@ -388,5 +413,16 @@ public:
};
};
+struct ColumnIdResult {
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ ColumnIdResult() = default; // Add default constructor
+
+ ColumnIdResult(std::set<uint64_t> column_ids_, std::set<uint64_t> filter_column_ids_)
+ : column_ids(std::move(column_ids_)),
+ filter_column_ids(std::move(filter_column_ids_)) {}
+};
+
#include "common/compile_check_end.h"
} // namespace doris::vectorized
diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp
index e5c2104d7d0..ba8365616d5 100644
--- a/be/src/vec/exec/scan/file_scanner.cpp
+++ b/be/src/vec/exec/scan/file_scanner.cpp
@@ -335,7 +335,7 @@ Status FileScanner::_process_runtime_filters_partition_prune(bool& can_filter_al
return Status::OK();
}
-Status FileScanner::_process_conjuncts_for_dict_filter() {
+Status FileScanner::_process_conjuncts() {
_slot_id_to_filter_conjuncts.clear();
_not_single_slot_filter_conjuncts.clear();
for (auto& conjunct : _push_down_conjuncts) {
@@ -373,7 +373,7 @@ Status FileScanner::_process_late_arrival_conjuncts() {
for (size_t i = 0; i != _conjuncts.size(); ++i) {
RETURN_IF_ERROR(_conjuncts[i]->clone(_state, _push_down_conjuncts[i]));
}
- RETURN_IF_ERROR(_process_conjuncts_for_dict_filter());
+ RETURN_IF_ERROR(_process_conjuncts());
_discard_conjuncts();
}
if (_applied_rf_num == _total_rf_num) {
@@ -386,6 +386,8 @@ void FileScanner::_get_slot_ids(VExpr* expr, std::vector<int>* slot_ids) {
for (auto& child_expr : expr->children()) {
if (child_expr->is_slot_ref()) {
VSlotRef* slot_ref = reinterpret_cast<VSlotRef*>(child_expr.get());
+ SlotDescriptor* slot_desc = _state->desc_tbl().get_slot_descriptor(slot_ref->slot_id());
+ slot_desc->set_is_predicate(true);
slot_ids->emplace_back(slot_ref->slot_id());
} else {
_get_slot_ids(child_expr.get(), slot_ids);
diff --git a/be/src/vec/exec/scan/file_scanner.h b/be/src/vec/exec/scan/file_scanner.h
index 1a44d06c77b..8c4682ca89f 100644
--- a/be/src/vec/exec/scan/file_scanner.h
+++ b/be/src/vec/exec/scan/file_scanner.h
@@ -252,7 +252,7 @@ private:
void _init_runtime_filter_partition_prune_ctxs();
void _init_runtime_filter_partition_prune_block();
Status _process_runtime_filters_partition_prune(bool& is_partition_pruned);
- Status _process_conjuncts_for_dict_filter();
+ Status _process_conjuncts();
Status _process_late_arrival_conjuncts();
void _get_slot_ids(VExpr* expr, std::vector<int>* slot_ids);
Status _generate_truncate_columns(bool need_to_get_parsed_schema);
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00000-0-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00000-0-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc
new file mode 100644
index 00000000000..832aaef64a0
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00000-0-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00001-1-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00001-1-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc
new file mode 100644
index 00000000000..36f589e393e
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00001-1-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00000-aa52c4c8-e0bd-4db6-b550-6323977446a4.metadata.json b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00000-aa52c4c8-e0bd-4db6-b550-6323977446a4.metadata.json
new file mode 100644
index 00000000000..97208990039
--- /dev/null
+++ b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00000-aa52c4c8-e0bd-4db6-b550-6323977446a4.metadata.json
@@ -0,0 +1,145 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "daf54708-3539-4de4-a130-322cd4c3fdc4",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg_orc",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1758865887090,
+ "last-column-id" : 18,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 4,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 7,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 8,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 9,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 10,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 11,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 5,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 14,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 15,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 6,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 16,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "orc",
+ "write.parquet.compression-codec" : "zlib"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00001-88a66011-3129-4bfb-9b4b-c1510c566d73.metadata.json b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00001-88a66011-3129-4bfb-9b4b-c1510c566d73.metadata.json
new file mode 100644
index 00000000000..12ec38bbdcd
--- /dev/null
+++ b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00001-88a66011-3129-4bfb-9b4b-c1510c566d73.metadata.json
@@ -0,0 +1,176 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "daf54708-3539-4de4-a130-322cd4c3fdc4",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg_orc",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1758865954986,
+ "last-column-id" : 18,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 4,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 7,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 8,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 9,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 10,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 11,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 5,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 14,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 15,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 6,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 16,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "orc",
+ "write.parquet.compression-codec" : "zlib"
+ },
+ "current-snapshot-id" : 2197982135324349212,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 2197982135324349212,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 2197982135324349212,
+ "timestamp-ms" : 1758865954986,
+ "summary" : {
+ "operation" : "append",
+ "spark.app.id" : "application_1738810850199_0674",
+ "added-data-files" : "2",
+ "added-records" : "10",
+ "added-files-size" : "4611",
+ "changed-partition-count" : "1",
+ "total-records" : "10",
+ "total-files-size" : "4611",
+ "total-data-files" : "2",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg_orc/metadata/snap-2197982135324349212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1758865954986,
+ "snapshot-id" : 2197982135324349212
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1758865887090,
+ "metadata-file" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg_orc/metadata/00000-aa52c4c8-e0bd-4db6-b550-6323977446a4.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/6a876742-85c2-4f84-ae24-292482fa9288-m0.avro b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/6a876742-85c2-4f84-ae24-292482fa9288-m0.avro
new file mode 100644
index 00000000000..1887220ebf0
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/6a876742-85c2-4f84-ae24-292482fa9288-m0.avro differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/snap-2197982135324349212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/snap-2197982135324349212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro
new file mode 100644
index 00000000000..7218d1326d3
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/snap-2197982135324349212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00000-0-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00000-0-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..160d000fed6
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00000-0-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00001-1-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00001-1-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..3ec61d58469
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00001-1-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00002-2-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00002-2-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..b0f56644ae5
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00002-2-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00003-3-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00003-3-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..dee411d7ef7
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00003-3-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00004-4-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00004-4-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..4264e841218
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00004-4-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00005-5-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00005-5-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..0264972b11d
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00005-5-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00006-6-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00006-6-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..3db75d8c01a
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00006-6-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00007-7-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00007-7-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..4838bb7d6e7
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00007-7-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00000-30938ed9-31af-46c8-9274-506c7f2709c9.metadata.json b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00000-30938ed9-31af-46c8-9274-506c7f2709c9.metadata.json
new file mode 100644
index 00000000000..e6fc794e81f
--- /dev/null
+++ b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00000-30938ed9-31af-46c8-9274-506c7f2709c9.metadata.json
@@ -0,0 +1,145 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "baf92432-1443-47fd-971a-0a9df2432e4a",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1758188914000,
+ "last-column-id" : 18,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 4,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 7,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 8,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 9,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 10,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 11,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 5,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 14,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 15,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 6,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 16,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "parquet",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00001-b1b4732d-f823-4900-8185-3eb3cb23e4a9.metadata.json b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00001-b1b4732d-f823-4900-8185-3eb3cb23e4a9.metadata.json
new file mode 100644
index 00000000000..d52c580cbf2
--- /dev/null
+++ b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00001-b1b4732d-f823-4900-8185-3eb3cb23e4a9.metadata.json
@@ -0,0 +1,176 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "baf92432-1443-47fd-971a-0a9df2432e4a",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1758188959726,
+ "last-column-id" : 18,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 4,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 7,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 8,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 9,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 10,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 11,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 5,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 14,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 15,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 6,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 16,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "parquet",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : 1689809712605583552,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 1689809712605583552,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 1689809712605583552,
+ "timestamp-ms" : 1758188959726,
+ "summary" : {
+ "operation" : "append",
+ "spark.app.id" : "application_1738810850199_0668",
+ "added-data-files" : "8",
+ "added-records" : "10",
+ "added-files-size" : "32551",
+ "changed-partition-count" : "1",
+ "total-records" : "10",
+ "total-files-size" : "32551",
+ "total-data-files" : "8",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg/metadata/snap-1689809712605583552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1758188959726,
+ "snapshot-id" : 1689809712605583552
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1758188914000,
+ "metadata-file" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg/metadata/00000-30938ed9-31af-46c8-9274-506c7f2709c9.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/51fb6058-7fc2-4791-96a9-f3111933728a-m0.avro b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/51fb6058-7fc2-4791-96a9-f3111933728a-m0.avro
new file mode 100644
index 00000000000..201c7908bc4
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/51fb6058-7fc2-4791-96a9-f3111933728a-m0.avro differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/snap-1689809712605583552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/snap-1689809712605583552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro
new file mode 100644
index 00000000000..1f6548d7ba8
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/snap-1689809712605583552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro differ
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/data/00000-8-5a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/data/00000-8-5a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc
new file mode 100644
index 00000000000..2f63195ff34
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/data/00000-8-5a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc differ
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00000-daa20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00000-daa20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json
new file mode 100644
index 00000000000..174f7a92c79
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00000-daa20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json
@@ -0,0 +1,548 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "a93bcb6d-568f-4fc2-8235-dc8e218c1beb",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_orc",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1760884376565,
+ "last-column-id" : 92,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 9,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 14,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 15,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 16,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 10,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 19,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 20,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 11,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 21,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 22,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 23,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ }, {
+ "id" : 4,
+ "name" : "tags",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 24,
+ "element" : "string",
+ "element-required" : false
+ }
+ }, {
+ "id" : 5,
+ "name" : "friends",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 25,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 26,
+ "name" : "user_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 27,
+ "name" : "nickname",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 28,
+ "name" : "friendship_level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 6,
+ "name" : "recent_activity",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 29,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 30,
+ "name" : "action",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 31,
+ "name" : "details",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 32,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 33,
+ "name" : "key",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 34,
+ "name" : "value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 7,
+ "name" : "attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 35,
+ "key" : "string",
+ "value-id" : 36,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 8,
+ "name" : "complex_attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 37,
+ "key" : "string",
+ "value-id" : 38,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 39,
+ "name" : "metadata",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 43,
+ "name" : "version",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 44,
+ "name" : "created_time",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 45,
+ "name" : "last_updated",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 46,
+ "name" : "quality_score",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ }, {
+ "id" : 40,
+ "name" : "historical_scores",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 47,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 48,
+ "name" : "period",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 49,
+ "name" : "score",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 50,
+ "name" : "components",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 52,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 53,
+ "name" : "component_name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 54,
+ "name" : "weight",
+ "required" : false,
+ "type" : "float"
+ }, {
+ "id" : 55,
+ "name" : "sub_scores",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 56,
+ "key" : "string",
+ "value-id" : 57,
+ "value" : "double",
+ "value-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 51,
+ "name" : "trends",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 58,
+ "name" : "direction",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 59,
+ "name" : "magnitude",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 60,
+ "name" : "confidence_interval",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 61,
+ "name" : "lower",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 62,
+ "name" : "upper",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 41,
+ "name" : "hierarchical_data",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 63,
+ "key" : "string",
+ "value-id" : 64,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 65,
+ "name" : "category",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 66,
+ "name" : "sub_items",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 68,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 69,
+ "name" : "item_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 70,
+ "name" : "properties",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 72,
+ "key" : "string",
+ "value-id" : 73,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 71,
+ "name" : "metrics",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 74,
+ "name" : "value",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 75,
+ "name" : "unit",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 67,
+ "name" : "summary",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 76,
+ "name" : "total_count",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 77,
+ "name" : "average_value",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ }, {
+ "id" : 42,
+ "name" : "validation_rules",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 78,
+ "name" : "required",
+ "required" : false,
+ "type" : "boolean"
+ }, {
+ "id" : 79,
+ "name" : "constraints",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 81,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 82,
+ "name" : "rule_type",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 83,
+ "name" : "parameters",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 85,
+ "key" : "string",
+ "value-id" : 86,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 84,
+ "name" : "error_message",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 80,
+ "name" : "complex_constraint",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 87,
+ "name" : "logic_operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 88,
+ "name" : "operands",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 89,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 90,
+ "name" : "field_path",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 91,
+ "name" : "operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 92,
+ "name" : "comparison_value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "orc",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00001-5c6ddc20-6aa8-4864-9d15-546dd5a6bae4.metadata.json b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00001-5c6ddc20-6aa8-4864-9d15-546dd5a6bae4.metadata.json
new file mode 100644
index 00000000000..3fc692fea56
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00001-5c6ddc20-6aa8-4864-9d15-546dd5a6bae4.metadata.json
@@ -0,0 +1,579 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "a93bcb6d-568f-4fc2-8235-dc8e218c1beb",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_orc",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1760884424603,
+ "last-column-id" : 92,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 9,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 14,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 15,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 16,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 10,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 19,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 20,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 11,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 21,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 22,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 23,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ }, {
+ "id" : 4,
+ "name" : "tags",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 24,
+ "element" : "string",
+ "element-required" : false
+ }
+ }, {
+ "id" : 5,
+ "name" : "friends",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 25,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 26,
+ "name" : "user_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 27,
+ "name" : "nickname",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 28,
+ "name" : "friendship_level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 6,
+ "name" : "recent_activity",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 29,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 30,
+ "name" : "action",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 31,
+ "name" : "details",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 32,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 33,
+ "name" : "key",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 34,
+ "name" : "value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 7,
+ "name" : "attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 35,
+ "key" : "string",
+ "value-id" : 36,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 8,
+ "name" : "complex_attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 37,
+ "key" : "string",
+ "value-id" : 38,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 39,
+ "name" : "metadata",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 43,
+ "name" : "version",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 44,
+ "name" : "created_time",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 45,
+ "name" : "last_updated",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 46,
+ "name" : "quality_score",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ }, {
+ "id" : 40,
+ "name" : "historical_scores",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 47,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 48,
+ "name" : "period",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 49,
+ "name" : "score",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 50,
+ "name" : "components",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 52,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 53,
+ "name" : "component_name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 54,
+ "name" : "weight",
+ "required" : false,
+ "type" : "float"
+ }, {
+ "id" : 55,
+ "name" : "sub_scores",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 56,
+ "key" : "string",
+ "value-id" : 57,
+ "value" : "double",
+ "value-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 51,
+ "name" : "trends",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 58,
+ "name" : "direction",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 59,
+ "name" : "magnitude",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 60,
+ "name" : "confidence_interval",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 61,
+ "name" : "lower",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 62,
+ "name" : "upper",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 41,
+ "name" : "hierarchical_data",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 63,
+ "key" : "string",
+ "value-id" : 64,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 65,
+ "name" : "category",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 66,
+ "name" : "sub_items",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 68,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 69,
+ "name" : "item_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 70,
+ "name" : "properties",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 72,
+ "key" : "string",
+ "value-id" : 73,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 71,
+ "name" : "metrics",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 74,
+ "name" : "value",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 75,
+ "name" : "unit",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 67,
+ "name" : "summary",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 76,
+ "name" : "total_count",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 77,
+ "name" : "average_value",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ }, {
+ "id" : 42,
+ "name" : "validation_rules",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 78,
+ "name" : "required",
+ "required" : false,
+ "type" : "boolean"
+ }, {
+ "id" : 79,
+ "name" : "constraints",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 81,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 82,
+ "name" : "rule_type",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 83,
+ "name" : "parameters",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 85,
+ "key" : "string",
+ "value-id" : 86,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 84,
+ "name" : "error_message",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 80,
+ "name" : "complex_constraint",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 87,
+ "name" : "logic_operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 88,
+ "name" : "operands",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 89,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 90,
+ "name" : "field_path",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 91,
+ "name" : "operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 92,
+ "name" : "comparison_value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "orc",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : 514179801967160228,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 514179801967160228,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 514179801967160228,
+ "timestamp-ms" : 1760884424603,
+ "summary" : {
+ "operation" : "append",
+ "spark.app.id" : "application_1738810850199_0681",
+ "added-data-files" : "1",
+ "added-records" : "1",
+ "added-files-size" : "7181",
+ "changed-partition-count" : "1",
+ "total-records" : "1",
+ "total-files-size" : "7181",
+ "total-data-files" : "1",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_orc/metadata/snap-514179801967160228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1760884424603,
+ "snapshot-id" : 514179801967160228
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1760884376565,
+ "metadata-file" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_orc/metadata/00000-daa20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/d9aed717-61ef-422b-b044-a16ca31f9e86-m0.avro b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/d9aed717-61ef-422b-b044-a16ca31f9e86-m0.avro
new file mode 100644
index 00000000000..d8b578a8f86
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/d9aed717-61ef-422b-b044-a16ca31f9e86-m0.avro
@@ -0,0 +1 @@
+Obj schema�]{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"long"},{"id":2,"name":"name","required":false,"type":"string"},{"id":3,"name":"profile","required":false,"type":{"type":"struct","fields":[{"id":9,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":12,"name":"street","required":false,"type":"string"},{"id":13,"name":"city","required":false,"type":"string"},{"id":14,"name":"coordinates","required":false,"type":{"type":"struc [...]
format-version2"partition-spec-id0 iceberg.schema�+{"type":"struct","schema-id":0,"fields":[{"id":0,"name":"status","required":true,"type":"int"},{"id":1,"name":"snapshot_id","required":false,"type":"long"},{"id":3,"name":"sequence_number","required":false,"type":"long"},{"id":4,"name":"file_sequence_number","required":false,"type":"long"},{"id":2,"name":"data_file","required":true,"type":{"type":"struct","fields":[{"id":134,"name":"content","required":true,"type":"int","doc":"Contents of the file: 0=data, 1=position delet [...]
partition-spec[]contentdata v�7�l���r���P�Kq=��l��XY~,� ð,�5;3� *baia��6_�kێͮ"!!!�co��{�B=��l��}8�s� 8̇�����m7 `��bg�4{dx��N7�̕N ��j`�9Q�cmz��A�[�]� s��{��r.6[A���7�n���3}��Vo�����7\3�S6�5�j�V}�5�Vq+ިeX=i٥�;y~�o/��H�I�"-я ��s� T�I��GA�A �J�* �,��1��'�8&�����}�:�\��.箨���������"X��d[,OD��\����ܐw7 ��p���C�#J?&����H�"���/(�M�%�^��!�^����[��H{G�}ʾ'��>�~@\dNp��8łӬp?gXr�s<�*k�s� ��gEqHA *@9�@����\�a���ul��������봻9����ѵ���.�a0 �r����P��6��Q��T*ò-;+��am�i���ݜ2�iW�親,�=<rbnr�_��O s2�v�7�
�l���r�
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/snap-514179801967160228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/snap-514179801967160228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro
new file mode 100644
index 00000000000..7621edf3b7f
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/snap-514179801967160228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro differ
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/data/00000-9-a7e0135f-d581-40e4-8d56-a929aded99e4-0-00001.parquet b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/data/00000-9-a7e0135f-d581-40e4-8d56-a929aded99e4-0-00001.parquet
new file mode 100644
index 00000000000..014ad377122
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/data/00000-9-a7e0135f-d581-40e4-8d56-a929aded99e4-0-00001.parquet differ
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00000-ed5ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00000-ed5ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json
new file mode 100644
index 00000000000..8bf743851ff
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00000-ed5ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json
@@ -0,0 +1,548 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "d3b72ae1-4880-4c3e-9111-ca1af3cc9c7c",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_parquet",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1760884395135,
+ "last-column-id" : 92,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 9,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 14,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 15,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 16,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 10,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 19,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 20,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 11,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 21,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 22,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 23,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ }, {
+ "id" : 4,
+ "name" : "tags",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 24,
+ "element" : "string",
+ "element-required" : false
+ }
+ }, {
+ "id" : 5,
+ "name" : "friends",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 25,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 26,
+ "name" : "user_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 27,
+ "name" : "nickname",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 28,
+ "name" : "friendship_level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 6,
+ "name" : "recent_activity",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 29,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 30,
+ "name" : "action",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 31,
+ "name" : "details",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 32,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 33,
+ "name" : "key",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 34,
+ "name" : "value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 7,
+ "name" : "attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 35,
+ "key" : "string",
+ "value-id" : 36,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 8,
+ "name" : "complex_attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 37,
+ "key" : "string",
+ "value-id" : 38,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 39,
+ "name" : "metadata",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 43,
+ "name" : "version",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 44,
+ "name" : "created_time",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 45,
+ "name" : "last_updated",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 46,
+ "name" : "quality_score",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ }, {
+ "id" : 40,
+ "name" : "historical_scores",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 47,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 48,
+ "name" : "period",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 49,
+ "name" : "score",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 50,
+ "name" : "components",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 52,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 53,
+ "name" : "component_name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 54,
+ "name" : "weight",
+ "required" : false,
+ "type" : "float"
+ }, {
+ "id" : 55,
+ "name" : "sub_scores",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 56,
+ "key" : "string",
+ "value-id" : 57,
+ "value" : "double",
+ "value-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 51,
+ "name" : "trends",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 58,
+ "name" : "direction",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 59,
+ "name" : "magnitude",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 60,
+ "name" : "confidence_interval",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 61,
+ "name" : "lower",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 62,
+ "name" : "upper",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 41,
+ "name" : "hierarchical_data",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 63,
+ "key" : "string",
+ "value-id" : 64,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 65,
+ "name" : "category",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 66,
+ "name" : "sub_items",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 68,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 69,
+ "name" : "item_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 70,
+ "name" : "properties",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 72,
+ "key" : "string",
+ "value-id" : 73,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 71,
+ "name" : "metrics",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 74,
+ "name" : "value",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 75,
+ "name" : "unit",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 67,
+ "name" : "summary",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 76,
+ "name" : "total_count",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 77,
+ "name" : "average_value",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ }, {
+ "id" : 42,
+ "name" : "validation_rules",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 78,
+ "name" : "required",
+ "required" : false,
+ "type" : "boolean"
+ }, {
+ "id" : 79,
+ "name" : "constraints",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 81,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 82,
+ "name" : "rule_type",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 83,
+ "name" : "parameters",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 85,
+ "key" : "string",
+ "value-id" : 86,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 84,
+ "name" : "error_message",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 80,
+ "name" : "complex_constraint",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 87,
+ "name" : "logic_operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 88,
+ "name" : "operands",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 89,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 90,
+ "name" : "field_path",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 91,
+ "name" : "operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 92,
+ "name" : "comparison_value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "parquet",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00001-a1156265-e89e-4c9f-a0fb-7f3075d4e8f9.metadata.json b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00001-a1156265-e89e-4c9f-a0fb-7f3075d4e8f9.metadata.json
new file mode 100644
index 00000000000..de90e000742
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00001-a1156265-e89e-4c9f-a0fb-7f3075d4e8f9.metadata.json
@@ -0,0 +1,579 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "d3b72ae1-4880-4c3e-9111-ca1af3cc9c7c",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_parquet",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1760884432817,
+ "last-column-id" : 92,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 9,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 14,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 15,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 16,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 10,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 19,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 20,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 11,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 21,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 22,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 23,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ }, {
+ "id" : 4,
+ "name" : "tags",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 24,
+ "element" : "string",
+ "element-required" : false
+ }
+ }, {
+ "id" : 5,
+ "name" : "friends",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 25,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 26,
+ "name" : "user_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 27,
+ "name" : "nickname",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 28,
+ "name" : "friendship_level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 6,
+ "name" : "recent_activity",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 29,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 30,
+ "name" : "action",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 31,
+ "name" : "details",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 32,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 33,
+ "name" : "key",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 34,
+ "name" : "value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 7,
+ "name" : "attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 35,
+ "key" : "string",
+ "value-id" : 36,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 8,
+ "name" : "complex_attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 37,
+ "key" : "string",
+ "value-id" : 38,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 39,
+ "name" : "metadata",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 43,
+ "name" : "version",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 44,
+ "name" : "created_time",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 45,
+ "name" : "last_updated",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 46,
+ "name" : "quality_score",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ }, {
+ "id" : 40,
+ "name" : "historical_scores",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 47,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 48,
+ "name" : "period",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 49,
+ "name" : "score",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 50,
+ "name" : "components",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 52,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 53,
+ "name" : "component_name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 54,
+ "name" : "weight",
+ "required" : false,
+ "type" : "float"
+ }, {
+ "id" : 55,
+ "name" : "sub_scores",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 56,
+ "key" : "string",
+ "value-id" : 57,
+ "value" : "double",
+ "value-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 51,
+ "name" : "trends",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 58,
+ "name" : "direction",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 59,
+ "name" : "magnitude",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 60,
+ "name" : "confidence_interval",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 61,
+ "name" : "lower",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 62,
+ "name" : "upper",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 41,
+ "name" : "hierarchical_data",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 63,
+ "key" : "string",
+ "value-id" : 64,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 65,
+ "name" : "category",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 66,
+ "name" : "sub_items",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 68,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 69,
+ "name" : "item_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 70,
+ "name" : "properties",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 72,
+ "key" : "string",
+ "value-id" : 73,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 71,
+ "name" : "metrics",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 74,
+ "name" : "value",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 75,
+ "name" : "unit",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 67,
+ "name" : "summary",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 76,
+ "name" : "total_count",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 77,
+ "name" : "average_value",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ }, {
+ "id" : 42,
+ "name" : "validation_rules",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 78,
+ "name" : "required",
+ "required" : false,
+ "type" : "boolean"
+ }, {
+ "id" : 79,
+ "name" : "constraints",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 81,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 82,
+ "name" : "rule_type",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 83,
+ "name" : "parameters",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 85,
+ "key" : "string",
+ "value-id" : 86,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 84,
+ "name" : "error_message",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 80,
+ "name" : "complex_constraint",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 87,
+ "name" : "logic_operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 88,
+ "name" : "operands",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 89,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 90,
+ "name" : "field_path",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 91,
+ "name" : "operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 92,
+ "name" : "comparison_value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "parquet",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : 6622777947381868002,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 6622777947381868002,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 6622777947381868002,
+ "timestamp-ms" : 1760884432817,
+ "summary" : {
+ "operation" : "append",
+ "spark.app.id" : "application_1738810850199_0681",
+ "added-data-files" : "1",
+ "added-records" : "1",
+ "added-files-size" : "21586",
+ "changed-partition-count" : "1",
+ "total-records" : "1",
+ "total-files-size" : "21586",
+ "total-data-files" : "1",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_parquet/metadata/snap-6622777947381868002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1760884432817,
+ "snapshot-id" : 6622777947381868002
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1760884395135,
+ "metadata-file" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_parquet/metadata/00000-ed5ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/bee22dff-a1d3-420d-bfba-c1b4193ab0df-m0.avro b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/bee22dff-a1d3-420d-bfba-c1b4193ab0df-m0.avro
new file mode 100644
index 00000000000..6071b684120
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/bee22dff-a1d3-420d-bfba-c1b4193ab0df-m0.avro
@@ -0,0 +1,5 @@
+Obj schema�]{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"long"},{"id":2,"name":"name","required":false,"type":"string"},{"id":3,"name":"profile","required":false,"type":{"type":"struct","fields":[{"id":9,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":12,"name":"street","required":false,"type":"string"},{"id":13,"name":"city","required":false,"type":"string"},{"id":14,"name":"coordinates","required":false,"type":{"type":"struc [...]
format-version2"partition-spec-id0 iceberg.schema�+{"type":"struct","schema-id":0,"fields":[{"id":0,"name":"status","required":true,"type":"int"},{"id":1,"name":"snapshot_id","required":false,"type":"long"},{"id":3,"name":"sequence_number","required":false,"type":"long"},{"id":4,"name":"file_sequence_number","required":false,"type":"long"},{"id":2,"name":"data_file","required":true,"type":{"type":"struct","fields":[{"id":134,"name":"content","required":true,"type":"int","doc":"Contents of the file: 0=data, 1=position delet [...]
partition-spec[]contentdata ��b���}zd�ɤb����
+Œ�kaǟ���Lr���! AB�^ri��H�g��D*�?bC�K�br���rM%� �A�C)����!H)����M�:��(����{����wx����~����{�����߾�F xIe�T� :�X4V�fW�,�+ -��XZ���`q���v@-���Լ3�7�Z�bh�|��4�n�T��U����%Gb\T.iRh,R�H,$�%-,��HTT�qE��x�U$љN gg���Z��@��D:�ri�1\�䄜����K#] R;�G�Xaܘl'�qʚ��픑62F�X�
+��C�4_��m�Qm�-���`���:>� �`�M�na���m� ��a�]l�ak �|������lGn& �% �i� p�(�h�&�7EӔ��)CYZ����.�eS��Ԣ6u��H��m �C�D�B�#m# �v�v����������߁7 ����0� Q��8LB�`���4d �P 0�Є����EXG�@�!l"l!��;�{��GlShH`,�q� �泌�a� �;��㓵rUHԴ �+�
+�� 3J�*����MmMX�Y��ȗ �H�ٓ[�DŽ���x�ʝ�,�??� ν�
+:�Ԛ&k���(N�)��] ��H$"J!)d�> _�N�� 7x�3�������E�uR�����s������O�q~�_�����b���}zd�ɤb���
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/snap-6622777947381868002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/snap-6622777947381868002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro
new file mode 100644
index 00000000000..a3eee4e6a11
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/snap-6622777947381868002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro differ
diff --git a/be/test/exec/test_data/nested_user_profiles_orc/_SUCCESS b/be/test/exec/test_data/nested_user_profiles_orc/_SUCCESS
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/be/test/exec/test_data/nested_user_profiles_orc/part-00000-62614f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc b/be/test/exec/test_data/nested_user_profiles_orc/part-00000-62614f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc
new file mode 100644
index 00000000000..8a0f8bc17b4
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_orc/part-00000-62614f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc differ
diff --git a/be/test/exec/test_data/nested_user_profiles_parquet/_SUCCESS b/be/test/exec/test_data/nested_user_profiles_parquet/_SUCCESS
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/be/test/exec/test_data/nested_user_profiles_parquet/part-00000-64a7a390-1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet b/be/test/exec/test_data/nested_user_profiles_parquet/part-00000-64a7a390-1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet
new file mode 100644
index 00000000000..73ae0061c21
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_parquet/part-00000-64a7a390-1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet differ
diff --git a/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp b/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
index 797ab9fc7db..edd131a2410 100644
--- a/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
+++ b/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
@@ -140,20 +140,18 @@ TEST_F(ParquetThriftReaderTest, complex_nested_file) {
ASSERT_EQ(schemaDescriptor.get_column_index("hobby"), 2);
auto hobby = schemaDescriptor.get_column("hobby");
- // should be parsed as ARRAY<MAP<STRUCT<STRING,STRING>>>
- ASSERT_TRUE(hobby->children.size() == 1 && hobby->children[0].children.size() == 1 &&
- hobby->children[0].children[0].children.size() == 2);
+ // should be parsed as ARRAY<MAP<KEY,VALUE>>
+ ASSERT_TRUE(hobby->children.size() == 1 && hobby->children[0].children.size() == 2);
ASSERT_TRUE(hobby->data_type->get_primitive_type() == TYPE_ARRAY &&
- hobby->children[0].data_type->get_primitive_type() == TYPE_MAP &&
- hobby->children[0].children[0].data_type->get_primitive_type() == TYPE_STRUCT);
+ hobby->children[0].data_type->get_primitive_type() == TYPE_MAP);
// hobby(opt) --- bag(rep) --- array_element(opt) --- map(rep)
// \------- key(req)
// \------- value(opt)
// R=0,D=1 R=1,D=2 R=1,D=3 R=2,D=4
// \------ R=2,D=4
// \------ R=2,D=5
- auto h_key = hobby->children[0].children[0].children[0];
- auto h_value = hobby->children[0].children[0].children[1];
+ auto h_key = hobby->children[0];
+ auto h_value = hobby->children[1];
ASSERT_TRUE(h_key.repetition_level == 2 && h_key.definition_level == 4);
ASSERT_TRUE(h_value.repetition_level == 2 && h_value.definition_level == 5);
diff --git a/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp b/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp
new file mode 100644
index 00000000000..7efca721fcd
--- /dev/null
+++ b/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp
@@ -0,0 +1,1179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cctz/time_zone.h>
+#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+#include <gen_cpp/Types_types.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/object_pool.h"
+#include "exec/olap_common.h"
+#include "io/fs/file_meta_cache.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_system.h"
+#include "io/fs/local_file_system.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_state.h"
+#include "util/timezone_utils.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_struct.h"
+#include "vec/core/block.h"
+#include "vec/core/column_with_type_and_name.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_struct.h"
+#include "vec/exec/format/parquet/vparquet_reader.h"
+#include "vec/exec/format/table/hive_reader.h"
+
+namespace doris::vectorized {
+
+// Define the column access path configuration structure
+struct ColumnAccessPathConfig {
+ std::string column_name;
+ std::vector<std::vector<std::string>> all_column_paths; // For all_column_access_paths
+ std::vector<std::vector<std::string>> predicate_paths; // For predicate_column_access_paths
+ bool use_name_paths = true; // true=use field names, false=use indices
+};
+
+// ORC column IDs are assigned in a tree-increment order, root is 0 and children increment sequentially.
+// 0: struct (table/root)
+// 1: id (bigint)
+// 2: name (string)
+// 3: profile (struct)
+// 4: address (struct)
+// 5: street (string)
+// 6: city (string)
+// 7: coordinates (struct)
+// 8: lat (double)
+// 9: lng (double)
+// 10: contact (struct)
+// 11: email (string)
+// 12: phone (struct)
+// 13: country_code (string)
+// 14: number (string)
+// 15: hobbies (array)
+// 16: element (struct)
+// 17: name (string)
+// 18: level (int)
+// 19: tags (array)
+// 20: element (string)
+// 21: friends (array)
+// 22: element (struct)
+// 23: user_id (bigint)
+// 24: nickname (string)
+// 25: friendship_level (int)
+// 26: recent_activity (array)
+// 27: element (struct)
+// 28: action (string)
+// 29: details (array)
+// 30: element (struct)
+// 31: key (string)
+// 32: value (string)
+// 33: attributes (map)
+// 34: key (string)
+// 35: value (string)
+// 36: complex_attributes (map)
+// 37: key (string)
+// 38: value (struct)
+// 39: metadata (struct)
+// 40: version (string)
+// 41: created_time (timestamp)
+// 42: last_updated (timestamp)
+// 43: quality_score (double)
+// 44: historical_scores (array)
+// 45: element (struct)
+// 46: period (string)
+// 47: score (double)
+// 48: components (array)
+// 49: element (struct)
+// 50: component_name (string)
+// 51: weight (float)
+// 52: sub_scores (map)
+// 53: key (string)
+// 54: value (double)
+// 55: trends (struct)
+// 56: direction (string)
+// 57: magnitude (double)
+// 58: confidence_interval (struct)
+// 59: lower (double)
+// 60: upper (double)
+// 61: hierarchical_data (map)
+// 62: key (string)
+// 63: value (struct)
+// 64: category (string)
+// 65: sub_items (array)
+// 66: element (struct)
+// 67: item_id (bigint)
+// 68: properties (map)
+// 69: key (string)
+// 70: value (string)
+// 71: metrics (struct)
+// 72: value (double)
+// 73: unit (string)
+// 74: summary (struct)
+// 75: total_count (int)
+// 76: average_value (double)
+// 77: validation_rules (struct)
+// 78: required (boolean)
+// 79: constraints (array)
+// 80: element (struct)
+// 81: rule_type (string)
+// 82: parameters (map)
+// 83: key (string)
+// 84: value (string)
+// 85: error_message (string)
+// 86: complex_constraint (struct)
+// 87: logic_operator (string)
+// 88: operands (array)
+// 89: element (struct)
+// 90: field_path (string)
+// 91: operator (string)
+// 92: comparison_value (string)
+
+class HiveReaderCreateColumnIdsTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ cache = std::make_unique<doris::FileMetaCache>(1024);
+
+ // Setup timezone
+ doris::TimezoneUtils::find_cctz_time_zone(doris::TimezoneUtils::default_time_zone,
+ timezone_obj);
+ }
+
+ void TearDown() override { cache.reset(); }
+
+ // Helper function to create tuple descriptor
+ const TupleDescriptor* create_tuple_descriptor(
+ DescriptorTbl** desc_tbl, ObjectPool& obj_pool, TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc, const std::vector<std::string>& column_names,
+ const std::vector<int>& column_positions,
+ const std::vector<TPrimitiveType::type>& column_types,
+ const std::vector<ColumnAccessPathConfig>& access_configs = {}) {
+ // Create table descriptor with complex schema
+ auto create_table_desc = [this, &access_configs](
+ TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc,
+ const std::vector<std::string>& table_column_names,
+ const std::vector<int>& table_column_positions,
+ const std::vector<TPrimitiveType::type>& types) {
+ t_table_desc.__set_id(0);
+ t_table_desc.__set_tableType(TTableType::HIVE_TABLE);
+ t_table_desc.__set_numCols(0);
+ t_table_desc.__set_numClusteringCols(0);
+ t_desc_table.tableDescriptors.push_back(t_table_desc);
+ t_desc_table.__isset.tableDescriptors = true;
+ for (int i = 0; i < table_column_names.size(); i++) {
+ TSlotDescriptor tslot_desc;
+ TTypeDesc type;
+ if (table_column_names[i] == "id") {
+ // id: bigint
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(TPrimitiveType::BIGINT);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "name") {
+ // name: string
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(TPrimitiveType::STRING);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "tags") {
+ // tags: array<string>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_node;
+ element_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType element_scalar;
+ element_scalar.__set_type(TPrimitiveType::STRING);
+ element_node.__set_scalar_type(element_scalar);
+ type.types.push_back(element_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "friends") {
+ // friends: array<struct<user_id:bigint, nickname:string, friendship_level:int>>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_struct_node;
+ element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> element_fields;
+ TStructField user_id_field;
+ user_id_field.__set_name("user_id");
+ user_id_field.__set_contains_null(true);
+ element_fields.push_back(user_id_field);
+ TStructField nickname_field;
+ nickname_field.__set_name("nickname");
+ nickname_field.__set_contains_null(true);
+ element_fields.push_back(nickname_field);
+ TStructField friendship_level_field;
+ friendship_level_field.__set_name("friendship_level");
+ friendship_level_field.__set_contains_null(true);
+ element_fields.push_back(friendship_level_field);
+ element_struct_node.__set_struct_fields(element_fields);
+ type.types.push_back(element_struct_node);
+ // user_id: bigint
+ TTypeNode user_id_node;
+ user_id_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType user_id_scalar;
+ user_id_scalar.__set_type(TPrimitiveType::BIGINT);
+ user_id_node.__set_scalar_type(user_id_scalar);
+ type.types.push_back(user_id_node);
+ // nickname: string
+ TTypeNode nickname_node;
+ nickname_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType nickname_scalar;
+ nickname_scalar.__set_type(TPrimitiveType::STRING);
+ nickname_node.__set_scalar_type(nickname_scalar);
+ type.types.push_back(nickname_node);
+ // friendship_level: int
+ TTypeNode friendship_level_node;
+ friendship_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType friendship_level_scalar;
+ friendship_level_scalar.__set_type(TPrimitiveType::INT);
+ friendship_level_node.__set_scalar_type(friendship_level_scalar);
+ type.types.push_back(friendship_level_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "recent_activity") {
+ // recent_activity: array<struct<action:string, details:array<struct<key:string, value:string>>>>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_struct_node;
+ element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> element_fields;
+ TStructField action_field;
+ action_field.__set_name("action");
+ action_field.__set_contains_null(true);
+ element_fields.push_back(action_field);
+ TStructField details_field;
+ details_field.__set_name("details");
+ details_field.__set_contains_null(true);
+ element_fields.push_back(details_field);
+ element_struct_node.__set_struct_fields(element_fields);
+ type.types.push_back(element_struct_node);
+ // action: string
+ TTypeNode action_node;
+ action_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType action_scalar;
+ action_scalar.__set_type(TPrimitiveType::STRING);
+ action_node.__set_scalar_type(action_scalar);
+ type.types.push_back(action_node);
+ // details: array<struct<key:string, value:string>>
+ TTypeNode details_array_node;
+ details_array_node.__set_type(TTypeNodeType::ARRAY);
+ details_array_node.__set_contains_nulls({true});
+ type.types.push_back(details_array_node);
+ TTypeNode details_element_struct_node;
+ details_element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> details_element_fields;
+ TStructField key_field;
+ key_field.__set_name("key");
+ key_field.__set_contains_null(true);
+ details_element_fields.push_back(key_field);
+ TStructField value_field;
+ value_field.__set_name("value");
+ value_field.__set_contains_null(true);
+ details_element_fields.push_back(value_field);
+ details_element_struct_node.__set_struct_fields(details_element_fields);
+ type.types.push_back(details_element_struct_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: string
+ TTypeNode value_node;
+ value_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType value_scalar;
+ value_scalar.__set_type(TPrimitiveType::STRING);
+ value_node.__set_scalar_type(value_scalar);
+ type.types.push_back(value_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "attributes") {
+ // attributes: map<string, string>
+ TTypeNode map_node;
+ map_node.__set_type(TTypeNodeType::MAP);
+ map_node.__set_contains_nulls({true, true});
+ type.types.push_back(map_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: string
+ TTypeNode value_node;
+ value_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType value_scalar;
+ value_scalar.__set_type(TPrimitiveType::STRING);
+ value_node.__set_scalar_type(value_scalar);
+ type.types.push_back(value_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "complex_attributes") {
+ // complex_attributes: map<string, struct<metadata:struct<version:string, created_time:timestamp, last_updated:timestamp>>>
+ TTypeNode map_node;
+ map_node.__set_type(TTypeNodeType::MAP);
+ map_node.__set_contains_nulls({true, true});
+ type.types.push_back(map_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: struct<metadata:struct<version:string, created_time:timestamp, last_updated:timestamp>>
+ TTypeNode value_struct_node;
+ value_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> value_fields;
+ TStructField metadata_field;
+ metadata_field.__set_name("metadata");
+ metadata_field.__set_contains_null(true);
+ value_fields.push_back(metadata_field);
+ value_struct_node.__set_struct_fields(value_fields);
+ type.types.push_back(value_struct_node);
+ // metadata: struct<version:string, created_time:timestamp, last_updated:timestamp>
+ TTypeNode metadata_struct_node;
+ metadata_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> metadata_fields;
+ TStructField version_field;
+ version_field.__set_name("version");
+ version_field.__set_contains_null(true);
+ metadata_fields.push_back(version_field);
+ TStructField created_time_field;
+ created_time_field.__set_name("created_time");
+ created_time_field.__set_contains_null(true);
+ metadata_fields.push_back(created_time_field);
+ TStructField last_updated_field;
+ last_updated_field.__set_name("last_updated");
+ last_updated_field.__set_contains_null(true);
+ metadata_fields.push_back(last_updated_field);
+ metadata_struct_node.__set_struct_fields(metadata_fields);
+ type.types.push_back(metadata_struct_node);
+ // version: string
+ TTypeNode version_node;
+ version_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType version_scalar;
+ version_scalar.__set_type(TPrimitiveType::STRING);
+ version_node.__set_scalar_type(version_scalar);
+ type.types.push_back(version_node);
+ // created_time: timestamp
+ TTypeNode created_time_node;
+ created_time_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType created_time_scalar;
+ created_time_scalar.__set_type(TPrimitiveType::DATETIME);
+ created_time_node.__set_scalar_type(created_time_scalar);
+ type.types.push_back(created_time_node);
+ // last_updated: timestamp
+ TTypeNode last_updated_node;
+ last_updated_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType last_updated_scalar;
+ last_updated_scalar.__set_type(TPrimitiveType::DATETIME);
+ last_updated_node.__set_scalar_type(last_updated_scalar);
+ type.types.push_back(last_updated_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "profile") {
+ // STRUCT/ARRAY nodes need contains_nulls, SCALAR nodes do not
+ TTypeNode struct_node;
+ struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> struct_fields;
+ TStructField address_field;
+ address_field.__set_name("address");
+ address_field.__set_contains_null(true);
+ struct_fields.push_back(address_field);
+ TStructField contact_field;
+ contact_field.__set_name("contact");
+ contact_field.__set_contains_null(true);
+ struct_fields.push_back(contact_field);
+ TStructField hobbies_field;
+ hobbies_field.__set_name("hobbies");
+ hobbies_field.__set_contains_null(true);
+ struct_fields.push_back(hobbies_field);
+ struct_node.__set_struct_fields(struct_fields);
+ type.types.push_back(struct_node);
+ TTypeNode address_node;
+ address_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> address_fields;
+ TStructField street_field;
+ street_field.__set_name("street");
+ street_field.__set_contains_null(true);
+ address_fields.push_back(street_field);
+ TStructField city_field;
+ city_field.__set_name("city");
+ city_field.__set_contains_null(true);
+ address_fields.push_back(city_field);
+ TStructField coordinates_field;
+ coordinates_field.__set_name("coordinates");
+ coordinates_field.__set_contains_null(true);
+ address_fields.push_back(coordinates_field);
+ address_node.__set_struct_fields(address_fields);
+ type.types.push_back(address_node);
+ TTypeNode street_node;
+ street_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType street_scalar;
+ street_scalar.__set_type(TPrimitiveType::STRING);
+ street_node.__set_scalar_type(street_scalar);
+ type.types.push_back(street_node);
+ TTypeNode city_node;
+ city_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType city_scalar;
+ city_scalar.__set_type(TPrimitiveType::STRING);
+ city_node.__set_scalar_type(city_scalar);
+ type.types.push_back(city_node);
+ TTypeNode coordinates_node;
+ coordinates_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> coordinates_fields;
+ TStructField lat_field;
+ lat_field.__set_name("lat");
+ lat_field.__set_contains_null(true);
+ coordinates_fields.push_back(lat_field);
+ TStructField lng_field;
+ lng_field.__set_name("lng");
+ lng_field.__set_contains_null(true);
+ coordinates_fields.push_back(lng_field);
+ coordinates_node.__set_struct_fields(coordinates_fields);
+ type.types.push_back(coordinates_node);
+ TTypeNode lat_node;
+ lat_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lat_scalar;
+ lat_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lat_node.__set_scalar_type(lat_scalar);
+ type.types.push_back(lat_node);
+ TTypeNode lng_node;
+ lng_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lng_scalar;
+ lng_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lng_node.__set_scalar_type(lng_scalar);
+ type.types.push_back(lng_node);
+ TTypeNode contact_node;
+ contact_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> contact_fields;
+ TStructField email_field;
+ email_field.__set_name("email");
+ email_field.__set_contains_null(true);
+ contact_fields.push_back(email_field);
+ TStructField phone_field;
+ phone_field.__set_name("phone");
+ phone_field.__set_contains_null(true);
+ contact_fields.push_back(phone_field);
+ contact_node.__set_struct_fields(contact_fields);
+ type.types.push_back(contact_node);
+ TTypeNode email_node;
+ email_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType email_scalar;
+ email_scalar.__set_type(TPrimitiveType::STRING);
+ email_node.__set_scalar_type(email_scalar);
+ type.types.push_back(email_node);
+ TTypeNode phone_node;
+ phone_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> phone_fields;
+ TStructField country_code_field;
+ country_code_field.__set_name("country_code");
+ country_code_field.__set_contains_null(true);
+ phone_fields.push_back(country_code_field);
+ TStructField number_field;
+ number_field.__set_name("number");
+ number_field.__set_contains_null(true);
+ phone_fields.push_back(number_field);
+ phone_node.__set_struct_fields(phone_fields);
+ type.types.push_back(phone_node);
+ TTypeNode country_code_node;
+ country_code_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType country_code_scalar;
+ country_code_scalar.__set_type(TPrimitiveType::STRING);
+ country_code_node.__set_scalar_type(country_code_scalar);
+ type.types.push_back(country_code_node);
+ TTypeNode number_node;
+ number_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType number_scalar;
+ number_scalar.__set_type(TPrimitiveType::STRING);
+ number_node.__set_scalar_type(number_scalar);
+ type.types.push_back(number_node);
+ TTypeNode hobbies_node;
+ hobbies_node.__set_type(TTypeNodeType::ARRAY);
+ hobbies_node.__set_contains_nulls({true});
+ type.types.push_back(hobbies_node);
+ TTypeNode hobby_element_node;
+ hobby_element_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> hobby_element_fields;
+ TStructField hobby_name_field;
+ hobby_name_field.__set_name("name");
+ hobby_name_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_name_field);
+ TStructField hobby_level_field;
+ hobby_level_field.__set_name("level");
+ hobby_level_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_level_field);
+ hobby_element_node.__set_struct_fields(hobby_element_fields);
+ type.types.push_back(hobby_element_node);
+ TTypeNode hobby_name_node;
+ hobby_name_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_name_scalar;
+ hobby_name_scalar.__set_type(TPrimitiveType::STRING);
+ hobby_name_node.__set_scalar_type(hobby_name_scalar);
+ type.types.push_back(hobby_name_node);
+ TTypeNode hobby_level_node;
+ hobby_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_level_scalar;
+ hobby_level_scalar.__set_type(TPrimitiveType::INT);
+ hobby_level_node.__set_scalar_type(hobby_level_scalar);
+ type.types.push_back(hobby_level_node);
+ tslot_desc.__set_slotType(type);
+ } else {
+ // Regular types
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(types[i]);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ }
+ tslot_desc.__set_id(i);
+ tslot_desc.__set_parent(0);
+ tslot_desc.__set_colName(table_column_names[i]);
+ tslot_desc.__set_columnPos(table_column_positions[i]);
+ tslot_desc.__set_byteOffset(0);
+ tslot_desc.__set_nullIndicatorByte(0);
+ tslot_desc.__set_nullIndicatorBit(-1);
+ tslot_desc.__set_slotIdx(0);
+ tslot_desc.__set_isMaterialized(true);
+
+ // Use configuration to set column_access_paths
+ for (const auto& config : access_configs) {
+ set_column_access_paths(tslot_desc, config);
+ }
+
+ t_desc_table.slotDescriptors.push_back(tslot_desc);
+ }
+ t_desc_table.__isset.slotDescriptors = true;
+ TTupleDescriptor t_tuple_desc;
+ t_tuple_desc.__set_id(0);
+ t_tuple_desc.__set_byteSize(16);
+ t_tuple_desc.__set_numNullBytes(0);
+ t_tuple_desc.__set_tableId(0);
+ t_tuple_desc.__isset.tableId = true;
+ t_desc_table.tupleDescriptors.push_back(t_tuple_desc);
+ };
+
+ create_table_desc(t_desc_table, t_table_desc, column_names, column_positions, column_types);
+ EXPECT_TRUE(DescriptorTbl::create(&obj_pool, t_desc_table, desc_tbl).ok());
+ return (*desc_tbl)->get_tuple_descriptor(0);
+ }
+
+ // Helper function: Set column access paths
+ void set_column_access_paths(TSlotDescriptor& tslot_desc,
+ const ColumnAccessPathConfig& config) {
+ if (config.column_name != tslot_desc.colName) {
+ return; // Not the target column, skip
+ }
+
+ // Set all_column_access_paths
+ if (!config.all_column_paths.empty()) {
+ std::vector<TColumnAccessPath> access_paths;
+ for (const auto& path_vector : config.all_column_paths) {
+ TColumnAccessPath access_path;
+ access_path.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path;
+ data_path.__set_path(path_vector);
+ access_path.__set_data_access_path(data_path);
+ access_paths.push_back(access_path);
+ }
+ tslot_desc.__set_all_access_paths(access_paths);
+ }
+
+ // Set predicate_column_access_paths
+ if (!config.predicate_paths.empty()) {
+ std::vector<TColumnAccessPath> access_paths;
+ for (const auto& path_vector : config.predicate_paths) {
+ TColumnAccessPath access_path;
+ access_path.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path;
+ data_path.__set_path(path_vector);
+ access_path.__set_data_access_path(data_path);
+ access_paths.push_back(access_path);
+ }
+ tslot_desc.__set_predicate_access_paths(access_paths);
+ }
+ }
+
+ std::unique_ptr<doris::FileMetaCache> cache;
+ cctz::time_zone timezone_obj;
+
+ // Helper function: Create and setup ParquetReader
+ std::tuple<std::unique_ptr<HiveParquetReader>, const FieldDescriptor*> create_parquet_reader(
+ const std::string& test_file) {
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ return {nullptr, nullptr};
+ }
+
+ RuntimeState runtime_state((TQueryGlobals()));
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_PARQUET;
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size();
+ scan_range.path = test_file;
+ RuntimeProfile profile("test_profile");
+
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+ auto generic_reader =
+ ParquetReader::create_unique(&profile, scan_params, scan_range, 1024, &ctz, nullptr,
+ &runtime_state, cache.get());
+ if (!generic_reader) {
+ return {nullptr, nullptr};
+ }
+
+ auto parquet_reader = static_cast<ParquetReader*>(generic_reader.get());
+ parquet_reader->set_file_reader(file_reader);
+
+ const FieldDescriptor* field_desc = nullptr;
+ st = parquet_reader->get_file_metadata_schema(&field_desc);
+ if (!st.ok() || !field_desc) {
+ return {nullptr, nullptr};
+ }
+
+ auto hive_reader = std::make_unique<HiveParquetReader>(
+ std::move(generic_reader), &profile, &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ return {std::move(hive_reader), field_desc};
+ }
+
+ // Helper function: Create and setup OrcReader
+ std::tuple<std::unique_ptr<HiveOrcReader>, const orc::Type*> create_orc_reader(
+ const std::string& test_file) {
+ // Open the Hive Orc test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ return {nullptr, nullptr};
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_ORC;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create OrcReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader =
+ OrcReader::create_unique(&profile, &runtime_state, scan_params, scan_range, 1024,
+ "CST", nullptr, cache.get());
+ if (!generic_reader) {
+ return {nullptr, nullptr};
+ }
+
+ auto orc_reader = static_cast<OrcReader*>(generic_reader.get());
+ // Get FieldDescriptor from Orc file
+ const orc::Type* orc_type_ptr = nullptr;
+ st = orc_reader->get_file_type(&orc_type_ptr);
+ if (!st.ok() || !orc_type_ptr) {
+ return {nullptr, nullptr};
+ }
+
+ // Create HiveOrcReader
+ auto hive_reader = std::make_unique<HiveOrcReader>(std::move(generic_reader), &profile,
+ &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ return {std::move(hive_reader), orc_type_ptr};
+ }
+
+ // Helper function: Run Parquet test with different column ID extraction methods
+ void run_parquet_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false,
+ bool should_skip_assertion = false) {
+ std::string test_file =
+ "./be/test/exec/test_data/nested_user_profiles_parquet/"
+ "part-00000-64a7a390-1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet";
+
+ auto [hive_reader, field_desc] = create_parquet_reader(test_file);
+ if (!hive_reader || !field_desc) {
+ GTEST_SKIP() << "Test file not found or failed to create reader: " << test_file;
+ return;
+ }
+
+ // Create tuple descriptor based on full schema
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+
+ // Define all columns according to the schema
+ std::vector<std::string> table_column_names = {
+ "name", "profile", "tags", "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ // TPrimitiveType::BIGINT, // id
+ TPrimitiveType::STRING, // name
+ TPrimitiveType::STRUCT, // profile
+ TPrimitiveType::ARRAY, // tags
+ TPrimitiveType::ARRAY, // friends
+ TPrimitiveType::ARRAY, // recent_activity
+ TPrimitiveType::MAP, // attributes
+ TPrimitiveType::MAP // complex_attributes
+ };
+
+ const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
+ &desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
+ table_column_positions, table_column_types, access_configs);
+
+ // Execute test based on method choice
+ ColumnIdResult actual_result;
+ if (use_top_level_method) {
+ actual_result = HiveParquetReader::_create_column_ids_by_top_level_col_index(
+ field_desc, tuple_descriptor);
+ } else {
+ actual_result = HiveParquetReader::_create_column_ids(field_desc, tuple_descriptor);
+ }
+
+ if (!should_skip_assertion) {
+ EXPECT_EQ(actual_result.column_ids, expected_column_ids);
+ EXPECT_EQ(actual_result.filter_column_ids, expected_filter_column_ids);
+ }
+ }
+
+ // Helper function: Run ORC test with different column ID extraction methods
+ void run_orc_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false,
+ bool should_skip_assertion = false) {
+ std::string test_file =
+ "./be/test/exec/test_data/nested_user_profiles_orc/"
+ "part-00000-62614f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc";
+
+ auto [hive_reader, orc_type] = create_orc_reader(test_file);
+ if (!hive_reader || !orc_type) {
+ GTEST_SKIP() << "Test file not found or failed to create reader: " << test_file;
+ return;
+ }
+
+ // Create tuple descriptor based on full schema
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+
+ // Define all columns according to the schema
+ std::vector<std::string> table_column_names = {
+ "name", "profile", "tags", "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ // TPrimitiveType::BIGINT, // id
+ TPrimitiveType::STRING, // name
+ TPrimitiveType::STRUCT, // profile
+ TPrimitiveType::ARRAY, // tags
+ TPrimitiveType::ARRAY, // friends
+ TPrimitiveType::ARRAY, // recent_activity
+ TPrimitiveType::MAP, // attributes
+ TPrimitiveType::MAP // complex_attributes
+ };
+
+ const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
+ &desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
+ table_column_positions, table_column_types, access_configs);
+
+ // Execute test based on method choice
+ ColumnIdResult actual_result;
+ if (use_top_level_method) {
+ actual_result = HiveOrcReader::_create_column_ids_by_top_level_col_index(
+ orc_type, tuple_descriptor);
+ } else {
+ actual_result = HiveOrcReader::_create_column_ids(orc_type, tuple_descriptor);
+ }
+
+ if (!should_skip_assertion) {
+ EXPECT_EQ(actual_result.column_ids, expected_column_ids);
+ EXPECT_EQ(actual_result.filter_column_ids, expected_filter_column_ids);
+ }
+ }
+};
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_1) {
+ // Properties:
+
+ // Configure access paths for profile column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"profile", "address", "coordinates", "lat"},
+ {"profile", "address", "coordinates", "lng"},
+ {"profile", "contact", "email"},
+ {"profile", "hobbies", "*", "level"}};
+ access_config.predicate_paths = {{"profile", "address", "coordinates", "lat"},
+ {"profile", "contact", "email"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 7, 8, 9, 10, 11, 15, 16, 18};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_2) {
+ // Properties:
+ // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
+ // ORC column IDs are assigned in a tree-increment order, root is 0 and children increment sequentially.
+ // 0: struct (table/root)
+ // 1: id (int64)
+ // 2: name (string)
+ // 3: profile (struct)
+ // 4: address (struct)
+ // 5: street (string)
+ // 6: city (string)
+ // 7: coordinates (struct)
+ // 8: lat (double)
+ // 9: lng (double)
+ // 10: contact (struct)
+ // 11: email (string)
+ // 12: phone (struct)
+ // 13: country_code (string)
+ // 14: number (string)
+ // 15: hobbies (list/array)
+ // 16: element (struct)
+ // 17: name (string)
+ // 18: level (int32)
+
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"profile"}};
+ access_config.predicate_paths = {{"profile", "address", "coordinates", "lat"},
+ {"profile", "contact", "email"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10,
+ 11, 12, 13, 14, 15, 16, 17, 18};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_3) {
+ // Properties:
+ // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
+ // ORC column IDs are assigned in a tree-increment order, root is 0 and children increment sequentially.
+ // 0: struct (table/root)
+ // 1: id (int64)
+ // 2: name (string)
+ // 3: profile (struct)
+ // 4: address (struct)
+ // 5: street (string)
+ // 6: city (string)
+ // 7: coordinates (struct)
+ // 8: lat (double)
+ // 9: lng (double)
+ // 10: contact (struct)
+ // 11: email (string)
+ // 12: phone (struct)
+ // 13: country_code (string)
+ // 14: number (string)
+ // 15: hobbies (list/array)
+ // 16: element (struct)
+ // 17: name (string)
+ // 18: level (int32)
+
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"profile", "contact"}, {"profile", "address"}};
+ access_config.predicate_paths = {{"profile", "address", "coordinates"},
+ {"profile", "contact", "email"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 9, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_4) {
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {};
+ access_config.predicate_paths = {};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_5) {
+ // 19: tags (array)
+ // 20: element (string)
+ // 21: friends (array)
+ // 22: element (struct)
+ // 23: user_id (bigint)
+ // 24: nickname (string)
+ // 25: friendship_level (int)
+ // 26: recent_activity (array)
+ // 27: element (struct)
+ // 28: action (string)
+ // 29: details (array)
+ // 30: element (struct)
+ // 31: key (string)
+ // 32: value (string)
+
+ std::vector<ColumnAccessPathConfig> access_configs;
+
+ {
+ // Configure access paths for friends column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "friends";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"friends", "*", "nickname"},
+ {"friends", "*", "friendship_level"}};
+ access_config.predicate_paths = {
+ {"friends", "*", "nickname"},
+ };
+ access_configs.push_back(access_config);
+ }
+
+ {
+ // Configure access paths for recent_activity column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "recent_activity";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"recent_activity", "*", "action"},
+ {"recent_activity", "*", "details", "*", "value"}};
+ access_config.predicate_paths = {
+ {"recent_activity", "*", "action"},
+ };
+ access_configs.push_back(access_config);
+ }
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 21, 22, 24, 25, 26, 27, 28, 29, 30, 32};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {21, 22, 24, 26, 27, 28};
+
+ run_parquet_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids, true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_6) {
+ // 33: attributes (map)
+ // 34: key (string)
+ // 35: value (string)
+ // 36: complex_attributes (map)
+ // 37: key (string)
+ // 38: value (struct)
+ // 39: metadata (struct)
+ // 40: version (string)
+ // 41: created_time (timestamp)
+ // 42: last_updated (timestamp)
+ // 43: quality_score (double)
+ // 44: historical_scores (array)
+ // 45: element (struct)
+ // 46: period (string)
+ // 47: score (double)
+ // 48: components (array)
+ // 49: element (struct)
+ // 50: component_name (string)
+ // 51: weight (float)
+ // 52: sub_scores (map)
+ // 53: key (string)
+ // 54: value (double)
+ // 55: trends (struct)
+ // 56: direction (string)
+ // 57: magnitude (double)
+ // 58: confidence_interval (struct)
+ // 59: lower (double)
+ // 60: upper (double)
+ // 61: hierarchical_data (map)
+ // 62: key (string)
+ // 63: value (struct)
+ // 64: category (string)
+ // 65: sub_items (array)
+ // 66: element (struct)
+ // 67: item_id (bigint)
+ // 68: properties (map)
+ // 69: key (string)
+ // 70: value (string)
+ // 71: metrics (struct)
+ // 72: value (double)
+ // 73: unit (string)
+ // 74: summary (struct)
+ // 75: total_count (int)
+ // 76: average_value (double)
+ // 77: validation_rules (struct)
+ // 78: required (boolean)
+ // 79: constraints (array)
+ // 80: element (struct)
+ // 81: rule_type (string)
+ // 82: parameters (map)
+ // 83: key (string)
+ // 84: value (string)
+ // 85: error_message (string)
+ // 86: complex_constraint (struct)
+ // 87: logic_operator (string)
+ // 88: operands (array)
+ // 89: element (struct)
+ // 90: field_path (string)
+ // 91: operator (string)
+ // 92: comparison_value (string)
+ std::vector<ColumnAccessPathConfig> access_configs;
+
+ {
+ // Configure access paths for complex_attributes column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "complex_attributes";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {
+ {"complex_attributes", "*", "metadata", "version"},
+ {"complex_attributes", "*", "historical_scores", "*", "components", "*",
+ "sub_scores"},
+ {"complex_attributes", "*", "hierarchical_data", "VALUES"},
+ {"complex_attributes", "VALUES", "validation_rules", "constraints", "*",
+ "parameters", "KEYS"}};
+ access_config.predicate_paths = {{"complex_attributes", "*", "metadata", "version"}
+
+ };
+ access_configs.push_back(access_config);
+ }
+
+ {
+ // parquet values should access keys
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> parquet_expected_column_ids = {
+ 2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53, 54, 61, 62, 63, 64,
+ 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> parquet_expected_filter_column_ids = {36, 37, 38, 39, 40};
+
+ run_parquet_test_with_method(access_configs, parquet_expected_column_ids,
+ parquet_expected_filter_column_ids);
+ run_parquet_test_with_method(access_configs, parquet_expected_column_ids,
+ parquet_expected_filter_column_ids, true);
+ }
+
+ {
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> orc_expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52,
+ 53, 54, 61, 63, 64, 65, 66, 67, 68, 69, 70,
+ 71, 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> orc_expected_filter_column_ids = {36, 37, 38, 39, 40};
+ run_orc_test_with_method(access_configs, orc_expected_column_ids,
+ orc_expected_filter_column_ids);
+ run_orc_test_with_method(access_configs, orc_expected_column_ids,
+ orc_expected_filter_column_ids, true);
+ }
+}
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/test/vec/exec/format/table/hive/hive_reader_test.cpp b/be/test/vec/exec/format/table/hive/hive_reader_test.cpp
new file mode 100644
index 00000000000..16bde5c7bae
--- /dev/null
+++ b/be/test/vec/exec/format/table/hive/hive_reader_test.cpp
@@ -0,0 +1,743 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/hive_reader.h"
+
+#include <cctz/time_zone.h>
+#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+#include <gen_cpp/Types_types.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/object_pool.h"
+#include "exec/olap_common.h"
+#include "io/fs/file_meta_cache.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_system.h"
+#include "io/fs/local_file_system.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_state.h"
+#include "util/timezone_utils.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_struct.h"
+#include "vec/core/block.h"
+#include "vec/core/column_with_type_and_name.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_struct.h"
+#include "vec/exec/format/parquet/vparquet_reader.h"
+
+namespace doris::vectorized::table {
+
+class HiveReaderTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ cache = std::make_unique<doris::FileMetaCache>(1024);
+
+ // Setup timezone
+ doris::TimezoneUtils::find_cctz_time_zone(doris::TimezoneUtils::default_time_zone,
+ timezone_obj);
+ }
+
+ void TearDown() override { cache.reset(); }
+
+ // Helper function to create complex struct types for testing
+ void create_complex_struct_types(DataTypePtr& coordinates_struct_type,
+ DataTypePtr& address_struct_type,
+ DataTypePtr& phone_struct_type,
+ DataTypePtr& contact_struct_type,
+ DataTypePtr& hobby_element_struct_type,
+ DataTypePtr& hobbies_array_type,
+ DataTypePtr& profile_struct_type, DataTypePtr& name_type) {
+ // Create name column type (direct field)
+ name_type = make_nullable(std::make_shared<DataTypeString>());
+
+ // First create coordinates struct type
+ std::vector<DataTypePtr> coordinates_types = {
+ make_nullable(std::make_shared<DataTypeFloat64>()), // lat (field ID 10)
+ make_nullable(std::make_shared<DataTypeFloat64>()) // lng (field ID 11)
+ };
+ std::vector<std::string> coordinates_names = {"lat", "lng"};
+ coordinates_struct_type = make_nullable(
+ std::make_shared<DataTypeStruct>(coordinates_types, coordinates_names));
+
+ // Create address struct type (with street, city, coordinates)
+ std::vector<DataTypePtr> address_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // street (field ID 7)
+ make_nullable(std::make_shared<DataTypeString>()), // city (field ID 8)
+ coordinates_struct_type // coordinates (field ID 9)
+ };
+ std::vector<std::string> address_names = {"street", "city", "coordinates"};
+ address_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(address_types, address_names));
+
+ // Create phone struct type
+ std::vector<DataTypePtr> phone_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // country_code (field ID 14)
+ make_nullable(std::make_shared<DataTypeString>()) // number (field ID 15)
+ };
+ std::vector<std::string> phone_names = {"country_code", "number"};
+ phone_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(phone_types, phone_names));
+
+ // Create contact struct type (with email, phone)
+ std::vector<DataTypePtr> contact_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // email (field ID 12)
+ phone_struct_type // phone (field ID 13)
+ };
+ std::vector<std::string> contact_names = {"email", "phone"};
+ contact_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(contact_types, contact_names));
+
+ // Create hobby element struct type for array elements
+ std::vector<DataTypePtr> hobby_element_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // name (field ID 17)
+ make_nullable(std::make_shared<DataTypeInt32>()) // level (field ID 18)
+ };
+ std::vector<std::string> hobby_element_names = {"name", "level"};
+ hobby_element_struct_type = make_nullable(
+ std::make_shared<DataTypeStruct>(hobby_element_types, hobby_element_names));
+
+ // Create hobbies array type
+ hobbies_array_type =
+ make_nullable(std::make_shared<DataTypeArray>(hobby_element_struct_type));
+
+ // Create complete profile struct type (with address, contact, hobbies)
+ std::vector<DataTypePtr> profile_types = {
+ address_struct_type, // address (field ID 4)
+ contact_struct_type, // contact (field ID 5)
+ hobbies_array_type // hobbies (field ID 6)
+ };
+ std::vector<std::string> profile_names = {"address", "contact", "hobbies"};
+ profile_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(profile_types, profile_names));
+ }
+
+ // Helper function to create tuple descriptor
+ const TupleDescriptor* create_tuple_descriptor(
+ DescriptorTbl** desc_tbl, ObjectPool& obj_pool, TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc, const std::vector<std::string>& column_names,
+ const std::vector<int>& column_positions,
+ const std::vector<TPrimitiveType::type>& column_types) {
+ // Create table descriptor with complex schema
+ auto create_table_desc = [](TDescriptorTable& t_desc_table, TTableDescriptor& t_table_desc,
+ const std::vector<std::string>& table_column_names,
+ const std::vector<int>& table_column_positions,
+ const std::vector<TPrimitiveType::type>& types) {
+ t_table_desc.__set_id(0);
+ t_table_desc.__set_tableType(TTableType::OLAP_TABLE);
+ t_table_desc.__set_numCols(0);
+ t_table_desc.__set_numClusteringCols(0);
+ t_desc_table.tableDescriptors.push_back(t_table_desc);
+ t_desc_table.__isset.tableDescriptors = true;
+ for (int i = 0; i < table_column_names.size(); i++) {
+ TSlotDescriptor tslot_desc;
+ TTypeDesc type;
+ if (table_column_names[i] == "profile") {
+ // STRUCT/ARRAY nodes set contains_nulls; SCALAR nodes do not
+ TTypeNode struct_node;
+ struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> struct_fields;
+ TStructField address_field;
+ address_field.__set_name("address");
+ address_field.__set_contains_null(true);
+ struct_fields.push_back(address_field);
+ TStructField contact_field;
+ contact_field.__set_name("contact");
+ contact_field.__set_contains_null(true);
+ struct_fields.push_back(contact_field);
+ TStructField hobbies_field;
+ hobbies_field.__set_name("hobbies");
+ hobbies_field.__set_contains_null(true);
+ struct_fields.push_back(hobbies_field);
+ struct_node.__set_struct_fields(struct_fields);
+ type.types.push_back(struct_node);
+ TTypeNode address_node;
+ address_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> address_fields;
+ TStructField street_field;
+ street_field.__set_name("street");
+ street_field.__set_contains_null(true);
+ address_fields.push_back(street_field);
+ TStructField city_field;
+ city_field.__set_name("city");
+ city_field.__set_contains_null(true);
+ address_fields.push_back(city_field);
+ TStructField coordinates_field;
+ coordinates_field.__set_name("coordinates");
+ coordinates_field.__set_contains_null(true);
+ address_fields.push_back(coordinates_field);
+ address_node.__set_struct_fields(address_fields);
+ type.types.push_back(address_node);
+ TTypeNode street_node;
+ street_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType street_scalar;
+ street_scalar.__set_type(TPrimitiveType::STRING);
+ street_node.__set_scalar_type(street_scalar);
+ type.types.push_back(street_node);
+ TTypeNode city_node;
+ city_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType city_scalar;
+ city_scalar.__set_type(TPrimitiveType::STRING);
+ city_node.__set_scalar_type(city_scalar);
+ type.types.push_back(city_node);
+ TTypeNode coordinates_node;
+ coordinates_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> coordinates_fields;
+ TStructField lat_field;
+ lat_field.__set_name("lat");
+ lat_field.__set_contains_null(true);
+ coordinates_fields.push_back(lat_field);
+ TStructField lng_field;
+ lng_field.__set_name("lng");
+ lng_field.__set_contains_null(true);
+ coordinates_fields.push_back(lng_field);
+ coordinates_node.__set_struct_fields(coordinates_fields);
+ type.types.push_back(coordinates_node);
+ TTypeNode lat_node;
+ lat_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lat_scalar;
+ lat_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lat_node.__set_scalar_type(lat_scalar);
+ type.types.push_back(lat_node);
+ TTypeNode lng_node;
+ lng_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lng_scalar;
+ lng_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lng_node.__set_scalar_type(lng_scalar);
+ type.types.push_back(lng_node);
+ TTypeNode contact_node;
+ contact_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> contact_fields;
+ TStructField email_field;
+ email_field.__set_name("email");
+ email_field.__set_contains_null(true);
+ contact_fields.push_back(email_field);
+ TStructField phone_field;
+ phone_field.__set_name("phone");
+ phone_field.__set_contains_null(true);
+ contact_fields.push_back(phone_field);
+ contact_node.__set_struct_fields(contact_fields);
+ type.types.push_back(contact_node);
+ TTypeNode email_node;
+ email_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType email_scalar;
+ email_scalar.__set_type(TPrimitiveType::STRING);
+ email_node.__set_scalar_type(email_scalar);
+ type.types.push_back(email_node);
+ TTypeNode phone_node;
+ phone_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> phone_fields;
+ TStructField country_code_field;
+ country_code_field.__set_name("country_code");
+ country_code_field.__set_contains_null(true);
+ phone_fields.push_back(country_code_field);
+ TStructField number_field;
+ number_field.__set_name("number");
+ number_field.__set_contains_null(true);
+ phone_fields.push_back(number_field);
+ phone_node.__set_struct_fields(phone_fields);
+ type.types.push_back(phone_node);
+ TTypeNode country_code_node;
+ country_code_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType country_code_scalar;
+ country_code_scalar.__set_type(TPrimitiveType::STRING);
+ country_code_node.__set_scalar_type(country_code_scalar);
+ type.types.push_back(country_code_node);
+ TTypeNode number_node;
+ number_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType number_scalar;
+ number_scalar.__set_type(TPrimitiveType::STRING);
+ number_node.__set_scalar_type(number_scalar);
+ type.types.push_back(number_node);
+ TTypeNode hobbies_node;
+ hobbies_node.__set_type(TTypeNodeType::ARRAY);
+ hobbies_node.__set_contains_nulls({true});
+ type.types.push_back(hobbies_node);
+ TTypeNode hobby_element_node;
+ hobby_element_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> hobby_element_fields;
+ TStructField hobby_name_field;
+ hobby_name_field.__set_name("name");
+ hobby_name_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_name_field);
+ TStructField hobby_level_field;
+ hobby_level_field.__set_name("level");
+ hobby_level_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_level_field);
+ hobby_element_node.__set_struct_fields(hobby_element_fields);
+ type.types.push_back(hobby_element_node);
+ TTypeNode hobby_name_node;
+ hobby_name_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_name_scalar;
+ hobby_name_scalar.__set_type(TPrimitiveType::STRING);
+ hobby_name_node.__set_scalar_type(hobby_name_scalar);
+ type.types.push_back(hobby_name_node);
+ TTypeNode hobby_level_node;
+ hobby_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_level_scalar;
+ hobby_level_scalar.__set_type(TPrimitiveType::INT);
+ hobby_level_node.__set_scalar_type(hobby_level_scalar);
+ type.types.push_back(hobby_level_node);
+ tslot_desc.__set_slotType(type);
+ } else {
+ // Regular type
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(types[i]);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ }
+ tslot_desc.__set_id(i);
+ tslot_desc.__set_parent(0);
+ tslot_desc.__set_colName(table_column_names[i]);
+ tslot_desc.__set_columnPos(table_column_positions[i]);
+ tslot_desc.__set_byteOffset(0);
+ tslot_desc.__set_nullIndicatorByte(0);
+ tslot_desc.__set_nullIndicatorBit(-1);
+ tslot_desc.__set_slotIdx(0);
+ tslot_desc.__set_isMaterialized(true);
+ // Set column_access_paths only for the profile field
+ if (table_column_names[i] == "profile") {
+ {
+ std::vector<TColumnAccessPath> access_paths;
+ // address.coordinates.lat
+ TColumnAccessPath path1;
+ path1.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path1;
+ data_path1.__set_path({"profile", "address", "coordinates", "lat"});
+ path1.__set_data_access_path(data_path1);
+ access_paths.push_back(path1);
+ // address.coordinates.lng
+ TColumnAccessPath path2;
+ path2.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path2;
+ data_path2.__set_path({"profile", "address", "coordinates", "lng"});
+ path2.__set_data_access_path(data_path2);
+ access_paths.push_back(path2);
+ // contact.email
+ TColumnAccessPath path3;
+ path3.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path3;
+ data_path3.__set_path({"profile", "contact", "email"});
+ path3.__set_data_access_path(data_path3);
+ access_paths.push_back(path3);
+ // hobbies[].element.level
+ TColumnAccessPath path4;
+ path4.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path4;
+ data_path4.__set_path({"profile", "hobbies", "*", "level"});
+ path4.__set_data_access_path(data_path4);
+ access_paths.push_back(path4);
+ tslot_desc.__set_all_access_paths(access_paths);
+ }
+ }
+ t_desc_table.slotDescriptors.push_back(tslot_desc);
+ }
+ t_desc_table.__isset.slotDescriptors = true;
+ TTupleDescriptor t_tuple_desc;
+ t_tuple_desc.__set_id(0);
+ t_tuple_desc.__set_byteSize(16);
+ t_tuple_desc.__set_numNullBytes(0);
+ t_tuple_desc.__set_tableId(0);
+ t_tuple_desc.__isset.tableId = true;
+ t_desc_table.tupleDescriptors.push_back(t_tuple_desc);
+ };
+
+ create_table_desc(t_desc_table, t_table_desc, column_names, column_positions, column_types);
+ EXPECT_TRUE(DescriptorTbl::create(&obj_pool, t_desc_table, desc_tbl).ok());
+ return (*desc_tbl)->get_tuple_descriptor(0);
+ }
+
+ // Helper function to recursively print column row counts and check size > 0
+ void verify_test_results(Block& block, size_t read_rows) {
+ // Verify that we read some data
+ EXPECT_GT(read_rows, 0) << "Should read at least one row";
+ EXPECT_EQ(block.rows(), read_rows);
+
+ // Verify column count matches expected (2 columns: name, profile)
+ EXPECT_EQ(block.columns(), 2);
+
+ // Verify column names and types
+ auto columns_with_names = block.get_columns_with_type_and_name();
+ std::vector<std::string> expected_column_names = {"name", "profile"};
+ for (size_t i = 0; i < expected_column_names.size(); i++) {
+ EXPECT_EQ(columns_with_names[i].name, expected_column_names[i]);
+ }
+
+ // Verify column types
+ EXPECT_TRUE(columns_with_names[0].type->get_name().find("String") !=
+ std::string::npos); // name is STRING
+ EXPECT_TRUE(columns_with_names[1].type->get_name().find("Struct") !=
+ std::string::npos); // profile is STRUCT
+
+ // Print row count for each column and nested subcolumns
+ std::cout << "Block rows: " << block.rows() << std::endl;
+
+ // Helper function to recursively print column row counts
+ std::function<void(const ColumnPtr&, const DataTypePtr&, const std::string&, int)>
+ print_column_rows = [&](const ColumnPtr& col, const DataTypePtr& type,
+ const std::string& name, int depth) {
+ std::string indent(depth * 2, ' ');
+ std::cout << indent << name << " row count: " << col->size() << std::endl;
+ EXPECT_GT(col->size(), 0) << name << " column/subcolumn size should be > 0";
+
+ // Check if it's a nullable column
+ if (const auto* nullable_col = typeid_cast<const ColumnNullable*>(col.get())) {
+ auto nested_type =
+ assert_cast<const DataTypeNullable*>(type.get())->get_nested_type();
+
+ // Only add ".nested" suffix for non-leaf (complex) nullable columns
+ // Leaf columns like String, Int, etc. should not get the ".nested" suffix
+ bool is_complex_type =
+ (typeid_cast<const DataTypeStruct*>(nested_type.get()) !=
+ nullptr) ||
+ (typeid_cast<const DataTypeArray*>(nested_type.get()) != nullptr) ||
+ (typeid_cast<const DataTypeMap*>(nested_type.get()) != nullptr);
+
+ std::string nested_name = is_complex_type ? name + ".nested" : name;
+ print_column_rows(nullable_col->get_nested_column_ptr(), nested_type,
+ nested_name, depth + (is_complex_type ? 1 : 0));
+ }
+ // Check if it's a struct column
+ else if (const auto* struct_col = typeid_cast<const ColumnStruct*>(col.get())) {
+ auto struct_type = assert_cast<const DataTypeStruct*>(type.get());
+ for (size_t i = 0; i < struct_col->tuple_size(); ++i) {
+ std::string field_name = struct_type->get_element_name(i);
+ auto field_type = struct_type->get_element(i);
+ print_column_rows(struct_col->get_column_ptr(i), field_type,
+ name + "." + field_name, depth + 1);
+ }
+ }
+ // Check if it's an array column
+ else if (const auto* array_col = typeid_cast<const ColumnArray*>(col.get())) {
+ auto array_type = assert_cast<const DataTypeArray*>(type.get());
+ auto element_type = array_type->get_nested_type();
+ print_column_rows(array_col->get_data_ptr(), element_type, name + ".data",
+ depth + 1);
+ }
+ };
+
+ // Print row counts for all columns
+ for (size_t i = 0; i < block.columns(); ++i) {
+ const auto& column_with_name = block.get_by_position(i);
+ print_column_rows(column_with_name.column, column_with_name.type, column_with_name.name,
+ 0);
+ EXPECT_EQ(column_with_name.column->size(), block.rows())
+ << "Column " << column_with_name.name << " size mismatch";
+ }
+ }
+
+ std::unique_ptr<doris::FileMetaCache> cache;
+ cctz::time_zone timezone_obj;
+};
+
+// Test reading real Hive Parquet file using HiveTableReader
+TEST_F(HiveReaderTest, ReadHiveParquetFile) {
+ // Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email
+ // Setup table descriptor for test columns with new schema:
+ /**
+ Schema:
+ message table {
+ required int64 id = 1;
+ required binary name (STRING) = 2;
+ required group profile = 3 {
+ optional group address = 4 {
+ optional binary street (STRING) = 7;
+ optional binary city (STRING) = 8;
+ optional group coordinates = 9 {
+ optional double lat = 10;
+ optional double lng = 11;
+ }
+ }
+ optional group contact = 5 {
+ optional binary email (STRING) = 12;
+ optional group phone = 13 {
+ optional binary country_code (STRING) = 14;
+ optional binary number (STRING) = 15;
+ }
+ }
+ optional group hobbies (LIST) = 6 {
+ repeated group list {
+ optional group element = 16 {
+ optional binary name (STRING) = 17;
+ optional int32 level = 18;
+ }
+ }
+ }
+ }
+ }
+ */
+
+ // Open the Hive Parquet test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ std::string test_file =
+ "./be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/"
+ "00000-0-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet";
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ GTEST_SKIP() << "Test file not found: " << test_file;
+ return;
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_PARQUET;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create ParquetReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader = ParquetReader::create_unique(&profile, scan_params, scan_range, 1024,
+ &ctz, nullptr, &runtime_state, cache.get());
+ ASSERT_NE(generic_reader, nullptr);
+
+ // Set file reader for the generic reader
+ auto parquet_reader = static_cast<ParquetReader*>(generic_reader.get());
+ parquet_reader->set_file_reader(file_reader);
+
+ // Create HiveParquetReader
+ auto hive_reader = std::make_unique<HiveParquetReader>(std::move(generic_reader), &profile,
+ &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ // Create complex struct types using helper function
+ DataTypePtr coordinates_struct_type, address_struct_type, phone_struct_type;
+ DataTypePtr contact_struct_type, hobby_element_struct_type, hobbies_array_type;
+ DataTypePtr profile_struct_type, name_type;
+ create_complex_struct_types(coordinates_struct_type, address_struct_type, phone_struct_type,
+ contact_struct_type, hobby_element_struct_type, hobbies_array_type,
+ profile_struct_type, name_type);
+
+ // Create tuple descriptor using helper function
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+ std::vector<std::string> table_column_names = {"name", "profile"};
+ std::vector<int> table_column_positions = {1, 2};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ TPrimitiveType::STRING, TPrimitiveType::STRUCT // profile 用 STRUCT 类型
+ };
+ const TupleDescriptor* tuple_descriptor =
+ create_tuple_descriptor(&desc_tbl, obj_pool, t_desc_table, t_table_desc,
+ table_column_names, table_column_positions, table_column_types);
+
+ VExprContextSPtrs conjuncts; // Empty conjuncts for this test
+ std::vector<std::string> table_col_names = {"name", "profile"};
+ const std::unordered_map<std::string, ColumnValueRangeType>* colname_to_value_range = nullptr;
+ const RowDescriptor* row_descriptor = nullptr;
+ const std::unordered_map<std::string, int>* colname_to_slot_id = nullptr;
+ const VExprContextSPtrs* not_single_slot_filter_conjuncts = nullptr;
+ const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts = nullptr;
+
+ st = hive_reader->init_reader(table_col_names, colname_to_value_range, conjuncts,
+ tuple_descriptor, row_descriptor, colname_to_slot_id,
+ not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts);
+ ASSERT_TRUE(st.ok()) << st;
+
+ std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>
+ partition_columns;
+ std::unordered_map<std::string, VExprContextSPtr> missing_columns;
+ ASSERT_TRUE(hive_reader->set_fill_columns(partition_columns, missing_columns).ok());
+
+ // Create block for reading nested structure (not flattened)
+ Block block;
+ {
+ MutableColumnPtr name_column = name_type->create_column();
+ block.insert(ColumnWithTypeAndName(std::move(name_column), name_type, "name"));
+ // Add profile column (nested struct)
+ MutableColumnPtr profile_column = profile_struct_type->create_column();
+ block.insert(
+ ColumnWithTypeAndName(std::move(profile_column), profile_struct_type, "profile"));
+ }
+
+ // Read data from the file
+ size_t read_rows = 0;
+ bool eof = false;
+ st = hive_reader->get_next_block(&block, &read_rows, &eof);
+ ASSERT_TRUE(st.ok()) << st;
+
+ // Verify test results using helper function
+ verify_test_results(block, read_rows);
+}
+
+// Test reading real Hive Orc file using HiveTableReader
+TEST_F(HiveReaderTest, ReadHiveOrcFile) {
+ // Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email
+ // Setup table descriptor for test columns with new schema:
+ /**
+ Schema:
+ message table {
+ required int64 id = 1;
+ required binary name (STRING) = 2;
+ required group profile = 3 {
+ optional group address = 4 {
+ optional binary street (STRING) = 7;
+ optional binary city (STRING) = 8;
+ optional group coordinates = 9 {
+ optional double lat = 10;
+ optional double lng = 11;
+ }
+ }
+ optional group contact = 5 {
+ optional binary email (STRING) = 12;
+ optional group phone = 13 {
+ optional binary country_code (STRING) = 14;
+ optional binary number (STRING) = 15;
+ }
+ }
+ optional group hobbies (LIST) = 6 {
+ repeated group list {
+ optional group element = 16 {
+ optional binary name (STRING) = 17;
+ optional int32 level = 18;
+ }
+ }
+ }
+ }
+ }
+ */
+ // Open the Hive Orc test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ std::string test_file =
+ "./be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/"
+ "00000-0-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc";
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ GTEST_SKIP() << "Test file not found: " << test_file;
+ return;
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_ORC;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create OrcReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader = OrcReader::create_unique(&profile, &runtime_state, scan_params,
+ scan_range, 1024, "CST", nullptr, cache.get());
+ ASSERT_NE(generic_reader, nullptr);
+
+ // Create HiveOrcReader
+ auto hive_reader =
+ std::make_unique<HiveOrcReader>(std::move(generic_reader), &profile, &runtime_state,
+ scan_params, scan_range, nullptr, nullptr, cache.get());
+
+ // Create complex struct types using helper function
+ DataTypePtr coordinates_struct_type, address_struct_type, phone_struct_type;
+ DataTypePtr contact_struct_type, hobby_element_struct_type, hobbies_array_type;
+ DataTypePtr profile_struct_type, name_type;
+ create_complex_struct_types(coordinates_struct_type, address_struct_type, phone_struct_type,
+ contact_struct_type, hobby_element_struct_type, hobbies_array_type,
+ profile_struct_type, name_type);
+
+ // Create tuple descriptor using helper function
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+ std::vector<std::string> table_column_names = {"name", "profile"};
+ std::vector<int> table_column_positions = {1, 2};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ TPrimitiveType::STRING, TPrimitiveType::STRUCT // profile 用 STRUCT 类型
+ };
+ const TupleDescriptor* tuple_descriptor =
+ create_tuple_descriptor(&desc_tbl, obj_pool, t_desc_table, t_table_desc,
+ table_column_names, table_column_positions, table_column_types);
+
+ VExprContextSPtrs conjuncts; // Empty conjuncts for this test
+ std::vector<std::string> table_col_names = {"name", "profile"};
+ const std::unordered_map<std::string, ColumnValueRangeType>* colname_to_value_range = nullptr;
+ const RowDescriptor* row_descriptor = nullptr;
+ const VExprContextSPtrs* not_single_slot_filter_conjuncts = nullptr;
+ const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts = nullptr;
+
+ st = hive_reader->init_reader(table_col_names, colname_to_value_range, conjuncts,
+ tuple_descriptor, row_descriptor,
+ not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts);
+ ASSERT_TRUE(st.ok()) << st;
+
+ std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>
+ partition_columns;
+ std::unordered_map<std::string, VExprContextSPtr> missing_columns;
+ ASSERT_TRUE(hive_reader->set_fill_columns(partition_columns, missing_columns).ok());
+
+ // Create block for reading nested structure (not flattened)
+ Block block;
+
+ {
+ MutableColumnPtr name_column = name_type->create_column();
+ block.insert(ColumnWithTypeAndName(std::move(name_column), name_type, "name"));
+ // Add profile column (nested struct)
+ MutableColumnPtr profile_column = profile_struct_type->create_column();
+ block.insert(
+ ColumnWithTypeAndName(std::move(profile_column), profile_struct_type, "profile"));
+ }
+
+ // Read data from the file
+ size_t read_rows = 0;
+ bool eof = false;
+ st = hive_reader->get_next_block(&block, &read_rows, &eof);
+ ASSERT_TRUE(st.ok()) << st;
+
+ // Verify test results using helper function
+ verify_test_results(block, read_rows);
+}
+
+} // namespace doris::vectorized::table
diff --git a/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp b/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp
new file mode 100644
index 00000000000..47e0ad87a4a
--- /dev/null
+++ b/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp
@@ -0,0 +1,1177 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cctz/time_zone.h>
+#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+#include <gen_cpp/Types_types.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/object_pool.h"
+#include "exec/olap_common.h"
+#include "io/fs/file_meta_cache.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_system.h"
+#include "io/fs/local_file_system.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_state.h"
+#include "util/timezone_utils.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_struct.h"
+#include "vec/core/block.h"
+#include "vec/core/column_with_type_and_name.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_struct.h"
+#include "vec/exec/format/parquet/vparquet_reader.h"
+#include "vec/exec/format/table/iceberg_reader.h"
+
+namespace doris::vectorized {
+
+// Define the column access path configuration structure
+struct ColumnAccessPathConfig {
+ std::string column_name;
+ std::vector<std::vector<std::string>> all_column_paths; // For all_column_access_paths
+ std::vector<std::vector<std::string>> predicate_paths; // For predicate_column_access_paths
+ bool use_name_paths = true; // true=use field names, false=use indices
+};
+
+// Iceberg 列ID分配 (基于 Protobuf Schema 定义)
+// 1: id (int64)
+// 2: name (string)
+// 3: profile (struct)
+// 9: address (struct)
+// 12: street (string)
+// 13: city (string)
+// 14: coordinates (struct)
+// 15: lat (double)
+// 16: lng (double)
+// 10: contact (struct)
+// 17: email (string)
+// 18: phone (struct)
+// 19: country_code (string)
+// 20: number (string)
+// 11: hobbies (list)
+// 21: element (struct)
+// 22: name (string)
+// 23: level (int32)
+// 4: tags (list)
+// 24: element (string)
+// 5: friends (list)
+// 25: element (struct)
+// 26: user_id (int64)
+// 27: nickname (string)
+// 28: friendship_level (int32)
+// 6: recent_activity (list)
+// 29: element (struct)
+// 30: action (string)
+// 31: details (list)
+// 32: element (struct)
+// 33: key (string)
+// 34: value (string)
+// 7: attributes (map)
+// 35: key (string)
+// 36: value (string)
+// 8: complex_attributes (map)
+// 37: key (string)
+// 38: value (struct)
+// 39: metadata (struct)
+// 43: version (string)
+// 44: created_time (timestamp)
+// 45: last_updated (timestamp)
+// 46: quality_score (double)
+// 40: historical_scores (list)
+// 47: element (struct)
+// 48: period (string)
+// 49: score (double)
+// 50: components (list)
+// 52: element (struct)
+// 53: component_name (string)
+// 54: weight (float)
+// 55: sub_scores (map)
+// 56: key (string)
+// 57: value (double)
+// 51: trends (struct)
+// 58: direction (string)
+// 59: magnitude (double)
+// 60: confidence_interval (struct)
+// 61: lower (double)
+// 62: upper (double)
+// 41: hierarchical_data (map)
+// 63: key (string)
+// 64: value (struct)
+// 65: category (string)
+// 66: sub_items (list)
+// 68: element (struct)
+// 69: item_id (int64)
+// 70: properties (map)
+// 72: key (string)
+// 73: value (string)
+// 71: metrics (struct)
+// 74: value (double)
+// 75: unit (string)
+// 67: summary (struct)
+// 76: total_count (int32)
+// 77: average_value (double)
+// 42: validation_rules (struct)
+// 78: required (boolean)
+// 79: constraints (list)
+// 81: element (struct)
+// 82: rule_type (string)
+// 83: parameters (map)
+// 85: key (string)
+// 86: value (string)
+// 84: error_message (string)
+// 80: complex_constraint (struct)
+// 87: logic_operator (string)
+// 88: operands (list)
+// 89: element (struct)
+// 90: field_path (string)
+// 91: operator (string)
+// 92: comparison_value (string)
+
+class IcebergReaderCreateColumnIdsTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ cache = std::make_unique<doris::FileMetaCache>(1024);
+
+ // Setup timezone
+ doris::TimezoneUtils::find_cctz_time_zone(doris::TimezoneUtils::default_time_zone,
+ timezone_obj);
+ }
+
+ void TearDown() override { cache.reset(); }
+
+ // Helper function to map column name to Iceberg field ID
+ int get_iceberg_field_id(const std::string& column_name) {
+ // Mapping based on the Iceberg schema definition in comments
+ static const std::unordered_map<std::string, int> column_to_field_id = {
+ {"id", 1}, {"name", 2},
+ {"profile", 3}, {"tags", 4},
+ {"friends", 5}, {"recent_activity", 6},
+ {"attributes", 7}, {"complex_attributes", 8}};
+
+ auto it = column_to_field_id.find(column_name);
+ if (it != column_to_field_id.end()) {
+ return it->second;
+ }
+ return -1; // Invalid field ID
+ }
+
+ // Helper function to create tuple descriptor
+ const TupleDescriptor* create_tuple_descriptor(
+ DescriptorTbl** desc_tbl, ObjectPool& obj_pool, TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc, const std::vector<std::string>& column_names,
+ const std::vector<int>& column_positions,
+ const std::vector<TPrimitiveType::type>& column_types,
+ const std::vector<ColumnAccessPathConfig>& access_configs = {}) {
+ // Create table descriptor with complex schema
+ auto create_table_desc = [this, &access_configs](
+ TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc,
+ const std::vector<std::string>& table_column_names,
+ const std::vector<int>& table_column_positions,
+ const std::vector<TPrimitiveType::type>& types) {
+ t_table_desc.__set_id(0);
+ t_table_desc.__set_tableType(TTableType::HIVE_TABLE);
+ t_table_desc.__set_numCols(0);
+ t_table_desc.__set_numClusteringCols(0);
+ t_desc_table.tableDescriptors.push_back(t_table_desc);
+ t_desc_table.__isset.tableDescriptors = true;
+ for (int i = 0; i < table_column_names.size(); i++) {
+ TSlotDescriptor tslot_desc;
+ TTypeDesc type;
+ if (table_column_names[i] == "id") {
+ // id: bigint
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(TPrimitiveType::BIGINT);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "name") {
+ // name: string
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(TPrimitiveType::STRING);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "tags") {
+ // tags: array<string>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_node;
+ element_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType element_scalar;
+ element_scalar.__set_type(TPrimitiveType::STRING);
+ element_node.__set_scalar_type(element_scalar);
+ type.types.push_back(element_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "friends") {
+ // friends: array<struct<user_id:bigint, nickname:string, friendship_level:int>>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_struct_node;
+ element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> element_fields;
+ TStructField user_id_field;
+ user_id_field.__set_name("user_id");
+ user_id_field.__set_contains_null(true);
+ element_fields.push_back(user_id_field);
+ TStructField nickname_field;
+ nickname_field.__set_name("nickname");
+ nickname_field.__set_contains_null(true);
+ element_fields.push_back(nickname_field);
+ TStructField friendship_level_field;
+ friendship_level_field.__set_name("friendship_level");
+ friendship_level_field.__set_contains_null(true);
+ element_fields.push_back(friendship_level_field);
+ element_struct_node.__set_struct_fields(element_fields);
+ type.types.push_back(element_struct_node);
+ // user_id: bigint
+ TTypeNode user_id_node;
+ user_id_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType user_id_scalar;
+ user_id_scalar.__set_type(TPrimitiveType::BIGINT);
+ user_id_node.__set_scalar_type(user_id_scalar);
+ type.types.push_back(user_id_node);
+ // nickname: string
+ TTypeNode nickname_node;
+ nickname_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType nickname_scalar;
+ nickname_scalar.__set_type(TPrimitiveType::STRING);
+ nickname_node.__set_scalar_type(nickname_scalar);
+ type.types.push_back(nickname_node);
+ // friendship_level: int
+ TTypeNode friendship_level_node;
+ friendship_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType friendship_level_scalar;
+ friendship_level_scalar.__set_type(TPrimitiveType::INT);
+ friendship_level_node.__set_scalar_type(friendship_level_scalar);
+ type.types.push_back(friendship_level_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "recent_activity") {
+ // recent_activity: array<struct<action:string, details:array<struct<key:string, value:string>>>>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_struct_node;
+ element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> element_fields;
+ TStructField action_field;
+ action_field.__set_name("action");
+ action_field.__set_contains_null(true);
+ element_fields.push_back(action_field);
+ TStructField details_field;
+ details_field.__set_name("details");
+ details_field.__set_contains_null(true);
+ element_fields.push_back(details_field);
+ element_struct_node.__set_struct_fields(element_fields);
+ type.types.push_back(element_struct_node);
+ // action: string
+ TTypeNode action_node;
+ action_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType action_scalar;
+ action_scalar.__set_type(TPrimitiveType::STRING);
+ action_node.__set_scalar_type(action_scalar);
+ type.types.push_back(action_node);
+ // details: array<struct<key:string, value:string>>
+ TTypeNode details_array_node;
+ details_array_node.__set_type(TTypeNodeType::ARRAY);
+ details_array_node.__set_contains_nulls({true});
+ type.types.push_back(details_array_node);
+ TTypeNode details_element_struct_node;
+ details_element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> details_element_fields;
+ TStructField key_field;
+ key_field.__set_name("key");
+ key_field.__set_contains_null(true);
+ details_element_fields.push_back(key_field);
+ TStructField value_field;
+ value_field.__set_name("value");
+ value_field.__set_contains_null(true);
+ details_element_fields.push_back(value_field);
+ details_element_struct_node.__set_struct_fields(details_element_fields);
+ type.types.push_back(details_element_struct_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: string
+ TTypeNode value_node;
+ value_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType value_scalar;
+ value_scalar.__set_type(TPrimitiveType::STRING);
+ value_node.__set_scalar_type(value_scalar);
+ type.types.push_back(value_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "attributes") {
+ // attributes: map<string, string>
+ TTypeNode map_node;
+ map_node.__set_type(TTypeNodeType::MAP);
+ map_node.__set_contains_nulls({true, true});
+ type.types.push_back(map_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: string
+ TTypeNode value_node;
+ value_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType value_scalar;
+ value_scalar.__set_type(TPrimitiveType::STRING);
+ value_node.__set_scalar_type(value_scalar);
+ type.types.push_back(value_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "complex_attributes") {
+ // complex_attributes: map<string, struct<metadata:struct<version:string, created_time:timestamp, last_updated:timestamp>>>
+ TTypeNode map_node;
+ map_node.__set_type(TTypeNodeType::MAP);
+ map_node.__set_contains_nulls({true, true});
+ type.types.push_back(map_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: struct<metadata:struct<version:string, created_time:timestamp, last_updated:timestamp>>
+ TTypeNode value_struct_node;
+ value_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> value_fields;
+ TStructField metadata_field;
+ metadata_field.__set_name("metadata");
+ metadata_field.__set_contains_null(true);
+ value_fields.push_back(metadata_field);
+ value_struct_node.__set_struct_fields(value_fields);
+ type.types.push_back(value_struct_node);
+ // metadata: struct<version:string, created_time:timestamp, last_updated:timestamp>
+ TTypeNode metadata_struct_node;
+ metadata_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> metadata_fields;
+ TStructField version_field;
+ version_field.__set_name("version");
+ version_field.__set_contains_null(true);
+ metadata_fields.push_back(version_field);
+ TStructField created_time_field;
+ created_time_field.__set_name("created_time");
+ created_time_field.__set_contains_null(true);
+ metadata_fields.push_back(created_time_field);
+ TStructField last_updated_field;
+ last_updated_field.__set_name("last_updated");
+ last_updated_field.__set_contains_null(true);
+ metadata_fields.push_back(last_updated_field);
+ metadata_struct_node.__set_struct_fields(metadata_fields);
+ type.types.push_back(metadata_struct_node);
+ // version: string
+ TTypeNode version_node;
+ version_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType version_scalar;
+ version_scalar.__set_type(TPrimitiveType::STRING);
+ version_node.__set_scalar_type(version_scalar);
+ type.types.push_back(version_node);
+ // created_time: timestamp
+ TTypeNode created_time_node;
+ created_time_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType created_time_scalar;
+ created_time_scalar.__set_type(TPrimitiveType::DATETIME);
+ created_time_node.__set_scalar_type(created_time_scalar);
+ type.types.push_back(created_time_node);
+ // last_updated: timestamp
+ TTypeNode last_updated_node;
+ last_updated_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType last_updated_scalar;
+ last_updated_scalar.__set_type(TPrimitiveType::DATETIME);
+ last_updated_node.__set_scalar_type(last_updated_scalar);
+ type.types.push_back(last_updated_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "profile") {
+ // STRUCT/ARRAY 节点设置 contains_nulls，SCALAR 节点不设置
+ TTypeNode struct_node;
+ struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> struct_fields;
+ TStructField address_field;
+ address_field.__set_name("address");
+ address_field.__set_contains_null(true);
+ struct_fields.push_back(address_field);
+ TStructField contact_field;
+ contact_field.__set_name("contact");
+ contact_field.__set_contains_null(true);
+ struct_fields.push_back(contact_field);
+ TStructField hobbies_field;
+ hobbies_field.__set_name("hobbies");
+ hobbies_field.__set_contains_null(true);
+ struct_fields.push_back(hobbies_field);
+ struct_node.__set_struct_fields(struct_fields);
+ type.types.push_back(struct_node);
+ TTypeNode address_node;
+ address_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> address_fields;
+ TStructField street_field;
+ street_field.__set_name("street");
+ street_field.__set_contains_null(true);
+ address_fields.push_back(street_field);
+ TStructField city_field;
+ city_field.__set_name("city");
+ city_field.__set_contains_null(true);
+ address_fields.push_back(city_field);
+ TStructField coordinates_field;
+ coordinates_field.__set_name("coordinates");
+ coordinates_field.__set_contains_null(true);
+ address_fields.push_back(coordinates_field);
+ address_node.__set_struct_fields(address_fields);
+ type.types.push_back(address_node);
+ TTypeNode street_node;
+ street_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType street_scalar;
+ street_scalar.__set_type(TPrimitiveType::STRING);
+ street_node.__set_scalar_type(street_scalar);
+ type.types.push_back(street_node);
+ TTypeNode city_node;
+ city_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType city_scalar;
+ city_scalar.__set_type(TPrimitiveType::STRING);
+ city_node.__set_scalar_type(city_scalar);
+ type.types.push_back(city_node);
+ TTypeNode coordinates_node;
+ coordinates_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> coordinates_fields;
+ TStructField lat_field;
+ lat_field.__set_name("lat");
+ lat_field.__set_contains_null(true);
+ coordinates_fields.push_back(lat_field);
+ TStructField lng_field;
+ lng_field.__set_name("lng");
+ lng_field.__set_contains_null(true);
+ coordinates_fields.push_back(lng_field);
+ coordinates_node.__set_struct_fields(coordinates_fields);
+ type.types.push_back(coordinates_node);
+ TTypeNode lat_node;
+ lat_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lat_scalar;
+ lat_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lat_node.__set_scalar_type(lat_scalar);
+ type.types.push_back(lat_node);
+ TTypeNode lng_node;
+ lng_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lng_scalar;
+ lng_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lng_node.__set_scalar_type(lng_scalar);
+ type.types.push_back(lng_node);
+ TTypeNode contact_node;
+ contact_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> contact_fields;
+ TStructField email_field;
+ email_field.__set_name("email");
+ email_field.__set_contains_null(true);
+ contact_fields.push_back(email_field);
+ TStructField phone_field;
+ phone_field.__set_name("phone");
+ phone_field.__set_contains_null(true);
+ contact_fields.push_back(phone_field);
+ contact_node.__set_struct_fields(contact_fields);
+ type.types.push_back(contact_node);
+ TTypeNode email_node;
+ email_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType email_scalar;
+ email_scalar.__set_type(TPrimitiveType::STRING);
+ email_node.__set_scalar_type(email_scalar);
+ type.types.push_back(email_node);
+ TTypeNode phone_node;
+ phone_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> phone_fields;
+ TStructField country_code_field;
+ country_code_field.__set_name("country_code");
+ country_code_field.__set_contains_null(true);
+ phone_fields.push_back(country_code_field);
+ TStructField number_field;
+ number_field.__set_name("number");
+ number_field.__set_contains_null(true);
+ phone_fields.push_back(number_field);
+ phone_node.__set_struct_fields(phone_fields);
+ type.types.push_back(phone_node);
+ TTypeNode country_code_node;
+ country_code_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType country_code_scalar;
+ country_code_scalar.__set_type(TPrimitiveType::STRING);
+ country_code_node.__set_scalar_type(country_code_scalar);
+ type.types.push_back(country_code_node);
+ TTypeNode number_node;
+ number_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType number_scalar;
+ number_scalar.__set_type(TPrimitiveType::STRING);
+ number_node.__set_scalar_type(number_scalar);
+ type.types.push_back(number_node);
+ TTypeNode hobbies_node;
+ hobbies_node.__set_type(TTypeNodeType::ARRAY);
+ hobbies_node.__set_contains_nulls({true});
+ type.types.push_back(hobbies_node);
+ TTypeNode hobby_element_node;
+ hobby_element_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> hobby_element_fields;
+ TStructField hobby_name_field;
+ hobby_name_field.__set_name("name");
+ hobby_name_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_name_field);
+ TStructField hobby_level_field;
+ hobby_level_field.__set_name("level");
+ hobby_level_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_level_field);
+ hobby_element_node.__set_struct_fields(hobby_element_fields);
+ type.types.push_back(hobby_element_node);
+ TTypeNode hobby_name_node;
+ hobby_name_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_name_scalar;
+ hobby_name_scalar.__set_type(TPrimitiveType::STRING);
+ hobby_name_node.__set_scalar_type(hobby_name_scalar);
+ type.types.push_back(hobby_name_node);
+ TTypeNode hobby_level_node;
+ hobby_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_level_scalar;
+ hobby_level_scalar.__set_type(TPrimitiveType::INT);
+ hobby_level_node.__set_scalar_type(hobby_level_scalar);
+ type.types.push_back(hobby_level_node);
+ tslot_desc.__set_slotType(type);
+ } else {
+ // 普通类型
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(types[i]);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ }
+ tslot_desc.__set_id(i);
+ tslot_desc.__set_parent(0);
+ tslot_desc.__set_colName(table_column_names[i]);
+ tslot_desc.__set_columnPos(table_column_positions[i]);
+ tslot_desc.__set_byteOffset(0);
+ tslot_desc.__set_nullIndicatorByte(0);
+ tslot_desc.__set_nullIndicatorBit(-1);
+ tslot_desc.__set_slotIdx(0);
+ tslot_desc.__set_isMaterialized(true);
+
+ // Set Iceberg field ID (col_unique_id)
+ int iceberg_field_id = get_iceberg_field_id(table_column_names[i]);
+ if (iceberg_field_id > 0) {
+ tslot_desc.__set_col_unique_id(iceberg_field_id);
+ }
+
+ // Use configuration to set column_access_paths
+ for (const auto& config : access_configs) {
+ set_column_access_paths(tslot_desc, config);
+ }
+
+ t_desc_table.slotDescriptors.push_back(tslot_desc);
+ }
+ t_desc_table.__isset.slotDescriptors = true;
+ TTupleDescriptor t_tuple_desc;
+ t_tuple_desc.__set_id(0);
+ t_tuple_desc.__set_byteSize(16);
+ t_tuple_desc.__set_numNullBytes(0);
+ t_tuple_desc.__set_tableId(0);
+ t_tuple_desc.__isset.tableId = true;
+ t_desc_table.tupleDescriptors.push_back(t_tuple_desc);
+ };
+
+ create_table_desc(t_desc_table, t_table_desc, column_names, column_positions, column_types);
+ EXPECT_TRUE(DescriptorTbl::create(&obj_pool, t_desc_table, desc_tbl).ok());
+ return (*desc_tbl)->get_tuple_descriptor(0);
+ }
+
+ // Helper function: set column access paths on a slot descriptor
+ void set_column_access_paths(TSlotDescriptor& tslot_desc,
+ const ColumnAccessPathConfig& config) {
+ if (config.column_name != tslot_desc.colName) {
+ return; // Not the target column, skip
+ }
+
+ // Set all_column_access_paths
+ if (!config.all_column_paths.empty()) {
+ std::vector<TColumnAccessPath> access_paths;
+ for (const auto& path_vector : config.all_column_paths) {
+ TColumnAccessPath access_path;
+ access_path.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path;
+ data_path.__set_path(path_vector);
+ access_path.__set_data_access_path(data_path);
+ access_paths.push_back(access_path);
+ }
+ tslot_desc.__set_all_access_paths(access_paths);
+ }
+
+ // Set predicate_column_access_paths
+ if (!config.predicate_paths.empty()) {
+ std::vector<TColumnAccessPath> access_paths;
+ for (const auto& path_vector : config.predicate_paths) {
+ TColumnAccessPath access_path;
+ access_path.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path;
+ data_path.__set_path(path_vector);
+ access_path.__set_data_access_path(data_path);
+ access_paths.push_back(access_path);
+ }
+ tslot_desc.__set_predicate_access_paths(access_paths);
+ }
+ }
+
+ std::unique_ptr<doris::FileMetaCache> cache;
+ cctz::time_zone timezone_obj;
+
+ // Helper function: create and setup ParquetReader
+ std::tuple<std::unique_ptr<IcebergParquetReader>, const FieldDescriptor*> create_parquet_reader(
+ const std::string& test_file) {
+ // Open the Iceberg Parquet test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ return {nullptr, nullptr};
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_PARQUET;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create ParquetReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader =
+ ParquetReader::create_unique(&profile, scan_params, scan_range, 1024, &ctz, nullptr,
+ &runtime_state, cache.get());
+ if (!generic_reader) {
+ return {nullptr, nullptr};
+ }
+
+ // Set file reader for the generic reader
+ auto parquet_reader = static_cast<ParquetReader*>(generic_reader.get());
+ parquet_reader->set_file_reader(file_reader);
+
+ const FieldDescriptor* field_desc = nullptr;
+ st = parquet_reader->get_file_metadata_schema(&field_desc);
+ if (!st.ok() || !field_desc) {
+ return {nullptr, nullptr};
+ }
+
+ // Create IcebergParquetReader
+ auto iceberg_reader = std::make_unique<IcebergParquetReader>(
+ std::move(generic_reader), &profile, &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ return {std::move(iceberg_reader), field_desc};
+ }
+
+ // Helper function: create and setup OrcReader
+ std::tuple<std::unique_ptr<IcebergOrcReader>, const orc::Type*> create_orc_reader(
+ const std::string& test_file) {
+ // Open the Iceberg Orc test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ return {nullptr, nullptr};
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_ORC;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create OrcReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader =
+ OrcReader::create_unique(&profile, &runtime_state, scan_params, scan_range, 1024,
+ "CST", nullptr, cache.get());
+ if (!generic_reader) {
+ return {nullptr, nullptr};
+ }
+
+ auto orc_reader = static_cast<OrcReader*>(generic_reader.get());
+ // Get FieldDescriptor from Orc file
+ const orc::Type* orc_type_ptr = nullptr;
+ st = orc_reader->get_file_type(&orc_type_ptr);
+ if (!st.ok() || !orc_type_ptr) {
+ return {nullptr, nullptr};
+ }
+
+ // Create IcebergOrcReader
+ auto iceberg_reader = std::make_unique<IcebergOrcReader>(
+ std::move(generic_reader), &profile, &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ return {std::move(iceberg_reader), orc_type_ptr};
+ }
+
+ // Helper function: run Parquet test with different column ID extraction methods
+ void run_parquet_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false,
+ bool should_skip_assertion = false) {
+ std::string test_file =
+ "./be/test/exec/test_data/nested_user_profiles_iceberg_parquet/data/"
+ "00000-9-a7e0135f-d581-40e4-8d56-a929aded99e4-0-00001.parquet";
+
+ auto [iceberg_reader, field_desc] = create_parquet_reader(test_file);
+ if (!iceberg_reader || !field_desc) {
+ GTEST_SKIP() << "Test file not found or failed to create reader: " << test_file;
+ return;
+ }
+
+ // Create tuple descriptor based on full schema
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+
+ // Define all columns according to the schema
+ std::vector<std::string> table_column_names = {
+ "name", "profile", "tags", "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ // TPrimitiveType::BIGINT, // id
+ TPrimitiveType::STRING, // name
+ TPrimitiveType::STRUCT, // profile
+ TPrimitiveType::ARRAY, // tags
+ TPrimitiveType::ARRAY, // friends
+ TPrimitiveType::ARRAY, // recent_activity
+ TPrimitiveType::MAP, // attributes
+ TPrimitiveType::MAP // complex_attributes
+ };
+
+ const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
+ &desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
+ table_column_positions, table_column_types, access_configs);
+
+ // Execute test based on method choice
+ ColumnIdResult actual_result;
+ if (use_top_level_method) {
+ // actual_result = IcebergParquetReader::_create_column_ids_by_top_level_col_index(
+ // field_desc, tuple_descriptor);
+ } else {
+ actual_result = IcebergParquetReader::_create_column_ids(field_desc, tuple_descriptor);
+ }
+
+ if (!should_skip_assertion) {
+ EXPECT_EQ(actual_result.column_ids, expected_column_ids);
+ EXPECT_EQ(actual_result.filter_column_ids, expected_filter_column_ids);
+ }
+ }
+
+ // Helper function: run Orc test with different column ID extraction methods
+ void run_orc_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false,
+ bool should_skip_assertion = false) {
+ std::string test_file =
+ "./be/test/exec/test_data/nested_user_profiles_iceberg_orc/data/"
+ "00000-8-5a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc";
+
+ auto [iceberg_reader, orc_type] = create_orc_reader(test_file);
+ if (!iceberg_reader || !orc_type) {
+ GTEST_SKIP() << "Test file not found or failed to create reader: " << test_file;
+ return;
+ }
+
+ // Create tuple descriptor based on full schema
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+
+ // Define all columns according to the schema
+ std::vector<std::string> table_column_names = {
+ "name", "profile", "tags", "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ // TPrimitiveType::BIGINT, // id
+ TPrimitiveType::STRING, // name
+ TPrimitiveType::STRUCT, // profile
+ TPrimitiveType::ARRAY, // tags
+ TPrimitiveType::ARRAY, // friends
+ TPrimitiveType::ARRAY, // recent_activity
+ TPrimitiveType::MAP, // attributes
+ TPrimitiveType::MAP // complex_attributes
+ };
+
+ const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
+ &desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
+ table_column_positions, table_column_types, access_configs);
+
+ // Execute test based on method choice
+ ColumnIdResult actual_result;
+ if (use_top_level_method) {
+ // actual_result = IcebergOrcReader::_create_column_ids_by_top_level_col_index(
+ // orc_type, tuple_descriptor);
+ } else {
+ actual_result = IcebergOrcReader::_create_column_ids(orc_type, tuple_descriptor);
+ }
+
+ if (!should_skip_assertion) {
+ EXPECT_EQ(actual_result.column_ids, expected_column_ids);
+ EXPECT_EQ(actual_result.filter_column_ids, expected_filter_column_ids);
+ }
+ }
+};
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_1) {
+ // Properties:
+
+ // Configure access paths for the profile column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"3", "9", "14", "15"},
+ {"3", "9", "14", "16"},
+ {"3", "10", "17"},
+ {"3", "11", "*", "23"}};
+ access_config.predicate_paths = {{"3", "9", "14", "15"}, {"3", "10", "17"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 7, 8, 9, 10, 11, 15, 16, 18};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_2) {
+ // Properties:
+ // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
+ // ORC 列ID分配为树型递增，根节点为0，子节点依次递增。
+ // 0: struct (table/root)
+ // 1: id (int64)
+ // 2: name (string)
+ // 3: profile (struct)
+ // 4: address (struct)
+ // 5: street (string)
+ // 6: city (string)
+ // 7: coordinates (struct)
+ // 8: lat (double)
+ // 9: lng (double)
+ // 10: contact (struct)
+ // 11: email (string)
+ // 12: phone (struct)
+ // 13: country_code (string)
+ // 14: number (string)
+ // 15: hobbies (list/array)
+ // 16: element (struct)
+ // 17: name (string)
+ // 18: level (int32)
+
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"3"}};
+ access_config.predicate_paths = {{"3", "9", "14", "15"}, {"3", "10", "17"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10,
+ 11, 12, 13, 14, 15, 16, 17, 18};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_3) {
+ // Properties:
+ // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
+ // ORC 列ID分配为树型递增，根节点为0，子节点依次递增。
+ // 0: struct (table/root)
+ // 1: id (int64)
+ // 2: name (string)
+ // 3: profile (struct)
+ // 4: address (struct)
+ // 5: street (string)
+ // 6: city (string)
+ // 7: coordinates (struct)
+ // 8: lat (double)
+ // 9: lng (double)
+ // 10: contact (struct)
+ // 11: email (string)
+ // 12: phone (struct)
+ // 13: country_code (string)
+ // 14: number (string)
+ // 15: hobbies (list/array)
+ // 16: element (struct)
+ // 17: name (string)
+ // 18: level (int32)
+
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"3", "10"}, {"3", "9"}};
+ access_config.predicate_paths = {{"3", "9", "14"}, {"3", "10", "17"}};
+
+ // access_config.all_column_paths = {{"profile", "contact"}, {"profile", "address"}};
+ // access_config.predicate_paths = {{"profile", "address", "coordinates"},
+ // {"profile", "contact", "email"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 9, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_4) {
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {};
+ access_config.predicate_paths = {};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_5) {
+ // 19: tags (array)
+ // 20: element (string)
+ // 21: friends (array)
+ // 22: element (struct)
+ // 23: user_id (bigint)
+ // 24: nickname (string)
+ // 25: friendship_level (int)
+ // 26: recent_activity (array)
+ // 27: element (struct)
+ // 28: action (string)
+ // 29: details (array)
+ // 30: element (struct)
+ // 31: key (string)
+ // 32: value (string)
+
+ std::vector<ColumnAccessPathConfig> access_configs;
+
+ {
+ // 配置friends列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "friends";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"5", "*", "27"}, {"5", "*", "28"}};
+ access_config.predicate_paths = {
+ {"5", "*", "27"},
+ };
+ access_configs.push_back(access_config);
+ }
+
+ {
+ // 配置recent_activity列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "recent_activity";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"6", "*", "30"}, {"6", "*", "31", "*", "34"}};
+ access_config.predicate_paths = {
+ {"6", "*", "30"},
+ };
+ access_configs.push_back(access_config);
+ }
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 21, 22, 24, 25, 26, 27, 28, 29, 30, 32};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {21, 22, 24, 26, 27, 28};
+
+ run_parquet_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_6) {
+ // 33: attributes (map)
+ // 34: key (string)
+ // 35: value (string)
+ // 36: complex_attributes (map)
+ // 37: key (string)
+ // 38: value (struct)
+ // 39: metadata (struct)
+ // 40: version (string)
+ // 41: created_time (timestamp)
+ // 42: last_updated (timestamp)
+ // 43: quality_score (double)
+ // 44: historical_scores (array)
+ // 45: element (struct)
+ // 46: period (string)
+ // 47: score (double)
+ // 48: components (array)
+ // 49: element (struct)
+ // 50: component_name (string)
+ // 51: weight (float)
+ // 52: sub_scores (map)
+ // 53: key (string)
+ // 54: value (double)
+ // 55: trends (struct)
+ // 56: direction (string)
+ // 57: magnitude (double)
+ // 58: confidence_interval (struct)
+ // 59: lower (double)
+ // 60: upper (double)
+ // 61: hierarchical_data (map)
+ // 62: key (string)
+ // 63: value (struct)
+ // 64: category (string)
+ // 65: sub_items (array)
+ // 66: element (struct)
+ // 67: item_id (bigint)
+ // 68: properties (map)
+ // 69: key (string)
+ // 70: value (string)
+ // 71: metrics (struct)
+ // 72: value (double)
+ // 73: unit (string)
+ // 74: summary (struct)
+ // 75: total_count (int)
+ // 76: average_value (double)
+ // 77: validation_rules (struct)
+ // 78: required (boolean)
+ // 79: constraints (array)
+ // 80: element (struct)
+ // 81: rule_type (string)
+ // 82: parameters (map)
+ // 83: key (string)
+ // 84: value (string)
+ // 85: error_message (string)
+ // 86: complex_constraint (struct)
+ // 87: logic_operator (string)
+ // 88: operands (array)
+ // 89: element (struct)
+ // 90: field_path (string)
+ // 91: operator (string)
+ // 92: comparison_value (string)
+ std::vector<ColumnAccessPathConfig> access_configs;
+
+ {
+ // 配置complex_attributes列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "complex_attributes";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"8", "*", "39", "43"},
+ {"8", "*", "40", "*", "50", "*", "55"},
+ {"8", "*", "41", "VALUES"},
+ {"8", "VALUES", "42", "79", "*", "83", "KEYS"}};
+ access_config.predicate_paths = {{"8", "*", "39", "43"}};
+ access_configs.push_back(access_config);
+ }
+
+ {
+ // parquet values should access keys
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53,
+ 54, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
+ 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {36, 37, 38, 39, 40};
+
+ run_parquet_test_with_method(access_configs, expected_column_ids,
+ expected_filter_column_ids);
+ }
+
+ {
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52,
+ 53, 54, 61, 63, 64, 65, 66, 67, 68, 69, 70,
+ 71, 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {36, 37, 38, 39, 40};
+
+ run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+ }
+}
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/test/vec/exec/format/table/iceberg/iceberg_reader_test.cpp b/be/test/vec/exec/format/table/iceberg/iceberg_reader_test.cpp
new file mode 100644
index 00000000000..12d6bb0299f
--- /dev/null
+++ b/be/test/vec/exec/format/table/iceberg/iceberg_reader_test.cpp
@@ -0,0 +1,739 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/iceberg_reader.h"
+
+#include <cctz/time_zone.h>
+#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+#include <gen_cpp/Types_types.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/object_pool.h"
+#include "exec/olap_common.h"
+#include "io/fs/file_meta_cache.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_system.h"
+#include "io/fs/local_file_system.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_state.h"
+#include "util/timezone_utils.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_struct.h"
+#include "vec/core/block.h"
+#include "vec/core/column_with_type_and_name.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_struct.h"
+#include "vec/exec/format/parquet/vparquet_reader.h"
+
+namespace doris::vectorized {
+
+class IcebergReaderTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ cache = std::make_unique<doris::FileMetaCache>(1024);
+
+ // Setup timezone
+ doris::TimezoneUtils::find_cctz_time_zone(doris::TimezoneUtils::default_time_zone,
+ timezone_obj);
+ }
+
+ void TearDown() override { cache.reset(); }
+
+ // Helper function to create complex struct types for testing
+ void create_complex_struct_types(DataTypePtr& coordinates_struct_type,
+ DataTypePtr& address_struct_type,
+ DataTypePtr& phone_struct_type,
+ DataTypePtr& contact_struct_type,
+ DataTypePtr& hobby_element_struct_type,
+ DataTypePtr& hobbies_array_type,
+ DataTypePtr& profile_struct_type, DataTypePtr& name_type) {
+ // Create name column type (direct field)
+ name_type = make_nullable(std::make_shared<DataTypeString>());
+
+ // First create coordinates struct type
+ std::vector<DataTypePtr> coordinates_types = {
+ make_nullable(std::make_shared<DataTypeFloat64>()), // lat (field ID 10)
+ make_nullable(std::make_shared<DataTypeFloat64>()) // lng (field ID 11)
+ };
+ std::vector<std::string> coordinates_names = {"lat", "lng"};
+ coordinates_struct_type = make_nullable(
+ std::make_shared<DataTypeStruct>(coordinates_types, coordinates_names));
+
+ // Create address struct type (with street, city, coordinates)
+ std::vector<DataTypePtr> address_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // street (field ID 7)
+ make_nullable(std::make_shared<DataTypeString>()), // city (field ID 8)
+ coordinates_struct_type // coordinates (field ID 9)
... 739 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 05/18: fix "Index 2 out of bounds for length 2" when use tvf
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit b7f17ea882062bba0a7cfe595b32921c613c0d60
Author: 924060929 <la...@selectdb.com>
AuthorDate: Wed Oct 29 11:17:41 2025 +0800
fix "Index 2 out of bounds for length 2" when use tvf
---
.../java/org/apache/doris/datasource/FileScanNode.java | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
index 05a843f0b6c..f1205069e7c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
@@ -50,6 +50,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import java.util.Collections;
@@ -212,10 +213,12 @@ public abstract class FileScanNode extends ExternalScanNode {
TExpr tExpr = new TExpr();
tExpr.setNodes(Lists.newArrayList());
- int index = 0;
- List<SlotDescriptor> slots = desc.getSlots();
+ Map<String, SlotDescriptor> nameToSlotDesc = Maps.newLinkedHashMap();
+ for (SlotDescriptor slot : desc.getSlots()) {
+ nameToSlotDesc.put(slot.getColumn().getName(), slot);
+ }
+
for (Column column : getColumns()) {
- SlotDescriptor slotDescriptor = slots.get(index++);
Expr expr;
Expression expression;
if (column.getDefaultValue() != null) {
@@ -230,9 +233,12 @@ public abstract class FileScanNode extends ExternalScanNode {
if (useVarcharAsNull) {
expression = new NullLiteral(VarcharType.SYSTEM_DEFAULT);
} else {
+ SlotDescriptor slotDescriptor = nameToSlotDesc.get(column.getName());
// the nested type(map/array/struct) maybe pruned,
// we should use the pruned type to avoid be core
- expression = new NullLiteral(DataType.fromCatalogType(slotDescriptor.getType()));
+ expression = new NullLiteral(DataType.fromCatalogType(
+ slotDescriptor == null ? column.getType() : slotDescriptor.getType()
+ ));
}
} else {
expression = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 17/18: fix: missed pruned type with map/array types
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 05c28879f0b13f467503487c876a21a0781a8222
Author: Hu Shenggang <hu...@selectdb.com>
AuthorDate: Tue Nov 4 14:26:08 2025 +0800
fix: missed pruned type with map/array types
---
be/src/http/action/debug_point_action.cpp | 4 +++-
be/src/olap/rowset/segment_v2/column_reader.cpp | 2 ++
be/src/olap/tablet_schema.cpp | 5 +++-
be/src/vec/exec/scan/olap_scanner.cpp | 4 +++-
.../complex_types/test_pruned_columns.out | 7 ++++++
.../complex_types/test_pruned_columns.groovy | 27 ++++++++++++++++++++++
6 files changed, 46 insertions(+), 3 deletions(-)
diff --git a/be/src/http/action/debug_point_action.cpp b/be/src/http/action/debug_point_action.cpp
index 04aa38efaa4..600ae645341 100644
--- a/be/src/http/action/debug_point_action.cpp
+++ b/be/src/http/action/debug_point_action.cpp
@@ -35,11 +35,13 @@ void BaseDebugPointAction::handle(HttpRequest* req) {
"Disable debug points. please check config::enable_debug_points");
}
std::string result = status.to_json();
- LOG(INFO) << "handle request result:" << result;
+ LOG(INFO) << "handle request result:" << result << ", status: " << status.to_string();
if (status.ok()) {
+ LOG(INFO) << "***** send ok";
HttpChannel::send_reply(req, HttpStatus::OK, result);
} else {
HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, result);
+ LOG(INFO) << "***** send error";
}
}
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 0cecd06e60e..822b5d31332 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -945,6 +945,8 @@ Result<TColumnAccessPaths> ColumnIterator::_get_sub_access_paths(
if (!name_path.data_access_path.path.empty()) {
++it;
} else {
+ LOG(INFO) << "********* set _column_name: " << _column_name
+ << " need_to_read true **********";
set_need_to_read();
it = sub_access_paths.erase(it);
}
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index cee5f53ae6b..e231ba88106 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -1684,11 +1684,14 @@ vectorized::Block TabletSchema::create_block(
tablet_columns_need_convert_null->find(cid) !=
tablet_columns_need_convert_null->end());
auto data_type = vectorized::DataTypeFactory::instance().create_data_type(col, is_nullable);
- if (col.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ if (col.type() == FieldType::OLAP_FIELD_TYPE_STRUCT ||
+ col.type() == FieldType::OLAP_FIELD_TYPE_MAP ||
+ col.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) {
if (_pruned_columns_data_type.contains(col.unique_id())) {
data_type = _pruned_columns_data_type.at(col.unique_id());
}
}
+
if (_vir_col_idx_to_unique_id.contains(cid)) {
block.insert({vectorized::ColumnNothing::create(0), data_type, col.name()});
VLOG_DEBUG << fmt::format(
diff --git a/be/src/vec/exec/scan/olap_scanner.cpp b/be/src/vec/exec/scan/olap_scanner.cpp
index 51fc4daa669..df48549895d 100644
--- a/be/src/vec/exec/scan/olap_scanner.cpp
+++ b/be/src/vec/exec/scan/olap_scanner.cpp
@@ -556,7 +556,9 @@ Status OlapScanner::_init_return_columns() {
{column.unique_id(), slot->predicate_access_paths()});
}
- if (slot->type()->get_primitive_type() == PrimitiveType::TYPE_STRUCT &&
+ if ((slot->type()->get_primitive_type() == PrimitiveType::TYPE_STRUCT ||
+ slot->type()->get_primitive_type() == PrimitiveType::TYPE_MAP ||
+ slot->type()->get_primitive_type() == PrimitiveType::TYPE_ARRAY) &&
!slot->all_access_paths().empty()) {
tablet_schema->add_pruned_columns_data_type(column.unique_id(), slot->type());
}
diff --git a/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out b/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out
index 74f336d27e5..870a5763074 100644
--- a/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out
+++ b/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out
@@ -22,3 +22,10 @@
1 beijing
2 shanghai
+-- !sql6 --
+2
+
+-- !sql7 --
+0.41
+0.99
+
diff --git a/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy b/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy
index ad032fab8f2..d54d924e97f 100644
--- a/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy
+++ b/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy
@@ -58,4 +58,31 @@ suite("test_pruned_columns") {
qt_sql5 """
select id, struct_element(s, 'city') from `tbl_test_pruned_columns` where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 order by 1;
"""
+
+ sql """DROP TABLE IF EXISTS `tbl_test_pruned_columns_map`"""
+ sql """
+ CREATE TABLE `tbl_test_pruned_columns_map` (
+ `id` bigint NULL,
+ `dynamic_attributes` map<text,struct<attribute_value:text,confidence_score:double,last_updated:datetime>> NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY RANDOM BUCKETS AUTO
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """
+ insert into `tbl_test_pruned_columns_map` values
+ (1, '{"theme_preference":{"attribute_value":"light", "confidence_score":0.41, "last_updated":"2025-11-03 15:32:33"}, "language_setting":{"attribute_value":"es", "confidence_score":0.65, "last_updated":"2025-11-03 15:32:33"}}'),
+ (2, '{"theme_preference":{"attribute_value":"light", "confidence_score":0.99, "last_updated":"2025-11-03 15:32:33"}, "language_setting":{"attribute_value":"es", "confidence_score":0.92, "last_updated":"2025-11-03 15:32:33"}}');
+ """
+
+ qt_sql6 """
+ select count(struct_element(dynamic_attributes['theme_preference'], 'confidence_score')) from `tbl_test_pruned_columns_map`;
+ """
+
+ qt_sql7 """
+ select struct_element(dynamic_attributes['theme_preference'], 'confidence_score') from `tbl_test_pruned_columns_map` order by id;
+ """
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 18/18: remove debug log
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5119904664cae4fb35676d0226ea9585e17b93cd
Author: Hu Shenggang <hu...@selectdb.com>
AuthorDate: Tue Nov 4 14:28:15 2025 +0800
remove debug log
---
be/src/http/action/debug_point_action.cpp | 3 ---
be/src/olap/rowset/segment_v2/column_reader.cpp | 2 --
2 files changed, 5 deletions(-)
diff --git a/be/src/http/action/debug_point_action.cpp b/be/src/http/action/debug_point_action.cpp
index 600ae645341..babea96268a 100644
--- a/be/src/http/action/debug_point_action.cpp
+++ b/be/src/http/action/debug_point_action.cpp
@@ -35,13 +35,10 @@ void BaseDebugPointAction::handle(HttpRequest* req) {
"Disable debug points. please check config::enable_debug_points");
}
std::string result = status.to_json();
- LOG(INFO) << "handle request result:" << result << ", status: " << status.to_string();
if (status.ok()) {
- LOG(INFO) << "***** send ok";
HttpChannel::send_reply(req, HttpStatus::OK, result);
} else {
HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, result);
- LOG(INFO) << "***** send error";
}
}
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 822b5d31332..0cecd06e60e 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -945,8 +945,6 @@ Result<TColumnAccessPaths> ColumnIterator::_get_sub_access_paths(
if (!name_path.data_access_path.path.empty()) {
++it;
} else {
- LOG(INFO) << "********* set _column_name: " << _column_name
- << " need_to_read true **********";
set_need_to_read();
it = sub_access_paths.erase(it);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 08/18: support push project in aggregate/join
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit bbc61b4cdab71e64e7b27376eb8b7ab3cbc4925d
Author: 924060929 <la...@selectdb.com>
AuthorDate: Wed Oct 29 19:24:51 2025 +0800
support push project in aggregate/join
---
.../org/apache/doris/datasource/FileScanNode.java | 2 +-
.../org/apache/doris/nereids/rules/RuleSet.java | 4 +-
.../org/apache/doris/nereids/rules/RuleType.java | 11 ++-
.../nereids/rules/analysis/NormalizeAggregate.java | 3 +-
...hDownNestedColumn.java => PushDownProject.java} | 107 +++++++++++++++++---
.../apache/doris/planner/MaterializationNode.java | 2 +
.../org/apache/doris/planner/OlapScanNode.java | 2 +-
.../java/org/apache/doris/planner/PlanNode.java | 109 +++++++++++++++++++++
.../java/org/apache/doris/planner/ScanNode.java | 109 ---------------------
.../rules/rewrite/PruneNestedColumnTest.java | 27 +++++
10 files changed, 245 insertions(+), 131 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
index f1205069e7c..46c69247bf1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
@@ -186,7 +186,7 @@ public abstract class FileScanNode extends ExternalScanNode {
}
output.append(String.format("numNodes=%s", numNodes)).append("\n");
- printNestedColumns(output, prefix);
+ printNestedColumns(output, prefix, getTupleDesc());
// pushdown agg
output.append(prefix).append(String.format("pushdown agg=%s", pushDownAggNoGroupingOp));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index a2268581a47..3d490930b97 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -115,7 +115,7 @@ import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughSort;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughWindow;
import org.apache.doris.nereids.rules.rewrite.PushDownJoinOtherCondition;
import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughJoin;
-import org.apache.doris.nereids.rules.rewrite.PushDownNestedColumn;
+import org.apache.doris.nereids.rules.rewrite.PushDownProject;
import org.apache.doris.nereids.rules.rewrite.PushDownProjectThroughLimit;
import org.apache.doris.nereids.rules.rewrite.PushDownTopNDistinctThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughJoin;
@@ -167,7 +167,7 @@ public class RuleSet {
new PushDownFilterThroughSetOperation(),
new PushDownFilterThroughGenerate(),
new PushDownProjectThroughLimit(),
- new PushDownNestedColumn(),
+ new PushDownProject(),
new EliminateOuterJoin(),
new MergeProjectable(),
new MergeFilters(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 70bedd1b85b..0e187084b4c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -196,11 +196,12 @@ public enum RuleType {
PUSH_DOWN_PREDICATE_THROUGH_REPEAT(RuleTypeClass.REWRITE),
PUSH_DOWN_EXPRESSIONS_IN_HASH_CONDITIONS(RuleTypeClass.REWRITE),
// Pushdown nested column project
- PUSH_DOWN_NESTED_COLUMN_THROUGH_JOIN(RuleTypeClass.REWRITE),
- PUSH_DOWN_NESTED_COLUMN_THROUGH_UNION(RuleTypeClass.REWRITE),
- PUSH_DOWN_NESTED_COLUMN_THROUGH_WINDOW(RuleTypeClass.REWRITE),
- PUSH_DOWN_NESTED_COLUMN_THROUGH_PARTITION_TOP_N(RuleTypeClass.REWRITE),
- PUSH_DOWN_NESTED_COLUMN_THROUGH_DEFER_MATERIALIZE_TOP_N(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_AGGREGATE(RuleTypeClass.REWRITE),
+ PUSH_DOWN_PROJECT_THROUGH_JOIN(RuleTypeClass.REWRITE),
+ PUSH_DOWN_PROJECT_THROUGH_UNION(RuleTypeClass.REWRITE),
+ PUSH_DOWN_PROJECT_THROUGH_WINDOW(RuleTypeClass.REWRITE),
+ PUSH_DOWN_PROJECT_THROUGH_PARTITION_TOP_N(RuleTypeClass.REWRITE),
+ // PUSH_DOWN_PROJECT_THROUGH_DEFER_MATERIALIZE_TOP_N(RuleTypeClass.REWRITE),
// Pushdown filter
PUSH_DOWN_FILTER_THROUGH_JOIN(RuleTypeClass.REWRITE),
PUSH_DOWN_FILTER_THROUGH_LEFT_SEMI_JOIN(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
index fd5ae6931d9..8af7236860c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
@@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.OrderExpression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotNotFromChildren;
import org.apache.doris.nereids.trees.expressions.SlotReference;
@@ -175,7 +176,7 @@ public class NormalizeAggregate implements RewriteRuleFactory, NormalizeToSlot {
if (arg instanceof Literal) {
continue;
}
- if (arg.containsType(SubqueryExpr.class, WindowExpression.class)) {
+ if (arg.containsType(SubqueryExpr.class, WindowExpression.class, PreferPushDownProject.class)) {
needPushDownSelfExprs.add(arg);
} else {
needPushDownInputs.add(arg);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java
similarity index 71%
rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java
rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java
index 6ef42183c7b..3a307d26ec8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java
@@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@@ -50,24 +51,27 @@ import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
-/** PushDownNestedColumnThroughJoin */
-public class PushDownNestedColumn implements RewriteRuleFactory {
+/** push down project if the expression instance of PreferPushDownProject */
+public class PushDownProject implements RewriteRuleFactory, NormalizeToSlot {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_JOIN.build(
- logicalProject(logicalJoin()).thenApply(this::defaultPushDown)
+ RuleType.PUSH_DOWN_PROJECT_THROUGH_JOIN.build(
+ logicalJoin().thenApply(this::pushDownJoinExpressions)
),
- RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_WINDOW.build(
- logicalProject(logicalWindow()).thenApply(this::defaultPushDown)
+ RuleType.PUSH_DOWN_PROJECT_THROUGH_JOIN.build(
+ logicalProject(logicalJoin()).thenApply(this::defaultPushDownProject)
),
- RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_PARTITION_TOP_N.build(
- logicalProject(logicalPartitionTopN()).thenApply(this::defaultPushDown)
+ RuleType.PUSH_DOWN_PROJECT_THROUGH_WINDOW.build(
+ logicalProject(logicalWindow()).thenApply(this::defaultPushDownProject)
),
- // RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_DEFER_MATERIALIZE_TOP_N.build(
- // logicalProject(logicalDeferMaterializeTopN()).thenApply(this::defaultPushDown)
+ RuleType.PUSH_DOWN_PROJECT_THROUGH_PARTITION_TOP_N.build(
+ logicalProject(logicalPartitionTopN()).thenApply(this::defaultPushDownProject)
+ ),
+ // RuleType.PUSH_DOWN_PROJECT_THROUGH_DEFER_MATERIALIZE_TOP_N.build(
+ // logicalProject(logicalDeferMaterializeTopN()).thenApply(this::defaultPushDownProject)
// ),
- RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_UNION.build(
+ RuleType.PUSH_DOWN_PROJECT_THROUGH_UNION.build(
logicalProject(
logicalUnion().when(u -> u.getQualifier() == Qualifier.ALL)
).thenApply(this::pushThroughUnion)
@@ -75,7 +79,86 @@ public class PushDownNestedColumn implements RewriteRuleFactory {
);
}
- private <C extends LogicalPlan> Plan defaultPushDown(MatchingContext<LogicalProject<C>> ctx) {
+ private Plan pushDownJoinExpressions(MatchingContext<LogicalJoin<Plan, Plan>> ctx) {
+ LogicalJoin<Plan, Plan> join = ctx.root;
+ Optional<Pair<List<Expression>, Map<Integer, List<NamedExpression>>>> rewriteHashJoinConjunctsResult
+ = pushDownProjectInExpressions(join, join.getHashJoinConjuncts(), ctx.statementContext);
+ Optional<Pair<List<Expression>, Map<Integer, List<NamedExpression>>>> rewriteOtherJoinConjunctsResult
+ = pushDownProjectInExpressions(join, join.getOtherJoinConjuncts(), ctx.statementContext);
+ if (!rewriteHashJoinConjunctsResult.isPresent() && !rewriteOtherJoinConjunctsResult.isPresent()) {
+ return join;
+ }
+
+ List<Expression> newHashJoinConjuncts = rewriteHashJoinConjunctsResult.isPresent()
+ ? rewriteHashJoinConjunctsResult.get().first : join.getHashJoinConjuncts();
+ List<Expression> newOtherJoinConjuncts = rewriteOtherJoinConjunctsResult.isPresent()
+ ? rewriteOtherJoinConjunctsResult.get().first : join.getOtherJoinConjuncts();
+
+ List<List<NamedExpression>> pushedOutput = new ArrayList<>();
+ pushedOutput.add(new ArrayList<>(join.left().getOutput()));
+ pushedOutput.add(new ArrayList<>(join.right().getOutput()));
+
+ if (rewriteHashJoinConjunctsResult.isPresent()) {
+ pushedOutput.get(0).addAll(rewriteHashJoinConjunctsResult.get().second.get(0));
+ pushedOutput.get(1).addAll(rewriteHashJoinConjunctsResult.get().second.get(1));
+ }
+ if (rewriteOtherJoinConjunctsResult.isPresent()) {
+ pushedOutput.get(0).addAll(rewriteOtherJoinConjunctsResult.get().second.get(0));
+ pushedOutput.get(1).addAll(rewriteOtherJoinConjunctsResult.get().second.get(1));
+ }
+
+ Plan newLeft = join.left();
+ Plan newRight = join.right();
+ if (pushedOutput.get(0).size() != newLeft.getOutput().size()) {
+ newLeft = new LogicalProject<>(pushedOutput.get(0), newLeft);
+ }
+ if (pushedOutput.get(1).size() != newRight.getOutput().size()) {
+ newRight = new LogicalProject<>(pushedOutput.get(1), newRight);
+ }
+
+ return join.withJoinConjuncts(
+ newHashJoinConjuncts, newOtherJoinConjuncts,
+ join.getMarkJoinConjuncts(), join.getJoinReorderContext()
+ ).withChildren(newLeft, newRight);
+ }
+
+ // return:
+ // key: rewrite the PreferPushDownProject to slot
+ // value: the pushed down project outputs which contains the Alias(PreferPushDownProject)
+ private Optional<Pair<List<Expression>, Map<Integer, List<NamedExpression>>>> pushDownProjectInExpressions(
+ Plan plan, Collection<Expression> expressions, StatementContext context) {
+
+ boolean changed = false;
+ Map<Integer, List<NamedExpression>> childIndexToPushedAlias = new LinkedHashMap<>();
+ List<Expression> newExpressions = new ArrayList<>();
+ for (Expression expression : expressions) {
+ Expression newExpression = expression.rewriteDownShortCircuit(e -> {
+ if (e instanceof PreferPushDownProject) {
+ List<Plan> children = plan.children();
+ for (int i = 0; i < children.size(); i++) {
+ Plan child = children.get(i);
+ if (child.getOutputSet().containsAll(e.getInputSlots())) {
+ Alias alias = new Alias(context.getNextExprId(), e);
+ Slot slot = alias.toSlot();
+ List<NamedExpression> namedExpressions
+ = childIndexToPushedAlias.computeIfAbsent(i, k -> new ArrayList<>());
+ namedExpressions.add(alias);
+ return slot;
+ }
+ }
+ }
+ return e;
+ });
+ newExpressions.add(newExpression);
+ changed |= newExpression != expression;
+ }
+ if (changed) {
+ return Optional.of(Pair.of(newExpressions, childIndexToPushedAlias));
+ }
+ return Optional.empty();
+ }
+
+ private <C extends LogicalPlan> Plan defaultPushDownProject(MatchingContext<LogicalProject<C>> ctx) {
if (!ctx.connectContext.getSessionVariable().enablePruneNestedColumns) {
return ctx.root;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java
index 89918c014d0..29f6baf0e77 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java
@@ -127,6 +127,8 @@ public class MaterializationNode extends PlanNode {
output.append(detailPrefix).append("table_idxs: ").append(idxs).append("\n");
output.append(detailPrefix).append("row_ids: ").append(rowIds).append("\n");
output.append(detailPrefix).append("isTopMaterializeNode: ").append(isTopMaterializeNode).append("\n");
+ printNestedColumns(output, detailPrefix, outputTupleDesc);
+
return output.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index f9c6ed0b237..658bc08add9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -1089,7 +1089,7 @@ public class OlapScanNode extends ScanNode {
getExplainString(rewrittenProjectList)).append("\n");
}
- printNestedColumns(output, prefix);
+ printNestedColumns(output, prefix, getTupleDesc());
return output.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index 382782c393a..ad5f0037a1e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -32,11 +32,14 @@ import org.apache.doris.common.Id;
import org.apache.doris.common.Pair;
import org.apache.doris.common.TreeNode;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.iceberg.source.IcebergScanNode;
import org.apache.doris.planner.normalize.Normalizer;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.PlanStats;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsDeriveResult;
+import org.apache.doris.thrift.TAccessPathType;
+import org.apache.doris.thrift.TColumnAccessPath;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TExpr;
import org.apache.doris.thrift.TNormalizedPlanNode;
@@ -50,6 +53,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -928,4 +932,109 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
}
return children.stream().anyMatch(PlanNode::hasSerialScanChildren);
}
+
+ protected void printNestedColumns(StringBuilder output, String prefix, TupleDescriptor tupleDesc) {
+ boolean printNestedColumnsHeader = true;
+ for (SlotDescriptor slot : tupleDesc.getSlots()) {
+ String prunedType = null;
+ if (!slot.getType().equals(slot.getColumn().getType())) {
+ prunedType = slot.getType().toString();
+ }
+ String displayAllAccessPathsString = null;
+ if (slot.getDisplayAllAccessPaths() != null
+ && slot.getDisplayAllAccessPaths() != null
+ && !slot.getDisplayAllAccessPaths().isEmpty()) {
+ if (this instanceof IcebergScanNode) {
+ displayAllAccessPathsString = mergeIcebergAccessPathsWithId(
+ slot.getAllAccessPaths(),
+ slot.getDisplayAllAccessPaths()
+ );
+ } else {
+ displayAllAccessPathsString = slot.getDisplayAllAccessPaths()
+ .stream()
+ .map(a -> {
+ if (a.type == TAccessPathType.DATA) {
+ return StringUtils.join(a.data_access_path.path, ".");
+ } else {
+ return StringUtils.join(a.meta_access_path.path, ".");
+ }
+ })
+ .collect(Collectors.joining(", "));
+ }
+ }
+ String displayPredicateAccessPathsString = null;
+ if (slot.getDisplayPredicateAccessPaths() != null
+ && slot.getDisplayPredicateAccessPaths() != null
+ && !slot.getDisplayPredicateAccessPaths().isEmpty()) {
+ if (this instanceof IcebergScanNode) {
+ displayPredicateAccessPathsString = mergeIcebergAccessPathsWithId(
+ slot.getPredicateAccessPaths(),
+ slot.getDisplayPredicateAccessPaths()
+ );
+ } else {
+ displayPredicateAccessPathsString = slot.getPredicateAccessPaths()
+ .stream()
+ .map(a -> {
+ if (a.type == TAccessPathType.DATA) {
+ return StringUtils.join(a.data_access_path.path, ".");
+ } else {
+ return StringUtils.join(a.meta_access_path.path, ".");
+ }
+ })
+ .collect(Collectors.joining(", "));
+ }
+ }
+
+
+ if (prunedType == null
+ && displayAllAccessPathsString == null
+ && displayPredicateAccessPathsString == null) {
+ continue;
+ }
+
+ if (printNestedColumnsHeader) {
+ output.append(prefix).append("nested columns:\n");
+ printNestedColumnsHeader = false;
+ }
+ output.append(prefix).append(" ").append(slot.getColumn().getName()).append(":\n");
+ output.append(prefix).append(" origin type: ").append(slot.getColumn().getType()).append("\n");
+ if (prunedType != null) {
+ output.append(prefix).append(" pruned type: ").append(prunedType).append("\n");
+ }
+ if (displayAllAccessPathsString != null) {
+ output.append(prefix).append(" all access paths: [")
+ .append(displayAllAccessPathsString).append("]\n");
+ }
+ if (displayPredicateAccessPathsString != null) {
+ output.append(prefix).append(" predicate access paths: [")
+ .append(displayPredicateAccessPathsString).append("]\n");
+ }
+ }
+ }
+
+ private String mergeIcebergAccessPathsWithId(
+ List<TColumnAccessPath> accessPaths, List<TColumnAccessPath> displayAccessPaths) {
+ List<String> mergeDisplayAccessPaths = Lists.newArrayList();
+ for (int i = 0; i < displayAccessPaths.size(); i++) {
+ TColumnAccessPath displayAccessPath = displayAccessPaths.get(i);
+ TColumnAccessPath idAccessPath = accessPaths.get(i);
+ List<String> nameAccessPathStrings = displayAccessPath.type == TAccessPathType.DATA
+ ? displayAccessPath.data_access_path.path : displayAccessPath.meta_access_path.path;
+ List<String> idAccessPathStrings = idAccessPath.type == TAccessPathType.DATA
+ ? idAccessPath.data_access_path.path : idAccessPath.meta_access_path.path;
+
+ List<String> mergedPath = new ArrayList<>();
+ for (int j = 0; j < idAccessPathStrings.size(); j++) {
+ String name = nameAccessPathStrings.get(j);
+ String id = idAccessPathStrings.get(j);
+ if (name.equals(id)) {
+ mergedPath.add(name);
+ } else {
+ mergedPath.add(name + "(" + id + ")");
+ }
+ }
+ mergeDisplayAccessPaths.add(StringUtils.join(mergedPath, "."));
+ }
+ return StringUtils.join(mergeDisplayAccessPaths, ", ");
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
index ee8270fbd79..fee9b0f8afc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
@@ -47,13 +47,10 @@ import org.apache.doris.datasource.FederationBackendPolicy;
import org.apache.doris.datasource.SplitAssignment;
import org.apache.doris.datasource.SplitGenerator;
import org.apache.doris.datasource.SplitSource;
-import org.apache.doris.datasource.iceberg.source.IcebergScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rpc.RpcException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.system.Backend;
-import org.apache.doris.thrift.TAccessPathType;
-import org.apache.doris.thrift.TColumnAccessPath;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TScanRange;
@@ -68,7 +65,6 @@ import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeRangeSet;
import org.apache.commons.collections.map.CaseInsensitiveMap;
-import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -543,111 +539,6 @@ public abstract class ScanNode extends PlanNode implements SplitGenerator {
.addValue(super.debugString()).toString();
}
- protected void printNestedColumns(StringBuilder output, String prefix) {
- boolean printNestedColumnsHeader = true;
- for (SlotDescriptor slot : getTupleDesc().getSlots()) {
- String prunedType = null;
- if (!slot.getType().equals(slot.getColumn().getType())) {
- prunedType = slot.getType().toString();
- }
- String displayAllAccessPathsString = null;
- if (slot.getDisplayAllAccessPaths() != null
- && slot.getDisplayAllAccessPaths() != null
- && !slot.getDisplayAllAccessPaths().isEmpty()) {
- if (this instanceof IcebergScanNode) {
- displayAllAccessPathsString = mergeIcebergAccessPathsWithId(
- slot.getAllAccessPaths(),
- slot.getDisplayAllAccessPaths()
- );
- } else {
- displayAllAccessPathsString = slot.getDisplayAllAccessPaths()
- .stream()
- .map(a -> {
- if (a.type == TAccessPathType.DATA) {
- return StringUtils.join(a.data_access_path.path, ".");
- } else {
- return StringUtils.join(a.meta_access_path.path, ".");
- }
- })
- .collect(Collectors.joining(", "));
- }
- }
- String displayPredicateAccessPathsString = null;
- if (slot.getDisplayPredicateAccessPaths() != null
- && slot.getDisplayPredicateAccessPaths() != null
- && !slot.getDisplayPredicateAccessPaths().isEmpty()) {
- if (this instanceof IcebergScanNode) {
- displayPredicateAccessPathsString = mergeIcebergAccessPathsWithId(
- slot.getPredicateAccessPaths(),
- slot.getDisplayPredicateAccessPaths()
- );
- } else {
- displayPredicateAccessPathsString = slot.getPredicateAccessPaths()
- .stream()
- .map(a -> {
- if (a.type == TAccessPathType.DATA) {
- return StringUtils.join(a.data_access_path.path, ".");
- } else {
- return StringUtils.join(a.meta_access_path.path, ".");
- }
- })
- .collect(Collectors.joining(", "));
- }
- }
-
-
- if (prunedType == null
- && displayAllAccessPathsString == null
- && displayPredicateAccessPathsString == null) {
- continue;
- }
-
- if (printNestedColumnsHeader) {
- output.append(prefix).append("nested columns:\n");
- printNestedColumnsHeader = false;
- }
- output.append(prefix).append(" ").append(slot.getColumn().getName()).append(":\n");
- output.append(prefix).append(" origin type: ").append(slot.getColumn().getType()).append("\n");
- if (prunedType != null) {
- output.append(prefix).append(" pruned type: ").append(prunedType).append("\n");
- }
- if (displayAllAccessPathsString != null) {
- output.append(prefix).append(" all access paths: [")
- .append(displayAllAccessPathsString).append("]\n");
- }
- if (displayPredicateAccessPathsString != null) {
- output.append(prefix).append(" predicate access paths: [")
- .append(displayPredicateAccessPathsString).append("]\n");
- }
- }
- }
-
- private String mergeIcebergAccessPathsWithId(
- List<TColumnAccessPath> accessPaths, List<TColumnAccessPath> displayAccessPaths) {
- List<String> mergeDisplayAccessPaths = Lists.newArrayList();
- for (int i = 0; i < displayAccessPaths.size(); i++) {
- TColumnAccessPath displayAccessPath = displayAccessPaths.get(i);
- TColumnAccessPath idAccessPath = accessPaths.get(i);
- List<String> nameAccessPathStrings = displayAccessPath.type == TAccessPathType.DATA
- ? displayAccessPath.data_access_path.path : displayAccessPath.meta_access_path.path;
- List<String> idAccessPathStrings = idAccessPath.type == TAccessPathType.DATA
- ? idAccessPath.data_access_path.path : idAccessPath.meta_access_path.path;
-
- List<String> mergedPath = new ArrayList<>();
- for (int j = 0; j < idAccessPathStrings.size(); j++) {
- String name = nameAccessPathStrings.get(j);
- String id = idAccessPathStrings.get(j);
- if (name.equals(id)) {
- mergedPath.add(name);
- } else {
- mergedPath.add(name + "(" + id + ")");
- }
- }
- mergeDisplayAccessPaths.add(StringUtils.join(mergedPath, "."));
- }
- return StringUtils.join(mergeDisplayAccessPaths, ", ");
- }
-
public List<TupleId> getOutputTupleIds() {
if (outputTupleDesc != null) {
return Lists.newArrayList(outputTupleDesc.getId());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
index 8e8d7c44d16..d2240a2727e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -428,6 +428,33 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
);
}
+ @Test
+ public void testAggregate() throws Exception {
+ assertColumn("select count(struct_element(s, 'city')) from tbl",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testJoin() throws Exception {
+ assertColumns("select 100 from tbl t1 join tbl t2 on struct_element(t1.s, 'city')=struct_element(t2.s, 'city')",
+ ImmutableList.of(
+ Triple.of(
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ ),
+ Triple.of(
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ )
+ )
+ );
+ }
+
@Test
public void testPushDownThroughWindow() {
PlanChecker.from(connectContext)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 10/18: only support prune olapTable/iceberg/hive(parquet/orc)
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 37beb93061ff8f8c34fa30b1c2363ddb044edfcc
Author: 924060929 <la...@selectdb.com>
AuthorDate: Thu Oct 30 18:08:44 2025 +0800
only support prune olapTable/iceberg/hive(parquet/orc)
---
.../doris/datasource/hive/HMSExternalTable.java | 40 +++++++++++++++++++++
.../doris/datasource/hive/source/HiveScanNode.java | 37 +------------------
.../nereids/rules/rewrite/SlotTypeReplacer.java | 11 ++++++
.../trees/expressions/functions/table/File.java | 8 ++++-
.../trees/expressions/functions/table/Hdfs.java | 9 ++++-
.../trees/expressions/functions/table/Local.java | 9 ++++-
.../trees/expressions/functions/table/S3.java | 9 ++++-
.../trees/plans/logical/LogicalFileScan.java | 31 +++++++++++++++-
.../trees/plans/logical/LogicalOlapScan.java | 7 +++-
.../plans/logical/SupportPruneNestedColumn.java | 24 +++++++++++++
.../logical/SupportPruneNestedColumnFormats.java | 42 ++++++++++++++++++++++
11 files changed, 185 insertions(+), 42 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index babbb099372..526b4c3d071 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -58,6 +58,7 @@ import org.apache.doris.mtmv.MTMVSnapshotIf;
import org.apache.doris.nereids.exceptions.NotSupportedException;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
import org.apache.doris.qe.GlobalVariable;
+import org.apache.doris.qe.SessionVariable;
import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.ColumnStatistic;
@@ -65,6 +66,7 @@ import org.apache.doris.statistics.ColumnStatisticBuilder;
import org.apache.doris.statistics.HMSAnalysisTask;
import org.apache.doris.statistics.StatsType;
import org.apache.doris.statistics.util.StatisticsUtil;
+import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.thrift.THiveTable;
import org.apache.doris.thrift.TTableDescriptor;
import org.apache.doris.thrift.TTableType;
@@ -1188,6 +1190,44 @@ public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableI
}
}
+ public TFileFormatType getFileFormatType(SessionVariable sessionVariable) throws UserException {
+ TFileFormatType type = null;
+ Table table = getRemoteTable();
+ String inputFormatName = table.getSd().getInputFormat();
+ String hiveFormat = HiveMetaStoreClientHelper.HiveFileFormat.getFormat(inputFormatName);
+ if (hiveFormat.equals(HiveMetaStoreClientHelper.HiveFileFormat.PARQUET.getDesc())) {
+ type = TFileFormatType.FORMAT_PARQUET;
+ } else if (hiveFormat.equals(HiveMetaStoreClientHelper.HiveFileFormat.ORC.getDesc())) {
+ type = TFileFormatType.FORMAT_ORC;
+ } else if (hiveFormat.equals(HiveMetaStoreClientHelper.HiveFileFormat.TEXT_FILE.getDesc())) {
+ String serDeLib = table.getSd().getSerdeInfo().getSerializationLib();
+ if (serDeLib.equals(HiveMetaStoreClientHelper.HIVE_JSON_SERDE)
+ || serDeLib.equals(HiveMetaStoreClientHelper.LEGACY_HIVE_JSON_SERDE)) {
+ type = TFileFormatType.FORMAT_JSON;
+ } else if (serDeLib.equals(HiveMetaStoreClientHelper.OPENX_JSON_SERDE)) {
+ if (!sessionVariable.isReadHiveJsonInOneColumn()) {
+ type = TFileFormatType.FORMAT_JSON;
+ } else if (sessionVariable.isReadHiveJsonInOneColumn() && firstColumnIsString()) {
+ type = TFileFormatType.FORMAT_CSV_PLAIN;
+ } else {
+ throw new UserException("You set read_hive_json_in_one_column = true, but the first column of "
+ + "table " + getName()
+ + " is not a string column.");
+ }
+ } else if (serDeLib.equals(HiveMetaStoreClientHelper.HIVE_TEXT_SERDE)) {
+ type = TFileFormatType.FORMAT_TEXT;
+ } else if (serDeLib.equals(HiveMetaStoreClientHelper.HIVE_OPEN_CSV_SERDE)) {
+ type = TFileFormatType.FORMAT_CSV_PLAIN;
+ } else if (serDeLib.equals(HiveMetaStoreClientHelper.HIVE_MULTI_DELIMIT_SERDE)) {
+ type = TFileFormatType.FORMAT_TEXT;
+ } else {
+ throw new UserException("Unsupported hive table serde: " + serDeLib);
+ }
+ }
+ return type;
+ }
+
+
private Table loadHiveTable() {
HMSCachedClient client = ((HMSExternalCatalog) catalog).getClient();
return client.getTable(getRemoteDbName(), remoteName);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
index a11dc6d247e..2a524a4138d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
@@ -409,44 +409,9 @@ public class HiveScanNode extends FileQueryScanNode {
@Override
public TFileFormatType getFileFormatType() throws UserException {
- TFileFormatType type = null;
- Table table = hmsTable.getRemoteTable();
- String inputFormatName = table.getSd().getInputFormat();
- String hiveFormat = HiveMetaStoreClientHelper.HiveFileFormat.getFormat(inputFormatName);
- if (hiveFormat.equals(HiveMetaStoreClientHelper.HiveFileFormat.PARQUET.getDesc())) {
- type = TFileFormatType.FORMAT_PARQUET;
- } else if (hiveFormat.equals(HiveMetaStoreClientHelper.HiveFileFormat.ORC.getDesc())) {
- type = TFileFormatType.FORMAT_ORC;
- } else if (hiveFormat.equals(HiveMetaStoreClientHelper.HiveFileFormat.TEXT_FILE.getDesc())) {
- String serDeLib = table.getSd().getSerdeInfo().getSerializationLib();
- if (serDeLib.equals(HiveMetaStoreClientHelper.HIVE_JSON_SERDE)
- || serDeLib.equals(HiveMetaStoreClientHelper.LEGACY_HIVE_JSON_SERDE)) {
- type = TFileFormatType.FORMAT_JSON;
- } else if (serDeLib.equals(HiveMetaStoreClientHelper.OPENX_JSON_SERDE)) {
- if (!sessionVariable.isReadHiveJsonInOneColumn()) {
- type = TFileFormatType.FORMAT_JSON;
- } else if (sessionVariable.isReadHiveJsonInOneColumn()
- && hmsTable.firstColumnIsString()) {
- type = TFileFormatType.FORMAT_CSV_PLAIN;
- } else {
- throw new UserException("You set read_hive_json_in_one_column = true, but the first column of "
- + "table " + hmsTable.getName()
- + " is not a string column.");
- }
- } else if (serDeLib.equals(HiveMetaStoreClientHelper.HIVE_TEXT_SERDE)) {
- type = TFileFormatType.FORMAT_TEXT;
- } else if (serDeLib.equals(HiveMetaStoreClientHelper.HIVE_OPEN_CSV_SERDE)) {
- type = TFileFormatType.FORMAT_CSV_PLAIN;
- } else if (serDeLib.equals(HiveMetaStoreClientHelper.HIVE_MULTI_DELIMIT_SERDE)) {
- type = TFileFormatType.FORMAT_TEXT;
- } else {
- throw new UserException("Unsupported hive table serde: " + serDeLib);
- }
- }
- return type;
+ return hmsTable.getFileFormatType(sessionVariable);
}
-
@Override
protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
if (split instanceof HiveSplit) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
index 459c7362758..9ce5de48061 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
@@ -32,6 +32,7 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
+import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
@@ -56,6 +57,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
+import org.apache.doris.nereids.trees.plans.logical.SupportPruneNestedColumn;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.DataType;
@@ -374,6 +376,10 @@ public class SlotTypeReplacer extends DefaultPlanRewriter<Void> {
@Override
public Plan visitLogicalFileScan(LogicalFileScan fileScan, Void context) {
+ if (!fileScan.supportPruneNestedColumn()) {
+ return fileScan;
+ }
+
Pair<Boolean, List<Slot>> replaced = replaceExpressions(fileScan.getOutput(), false, true);
if (replaced.first) {
List<Slot> replaceSlots = new ArrayList<>(replaced.second);
@@ -407,6 +413,11 @@ public class SlotTypeReplacer extends DefaultPlanRewriter<Void> {
@Override
public Plan visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, Void context) {
+ TableValuedFunction function = tvfRelation.getFunction();
+ if (!(function instanceof SupportPruneNestedColumn)
+ || !((SupportPruneNestedColumn) function).supportPruneNestedColumn()) {
+ return tvfRelation;
+ }
Pair<Boolean, List<Slot>> replaced
= replaceExpressions(tvfRelation.getOutput(), false, true);
if (replaced.first) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/File.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/File.java
index aed17d7b5a8..660dfad4874 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/File.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/File.java
@@ -21,6 +21,8 @@ import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Properties;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.trees.plans.logical.SupportPruneNestedColumn;
+import org.apache.doris.nereids.trees.plans.logical.SupportPruneNestedColumnFormats;
import org.apache.doris.nereids.types.coercion.AnyDataType;
import org.apache.doris.tablefunction.FileTableValuedFunction;
import org.apache.doris.tablefunction.TableValuedFunctionIf;
@@ -30,7 +32,7 @@ import java.util.Map;
/**
* File Tvf
*/
-public class File extends TableValuedFunction {
+public class File extends TableValuedFunction implements SupportPruneNestedColumn {
public File(Properties properties) {
super("file", properties);
}
@@ -55,4 +57,8 @@ public class File extends TableValuedFunction {
return visitor.visitFile(this, context);
}
+ @Override
+ public boolean supportPruneNestedColumn() {
+ return SupportPruneNestedColumnFormats.supportFormat(getTVFProperties());
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java
index 5f8651ed61c..8389892f6cf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java
@@ -21,6 +21,8 @@ import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Properties;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.trees.plans.logical.SupportPruneNestedColumn;
+import org.apache.doris.nereids.trees.plans.logical.SupportPruneNestedColumnFormats;
import org.apache.doris.nereids.types.coercion.AnyDataType;
import org.apache.doris.tablefunction.HdfsTableValuedFunction;
import org.apache.doris.tablefunction.TableValuedFunctionIf;
@@ -29,7 +31,7 @@ import java.util.List;
import java.util.Map;
/** hdfs */
-public class Hdfs extends TableValuedFunction {
+public class Hdfs extends TableValuedFunction implements SupportPruneNestedColumn {
public Hdfs(Properties properties) {
super("hdfs", properties);
}
@@ -53,4 +55,9 @@ public class Hdfs extends TableValuedFunction {
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitHdfs(this, context);
}
+
+ @Override
+ public boolean supportPruneNestedColumn() {
+ return SupportPruneNestedColumnFormats.supportFormat(getTVFProperties());
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Local.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Local.java
index 4330980ee86..094324e895b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Local.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Local.java
@@ -21,6 +21,8 @@ import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Properties;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.trees.plans.logical.SupportPruneNestedColumn;
+import org.apache.doris.nereids.trees.plans.logical.SupportPruneNestedColumnFormats;
import org.apache.doris.nereids.types.coercion.AnyDataType;
import org.apache.doris.tablefunction.LocalTableValuedFunction;
import org.apache.doris.tablefunction.TableValuedFunctionIf;
@@ -30,7 +32,7 @@ import java.util.Map;
/**
* local
*/
-public class Local extends TableValuedFunction {
+public class Local extends TableValuedFunction implements SupportPruneNestedColumn {
public Local(Properties properties) {
super("local", properties);
}
@@ -54,4 +56,9 @@ public class Local extends TableValuedFunction {
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitLocal(this, context);
}
+
+ @Override
+ public boolean supportPruneNestedColumn() {
+ return SupportPruneNestedColumnFormats.supportFormat(getTVFProperties());
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java
index d2623d8fd3c..3c4c924d7f4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java
@@ -21,6 +21,8 @@ import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Properties;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.trees.plans.logical.SupportPruneNestedColumn;
+import org.apache.doris.nereids.trees.plans.logical.SupportPruneNestedColumnFormats;
import org.apache.doris.nereids.types.coercion.AnyDataType;
import org.apache.doris.tablefunction.S3TableValuedFunction;
import org.apache.doris.tablefunction.TableValuedFunctionIf;
@@ -28,7 +30,7 @@ import org.apache.doris.tablefunction.TableValuedFunctionIf;
import java.util.Map;
/** s3 */
-public class S3 extends TableValuedFunction {
+public class S3 extends TableValuedFunction implements SupportPruneNestedColumn {
public S3(Properties properties) {
super("s3", properties);
}
@@ -52,4 +54,9 @@ public class S3 extends TableValuedFunction {
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitS3(this, context);
}
+
+ @Override
+ public boolean supportPruneNestedColumn() {
+ return SupportPruneNestedColumnFormats.supportFormat(getTVFProperties());
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
index 8854800005e..5d4c1167a82 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
@@ -21,6 +21,8 @@ import org.apache.doris.analysis.TableScanParams;
import org.apache.doris.analysis.TableSnapshot;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.datasource.ExternalTable;
+import org.apache.doris.datasource.hive.HMSExternalTable;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.datasource.mvcc.MvccUtil;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
@@ -32,6 +34,9 @@ import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.thrift.TFileFormatType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -46,7 +51,7 @@ import java.util.Optional;
/**
* Logical file scan for external catalog.
*/
-public class LogicalFileScan extends LogicalCatalogRelation {
+public class LogicalFileScan extends LogicalCatalogRelation implements SupportPruneNestedColumn {
protected final SelectedPartitions selectedPartitions;
protected final Optional<TableSample> tableSample;
protected final Optional<TableSnapshot> tableSnapshot;
@@ -189,6 +194,30 @@ public class LogicalFileScan extends LogicalCatalogRelation {
return super.computeAsteriskOutput();
}
+ @Override
+ public boolean supportPruneNestedColumn() {
+ ExternalTable table = getTable();
+ if (table instanceof IcebergExternalTable) {
+ return true;
+ } else if (table instanceof HMSExternalTable) {
+ try {
+ ConnectContext connectContext = ConnectContext.get();
+ SessionVariable sessionVariable = connectContext.getSessionVariable();
+ TFileFormatType fileFormatType = ((HMSExternalTable) table).getFileFormatType(sessionVariable);
+ switch (fileFormatType) {
+ case FORMAT_PARQUET:
+ case FORMAT_ORC:
+ return true;
+ default:
+ return false;
+ }
+ } catch (Throwable t) {
+ // ignore and not prune
+ }
+ }
+ return false;
+ }
+
/**
* SelectedPartitions contains the selected partitions and the total partition number.
* Mainly for hive table partition pruning.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
index 394e414cc2e..cb1214cd7c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
@@ -72,7 +72,7 @@ import java.util.stream.Collectors;
/**
* Logical OlapScan.
*/
-public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan {
+public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan, SupportPruneNestedColumn {
private static final Logger LOG = LogManager.getLogger(LogicalOlapScan.class);
@@ -868,4 +868,9 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
manuallySpecifiedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit,
annOrderKeys, annLimit, tableAlias);
}
+
+ @Override
+ public boolean supportPruneNestedColumn() {
+ return true;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/SupportPruneNestedColumn.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/SupportPruneNestedColumn.java
new file mode 100644
index 00000000000..44f1b733dd1
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/SupportPruneNestedColumn.java
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+/** SupportPruneNestedColumn */
+public interface SupportPruneNestedColumn {
+ // return false will not prune the nested column
+ boolean supportPruneNestedColumn();
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/SupportPruneNestedColumnFormats.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/SupportPruneNestedColumnFormats.java
new file mode 100644
index 00000000000..a9ee8f64b49
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/SupportPruneNestedColumnFormats.java
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+import org.apache.doris.nereids.trees.expressions.Properties;
+
+import com.google.common.collect.ImmutableSet;
+
+import java.util.Map.Entry;
+import java.util.Set;
+
+/** SupportPruneNestedColumnFormats */
+public class SupportPruneNestedColumnFormats {
+ private static final Set<String> SUPPORTED_FORMATS = ImmutableSet.of(
+ "parquet", "orc"
+ );
+
+ /** supportFormat */
+ public static boolean supportFormat(Properties properties) {
+ for (Entry<String, String> kv : properties.getMap().entrySet()) {
+ if (kv.getKey().equalsIgnoreCase("format")) {
+ return SUPPORTED_FORMATS.contains(kv.getValue().toLowerCase());
+ }
+ }
+ return false;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 13/18: should not prune project nested column if the bottom table can not prune, e.g. hive(textfile)
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9efa81907b5b7e546c5fa268bd14b9712f007a42
Author: 924060929 <la...@selectdb.com>
AuthorDate: Fri Oct 31 12:36:22 2025 +0800
should not prune project nested column if the bottom table can not prune, e.g. hive(textfile)
---
.../nereids/rules/rewrite/NestedColumnPruning.java | 3 +-
.../nereids/rules/rewrite/SlotTypeReplacer.java | 53 ++++++++++++++++++++--
.../org/apache/doris/nereids/trees/TreeNode.java | 2 +-
3 files changed, 50 insertions(+), 8 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
index de22e9d2101..b03f183b115 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
@@ -92,8 +92,7 @@ public class NestedColumnPruning implements CustomRewriter {
AccessPathInfo accessPathInfo = kv.getValue();
slotIdToPruneType.put(slotId, accessPathInfo);
}
- SlotTypeReplacer typeReplacer = new SlotTypeReplacer(slotIdToPruneType);
- return plan.accept(typeReplacer, null);
+ return new SlotTypeReplacer(slotIdToPruneType, plan).replace();
}
return plan;
} catch (Throwable t) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
index 9ce5de48061..9f6a02391eb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
@@ -77,9 +77,11 @@ import com.google.common.collect.ImmutableMultimap.Builder;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.IdentityHashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -89,10 +91,32 @@ import java.util.Set;
/** SlotTypeReplacer */
public class SlotTypeReplacer extends DefaultPlanRewriter<Void> {
+ private Plan plan;
private Map<Integer, AccessPathInfo> replacedDataTypes;
+ private IdentityHashMap<Plan, Void> shouldPrunePlans = new IdentityHashMap();
- public SlotTypeReplacer(Map<Integer, AccessPathInfo> bottomReplacedDataTypes) {
+ public SlotTypeReplacer(Map<Integer, AccessPathInfo> bottomReplacedDataTypes, Plan plan) {
this.replacedDataTypes = Maps.newLinkedHashMap(bottomReplacedDataTypes);
+ this.plan = plan;
+ }
+
+ /** replace */
+ public Plan replace() {
+ Set<Integer> shouldReplaceSlots = Sets.newLinkedHashSet();
+ plan.foreachUp(p -> {
+ if (p instanceof LogicalTVFRelation) {
+ TableValuedFunction function = ((LogicalTVFRelation) p).getFunction();
+ tryRecordReplaceSlots((Plan) p, function, shouldReplaceSlots);
+ } else {
+ tryRecordReplaceSlots((Plan) p, p, shouldReplaceSlots);
+ }
+ });
+ replacedDataTypes.keySet().retainAll(shouldReplaceSlots);
+
+ if (replacedDataTypes.isEmpty()) {
+ return plan;
+ }
+ return plan.accept(this, null);
}
@Override
@@ -376,7 +400,7 @@ public class SlotTypeReplacer extends DefaultPlanRewriter<Void> {
@Override
public Plan visitLogicalFileScan(LogicalFileScan fileScan, Void context) {
- if (!fileScan.supportPruneNestedColumn()) {
+ if (!shouldPrunePlans.containsKey(fileScan)) {
return fileScan;
}
@@ -413,9 +437,7 @@ public class SlotTypeReplacer extends DefaultPlanRewriter<Void> {
@Override
public Plan visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, Void context) {
- TableValuedFunction function = tvfRelation.getFunction();
- if (!(function instanceof SupportPruneNestedColumn)
- || !((SupportPruneNestedColumn) function).supportPruneNestedColumn()) {
+ if (!shouldPrunePlans.containsKey(tvfRelation)) {
return tvfRelation;
}
Pair<Boolean, List<Slot>> replaced
@@ -428,6 +450,9 @@ public class SlotTypeReplacer extends DefaultPlanRewriter<Void> {
@Override
public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Void context) {
+ if (!shouldPrunePlans.containsKey(olapScan)) {
+ return olapScan;
+ }
Pair<Boolean, List<Slot>> replaced = replaceExpressions(olapScan.getOutput(), false, true);
if (replaced.first) {
return olapScan.withPrunedTypeSlots(replaced.second);
@@ -682,4 +707,22 @@ public class SlotTypeReplacer extends DefaultPlanRewriter<Void> {
}
}
}
+
+ private void tryRecordReplaceSlots(Plan plan, Object checkObj, Set<Integer> shouldReplaceSlots) {
+ if (checkObj instanceof SupportPruneNestedColumn
+ && ((SupportPruneNestedColumn) checkObj).supportPruneNestedColumn()) {
+ List<Slot> output = plan.getOutput();
+ boolean shouldPrune = false;
+ for (Slot slot : output) {
+ int slotId = slot.getExprId().asInt();
+ if (replacedDataTypes.containsKey(slotId)) {
+ shouldReplaceSlots.add(slotId);
+ shouldPrune = true;
+ }
+ }
+ if (shouldPrune) {
+ shouldPrunePlans.put(plan, null);
+ }
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
index a5e3ee9c5e1..2e59b283f93 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
@@ -236,7 +236,7 @@ public interface TreeNode<NODE_TYPE extends TreeNode<NODE_TYPE>> {
default void foreachUp(Consumer<TreeNode<NODE_TYPE>> func) {
for (NODE_TYPE child : children()) {
- child.foreach(func);
+ child.foreachUp(func);
}
func.accept(this);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 12/18: fix npe
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit b2a048deff98bb32d3463b29b6b153b21798ea7a
Author: 924060929 <la...@selectdb.com>
AuthorDate: Fri Oct 31 11:01:02 2025 +0800
fix npe
---
.../nereids/rules/rewrite/PushDownProject.java | 22 ++++++++++++++++++----
.../java/org/apache/doris/planner/PlanNode.java | 2 +-
2 files changed, 19 insertions(+), 5 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java
index 3a307d26ec8..8b1fbaac8ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java
@@ -99,12 +99,26 @@ public class PushDownProject implements RewriteRuleFactory, NormalizeToSlot {
pushedOutput.add(new ArrayList<>(join.right().getOutput()));
if (rewriteHashJoinConjunctsResult.isPresent()) {
- pushedOutput.get(0).addAll(rewriteHashJoinConjunctsResult.get().second.get(0));
- pushedOutput.get(1).addAll(rewriteHashJoinConjunctsResult.get().second.get(1));
+ Map<Integer, List<NamedExpression>> childIndexToConjuncts = rewriteHashJoinConjunctsResult.get().second;
+ List<NamedExpression> leftConjuncts = childIndexToConjuncts.get(0);
+ if (leftConjuncts != null) {
+ pushedOutput.get(0).addAll(leftConjuncts);
+ }
+ List<NamedExpression> rightConjuncts = childIndexToConjuncts.get(1);
+ if (rightConjuncts != null) {
+ pushedOutput.get(1).addAll(rightConjuncts);
+ }
}
if (rewriteOtherJoinConjunctsResult.isPresent()) {
- pushedOutput.get(0).addAll(rewriteOtherJoinConjunctsResult.get().second.get(0));
- pushedOutput.get(1).addAll(rewriteOtherJoinConjunctsResult.get().second.get(1));
+ Map<Integer, List<NamedExpression>> childIndexToConjuncts = rewriteOtherJoinConjunctsResult.get().second;
+ List<NamedExpression> leftOtherConjuncts = childIndexToConjuncts.get(0);
+ if (leftOtherConjuncts != null) {
+ pushedOutput.get(0).addAll(leftOtherConjuncts);
+ }
+ List<NamedExpression> rightOtherConjuncts = childIndexToConjuncts.get(1);
+ if (rightOtherConjuncts != null) {
+ pushedOutput.get(1).addAll(rightOtherConjuncts);
+ }
}
Plan newLeft = join.left();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index ad5f0037a1e..861780ead2b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -937,7 +937,7 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
boolean printNestedColumnsHeader = true;
for (SlotDescriptor slot : tupleDesc.getSlots()) {
String prunedType = null;
- if (!slot.getType().equals(slot.getColumn().getType())) {
+ if (slot.getColumn() != null && !slot.getType().equals(slot.getColumn().getType())) {
prunedType = slot.getType().toString();
}
String displayAllAccessPathsString = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org