You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/10/08 03:23:31 UTC
(doris) branch branch-4.2 updated (9030a3f87a8 -> 0e741f9162b)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
from 9030a3f87a8 branch-4.2: [feature](cloud) support single rowset grouped compaction (#65907) (#68361)
new 48f05f67dd1 branch-4.1: [improve](profile) Skip _init_profile in load_v1 when enable_profile is false #61131 (#62801)
new e3e6cdbfb71 [fix](regression) Wait for final Iceberg Variant STRING profile counters (#68597)
new c5ebba21dfd [feature](lance) Support time travel, tags, branches and REST Namespace managed versioning (#68453)
new aa2fa487ade [fix](iceberg) Preserve file specs after evolving to unpartitioned tables (#68481)
new ffa16d4b68d [fix](arrow) Validate Flight result arrays before publishing batches (#68596)
new 69e3bffa030 branch-4.1: [fix](function) Keep the leading NULLs of every row in array_cum_sum #68554 (#68605)
new d3f3f52ecaa add temp dir in testutil (#68638)
new 09df202391a branch-4.1: [fix](regexp) Apply extended regex fallback to non-constant patterns #68501 (#68591)
new 37352da41c1 [fix](be) Backport unlimited topn expansion rates to branch-4.1 (#68545)
new a8f3b9dd7dc [fix](regression) Backport regression stability fixes to branch-4.1 (#68449) (#68618)
new 8cbfea08b3e branch-4.1: [fix](fe) Apply the same checks to HTTP Basic and cookie requests on /rest/v1 (#68635)
new 7d0c1bf425e [fix](arrow-flight) Deduplicate parallel result endpoints by backend (#68642)
new 20a412ebe47 [fix](arrow) Reject out-of-range Flight timestamps (#68604)
new aa59d75da7f [fix](arrow-flight) Cancel queries when result streams close early (#68630)
new 761c243c71a branch-4.1: [fix](function) Skip the rows with a NULL argument in array_range #68620 (#68644)
new bd6fecbc979 branch-4.1: [fix](auth) Treat only root@'%' as root in SET PASSWORD and ALTER USER (#68634)
new 2f9f1e253fa branch-4.1: [fix](auth) Check column privileges held by grantor in GRANT/REVOKE (#68629)
new a7daed69925 branch-4.1: [fix](scan) Fix lost and duplicated rows when splitting CSV/JSON on multi-character line delimiters #68539 (#68600)
new 7606fcebd4e branch-4.1: [fix](cloud) Invalidate version caches on visible commit retries (#67813) (#68650)
new 37e65c49e08 branch-4.1: [fix](meta cache) Invalidate external row count cache after metadata changes #64160 (#68196)
new e98db0594f7 branch-4.1:[feat](paimon) support paimon-rust reader on BE (#66227)
new ce1bc2c43bd [fix](flight-sql) Return analyzed query schemas and table type metadata (#68631)
new d30e392db32 branch-4.1: [fix](be) Restore BOOLEAN support in linear_histogram #68478 (#68676)
new 810c92fb836 branch-4.1: [fix](cloud) Respect recycle type when deleting partial update rowsets (#68594) (#68665)
new ac940a48c12 branch-4.1: [opt](cloud) Add recycler rowset recycling benchmarks test #68043 (#68581)
new 62b9b6130f4 branch-4.1: [fix](auth) Reject privilege object names with more than three parts in GRANT/REVOKE #68660 (#68682)
new 710f324af72 [improvement](lance) Add ANN prefilter optimization and profiles (branch-4.1) (#68615)
new 5320dddf5dc [improvement](lance) Expose query parallelism and complete row fetch profiles (branch-4.1) (#68690)
new 0cd51660c4b branch-4.1: [fix](storage) Do not cache the failure of loading segment rows in BetaRowset #68675 (#68683)
new 26004ed0c6d branch-4.1: [fix](array-func) Handle outer-null containers with hidden nested data (#68022) (#68696)
new 98e5b46adf6 branch-4.1: [fix](jsonb) Stop truncating JSON text at a U+0000 inside a string #68244 (#68735)
new 238cf2c2538 [improvement](lance) Push down array membership and boolean scalar index predicates (#68687)
new 4c5f158be1c branch-4.1: [fix](be) Stop background workers before startup exit #68652 (#68653)
new 1bbc51ef3c5 [fix](arrow-flight) Preserve nested logical type metadata (#68614)
new 0e741f9162b [fix](arrow) Correct year-zero DATE conversion on branch-4.1 (#68649)
The 35 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/build-thirdparty.yml | 3 +
be/CMakeLists.txt | 3 +-
be/cmake/thirdparty.cmake | 7 +
be/src/agent/heartbeat_server.cpp | 2 +
.../cloud/cloud_engine_calc_delete_bitmap_task.cpp | 26 +
be/src/cloud/cloud_meta_mgr.cpp | 10 +
be/src/cloud/cloud_tablets_channel.cpp | 4 +-
.../data_type_serde/data_type_datev2_serde.cpp | 73 +-
be/src/core/value/vdatetime_value.h | 53 +-
be/src/exec/operator/materialization_opertor.cpp | 6 +-
.../exec/operator/memory_scratch_sink_operator.cpp | 3 +-
be/src/exec/operator/result_sink_operator.cpp | 23 +-
be/src/exec/operator/result_sink_operator.h | 3 +
be/src/exec/rowid_fetcher.cpp | 21 +
be/src/exec/rowid_fetcher.h | 2 +
be/src/exec/scan/file_scanner.cpp | 19 +
be/src/exec/scan/file_scanner_v2.cpp | 15 +-
.../sink/writer/iceberg/partition_transformers.h | 8 +-
.../sink/writer/varrow_flight_result_writer.cpp | 21 +-
.../exec/sink/writer/varrow_flight_result_writer.h | 13 +-
.../exprs/aggregate/aggregate_function_foreach.h | 181 +-
.../aggregate_function_linear_histogram.cpp | 7 +-
be/src/exprs/aggregate/aggregate_function_null.h | 114 +
.../aggregate/aggregate_function_python_udaf.cpp | 4 +-
be/src/exprs/aggregate/aggregate_function_topn.h | 3 +-
.../function/array/function_array_cum_sum.cpp | 14 +-
.../array/function_array_enumerate_uniq.cpp | 146 +-
.../exprs/function/array/function_array_range.cpp | 12 +
.../exprs/function/array/function_array_sortby.cpp | 35 +-
.../exprs/function/array/function_array_split.cpp | 127 +-
be/src/exprs/function/array/function_array_zip.cpp | 175 +-
be/src/exprs/function/like.cpp | 131 +-
be/src/exprs/function/like.h | 17 +
.../lambda_function/varray_filter_function.cpp | 17 +-
.../exprs/lambda_function/varray_map_function.cpp | 123 +-
.../exprs/lambda_function/varray_sort_function.cpp | 5 +
be/src/format/arrow/arrow_block_convertor.cpp | 171 +-
be/src/format/arrow/arrow_block_convertor.h | 17 +-
be/src/format/arrow/arrow_row_batch.cpp | 147 +-
be/src/format/arrow/arrow_row_batch.h | 71 +-
be/src/format/csv/csv_reader.cpp | 8 +
.../file_reader/new_plain_text_line_reader.cpp | 77 +
.../file_reader/new_plain_text_line_reader.h | 6 +
be/src/format/json/new_json_reader.cpp | 15 +-
be/src/format/json/new_json_reader.h | 5 +-
.../parquet/parquet_arrow_block_convertor.cpp | 18 +-
.../format/parquet/parquet_arrow_block_convertor.h | 20 +
be/src/format/table/iceberg/arrow_schema_util.cpp | 46 +-
be/src/format/table/iceberg/arrow_schema_util.h | 20 +-
.../iceberg/iceberg_arrow_block_convertor.cpp | 12 +-
.../table/paimon/paimon_arrow_block_convertor.cpp | 16 +-
.../table/paimon/paimon_arrow_block_convertor.h | 11 +
be/src/format_v2/delimited_text/csv_reader.cpp | 2 +
.../delimited_text/delimited_text_reader.cpp | 14 +
.../delimited_text/delimited_text_reader.h | 2 +
be/src/format_v2/json/json_reader.cpp | 15 +-
be/src/format_v2/json/json_reader.h | 4 +-
be/src/format_v2/orc/orc_reader.cpp | 164 +-
be/src/format_v2/parquet/parquet_reader.cpp | 43 +
be/src/format_v2/parquet/parquet_statistics.cpp | 10 +
be/src/format_v2/table/lance_reader.cpp | 62 +
be/src/format_v2/table/paimon_reader.cpp | 62 +-
be/src/format_v2/table/paimon_reader.h | 18 +-
.../table/paimon_rust_predicate_converter.cpp | 840 +++++++
.../table/paimon_rust_predicate_converter.h | 161 ++
.../format_v2/table/paimon_rust_table_reader.cpp | 924 ++++++++
be/src/format_v2/table/paimon_rust_table_reader.h | 188 ++
be/src/load/channel/load_channel.cpp | 19 +-
be/src/load/channel/tablets_channel.cpp | 10 +-
be/src/load/delta_writer/delta_writer.cpp | 6 +-
be/src/load/memtable/memtable_writer.cpp | 3 +
be/src/runtime/result_buffer_mgr.cpp | 43 +
be/src/runtime/result_buffer_mgr.h | 5 +
.../arrow_flight/arrow_flight_batch_reader.cpp | 154 +-
.../arrow_flight/arrow_flight_batch_reader.h | 23 +-
be/src/service/arrow_flight/flight_sql_service.cpp | 8 +-
be/src/service/doris_main.cpp | 69 +-
be/src/service/internal_service.cpp | 9 +-
be/src/storage/rowset/beta_rowset.cpp | 17 +-
be/src/storage/rowset/beta_rowset.h | 5 +-
be/src/storage/rowset_builder.cpp | 6 +-
be/src/udf/python/python_udf_meta.cpp | 3 +-
be/src/util/jsonb_utils.h | 9 +-
.../core/data_type/common_data_type_serder_test.h | 4 +-
.../data_type_datev2_serde_calendar_test.cpp | 358 +++
.../data_type_serde/data_type_serde_arrow_test.cpp | 31 +-
.../operator/materialization_shared_state_test.cpp | 49 +
be/test/exec/scan/file_scanner_v2_test.cpp | 42 +
.../writer/iceberg/partition_transformers_test.cpp | 27 +
.../orc_scanner/date_count_truncated_stats.orc | Bin 0 -> 230 bytes
.../{my-file.orc => date_large_ordinal.orc} | Bin
.../orc_scanner/date_metadata_fixtures.md | 39 +
.../test_data/orc_scanner/date_out_of_range.orc | Bin 0 -> 619 bytes
.../exec/test_data/orc_scanner/date_preflight.orc | Bin 0 -> 390 bytes
.../orc_scanner/date_preflight_bad_index.orc | Bin 0 -> 390 bytes
.../test_data/paimon_rust_batch_size}/README.md | 29 +-
...-3b6dc957-0664-4ec4-946c-9535d228957a-0.parquet | Bin 0 -> 772 bytes
.../paimon_rust_batch_size/schema/schema-0 | 22 +
.../test_data/paimon_rust_batch_size/split.bin | Bin 0 -> 557 bytes
.../exprs/aggregate/agg_linear_histogram_test.cpp | 31 +-
be/test/exprs/aggregate/agg_topn_test.cpp | 125 +
.../aggregate_function_exception_test.cpp | 324 ++-
.../exprs/function/function_array_cum_sum_test.cpp | 100 +
.../exprs/function/function_array_range_test.cpp | 12 +
.../exprs/function/function_array_split_test.cpp | 224 ++
be/test/exprs/function/function_array_zip_test.cpp | 396 ++++
be/test/exprs/function/function_like_test.cpp | 104 +-
.../lambda_function/array_map_function_test.cpp | 619 ++++-
.../format/arrow/arrow_block_convertor_test.cpp | 133 ++
.../format/arrow/arrow_flight_timestamp_test.cpp | 225 ++
be/test/format/arrow/arrow_row_batch_test.cpp | 305 +++
.../new_plain_text_line_reader_test.cpp | 133 ++
.../table/iceberg/arrow_schema_util_test.cpp | 16 +-
.../format_v2/delimited_text/csv_reader_test.cpp | 160 ++
be/test/format_v2/json/json_reader_test.cpp | 175 ++
be/test/format_v2/orc/orc_reader_test.cpp | 559 ++++-
be/test/format_v2/parquet/parquet_reader_test.cpp | 248 ++
.../format_v2/parquet/parquet_statistics_test.cpp | 174 ++
be/test/format_v2/table/lance_reader_test.cpp | 69 +
be/test/format_v2/table/paimon_reader_test.cpp | 167 +-
.../table/paimon_rust_predicate_converter_test.cpp | 674 ++++++
.../table/paimon_rust_table_reader_test.cpp | 712 ++++++
.../arrow_flight/arrow_flight_cancel_test.cpp | 567 +++++
be/test/storage/rowset/beta_rowset_test.cpp | 54 +
be/test/testutil/scoped_temp_dir.h | 60 +
be/test/util/jsonb_parser_simd_test.cpp | 37 +
build.sh | 34 +-
cloud/src/recycler/recycler.cpp | 4 +-
cloud/test/CMakeLists.txt | 1 +
cloud/test/recycler_benchmark_test.cpp | 1069 +++++++++
cloud/test/recycler_test.cpp | 34 +
docker/runtime/doris-compose/utils.py | 43 +-
.../docker-compose/iceberg/iceberg.yaml.tpl | 24 +-
.../scripts/lance_build_array_predicates.py | 106 +
.../scripts/lance_build_preinstalled_catalog.py | 20 +-
.../iceberg/scripts/lance_build_time_travel.py | 178 ++
.../iceberg/scripts/lance_fixture_requirements.txt | 7 +-
.../iceberg/scripts/lance_rest_server.py | 198 +-
.../bitmap_page_lookup.lance | Bin 0 -> 993 bytes
.../page_data.lance | Bin 0 -> 769 bytes
.../page_lookup.lance | Bin 0 -> 1259 bytes
.../0-a4640b8d-55a7-47f8-9392-5ec3d0ed19c9.txn | Bin 0 -> 325 bytes
.../1-d21db0fc-7e46-4699-9948-ab6f8d66098c.txn | Bin 0 -> 184 bytes
.../2-8498550d-87e2-42c8-931c-1ed0d8bc7371.txn | Bin 0 -> 197 bytes
.../_versions/18446744073709551612.manifest | Bin 0 -> 901 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 735 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 729 bytes
.../_versions/latest_version_hint.json | 0
...0010111110100106a0464445da0b5a4b7c3465eb5.lance | Bin 0 -> 1223 bytes
...101111100111010027ce24037932857b1f2b85e24.lance | Bin 0 -> 1223 bytes
.../bitmap_page_lookup.lance | Bin 0 -> 865 bytes
.../page_data.lance | Bin 0 -> 704 bytes
.../page_lookup.lance | Bin 0 -> 1259 bytes
.../0-58691d36-bff2-4a01-ad25-ae3642019067.txn | Bin 0 -> 244 bytes
.../1-c343d785-33b7-4448-b0da-160c970f78a1.txn | Bin 0 -> 182 bytes
.../2-bcdae8c1-73a4-4b25-ad8b-887bf393af0f.txn | Bin 0 -> 195 bytes
.../3-8a4b0277-3a05-4eca-8787-2f5ad24763fb.txn | Bin 0 -> 124 bytes
.../_versions/18446744073709551611.manifest | Bin 0 -> 824 bytes
.../_versions/18446744073709551612.manifest | Bin 0 -> 812 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 648 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 565 bytes
.../_versions/latest_version_hint.json | 0
...011101101000101d214d7401ba969a49532b0ae8b.lance | Bin 0 -> 1223 bytes
...00100001001011139a9b14af698a84d7cb598d6a7.lance | Bin 0 -> 1223 bytes
.../0-7c18f76d-3524-4079-835f-7404de102fa9.txn | Bin 0 -> 325 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 729 bytes
.../_versions/latest_version_hint.json | 0
...0010010101001015f6f6345d896c3324132a6d562.lance | Bin 0 -> 1223 bytes
...1100100010111008b02e34ab199273cee3ff3688e.lance | Bin 0 -> 1223 bytes
.../time_travel.lance/_refs/branches/dev.json | 15 +
.../lance/time_travel.lance/_refs/tags/rel.json | 8 +
.../lance/time_travel.lance/_refs/tags/v1.json | 8 +
.../lance/time_travel.lance/_refs/tags/v2.json | 8 +
.../lance/time_travel.lance/_refs/tags/v3.json | 8 +
.../0-c7a3192f-4b4a-4e8b-b3a3-b736534c10f3.txn | Bin 0 -> 185 bytes
.../1-79502d42-ca15-43d2-99e3-8c734698ae2b.txn | Bin 0 -> 122 bytes
.../2-e678f5d6-e017-494d-adaf-fd1c0e245182.txn | Bin 0 -> 122 bytes
.../_versions/18446744073709551612.manifest | Bin 0 -> 546 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 464 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 447 bytes
.../_versions/latest_version_hint.json | 0
...11001001000111077951d4cb09c0a747e1eddc25e.lance | Bin 0 -> 557 bytes
...010000110100100bcc907454fa3151c2176cb1bc4.lance | Bin 0 -> 557 bytes
...1100100111000010ab68a492fa5d2c0fa972154e5.lance | Bin 0 -> 557 bytes
.../2-ca6009ab-1db7-4777-9e3e-a9374ed947b9.txn | Bin 0 -> 122 bytes
.../2-d76280f6-7616-446b-9575-5b8ec5ea2df1.txn | 1 +
.../dev/_versions/18446744073709551612.manifest | Bin 0 -> 605 bytes
.../dev/_versions/18446744073709551613.manifest | Bin 0 -> 494 bytes
...0110110000010015e5eaf41b5892dff0697f4cbcd.lance | Bin 0 -> 414 bytes
docker/thirdparties/lance-jni-helpers.sh | 10 +-
docs/lance-ann-profile.md | 103 +
docs/lance-jni-packaging.md | 6 +-
.../main/java/org/apache/doris/catalog/Env.java | 2 +-
.../java/org/apache/doris/catalog/OlapTable.java | 20 +-
.../org/apache/doris/catalog/RefreshManager.java | 182 +-
.../cloud/catalog/CloudFEVersionSynchronizer.java | 59 +
.../apache/doris/cloud/catalog/CloudPartition.java | 43 +-
.../transaction/CloudGlobalTransactionMgr.java | 11 +
.../doris/common/profile/ExecutionProfile.java | 69 +-
.../org/apache/doris/common/profile/Profile.java | 68 +
.../doris/common/profile/SummaryProfile.java | 10 +
.../org/apache/doris/datasource/CatalogMgr.java | 96 +-
.../apache/doris/datasource/ExternalCatalog.java | 290 ++-
.../apache/doris/datasource/ExternalDatabase.java | 69 +-
.../doris/datasource/ExternalMetaCacheMgr.java | 237 +-
.../doris/datasource/ExternalRowCountCache.java | 350 ++-
.../org/apache/doris/datasource/ExternalTable.java | 2 +-
.../doris/datasource/doris/FeServiceClient.java | 11 +-
.../doris/datasource/hive/HMSExternalCatalog.java | 71 +-
.../doris/datasource/hive/HiveMetadataOps.java | 29 +-
.../datasource/hive/event/AddPartitionEvent.java | 3 +-
.../datasource/hive/event/AlterDatabaseEvent.java | 17 +-
.../datasource/hive/event/AlterPartitionEvent.java | 6 +-
.../datasource/hive/event/CreateDatabaseEvent.java | 3 +-
.../datasource/hive/event/DropDatabaseEvent.java | 2 +-
.../datasource/hive/event/DropPartitionEvent.java | 2 +-
.../doris/datasource/hive/event/InsertEvent.java | 4 +-
.../datasource/iceberg/IcebergExternalCatalog.java | 15 +-
.../datasource/iceberg/IcebergMetadataOps.java | 26 +-
.../doris/datasource/iceberg/IcebergUtils.java | 11 +-
.../datasource/iceberg/source/IcebergScanNode.java | 46 +-
.../doris/datasource/jdbc/JdbcExternalCatalog.java | 31 +-
.../doris/datasource/lance/LanceCatalogClient.java | 556 ++++-
.../datasource/lance/LanceExternalCatalog.java | 11 +-
.../doris/datasource/lance/LanceExternalTable.java | 75 +-
.../doris/datasource/lance/LanceManifestPaths.java | 141 ++
.../datasource/lance/LanceNamespaceClient.java | 154 +-
.../lance/metadata/LanceRefSelector.java | 75 +
.../lance/metadata/LanceSnapshotResolver.java | 141 +-
.../lance/metadata/LanceTableAccess.java | 60 +-
.../lance/metadata/LanceTableMetadata.java | 14 +
.../lance/source/LancePredicateConverter.java | 75 +-
.../lance/source/LanceScalarIndexPlanner.java | 233 +-
.../datasource/lance/source/LanceScanNode.java | 9 +
.../datasource/lance/source/LanceScanPlanner.java | 6 +-
.../lance/source/LanceSubstraitSerializer.java | 18 +-
.../maxcompute/MaxComputeMetadataOps.java | 22 +-
.../doris/datasource/metacache/MetaCache.java | 16 +
.../metacache/paimon/PaimonTableLoader.java | 9 +
.../datasource/operations/ExternalMetadataOps.java | 33 +-
.../paimon/DorisToPaimonTypeVisitor.java | 17 +-
.../datasource/paimon/PaimonExternalCatalog.java | 31 +-
.../datasource/paimon/PaimonExternalMetaCache.java | 4 +-
.../doris/datasource/paimon/PaimonMetadataOps.java | 52 +-
.../doris/datasource/paimon/PaimonScanParams.java | 59 +
.../apache/doris/datasource/paimon/PaimonUtil.java | 33 +
.../source/PaimonRustReaderCapabilities.java | 282 +++
.../datasource/paimon/source/PaimonScanNode.java | 683 +++++-
.../doris/httpv2/controller/BaseController.java | 9 +-
.../doris/httpv2/controller/LoginController.java | 11 +-
.../org/apache/doris/mysql/privilege/Auth.java | 10 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 68 +-
.../plans/commands/GrantTablePrivilegeCommand.java | 25 +-
.../nereids/trees/plans/commands/HelpCommand.java | 23 +
.../commands/RevokeTablePrivilegeCommand.java | 13 +-
.../trees/plans/commands/ShowConfigCommand.java | 6 +
.../plans/commands/ShowCreateProcedureCommand.java | 6 +
.../plans/commands/ShowCreateTableCommand.java | 13 +
.../trees/plans/commands/ShowProcCommand.java | 24 +
.../plans/commands/ShowProcedureStatusCommand.java | 6 +
.../trees/plans/commands/ShowSnapshotCommand.java | 18 +-
.../trees/plans/commands/info/AlterUserInfo.java | 6 +-
.../trees/plans/commands/info/SetPassVarOp.java | 3 +-
.../insert/BaseExternalTableInsertExecutor.java | 9 +-
.../plans/commands/insert/HiveInsertExecutor.java | 40 +-
.../plans/commands/insert/WarmupSelectCommand.java | 6 +
.../trees/plans/commands/use/UseCommand.java | 8 +
.../java/org/apache/doris/persist/DropDbInfo.java | 12 +
.../java/org/apache/doris/persist/DropInfo.java | 12 +
.../java/org/apache/doris/persist/EditLog.java | 2 +-
.../java/org/apache/doris/planner/ResultSink.java | 3 +
.../java/org/apache/doris/qe/ConnectContext.java | 48 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 5 +
.../org/apache/doris/qe/NereidsCoordinator.java | 14 +
.../java/org/apache/doris/qe/SessionVariable.java | 19 +
.../java/org/apache/doris/qe/StmtExecutor.java | 6 +
.../qe/runtime/PipelineExecutionTaskBuilder.java | 9 +-
.../apache/doris/service/FrontendServiceImpl.java | 6 +
.../arrowflight/DorisFlightSqlProducer.java | 213 +-
.../arrowflight/FlightSqlConnectProcessor.java | 9 +-
.../arrowflight/FlightSqlQueryCancellation.java | 146 ++
.../service/arrowflight/FlightSqlQuerySchema.java | 413 ++++
.../service/arrowflight/FlightSqlSchemaHelper.java | 49 +-
.../arrowflight/results/FlightSqlChannel.java | 14 +-
.../main/java/org/apache/doris/system/Backend.java | 11 +
.../org/apache/doris/system/BackendHbResponse.java | 10 +
.../java/org/apache/doris/system/HeartbeatMgr.java | 5 +-
.../LanceExternalSearchTableValuedFunction.java | 20 +-
.../VectorSearchTableValuedFunction.java | 11 +-
.../main/resources/substrait/lance_functions.yaml | 27 +
.../doris/catalog/RefreshManagerReplayTest.java | 23 +
.../apache/doris/catalog/RefreshManagerTest.java | 511 ++++
.../transaction/CloudGlobalTransactionMgrTest.java | 400 +++-
.../common/profile/ProfileCompletionStateTest.java | 217 ++
.../doris/common/profile/ProfileManagerTest.java | 29 +
.../apache/doris/datasource/CatalogMgrTest.java | 1127 ++++++++-
.../ExternalMetaCacheRouteResolverTest.java | 444 +++-
.../datasource/ExternalRowCountCacheTest.java | 557 ++++-
.../org/apache/doris/datasource/MetaCacheTest.java | 9 +
.../datasource/doris/FeServiceClientTest.java | 83 +
.../doris/datasource/hive/HiveMetadataOpsTest.java | 120 +
.../doris/datasource/iceberg/IcebergUtilsTest.java | 23 +
.../iceberg/source/IcebergScanNodeTest.java | 339 +++
.../datasource/jdbc/JdbcExternalCatalogTest.java | 10 +
.../lance/LanceCatalogLifecycleTest.java | 63 +
.../datasource/lance/LanceManagedAccessTest.java | 168 ++
.../lance/LanceManagedVersioningTest.java | 1460 ++++++++++++
.../datasource/lance/LanceManifestPathsTest.java | 120 +
.../lance/LancePredicateConverterTest.java | 115 +
.../datasource/lance/LanceSharedSessionTest.java | 11 +-
.../lance/LanceTableAccessCacheTest.java | 17 +
.../lance/metadata/LanceSnapshotTest.java | 211 ++
.../lance/source/LanceScalarIndexPlannerTest.java | 292 +++
.../datasource/lance/source/LanceScanNodeTest.java | 75 +-
.../paimon/DorisToPaimonTypeVisitorTest.java | 78 +
.../paimon/PaimonExternalMetaCacheTest.java | 130 +-
.../datasource/paimon/PaimonMetadataOpsTest.java | 16 +
.../source/PaimonRustBackendCapabilityTest.java | 118 +
.../source/PaimonRustReaderCompatibilityTest.java | 496 ++++
.../source/PaimonRustReaderEvolutionTest.java | 184 ++
.../source/PaimonRustReaderLegacySchemaTest.java | 101 +
.../paimon/source/PaimonScanNodeTest.java | 1545 ++++++++++++-
.../controller/BaseControllerBasicAuthTest.java | 188 ++
.../doris/mysql/privilege/SystemRootUserTest.java | 108 +
.../commands/GrantTablePrivilegeCommandTest.java | 116 +
.../commands/RevokeTablePrivilegeCommandTest.java | 29 +
.../commands/SchemaOnlyCommandMetadataTest.java | 123 +
.../commands/insert/HiveInsertExecutorTest.java | 343 +++
.../doris/persist/DropAndRecoverInfoTest.java | 13 +
.../org/apache/doris/persist/DropDbInfoTest.java | 13 +
.../doris/planner/ResultSinkMetadataTest.java | 38 +
.../qe/NereidsCoordinatorFlightEndpointsTest.java | 118 +
.../runtime/PipelineExecutionTaskBuilderTest.java | 96 +
.../arrowflight/DorisFlightSqlProducerTest.java | 114 -
.../arrowflight/DorisFlightSqlSchemaTest.java | 775 +++++++
.../FlightSqlConnectProcessorSchemaTest.java | 167 ++
.../arrowflight/FlightSqlExecutionSchemaTest.java | 178 ++
.../FlightSqlQueryCancellationTest.java | 109 +
.../FlightSqlSchemaHelperArrowTypeTest.java | 11 +
.../arrowflight/FlightSqlTableTypesTest.java | 91 +
.../system/BackendPaimonRustCapabilityTest.java | 70 +
.../VectorSearchTableValuedFunctionTest.java | 50 +
fe/pom.xml | 2 +-
gensrc/thrift/DataSinks.thrift | 2 +
gensrc/thrift/FrontendService.thrift | 3 +
gensrc/thrift/HeartbeatService.thrift | 1 +
gensrc/thrift/PaloInternalService.thrift | 3 +
gensrc/thrift/PlanNodes.thrift | 9 +
.../test_grant_col_priv.out} | 7 +-
.../arrow_flight_sql_p0/test_date_year_zero.out | 11 +
.../cache/ttl/st04_alter_ttl_n_to_0_runtime.out | 7 +-
.../cache/ttl/st06_warmup_ttl_type_assert.out | 4 +-
.../data/cloud_p0/cache/ttl/test_show_ttl.out | 16 -
.../outfile/test_outfile_date_year_zero.out | 37 +
.../lance/test_lance_rest_catalog.out | 10 +
.../lance/test_lance_rest_time_travel.out | 134 ++
.../lance/test_lance_time_travel.out | 141 ++
.../test_remote_doris_all_types_select.out | 4 +-
.../remote_doris/test_remote_doris_statistics.out | 2 +-
.../tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out | 34 -
.../agg_function/topn/topn_unlimited.out | 91 +
.../test_linear_histogram_boolean.out | 17 +
.../test_array_cum_sum_leading_null.out | 14 +
.../array_functions/test_array_map_function.out | 11 +
.../test_array_range_null_argument.out | 15 +
.../array_functions/test_array_split.out | 7 +
.../test_regexp_extended_nonconstant_pattern.out | 28 +
.../table_function/test_json_text_embedded_nul.out | 19 +
.../test_double_write_when_schema_change.out | 4 +-
.../org/apache/doris/regression/Config.groovy | 139 +-
.../doris/regression/action/ProfileAction.groovy | 38 +-
.../doris/regression/suite/ScriptContext.groovy | 11 +
.../org/apache/doris/regression/suite/Suite.groovy | 231 +-
.../doris/regression/suite/SuiteContext.groovy | 55 +-
.../apache/doris/regression/util/DebugPoint.groovy | 12 +-
.../org/apache/doris/regression/util/Http.groovy | 32 +-
.../regression/util/MultiClusterStateGuard.groovy | 195 ++
.../doris/regression/util/MySqlClient.groovy | 70 +
.../doris/regression/util/RowsetMetaUtils.groovy | 79 +
.../regression/util/WarmupMetricsUtils.groovy | 96 +-
.../regression/ConfigArrowFlightSqlUrlTest.groovy | 61 +
.../doris/regression/ConfigRootPasswordTest.groovy | 61 +
.../suite/CompactionPluginCompatibilityTest.groovy | 154 ++
.../regression/suite/SuiteSshExecTest.groovy} | 24 +-
.../regression/util/HttpAuthenticationTest.groovy | 72 +
.../util/MultiClusterStateGuardTest.groovy | 256 ++
.../doris/regression/util/MySqlClientTest.groovy | 96 +
.../regression/util/RowsetMetaUtilsTest.groovy | 81 +
.../nonConcurrent/conf/regression-conf.groovy | 2 +-
.../plugins/cloud_multi_cluster_state.groovy | 136 ++
regression-test/plugins/plugin_compaction.groovy | 134 +-
.../plugins/plugin_curl_requester.groovy | 3 +-
regression-test/plugins/plugin_index_change.groovy | 5 +-
.../suites/account_p0/test_auth_show.groovy | 4 +-
.../suites/account_p0/test_grant_col_priv.groovy | 109 +
.../suites/account_p0/test_grant_priv.groovy | 10 +-
.../account_p0/test_grant_priv_resource.groovy | 4 +-
.../account_p0/test_grant_priv_workload.groovy | 4 +-
.../test_grant_revoke_object_name.groovy | 51 +
.../account_p0/test_nereids_authentication.groovy | 14 +-
.../account_p0/test_nereids_row_policy.groovy | 4 +-
.../suites/account_p0/test_set_password.groovy | 6 +-
.../suites/account_p0/test_system_user.groovy | 31 +
.../suites/account_p0/test_view_row_policy.groovy | 6 +-
.../ann_topn_small_candidate_fallback.groovy | 21 +-
.../arrow_flight_sql_p0/test_auth_remote_ip.groovy | 3 +-
.../arrow_flight_sql_p0/test_date_year_zero.groovy | 95 +
.../test_flight_cancel_cleanup.groovy | 128 +
.../test_flight_int96_timestamp_range.groovy | 83 +
.../test_flight_parallel_partitions.groovy | 134 ++
.../test_flight_timestamp_range.groovy | 81 +
.../test_flight_utf8_validation.groovy | 71 +
.../arrow_flight_sql_p0/test_schema_only.groovy | 288 +++
.../arrow_flight_sql_p0/test_table_types.groovy | 73 +
.../suites/audit/test_audit_log_queue_time.groovy | 212 +-
.../auth_call/test_dml_select_udf_auth.groovy | 1 +
.../test_dml_stream_load_be_auth_docker.groovy | 49 +-
.../suites/auth_p0/test_http_rest_v1_auth.groovy | 90 +
.../auth_p0/test_role_mapping_system_table.groovy | 2 +-
.../cloud_p0/auth/test_set_default_cluster.groovy | 6 +-
.../balance/test_cross_cg_bvar_metrics.groovy | 27 +-
.../multi_cluster/read_write/sync_insert.groovy | 330 ++-
.../read_write/test_multi_stale_rowset.groovy | 70 +-
.../cluster/test_warm_up_cluster_batch.groovy | 409 ++--
...est_warm_up_cluster_event_cancel_expired.groovy | 92 +-
.../cluster/test_warm_up_compute_group.groovy | 209 +-
.../warm_up/table/test_warm_up_partition.groovy | 399 ++--
.../test_warm_up_same_table_multi_times.groovy | 43 +-
.../warm_up/table/test_warm_up_table.groovy | 200 +-
.../warm_up/table/test_warm_up_tables.groovy | 486 ++--
.../test_clear_file_cache_on_load_failure.groovy | 121 +-
.../cloud_p0/cache/test_file_cache_info.groovy | 318 ++-
.../suites/cloud_p0/cache/ttl/alter_ttl_2.groovy | 231 +-
.../suites/cloud_p0/cache/ttl/alter_ttl_4.groovy | 489 ++--
.../cloud_p0/cache/ttl/alter_ttl_random.groovy | 21 +-
.../cloud_p0/cache/ttl/alter_ttl_seconds.groovy | 170 --
.../cache/ttl/st04_alter_ttl_n_to_0_runtime.groovy | 233 +-
.../cache/ttl/st06_warmup_ttl_type_assert.groovy | 231 +-
.../suites/cloud_p0/cache/ttl/test_show_ttl.groovy | 57 +
.../suites/cloud_p0/cache/ttl/test_show_ttl.sql | 40 -
.../suites/cloud_p0/cache/ttl/test_ttl.groovy | 433 ++--
...st_file_cache_warmup_read_metrics_docker.groovy | 4 +-
.../cache/warm_up/test_warmup_table_docker.groovy | 6 +-
...test_cloud_alter_disable_auto_compaction.groovy | 8 +-
.../async_copy_into/async_load.groovy | 10 +-
.../multi_cluster/copy_into/sync_laod.groovy | 124 +-
.../cloud_p0/multi_cluster/default_cluster.groovy | 38 +-
.../multi_cluster_s3_load/load.groovy | 66 +-
.../cloud_p0/multi_cluster/rename_cluster.groovy | 10 +-
.../routine_load/test_routine_load.groovy | 6 +-
...test_materialized_view_with_drop_cluster.groovy | 8 +-
...est_materialized_view_with_readd_cluster.groovy | 8 +-
.../test_rollup_with_drop_cluster.groovy | 8 +-
.../test_rollup_with_readd_cluster.groovy | 8 +-
.../test_schema_change_with_drop_cluster.groovy | 8 +-
.../test_schema_change_with_readd_cluster.groovy | 8 +-
.../smooth_upgrade/smooth_upgrade.groovy | 10 +
.../multi_cluster/stream_load/stream_load.groovy | 67 +-
.../stream_load/stream_load_2pc.groovy | 3 +
.../stream_load/stream_load_lb.groovy | 12 +-
.../cloud_p0/multi_cluster/table_rebalance.groovy | 30 +-
.../test_apsaradb_internal_stage.groovy | 31 +-
.../multi_cluster/test_cloud_cluster.groovy | 19 +-
.../multi_cluster/test_drop_cluster.groovy | 98 +-
.../cloud_p0/multi_cluster/test_drop_userid.groovy | 2 +-
.../test_group_commit_multi_cluster.groovy | 12 +-
.../multi_cluster/test_overdue_instance.groovy | 54 +-
.../suites/cloud_p0/multi_cluster/test_tvf.groovy | 11 +-
.../default_vcg_auto_failover.groovy | 19 +-
.../use_vcg_read_write_routine_load.groovy | 722 +++---
.../cloud_p0/snapshot/test_snapshot_command.groovy | 12 +-
.../test_clean_tablet_when_drop_force_table.groovy | 144 +-
.../test_clean_tablet_when_rebalance.groovy | 109 +-
.../test_create_partition_idempotence.groovy | 3 +-
.../test_insert_overwrite_idempotence.groovy | 3 +-
.../test_schema_change_with_compaction10.groovy | 66 +-
.../test_schema_change_with_compaction11.groovy | 91 +-
.../test_schema_change_with_compaction5.groovy | 25 +-
.../test_schema_change_with_compaction6.groovy | 67 +-
.../test_schema_change_with_compaction9.groovy | 24 +-
.../test_inverted_index_sc_version1_race.groovy | 2 +-
.../test_sc_compaction_cross_v1_retry.groovy | 73 +-
...c_compaction_cross_v1_stale_base_refresh.groovy | 30 +-
.../test_sc_compaction_optimization.groovy | 50 +-
...est_sc_compaction_optimization_with_load.groovy | 52 +-
.../compaction_width_array_column.groovy | 2 +-
.../compaction/test_admin_compact_tablet.groovy | 88 +-
...paction_with_dup_key_max_file_size_limit.groovy | 148 +-
.../test_compaction_uniq_keys_row_store.groovy | 4 +-
.../test_compaction_uniq_keys_row_store_ck.groovy | 4 +-
...test_compaction_update_big_delete_bitmap.groovy | 17 +-
.../test_compaction_with_visible_version.groovy | 3 +
.../test_config_prune_delete_sign.groovy | 129 +-
...ilecache_with_base_compaction_thresthold.groovy | 150 +-
.../test_mow_compact_multi_segments.groovy | 48 +-
...paction_agg_and_remove_pre_delete_bitmap.groovy | 17 +-
...test_mow_compaction_and_rowset_not_exist.groovy | 17 +-
.../test_mow_compaction_and_schema_change.groovy | 17 +-
.../test_mow_stale_rowset_delete_bitmap.groovy | 17 +-
.../test_non_mow_key_bounds_aggregation.groovy | 25 +-
.../test_create_view_nereids.groovy | 1 +
.../suites/ddl_p0/test_drop_view_nereids.groovy | 1 +
.../delete/batch-delete-manual.md.groovy | 12 +-
.../import-way/group-commit-manual.md.groovy | 11 +-
.../import/import-way/mysql-load-manual.md.groovy | 20 +-
.../update/unique-update-transaction.md.groovy | 6 +-
.../update/update-of-aggregate-model.md.groovy | 2 +-
.../update/update-of-unique-model.md.groovy | 2 +-
.../outfile/test_outfile_date_year_zero.groovy | 85 +
.../test_iceberg_arrow_flight_split_source.groovy | 3 +-
.../iceberg/test_iceberg_variant_read.groovy | 19 +-
...t_iceberg_delete_unpartitioned_evolution.groovy | 198 ++
.../write/test_iceberg_write_date_year_zero.groovy | 79 +
.../select/test_mysql_varbinary_with_udf.groovy | 1 +
.../test_lance_array_predicate_pushdown.groovy | 154 ++
.../lance/test_lance_rest_catalog.groovy | 4 +-
.../lance/test_lance_rest_time_travel.groovy | 294 +++
.../lance/test_lance_time_travel.groovy | 267 +++
.../lance/test_lance_vector_search.groovy | 20 +
.../test_paimon_rust_reader_cast_predicates.groovy | 192 ++
.../test_paimon_rust_reader_compatibility.groovy | 308 +++
.../test_paimon_rust_reader_eq_for_null.groovy | 555 +++++
...test_paimon_rust_reader_schema_evolution.groovy | 134 ++
.../paimon/test_paimon_rust_reader_v2.groovy | 146 ++
.../test_remote_doris_all_types_select.groovy | 15 +-
.../test_remote_doris_statistics.groovy | 3 +-
.../test_remote_doris_table_stats.groovy | 3 +-
.../orc_tvf/test_hdfs_orc_group6_orc_files.groovy | 38 +-
.../test_cloud_full_compaction_do_lease.groovy | 142 +-
.../test_cloud_mow_delete_bitmap_lock_case.groovy | 412 +++-
.../test_cloud_mow_partial_update_retry.groovy | 99 +-
..._cloud_mow_stale_resp_load_load_conflict.groovy | 105 +-
.../test_delete_bitmap_lock_with_restart.groovy | 545 +++--
...est_cold_data_compaction_fault_injection.groovy | 146 +-
...ion_clone_missing_rowset_fault_injection.groovy | 29 +-
.../test_incomplete_commit_info.groovy | 117 +-
...st_skip_index_compaction_fault_injection.groovy | 90 +-
.../http_rest_api/get/test_schema_api.groovy | 4 +-
.../http_rest_api/post/test_query_stmt.groovy | 3 +-
.../insert_p0/insert_group_commit_into.groovy | 10 +-
.../mow_insert_with_partition_drop.groovy | 24 +-
.../test_insert_visible_timeout_return_mode.groovy | 16 +-
regression-test/suites/insert_p0/test_jdbc.groovy | 4 +-
.../transaction/test_publish_exception.groovy | 5 +
.../suites/insert_p0/transaction/txn_insert.groovy | 5 +-
.../operate_internal_schema.groovy | 2 +-
.../show_data/test_show_index_data_p2.groovy | 104 +-
.../collect/test_javaudf_groupcount.groovy | 1 +
.../collect/test_javaudf_murmurhash3.groovy | 1 +
.../collect/test_javaudf_sessionize.groovy | 2 +-
.../javaudf_p0/date/test_javaudf_adddays.groovy | 1 +
.../date/test_javaudf_addisoperioud.groovy | 1 +
.../javaudf_p0/date/test_javaudf_daydiff.groovy | 1 +
.../function_meta/function_query_test.groovy | 10 +-
.../suites/javaudf_p0/function_meta/load.groovy | 19 +-
.../json/test_javaudf_convertfromcamelcase.groovy | 1 +
.../json/test_javaudf_converttocamelcase.groovy | 1 +
regression-test/suites/javaudf_p0/null/load.groovy | 1 +
.../sanity/test_javaudf_assertequal.groovy | 1 +
.../sanity/test_javaudf_assertlessthan.groovy | 1 +
.../sanity/test_javaudf_assertudf.groovy | 1 +
.../javaudf_p0/sketch/test_javaudf_md5.groovy | 1 +
.../test_javaudaf_my_date_datetime.groovy | 1 +
.../test_javaudaf_mygroupconcat_string.groovy | 1 +
.../javaudf_p0/test_javaudaf_mysum_array.groovy | 1 +
.../javaudf_p0/test_javaudaf_mysum_decimal.groovy | 1 +
.../test_javaudaf_mysum_float_double.groovy | 1 +
.../javaudf_p0/test_javaudaf_mysum_int.groovy | 1 +
.../javaudf_p0/test_javaudaf_null_test.groovy | 1 +
.../javaudf_p0/test_javaudaf_return_map.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_agg_map.groovy | 1 +
.../javaudf_p0/test_javaudf_all_types.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_array.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_auth.groovy | 5 +-
.../suites/javaudf_p0/test_javaudf_boolean.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_case.groovy | 1 +
.../javaudf_p0/test_javaudf_const_test.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_date.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_decimal.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_float.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_int.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_map.groovy | 1 +
.../javaudf_p0/test_javaudf_multi_evaluate.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_no_input.groovy | 2 +-
.../suites/javaudf_p0/test_javaudf_override.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_ret_map.groovy | 1 +
.../test_javaudf_static_load_test.groovy | 2 +-
.../suites/javaudf_p0/test_javaudf_string.groovy | 1 +
.../javaudf_p0/test_javaudf_struct_type.groovy | 1 +
.../javaudf_p0/test_javaudf_with_decimal.groovy | 1 +
.../javaudf_p0/test_javaudtf_all_types.groovy | 1 +
.../javaudf_p0/test_javaudtf_arrayint.groovy | 1 +
.../suites/javaudf_p0/test_javaudtf_decimal.groovy | 1 +
.../suites/javaudf_p0/test_javaudtf_double.groovy | 1 +
.../suites/javaudf_p0/test_javaudtf_int.groovy | 1 +
.../suites/javaudf_p0/test_javaudtf_string.groovy | 1 +
.../test_streaming_insert_job_priv.groovy | 8 +-
.../test_insert_statistic_after_fe_restart.groovy | 6 +-
.../test_routine_load_advanced_mapping.groovy | 56 +-
.../routine_load/test_routine_load_with_udf.groovy | 1 +
.../scripts/stream_load_redirect_chunked_e2e.py | 16 +-
.../test_packed_file_stream_load_case1.groovy | 2 +-
.../test_packed_file_stream_load_case2.groovy | 2 +-
.../test_packed_file_stream_load_case3.groovy | 2 +-
.../test_packed_file_stream_load_case4.groovy | 2 +-
.../test_packed_file_stream_load_case5.groovy | 11 +-
.../test_packed_file_stream_load_case6.groovy | 11 +-
.../test_packed_file_stream_load_case7.groovy | 11 +-
.../test_packed_file_stream_load_case8.groovy | 11 +-
.../load_p0/stream_load/test_sink_tolerate.groovy | 3 +-
.../load_p0/stream_load/test_stream_load.groovy | 38 +-
.../stream_load/test_stream_load_endpoint.groovy | 141 +-
...test_stream_load_fe_redirect_chunked_e2e.groovy | 10 +-
.../stream_load/test_stream_load_with_udf.groovy | 1 +
.../suites/manager/test_manager_interface_3.groovy | 317 +--
.../suites/mtmv_p0/test_expand_star_mtmv.groovy | 1 +
.../mysql_ssl_p0/test_mysql_connection.groovy | 66 +-
.../agg_function/topn/topn_unlimited.groovy | 75 +
.../cache/parse_sql_from_sql_cache.groovy | 21 +-
.../insert_into_table/insert_auth.groovy | 6 +-
.../javaudf/collect/test_javaudf_groupcount.groovy | 1 +
.../collect/test_javaudf_murmurhash3.groovy | 1 +
.../javaudf/collect/test_javaudf_sessionize.groovy | 2 +-
.../javaudf/date/test_javaudf_adddays.groovy | 1 +
.../javaudf/date/test_javaudf_addisoperioud.groovy | 1 +
.../javaudf/date/test_javaudf_daydiff.groovy | 1 +
.../json/test_javaudf_convertfromcamelcase.groovy | 1 +
.../json/test_javaudf_converttocamelcase.groovy | 1 +
.../javaudf/sanity/test_javaudf_assertequal.groovy | 1 +
.../sanity/test_javaudf_assertlessthan.groovy | 1 +
.../javaudf/sanity/test_javaudf_assertudf.groovy | 1 +
.../javaudf/sketch/test_javaudf_md5.groovy | 1 +
.../javaudf/test_javaudaf_my_date_datetime.groovy | 1 +
.../test_javaudaf_mygroupconcat_string.groovy | 1 +
.../javaudf/test_javaudaf_mysum_decimal.groovy | 1 +
.../test_javaudaf_mysum_float_double.groovy | 1 +
.../javaudf/test_javaudaf_mysum_int.groovy | 1 +
.../javaudf/test_javaudaf_null_test.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_boolean.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_case.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_date.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_decimal.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_float.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_int.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_ip.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_null.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_string.groovy | 1 +
.../agg_state/test_outfile_agg_state.groovy | 2 +-
.../agg_state_array/test_outfile_agg_array.groovy | 2 +-
.../test_outfile_agg_state_bitmap.groovy | 2 +-
.../nereids_p0/outfile/hll/test_outfile_hll.groovy | 2 +-
.../test_outfile_quantile_state.groovy | 2 +-
.../show/test_nereids_show_functions.groovy | 13 +-
.../stats/col_stats/hot_value_analyze_sync.groovy | 2 +-
.../test_partition_create_tablet_rr.groovy | 1 +
.../auto_partition/sql/multi_thread_load.groovy | 30 +-
.../sql/stress_test_insert_into.groovy | 22 +-
.../diff_data/stress_test_diff_date_list.groovy | 16 +-
.../stress_test_high_concurrency_load.groovy | 19 +-
.../same_data/stress_test_same_date_range.groovy | 19 +-
.../stress_test_two_stream_load.groovy | 19 +-
.../suites/point_query_p0/test_point_query.groovy | 10 +-
.../point_query_p0/test_point_query_ck.groovy | 17 +-
.../test_point_query_partition.groovy | 6 +-
.../suites/point_query_p0/test_rowstore.groovy | 14 +-
.../suites/point_query_p0/test_rowstore_ck.groovy | 14 +-
.../test_python_raise_error_propagation.groovy | 3 +
.../pythonudaf_p0/test_pythonudaf_basic.groovy | 1 +
...st_pythonudaf_complex_aggregation_module.groovy | 1 +
..._pythonudaf_complex_state_objects_module.groovy | 1 +
.../pythonudaf_p0/test_pythonudaf_drop.groovy | 2 +
.../test_pythonudaf_forbidden_module.groovy | 1 +
.../test_pythonudaf_pkg_isolation.groovy | 2 +
.../pythonudaf_p0/test_pythonudaf_simple.groovy | 1 +
.../test_pythonudaf_window_advanced_module.groovy | 1 +
.../test_python_udaf_complex.groovy | 1 +
.../test_python_udf_business_logic.groovy | 1 +
.../test_python_udf_external_api.groovy | 3 +-
.../test_python_udf_nlp_chinese.groovy | 1 +
.../test_python_udtf_complex.groovy | 1 +
.../sanity/test_pythonudf_assertequal.groovy | 1 +
.../sanity/test_pythonudf_assertlessthan.groovy | 1 +
.../pythonudf_p0/test_pythonudf_array.groovy | 1 +
.../test_pythonudf_base_data_type.groovy | 1 +
.../pythonudf_p0/test_pythonudf_boolean.groovy | 1 +
.../test_pythonudf_complex_data_type.groovy | 1 +
.../suites/pythonudf_p0/test_pythonudf_drop.groovy | 2 +
.../test_pythonudf_file_protocol.groovy | 1 +
.../pythonudf_p0/test_pythonudf_float.groovy | 1 +
.../test_pythonudf_forbidden_module.groovy | 1 +
.../suites/pythonudf_p0/test_pythonudf_int.groovy | 1 +
.../suites/pythonudf_p0/test_pythonudf_map.groovy | 1 +
.../pythonudf_p0/test_pythonudf_module.groovy | 1 +
.../test_pythonudf_module_advanced.groovy | 1 +
.../test_pythonudf_module_scalar.groovy | 1 +
.../test_pythonudf_module_vector.groovy | 1 +
.../test_pythonudf_pkg_isolation.groovy | 2 +
.../pythonudf_p0/test_pythonudf_ret_map.groovy | 1 +
.../pythonudf_p0/test_pythonudf_string.groovy | 1 +
.../test_pythonudtf_basic_module.groovy | 1 +
.../test_pythonudtf_data_types_module.groovy | 1 +
.../pythonudtf_p0/test_pythonudtf_drop.groovy | 2 +
.../test_pythonudtf_edge_cases_module.groovy | 1 +
.../test_pythonudtf_exceptions_module.groovy | 1 +
.../test_pythonudtf_forbidden_module.groovy | 1 +
.../test_pythonudtf_io_patterns_module.groovy | 1 +
.../test_pythonudtf_pkg_isolation.groovy | 2 +
.../test_pythonudtf_sql_integration_module.groovy | 1 +
.../query_p0/cache/query_cache_with_context.groovy | 3 +-
.../test_linear_histogram_boolean.groovy | 63 +
.../test_array_cum_sum_leading_null.groovy | 51 +
.../array_functions/test_array_map_function.groovy | 44 +
.../test_array_range_null_argument.groovy | 51 +
.../array_functions/test_array_split.groovy | 16 +-
...test_regexp_extended_nonconstant_pattern.groovy | 85 +
.../test_json_text_embedded_nul.groovy | 62 +
.../query_p0/system/test_partitions_schema.groovy | 6 +-
.../query_p0/system/test_table_options.groovy | 6 +-
.../query_p0/system/test_table_properties.groovy | 6 +-
.../suites/query_p0/test_row_policy.groovy | 4 +-
.../query_p2/big_join_build/big_join_build.groovy | 2 +-
.../test_set_operation_large_string.groovy | 2 +-
.../suites/query_profile/profile_safe.groovy | 5 +-
.../query_profile/s3_load_profile_test.groovy | 55 +-
.../query_profile/test_load_channel_profile.groovy | 6 +-
...e_column_with_delete_drop_column_dup_key.groovy | 82 +-
...olumn_with_delete_drop_column_unique_key.groovy | 63 +-
.../test_filecache_with_alter_table.groovy | 36 +-
.../test_alter_table_column_rename.groovy | 2 +-
.../test_schema_change_fail.groovy | 11 +-
...test_show_create_table_and_views_nereids.groovy | 89 +-
.../suites/show_p0/test_show_data.groovy | 2 +-
.../suites/show_p0/test_show_processlist.groovy | 6 +-
.../suites/statistics/test_analyze_all_null.groovy | 2 +-
.../suites/statistics/test_hot_value.groovy | 2 +-
.../test_skip_agg_table_value_column.groovy | 2 +-
.../test_partial_update_2pc_schema_change.groovy | 38 +-
.../legacy/test_f_2pc_schema_change.groovy | 38 +-
.../publish/test_f_auto_inc_compaction.groovy | 35 +-
.../test_f_seq_publish_read_from_old.groovy | 33 +-
...xble_partial_update_publish_conflict_seq.groovy | 35 +-
...flexible_partial_update_publish_conflict.groovy | 35 +-
.../test_key_bounds_truncation_basic.groovy | 27 +-
...est_key_bounds_truncation_read_scenarios.groovy | 39 +-
.../test_partial_update_publish_all_del.groovy | 35 +-
.../publish/test_partial_update_publish_seq.groovy | 35 +-
...partial_update_delete_sign_with_conflict.groovy | 33 +-
.../test_partial_update_multi_stmt.groovy | 13 +-
.../test_double_write_when_schema_change.groovy | 4 +-
.../suites/variant_p0/variant_with_rowstore.groovy | 4 +-
samples/arrow-flight-sql/python/README.md | 22 +
.../python/test_nested_type_metadata.py | 108 +
thirdparty/build-thirdparty.sh | 189 +-
thirdparty/download-thirdparty.sh | 35 +-
thirdparty/lance-install.sh | 38 +
thirdparty/patches/lance-c-0.1.9-pr-74.patch | 1159 ----------
thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch | 2441 --------------------
thirdparty/patches/lance-c-0.1.9-pr-77.patch | 1863 ---------------
thirdparty/patches/lance-c-0.1.9-pr-79.patch | 1782 --------------
thirdparty/patches/lance-c-0.1.9-pr-80.patch | 225 --
thirdparty/patches/lance-c-0.1.9-pr-83.patch | 1914 ---------------
.../patches/lance-c-0.1.9-prefilter-fts.patch | 148 --
thirdparty/patches/lance-c-0.1.9-prefilter.patch | 147 --
...nce-c-0.1.9-pr-73.patch => lance-c-foyer.patch} | 884 ++++---
thirdparty/test/lance-install-test.sh | 176 ++
thirdparty/test/lance-prefilter-patch-test.sh | 124 +
thirdparty/test/paimon-build-test.py | 279 +++
thirdparty/vars.sh | 16 +-
tools/legacy_date_migration/README.md | 117 +
.../legacy_date_migration/migrate_legacy_dates.py | 162 ++
.../test_migrate_legacy_dates.py | 179 ++
771 files changed, 45602 insertions(+), 16591 deletions(-)
create mode 100644 be/src/format_v2/table/paimon_rust_predicate_converter.cpp
create mode 100644 be/src/format_v2/table/paimon_rust_predicate_converter.h
create mode 100644 be/src/format_v2/table/paimon_rust_table_reader.cpp
create mode 100644 be/src/format_v2/table/paimon_rust_table_reader.h
create mode 100644 be/test/core/data_type_serde/data_type_datev2_serde_calendar_test.cpp
create mode 100644 be/test/exec/test_data/orc_scanner/date_count_truncated_stats.orc
copy be/test/exec/test_data/orc_scanner/{my-file.orc => date_large_ordinal.orc} (100%)
create mode 100644 be/test/exec/test_data/orc_scanner/date_metadata_fixtures.md
create mode 100644 be/test/exec/test_data/orc_scanner/date_out_of_range.orc
create mode 100644 be/test/exec/test_data/orc_scanner/date_preflight.orc
create mode 100644 be/test/exec/test_data/orc_scanner/date_preflight_bad_index.orc
copy {samples/arrow-flight-sql/python => be/test/exec/test_data/paimon_rust_batch_size}/README.md (50%)
create mode 100644 be/test/exec/test_data/paimon_rust_batch_size/bucket-0/data-3b6dc957-0664-4ec4-946c-9535d228957a-0.parquet
create mode 100644 be/test/exec/test_data/paimon_rust_batch_size/schema/schema-0
create mode 100644 be/test/exec/test_data/paimon_rust_batch_size/split.bin
create mode 100644 be/test/exprs/aggregate/agg_topn_test.cpp
create mode 100644 be/test/exprs/function/function_array_cum_sum_test.cpp
create mode 100644 be/test/exprs/function/function_array_split_test.cpp
create mode 100644 be/test/exprs/function/function_array_zip_test.cpp
create mode 100644 be/test/format/arrow/arrow_flight_timestamp_test.cpp
create mode 100644 be/test/format/arrow/arrow_row_batch_test.cpp
create mode 100644 be/test/format_v2/table/paimon_rust_predicate_converter_test.cpp
create mode 100644 be/test/format_v2/table/paimon_rust_table_reader_test.cpp
create mode 100644 be/test/service/arrow_flight/arrow_flight_cancel_test.cpp
create mode 100644 be/test/testutil/scoped_temp_dir.h
create mode 100644 cloud/test/recycler_benchmark_test.cpp
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/lance_build_array_predicates.py
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/lance_build_time_travel.py
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_indices/026150a9-7d9e-4859-a412-fa0582e40071/bitmap_page_lookup.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_indices/79b28d78-5490-40ba-a969-399822c12126/page_data.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_indices/79b28d78-5490-40ba-a969-399822c12126/page_lookup.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_transactions/0-a4640b8d-55a7-47f8-9392-5ec3d0ed19c9.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_transactions/1-d21db0fc-7e46-4699-9948-ab6f8d66098c.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_transactions/2-8498550d-87e2-42c8-931c-1ed0d8bc7371.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/18446744073709551612.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/18446744073709551613.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/18446744073709551614.manifest
copy {be/test/format_v2/table/lance/data/all_types.lance => docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance}/_versions/latest_version_hint.json (100%)
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/data/1001010000010111110100106a0464445da0b5a4b7c3465eb5.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/data/111000011101111100111010027ce24037932857b1f2b85e24.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_indices/9b6b2f0a-bd44-450f-9a3e-ab3f4ace7990/bitmap_page_lookup.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_indices/db755621-13aa-4ca9-a495-c0745e34f7c4/page_data.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_indices/db755621-13aa-4ca9-a495-c0745e34f7c4/page_lookup.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/0-58691d36-bff2-4a01-ad25-ae3642019067.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/1-c343d785-33b7-4448-b0da-160c970f78a1.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/2-bcdae8c1-73a4-4b25-ad8b-887bf393af0f.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/3-8a4b0277-3a05-4eca-8787-2f5ad24763fb.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551611.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551612.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551613.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551614.manifest
copy docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/{multi_frag.lance => predicate_arrays/partial.lance}/_versions/latest_version_hint.json (100%)
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/data/011010010011101101000101d214d7401ba969a49532b0ae8b.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/data/11001100100100001001011139a9b14af698a84d7cb598d6a7.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/_transactions/0-7c18f76d-3524-4079-835f-7404de102fa9.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/_versions/18446744073709551614.manifest
copy {be/test/format_v2/table/lance/data/multivector_dimensions.lance => docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance}/_versions/latest_version_hint.json (100%)
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/data/0011111110010010101001015f6f6345d896c3324132a6d562.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/data/1001011111100100010111008b02e34ab199273cee3ff3688e.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/branches/dev.json
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/rel.json
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/v1.json
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/v2.json
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/v3.json
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_transactions/0-c7a3192f-4b4a-4e8b-b3a3-b736534c10f3.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_transactions/1-79502d42-ca15-43d2-99e3-8c734698ae2b.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_transactions/2-e678f5d6-e017-494d-adaf-fd1c0e245182.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/18446744073709551612.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/18446744073709551613.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/18446744073709551614.manifest
copy {be/test/format_v2/table/lance/data/all_types.lance => docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance}/_versions/latest_version_hint.json (100%)
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/data/01001111111001001000111077951d4cb09c0a747e1eddc25e.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/data/111001110010000110100100bcc907454fa3151c2176cb1bc4.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/data/1111000011100100111000010ab68a492fa5d2c0fa972154e5.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_transactions/2-ca6009ab-1db7-4777-9e3e-a9374ed947b9.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_transactions/2-d76280f6-7616-446b-9575-5b8ec5ea2df1.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_versions/18446744073709551612.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_versions/18446744073709551613.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/data/0010100010110110000010015e5eaf41b5892dff0697f4cbcd.lance
create mode 100644 docs/lance-ann-profile.md
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceManifestPaths.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceRefSelector.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQueryCancellation.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java
create mode 100644 fe/fe-core/src/main/resources/substrait/lance_functions.yaml
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfileCompletionStateTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/doris/FeServiceClientTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManagedAccessTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManagedVersioningTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManifestPathsTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlannerTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/DorisToPaimonTypeVisitorTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustBackendCapabilityTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderEvolutionTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderLegacySchemaTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/httpv2/controller/BaseControllerBasicAuthTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/SystemRootUserTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/SchemaOnlyCommandMetadataTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/planner/ResultSinkMetadataTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/NereidsCoordinatorFlightEndpointsTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilderTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlSchemaTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessorSchemaTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlExecutionSchemaTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlQueryCancellationTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlTableTypesTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java
copy regression-test/data/{cloud_p0/cache/ttl/st04_alter_ttl_n_to_0_runtime.out => account_p0/test_grant_col_priv.out} (68%)
create mode 100644 regression-test/data/arrow_flight_sql_p0/test_date_year_zero.out
delete mode 100644 regression-test/data/cloud_p0/cache/ttl/test_show_ttl.out
create mode 100644 regression-test/data/export_p0/outfile/test_outfile_date_year_zero.out
create mode 100644 regression-test/data/external_table_p0/lance/test_lance_rest_time_travel.out
create mode 100644 regression-test/data/external_table_p0/lance/test_lance_time_travel.out
create mode 100644 regression-test/data/nereids_function_p0/agg_function/topn/topn_unlimited.out
create mode 100644 regression-test/data/query_p0/sql_functions/aggregate_functions/test_linear_histogram_boolean.out
create mode 100644 regression-test/data/query_p0/sql_functions/array_functions/test_array_cum_sum_leading_null.out
create mode 100644 regression-test/data/query_p0/sql_functions/array_functions/test_array_range_null_argument.out
create mode 100644 regression-test/data/query_p0/sql_functions/string_functions/test_regexp_extended_nonconstant_pattern.out
create mode 100644 regression-test/data/query_p0/sql_functions/table_function/test_json_text_embedded_nul.out
create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MultiClusterStateGuard.groovy
create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MySqlClient.groovy
create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/util/RowsetMetaUtils.groovy
create mode 100644 regression-test/framework/src/test/groovy/org/apache/doris/regression/ConfigArrowFlightSqlUrlTest.groovy
create mode 100644 regression-test/framework/src/test/groovy/org/apache/doris/regression/ConfigRootPasswordTest.groovy
create mode 100644 regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/CompactionPluginCompatibilityTest.groovy
copy regression-test/{suites/show_p0/test_show_data.groovy => framework/src/test/groovy/org/apache/doris/regression/suite/SuiteSshExecTest.groovy} (61%)
create mode 100644 regression-test/framework/src/test/groovy/org/apache/doris/regression/util/HttpAuthenticationTest.groovy
create mode 100644 regression-test/framework/src/test/groovy/org/apache/doris/regression/util/MultiClusterStateGuardTest.groovy
create mode 100644 regression-test/framework/src/test/groovy/org/apache/doris/regression/util/MySqlClientTest.groovy
create mode 100644 regression-test/framework/src/test/groovy/org/apache/doris/regression/util/RowsetMetaUtilsTest.groovy
create mode 100644 regression-test/plugins/cloud_multi_cluster_state.groovy
create mode 100644 regression-test/suites/account_p0/test_grant_col_priv.groovy
create mode 100644 regression-test/suites/account_p0/test_grant_revoke_object_name.groovy
create mode 100644 regression-test/suites/arrow_flight_sql_p0/test_date_year_zero.groovy
create mode 100644 regression-test/suites/arrow_flight_sql_p0/test_flight_cancel_cleanup.groovy
create mode 100644 regression-test/suites/arrow_flight_sql_p0/test_flight_int96_timestamp_range.groovy
create mode 100644 regression-test/suites/arrow_flight_sql_p0/test_flight_parallel_partitions.groovy
create mode 100644 regression-test/suites/arrow_flight_sql_p0/test_flight_timestamp_range.groovy
create mode 100644 regression-test/suites/arrow_flight_sql_p0/test_flight_utf8_validation.groovy
create mode 100644 regression-test/suites/arrow_flight_sql_p0/test_schema_only.groovy
create mode 100644 regression-test/suites/arrow_flight_sql_p0/test_table_types.groovy
create mode 100644 regression-test/suites/auth_p0/test_http_rest_v1_auth.groovy
delete mode 100644 regression-test/suites/cloud_p0/cache/ttl/alter_ttl_seconds.groovy
create mode 100644 regression-test/suites/cloud_p0/cache/ttl/test_show_ttl.groovy
delete mode 100644 regression-test/suites/cloud_p0/cache/ttl/test_show_ttl.sql
create mode 100644 regression-test/suites/export_p0/outfile/test_outfile_date_year_zero.groovy
create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_delete_unpartitioned_evolution.groovy
create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_date_year_zero.groovy
create mode 100644 regression-test/suites/external_table_p0/lance/test_lance_array_predicate_pushdown.groovy
create mode 100644 regression-test/suites/external_table_p0/lance/test_lance_rest_time_travel.groovy
create mode 100644 regression-test/suites/external_table_p0/lance/test_lance_time_travel.groovy
create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_cast_predicates.groovy
create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_compatibility.groovy
create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_eq_for_null.groovy
create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_schema_evolution.groovy
create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_v2.groovy
create mode 100644 regression-test/suites/nereids_function_p0/agg_function/topn/topn_unlimited.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/aggregate_functions/test_linear_histogram_boolean.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/array_functions/test_array_cum_sum_leading_null.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/array_functions/test_array_range_null_argument.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/string_functions/test_regexp_extended_nonconstant_pattern.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/table_function/test_json_text_embedded_nul.groovy
create mode 100644 samples/arrow-flight-sql/python/test_nested_type_metadata.py
create mode 100644 thirdparty/lance-install.sh
delete mode 100644 thirdparty/patches/lance-c-0.1.9-pr-74.patch
delete mode 100644 thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch
delete mode 100644 thirdparty/patches/lance-c-0.1.9-pr-77.patch
delete mode 100644 thirdparty/patches/lance-c-0.1.9-pr-79.patch
delete mode 100644 thirdparty/patches/lance-c-0.1.9-pr-80.patch
delete mode 100644 thirdparty/patches/lance-c-0.1.9-pr-83.patch
delete mode 100644 thirdparty/patches/lance-c-0.1.9-prefilter-fts.patch
delete mode 100644 thirdparty/patches/lance-c-0.1.9-prefilter.patch
rename thirdparty/patches/{lance-c-0.1.9-pr-73.patch => lance-c-foyer.patch} (76%)
create mode 100644 thirdparty/test/lance-install-test.sh
create mode 100755 thirdparty/test/lance-prefilter-patch-test.sh
create mode 100644 thirdparty/test/paimon-build-test.py
create mode 100644 tools/legacy_date_migration/README.md
create mode 100644 tools/legacy_date_migration/migrate_legacy_dates.py
create mode 100644 tools/legacy_date_migration/test_migrate_legacy_dates.py
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 18/35: branch-4.1: [fix](scan) Fix lost and duplicated rows when splitting CSV/JSON on multi-character line delimiters #68539 (#68600)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a7daed69925b4ac63b1aac8f506e4ef736f90f14
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Sep 29 23:39:51 2026 +0800
branch-4.1: [fix](scan) Fix lost and duplicated rows when splitting CSV/JSON on multi-character line delimiters #68539 (#68600)
Cherry-picked from #68539
Co-authored-by: hui lai <la...@selectdb.com>
---
be/src/format/csv/csv_reader.cpp | 8 +
.../file_reader/new_plain_text_line_reader.cpp | 77 +++++++++
.../file_reader/new_plain_text_line_reader.h | 6 +
be/src/format/json/new_json_reader.cpp | 15 +-
be/src/format/json/new_json_reader.h | 5 +-
be/src/format_v2/delimited_text/csv_reader.cpp | 2 +
.../delimited_text/delimited_text_reader.cpp | 14 ++
.../delimited_text/delimited_text_reader.h | 2 +
be/src/format_v2/json/json_reader.cpp | 15 +-
be/src/format_v2/json/json_reader.h | 4 +-
.../new_plain_text_line_reader_test.cpp | 133 ++++++++++++++++
.../format_v2/delimited_text/csv_reader_test.cpp | 160 +++++++++++++++++++
be/test/format_v2/json/json_reader_test.cpp | 175 +++++++++++++++++++++
13 files changed, 598 insertions(+), 18 deletions(-)
diff --git a/be/src/format/csv/csv_reader.cpp b/be/src/format/csv/csv_reader.cpp
index 6518ab42459..250307b3348 100644
--- a/be/src/format/csv/csv_reader.cpp
+++ b/be/src/format/csv/csv_reader.cpp
@@ -345,6 +345,14 @@ Status CsvReader::get_next_block(Block* block, size_t* read_rows, bool* eof) {
bool success = false;
bool is_remove_bom = false;
+ if (_range.start_offset != 0 && _skip_lines > 0 && _enclose == 0 &&
+ _file_format_type == TFileFormatType::FORMAT_CSV_PLAIN) {
+ auto* text_reader = assert_cast<NewPlainTextLineReader*>(_line_reader.get());
+ RETURN_IF_ERROR(text_reader->skip_split_prefix(_range.start_offset, _line_delimiter,
+ &_line_reader_eof, _io_ctx));
+ _skip_lines = 0;
+ is_remove_bom = true;
+ }
if (_push_down_agg_type == TPushAggOp::type::COUNT) {
while (rows < batch_size && !_line_reader_eof) {
const uint8_t* ptr = nullptr;
diff --git a/be/src/format/file_reader/new_plain_text_line_reader.cpp b/be/src/format/file_reader/new_plain_text_line_reader.cpp
index 25a7a0a7ac2..cb18ad5f21b 100644
--- a/be/src/format/file_reader/new_plain_text_line_reader.cpp
+++ b/be/src/format/file_reader/new_plain_text_line_reader.cpp
@@ -25,6 +25,7 @@
#include <immintrin.h>
#endif
#include <algorithm>
+#include <array>
#include <cstddef>
#include <cstring>
#include <ostream>
@@ -287,6 +288,82 @@ inline bool NewPlainTextLineReader::update_eof() {
return _eof;
}
+Status NewPlainTextLineReader::skip_split_prefix(size_t split_start, const std::string& delimiter,
+ bool* eof, const io::IOContext* io_ctx,
+ size_t* skipped_lines) {
+ DCHECK_EQ(_total_read_bytes, 0);
+ DCHECK_EQ(_output_buf_limit, 0);
+ bool overlaps = false;
+ if (delimiter.size() > 1) {
+ // Compute the KMP prefix function once for this split in linear time. A nonempty
+ // proper prefix that is also a suffix of the whole delimiter permits overlapping matches.
+ std::vector<size_t> prefix_lengths(delimiter.size());
+ for (size_t i = 1, matched = 0; i < delimiter.size(); ++i) {
+ while (matched > 0 && delimiter[i] != delimiter[matched]) {
+ matched = prefix_lengths[matched - 1];
+ }
+ if (delimiter[i] == delimiter[matched]) {
+ ++matched;
+ }
+ prefix_lengths[i] = matched;
+ }
+ overlaps = prefix_lengths.back() > 0;
+ }
+
+ if (overlaps && _decompressor == nullptr) {
+ // A fixed lookbehind can start in an overlapping delimiter chain (e.g. three newlines
+ // with a two-newline delimiter). Find a byte that cannot belong to any delimiter, then
+ // replay greedy matches from immediately after it. Keep scratch bounded even for long
+ // delimiter runs; ordinary delimiters do not need this extra I/O.
+ std::array<bool, 256> delimiter_bytes {};
+ for (unsigned char byte : delimiter) {
+ delimiter_bytes[byte] = true;
+ }
+ constexpr size_t max_lookbehind_size = 64 * 1024;
+ std::vector<char> buffer(1024);
+ size_t sync_offset = _current_offset;
+ bool synchronized = false;
+ while (sync_offset > 0 && !synchronized) {
+ const size_t length = std::min(sync_offset, buffer.size());
+ const size_t offset = sync_offset - length;
+ size_t bytes_read = 0;
+ RETURN_IF_ERROR(_file_reader->read_at(offset, Slice(buffer.data(), length), &bytes_read,
+ io_ctx));
+ if (bytes_read != length) {
+ return Status::IOError("Short read while aligning text split at offset {}",
+ split_start);
+ }
+ sync_offset = offset;
+ for (size_t i = length; i > 0; --i) {
+ if (!delimiter_bytes[static_cast<unsigned char>(buffer[i - 1])]) {
+ sync_offset = offset + i;
+ synchronized = true;
+ break;
+ }
+ }
+ if (!synchronized && sync_offset > 0) {
+ // Extend backward into new bytes; do not reread the already searched suffix.
+ buffer.resize(std::min(buffer.size() * 2, max_lookbehind_size));
+ }
+ }
+ _min_length += _current_offset - sync_offset;
+ _current_offset = sync_offset;
+ }
+
+ const size_t prefix_length = split_start - _current_offset;
+ const uint8_t* line = nullptr;
+ size_t size = 0;
+ size_t skipped = 0;
+ do {
+ RETURN_IF_ERROR(read_line(&line, &size, eof, io_ctx));
+ skipped += !*eof;
+ } while (!*eof && _total_read_bytes < prefix_length);
+ if (skipped_lines != nullptr) {
+ *skipped_lines = skipped;
+ }
+ return Status::OK();
+}
+
// extend input buf if necessary only when _more_input_bytes > 0
void NewPlainTextLineReader::extend_input_buf() {
DCHECK(_more_input_bytes > 0);
diff --git a/be/src/format/file_reader/new_plain_text_line_reader.h b/be/src/format/file_reader/new_plain_text_line_reader.h
index ed7f80493b0..72c090556b6 100644
--- a/be/src/format/file_reader/new_plain_text_line_reader.h
+++ b/be/src/format/file_reader/new_plain_text_line_reader.h
@@ -248,6 +248,12 @@ public:
Status read_line(const uint8_t** ptr, size_t* size, bool* eof,
const io::IOContext* io_ctx) override;
+ // Called before the first read of a non-first plain-text split. Discard records owned by the
+ // preceding split, preserving greedy delimiter matching. Not suitable for enclosed CSV:
+ // finding a delimiter synchronization point does not recover quote/escape state.
+ Status skip_split_prefix(size_t split_start, const std::string& delimiter, bool* eof,
+ const io::IOContext* io_ctx, size_t* skipped_lines = nullptr);
+
inline TextLineReaderCtxPtr text_line_reader_ctx() { return _line_reader_ctx; }
void close() override;
diff --git a/be/src/format/json/new_json_reader.cpp b/be/src/format/json/new_json_reader.cpp
index 31cb50f4096..4dec574df88 100644
--- a/be/src/format/json/new_json_reader.cpp
+++ b/be/src/format/json/new_json_reader.cpp
@@ -152,6 +152,8 @@ NewJsonReader::NewJsonReader(RuntimeProfile* profile, const TFileScanRangeParams
_init_file_description();
}
+NewJsonReader::~NewJsonReader() = default;
+
void NewJsonReader::_init_system_properties() {
if (_range.__isset.file_type) {
// for compatibility
@@ -211,9 +213,8 @@ Status NewJsonReader::get_next_block(Block* block, size_t* read_rows, bool* eof)
while (block->rows() < batch_size && !_reader_eof && (block->bytes() < max_block_bytes)) {
if (UNLIKELY(_read_json_by_line && _skip_first_line)) {
- size_t size = 0;
- const uint8_t* line_ptr = nullptr;
- RETURN_IF_ERROR(_line_reader->read_line(&line_ptr, &size, &_reader_eof, _io_ctx));
+ RETURN_IF_ERROR(_line_reader->skip_split_prefix(_range.start_offset, _line_delimiter,
+ &_reader_eof, _io_ctx));
_skip_first_line = false;
continue;
}
@@ -445,7 +446,9 @@ void json_reader_detail::pop_back_last_inserted_value(Block& block, size_t colum
Status NewJsonReader::_open_file_reader(bool need_schema) {
int64_t start_offset = _range.start_offset;
if (start_offset != 0) {
- start_offset -= 1;
+ // Include the whole delimiter when the split starts inside it, so skipping the first
+ // partial line cannot discard the next complete JSON record.
+ start_offset -= std::min<int64_t>(start_offset, _line_delimiter_length);
}
_current_offset = start_offset;
@@ -482,8 +485,8 @@ Status NewJsonReader::_open_file_reader(bool need_schema) {
Status NewJsonReader::_open_line_reader() {
int64_t size = _range.size;
if (_range.start_offset != 0) {
- // When we fetch range doesn't start from 0, size will += 1.
- size += 1;
+ // Preserve the original range end after moving the start backwards.
+ size += _range.start_offset - _current_offset;
_skip_first_line = true;
} else {
_skip_first_line = false;
diff --git a/be/src/format/json/new_json_reader.h b/be/src/format/json/new_json_reader.h
index 15ac8f14c41..1179591ad44 100644
--- a/be/src/format/json/new_json_reader.h
+++ b/be/src/format/json/new_json_reader.h
@@ -62,6 +62,7 @@ struct IOContext;
struct ScannerCounter;
class Block;
class IColumn;
+class NewPlainTextLineReader;
namespace json_reader_detail {
Status append_null_for_malformed_json(Block& block);
@@ -83,7 +84,7 @@ public:
const TFileRangeDesc& range, const std::vector<SlotDescriptor*>& file_slot_descs,
size_t batch_size, io::IOContext* io_ctx,
std::shared_ptr<io::IOContext> io_ctx_holder = nullptr);
- ~NewJsonReader() override = default;
+ ~NewJsonReader() override;
Status init_reader(
const std::unordered_map<std::string, VExprContextSPtr>& col_default_value_ctx,
@@ -200,7 +201,7 @@ private:
const std::vector<SlotDescriptor*>& _file_slot_descs;
io::FileReaderSPtr _file_reader;
- std::unique_ptr<LineReader> _line_reader;
+ std::unique_ptr<NewPlainTextLineReader> _line_reader;
bool _reader_eof;
std::unique_ptr<Decompressor> _decompressor;
TFileCompressType::type _file_compress_type;
diff --git a/be/src/format_v2/delimited_text/csv_reader.cpp b/be/src/format_v2/delimited_text/csv_reader.cpp
index bb6c55d3084..e5eca54df6f 100644
--- a/be/src/format_v2/delimited_text/csv_reader.cpp
+++ b/be/src/format_v2/delimited_text/csv_reader.cpp
@@ -128,11 +128,13 @@ Status CsvReader::_create_decompressor() {
}
Status CsvReader::_create_line_reader() {
+ _align_split_prefix = false;
if (is_csv_text_format(_file_format_type)) {
std::shared_ptr<TextLineReaderContextIf> text_line_reader_ctx;
if (_enclose == 0) {
text_line_reader_ctx = std::make_shared<PlainTextLineReaderCtx>(
_line_delimiter, _line_delimiter.size(), _keep_cr);
+ _align_split_prefix = _file_description->range_start_offset != 0;
} else {
const size_t col_sep_num =
_source_file_slot_descs.size() > 1 ? _source_file_slot_descs.size() - 1 : 0;
diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.cpp b/be/src/format_v2/delimited_text/delimited_text_reader.cpp
index 63486d174ef..fa185600cc7 100644
--- a/be/src/format_v2/delimited_text/delimited_text_reader.cpp
+++ b/be/src/format_v2/delimited_text/delimited_text_reader.cpp
@@ -32,6 +32,7 @@
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_string.h"
#include "core/data_type/data_type_struct.h"
+#include "format/file_reader/new_plain_text_line_reader.h"
#include "format/line_reader.h"
#include "format_v2/column_mapper.h"
#include "format_v2/materialized_reader_util.h"
@@ -555,6 +556,19 @@ Status DelimitedTextReader::_open_file() {
Status DelimitedTextReader::_read_next_line(Slice* line, bool* eof) {
DORIS_CHECK(line != nullptr);
DORIS_CHECK(eof != nullptr);
+ if (_align_split_prefix) {
+ SCOPED_TIMER(_text_profile.read_line_time);
+ DCHECK_EQ(_skip_lines, 1);
+ size_t skipped_lines = 0;
+ auto* text_reader = assert_cast<NewPlainTextLineReader*>(_line_reader.get());
+ RETURN_IF_ERROR(text_reader->skip_split_prefix(_file_description->range_start_offset,
+ _line_delimiter, &_line_reader_eof,
+ _io_ctx.get(), &skipped_lines));
+ _align_split_prefix = false;
+ _skip_lines = 0;
+ _bom_removed = true;
+ update_counter(_text_profile.skipped_lines, skipped_lines);
+ }
while (true) {
const uint8_t* ptr = nullptr;
size_t size = 0;
diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.h b/be/src/format_v2/delimited_text/delimited_text_reader.h
index dff27980c9c..472f21f59c9 100644
--- a/be/src/format_v2/delimited_text/delimited_text_reader.h
+++ b/be/src/format_v2/delimited_text/delimited_text_reader.h
@@ -156,6 +156,8 @@ protected:
int64_t _start_offset = 0;
int64_t _size = -1;
int _skip_lines = 0;
+ // Enabled only by readers using plain, quote-independent delimiter matching.
+ bool _align_split_prefix = false;
char _escape = 0;
bool _line_reader_eof = false;
bool _bom_removed = false;
diff --git a/be/src/format_v2/json/json_reader.cpp b/be/src/format_v2/json/json_reader.cpp
index caf4205fa61..78601f90584 100644
--- a/be/src/format_v2/json/json_reader.cpp
+++ b/be/src/format_v2/json/json_reader.cpp
@@ -314,10 +314,8 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) {
while (file_block->rows() < batch_size && !_reader_eof &&
file_block->bytes() < max_block_bytes) {
if (_read_json_by_line && _skip_first_line) {
- size_t skipped_size = 0;
- const uint8_t* skipped_line = nullptr;
- RETURN_IF_ERROR(_line_reader->read_line(&skipped_line, &skipped_size, &_reader_eof,
- _io_ctx.get()));
+ RETURN_IF_ERROR(_line_reader->skip_split_prefix(
+ _reader_range.start_offset, _line_delimiter, &_reader_eof, _io_ctx.get()));
_skip_first_line = false;
continue;
}
@@ -445,7 +443,9 @@ TFileRangeDesc JsonReader::_json_range() const {
Status JsonReader::_open_file_reader() {
_current_offset = _reader_range.start_offset;
if (_current_offset != 0) {
- --_current_offset;
+ // Include the whole delimiter when the split starts inside it, so skipping the first
+ // partial line cannot discard the next complete JSON record.
+ _current_offset -= std::min<int64_t>(_current_offset, _line_delimiter_length);
}
if (_scan_params->file_type == TFileType::FILE_STREAM) {
if (!_stream_load_id.has_value()) {
@@ -478,9 +478,8 @@ Status JsonReader::_create_decompressor() {
Status JsonReader::_create_line_reader() {
int64_t size = _reader_range.size;
if (_reader_range.start_offset != 0) {
- // Start one byte earlier and discard the first partial line, matching split semantics used
- // by text readers.
- ++size;
+ // Preserve the original range end after moving the start backwards.
+ size += _reader_range.start_offset - _current_offset;
_skip_first_line = true;
} else {
_skip_first_line = false;
diff --git a/be/src/format_v2/json/json_reader.h b/be/src/format_v2/json/json_reader.h
index c7346cb1d66..f5e6614b2da 100644
--- a/be/src/format_v2/json/json_reader.h
+++ b/be/src/format_v2/json/json_reader.h
@@ -35,7 +35,7 @@
namespace doris {
class Decompressor;
-class LineReader;
+class NewPlainTextLineReader;
class SlotDescriptor;
class IColumn;
} // namespace doris
@@ -188,7 +188,7 @@ private:
io::FileReaderSPtr _physical_file_reader;
std::unique_ptr<Decompressor> _decompressor;
- std::unique_ptr<LineReader> _line_reader;
+ std::unique_ptr<NewPlainTextLineReader> _line_reader;
int64_t _current_offset = 0;
bool _reader_eof = false;
bool _skip_first_line = false;
diff --git a/be/test/format/file_reader/new_plain_text_line_reader_test.cpp b/be/test/format/file_reader/new_plain_text_line_reader_test.cpp
index 93d02067863..3d8ddd57f0c 100644
--- a/be/test/format/file_reader/new_plain_text_line_reader_test.cpp
+++ b/be/test/format/file_reader/new_plain_text_line_reader_test.cpp
@@ -21,8 +21,141 @@
#include <gtest/gtest.h>
+#include <algorithm>
+#include <cstring>
+#include <utility>
+
+#include "io/fs/file_reader.h"
+
namespace doris {
+namespace {
+class RecordingSplitFileReader : public io::FileReader {
+public:
+ explicit RecordingSplitFileReader(std::string content) : _content(std::move(content)) {}
+ Status close() override {
+ _closed = true;
+ return Status::OK();
+ }
+ const io::Path& path() const override { return _path; }
+ size_t size() const override { return _content.size(); }
+ bool closed() const override { return _closed; }
+ int64_t mtime() const override { return 0; }
+
+ std::vector<std::pair<size_t, size_t>> requests;
+
+protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const io::IOContext*) override {
+ requests.emplace_back(offset, result.size);
+ *bytes_read = std::min(result.size, _content.size() - std::min(offset, _content.size()));
+ if (*bytes_read > 0) {
+ std::memcpy(result.mutable_data(), _content.data() + offset, *bytes_read);
+ }
+ return Status::OK();
+ }
+
+private:
+ io::Path _path {"split-prefix-test"};
+ std::string _content;
+ bool _closed = false;
+};
+} // namespace
+
+TEST(PlainTextSplitPrefixTest, DelimiterOverlapControlsBackwardProbes) {
+ const std::vector<std::pair<std::string, bool>> delimiters = {
+ {"\n", false},
+ {"abc", false},
+ {"aaaaab", false},
+ {"||", true},
+ {"aba", true},
+ {"abcab", true},
+ {"ababcabab", true},
+ {std::string(100 * 1024 - 1, 'a') + "b", false},
+ {std::string(100 * 1024, 'a'), true}};
+ for (const auto& [delimiter, overlaps] : delimiters) {
+ SCOPED_TRACE(::testing::Message() << "delimiter length=" << delimiter.size()
+ << ", prefix=" << delimiter.substr(0, 16));
+ const std::string first = "xxx";
+ const std::string content = first + delimiter + "second" + delimiter + "third";
+ const size_t split = first.size() + delimiter.size();
+ auto file = std::make_shared<RecordingSplitFileReader>(content);
+ RuntimeProfile profile("split_prefix");
+ NewPlainTextLineReader reader(
+ &profile, file, nullptr,
+ std::make_shared<PlainTextLineReaderCtx>(delimiter, delimiter.size(), false),
+ content.size() - first.size(), first.size());
+ bool eof = false;
+ size_t skipped_lines = 0;
+ ASSERT_TRUE(reader.skip_split_prefix(split, delimiter, &eof, nullptr, &skipped_lines).ok());
+ ASSERT_FALSE(eof);
+ EXPECT_EQ(skipped_lines, 1);
+ ASSERT_FALSE(file->requests.empty());
+ // Only a self-overlapping delimiter requires a probe before the initial read offset.
+ EXPECT_EQ(file->requests.front().first < first.size(), overlaps);
+ const uint8_t* line = nullptr;
+ size_t size = 0;
+ ASSERT_TRUE(reader.read_line(&line, &size, &eof, nullptr).ok());
+ EXPECT_EQ(std::string(reinterpret_cast<const char*>(line), size), "second");
+ }
+}
+
+TEST(PlainTextSplitPrefixTest, NearbySynchronizationUsesSmallProbe) {
+ const std::string content = std::string(8192, 'x') + "a|||b||c";
+ const size_t split = 8192 + 4;
+ auto file = std::make_shared<RecordingSplitFileReader>(content);
+ RuntimeProfile profile("split_prefix");
+ NewPlainTextLineReader reader(&profile, file, nullptr,
+ std::make_shared<PlainTextLineReaderCtx>("||", 2, false),
+ content.size() - split + 2, split - 2);
+ bool eof = false;
+ size_t skipped_lines = 0;
+ ASSERT_TRUE(reader.skip_split_prefix(split, "||", &eof, nullptr, &skipped_lines).ok());
+ ASSERT_FALSE(eof);
+ ASSERT_GE(file->requests.size(), 2);
+ EXPECT_EQ(file->requests.front().second, 1024);
+ // The next read replays forward from the synchronization point; no larger probe was needed.
+ EXPECT_GT(file->requests[1].first, file->requests[0].first);
+ EXPECT_EQ(skipped_lines, 2);
+ const uint8_t* line = nullptr;
+ size_t size = 0;
+ ASSERT_TRUE(reader.read_line(&line, &size, &eof, nullptr).ok());
+ EXPECT_EQ(std::string(reinterpret_cast<const char*>(line), size), "c");
+}
+
+TEST(PlainTextSplitPrefixTest, LongRunGrowsProbesWithoutRereading) {
+ const std::string prefix = "a" + std::string(256 * 1024 + 1, '|');
+ const std::string content = prefix + "b||c";
+ const size_t split = prefix.size();
+ auto file = std::make_shared<RecordingSplitFileReader>(content);
+ RuntimeProfile profile("split_prefix");
+ NewPlainTextLineReader reader(&profile, file, nullptr,
+ std::make_shared<PlainTextLineReaderCtx>("||", 2, false),
+ content.size() - split + 2, split - 2);
+ bool eof = false;
+ ASSERT_TRUE(reader.skip_split_prefix(split, "||", &eof, nullptr).ok());
+ ASSERT_FALSE(eof);
+ size_t previous_offset = split - 2;
+ size_t probe_size = 1024;
+ size_t probes = 0;
+ for (const auto& [offset, length] : file->requests) {
+ if (offset >= previous_offset) {
+ break; // Forward replay has begun.
+ }
+ EXPECT_EQ(offset + length, previous_offset);
+ EXPECT_EQ(length, std::min(previous_offset, probe_size));
+ previous_offset = offset;
+ probe_size = std::min(probe_size * 2, size_t {64 * 1024});
+ ++probes;
+ }
+ EXPECT_GT(probes, 6);
+ EXPECT_EQ(previous_offset, 0);
+ const uint8_t* line = nullptr;
+ size_t size = 0;
+ ASSERT_TRUE(reader.read_line(&line, &size, &eof, nullptr).ok());
+ EXPECT_EQ(std::string(reinterpret_cast<const char*>(line), size), "c");
+}
+
// Base test class for text line reader tests
class PlainTextLineReaderTest : public testing::Test {
protected:
diff --git a/be/test/format_v2/delimited_text/csv_reader_test.cpp b/be/test/format_v2/delimited_text/csv_reader_test.cpp
index c04e17e07f6..8c537b1e60e 100644
--- a/be/test/format_v2/delimited_text/csv_reader_test.cpp
+++ b/be/test/format_v2/delimited_text/csv_reader_test.cpp
@@ -39,13 +39,16 @@
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
#include "core/data_type/data_type_struct.h"
+#include "exec/scan/scanner.h"
#include "exprs/vexpr.h"
#include "exprs/vexpr_context.h"
+#include "format/csv/csv_reader.h"
#include "format_v2/column_mapper.h"
#include "io/io_common.h"
#include "runtime/runtime_profile.h"
#include "testutil/desc_tbl_builder.h"
#include "testutil/mock/mock_runtime_state.h"
+#include "testutil/scoped_temp_dir.h"
#include "util/debug_points.h"
#include "util/defer_op.h"
@@ -324,6 +327,163 @@ VExprContextSPtr prepared_conjunct(RuntimeState* state, const VExprSPtr& expr) {
return context;
}
+class PlainCsvSplitTest : public testing::TestWithParam<bool> {
+protected:
+ void read_range(const std::string& content, const std::string& delimiter, int64_t start,
+ int64_t size, bool count_only, std::vector<std::string>* values,
+ size_t* total_rows, int header_mode = 0) {
+ const auto path = (_dir.path() / "split.csv").string();
+ std::ofstream(path, std::ios::binary) << content;
+ auto params = csv_scan_params();
+ params.__set_compress_type(TFileCompressType::PLAIN);
+ params.__set_column_idxs({0});
+ params.file_attributes.__isset.header_type = false;
+ params.file_attributes.text_params.__set_line_delimiter(delimiter);
+ if (header_mode == 1) {
+ params.file_attributes.__set_header_type(BeConsts::CSV_WITH_NAMES);
+ } else if (header_mode == 2) {
+ params.file_attributes.__set_header_type(BeConsts::CSV_WITH_NAMES_AND_TYPES);
+ } else if (header_mode == 3) {
+ params.file_attributes.__set_skip_lines(2);
+ }
+ ObjectPool pool;
+ auto type = make_nullable(std::make_shared<DataTypeString>());
+ std::vector<SlotDescriptor*> slots {make_test_slot(&pool, 0, 0, type, "id")};
+ MockRuntimeState state;
+ state._batch_size = 2;
+ RuntimeProfile profile("plain_csv_split_test");
+ auto read_blocks = [&](auto&& next_block) {
+ bool eof = false;
+ while (!eof) {
+ Block block;
+ block.insert({type->create_column(), type, "id"});
+ size_t rows = 0;
+ auto status = next_block(&block, &rows, &eof);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_EQ(rows, block.rows());
+ *total_rows += rows;
+ if (!count_only) {
+ for (size_t row = 0; row < rows; ++row) {
+ ASSERT_FALSE(is_null_at(*block.get_by_position(0).column, row));
+ values->push_back(
+ nullable_string_at(*block.get_by_position(0).column, row));
+ }
+ }
+ }
+ };
+ if (GetParam()) {
+ auto reader = create_reader(path, ¶ms, slots, &state, &profile, start, size);
+ auto request = std::make_shared<FileScanRequest>();
+ request->local_positions.emplace(LocalColumnId(0), LocalIndex(0));
+ ASSERT_TRUE(reader->open(request).ok());
+ if (count_only) {
+ FileAggregateRequest aggregate_request;
+ aggregate_request.agg_type = TPushAggOp::type::COUNT;
+ FileAggregateResult result;
+ ASSERT_TRUE(reader->get_aggregate_result(aggregate_request, &result).ok());
+ *total_rows += result.count;
+ } else {
+ read_blocks([&](Block* block, size_t* rows, bool* eof) {
+ return reader->get_block(block, rows, eof);
+ });
+ }
+ } else {
+ TFileRangeDesc range;
+ range.__set_path(path);
+ range.__set_start_offset(start);
+ range.__set_size(size);
+ range.__set_file_size(content.size());
+ ScannerCounter counter;
+ auto reader = ::doris::CsvReader::create_unique(
+ &state, &profile, &counter, params, range, slots, state.batch_size(), nullptr);
+ ASSERT_TRUE(reader->init_reader(true).ok());
+ if (count_only) {
+ reader->set_push_down_agg_type(TPushAggOp::type::COUNT);
+ }
+ read_blocks([&](Block* block, size_t* rows, bool* eof) {
+ return reader->get_next_block(block, rows, eof);
+ });
+ EXPECT_EQ(counter.num_rows_filtered, 0);
+ }
+ }
+
+ doris::test::ScopedTempDirectory _dir {"doris_plain_csv_split_test"};
+};
+
+TEST_P(PlainCsvSplitTest, EveryByteBoundaryMatchesUnsplitRowsAndCount) {
+ for (const std::string delimiter : {"\n", "\r\n", "ABCDE", "||", "aba"}) {
+ const std::string extra = delimiter == "||" ? "|" : delimiter == "aba" ? "ba" : "";
+ for (bool trailing : {false, true}) {
+ const std::string content =
+ "1" + delimiter + extra + "2" + delimiter + "3" + (trailing ? delimiter : "");
+ const auto file_size = static_cast<int64_t>(content.size());
+ const std::vector<std::string> expected {"1", extra + "2", "3"};
+ for (bool count_only : {false, true}) {
+ SCOPED_TRACE(testing::Message() << "delimiter=" << delimiter << ", trailing="
+ << trailing << ", count=" << count_only);
+ std::vector<std::string> unsplit;
+ size_t unsplit_rows = 0;
+ ASSERT_NO_FATAL_FAILURE(read_range(content, delimiter, 0, file_size, count_only,
+ &unsplit, &unsplit_rows));
+ ASSERT_EQ(unsplit_rows, expected.size());
+ if (!count_only) {
+ ASSERT_EQ(unsplit, expected);
+ }
+ for (int64_t split = 1; split < file_size; ++split) {
+ SCOPED_TRACE(testing::Message() << "split=" << split);
+ std::vector<std::string> values;
+ size_t rows = 0;
+ ASSERT_NO_FATAL_FAILURE(
+ read_range(content, delimiter, 0, split, count_only, &values, &rows));
+ ASSERT_NO_FATAL_FAILURE(read_range(content, delimiter, split, file_size - split,
+ count_only, &values, &rows));
+ ASSERT_EQ(rows, expected.size());
+ if (!count_only) {
+ ASSERT_EQ(values, expected);
+ }
+ }
+ std::vector<std::string> values;
+ size_t rows = 0;
+ for (int64_t start = 0; start < file_size; ++start) {
+ ASSERT_NO_FATAL_FAILURE(
+ read_range(content, delimiter, start, 1, count_only, &values, &rows));
+ }
+ EXPECT_EQ(rows, expected.size());
+ if (!count_only) {
+ EXPECT_EQ(values, expected);
+ }
+ }
+ }
+ }
+}
+
+TEST_P(PlainCsvSplitTest, FirstSplitStillHonorsHeadersAndSkipLines) {
+ for (int header_mode : {1, 2, 3}) {
+ const std::string header = header_mode == 1 ? "id||" : "id||String||";
+ const std::string content = header + "1|||2||3";
+ const auto split = static_cast<int64_t>(header.size() + 4);
+ for (bool count_only : {false, true}) {
+ SCOPED_TRACE(testing::Message()
+ << "header_mode=" << header_mode << ", count=" << count_only);
+ std::vector<std::string> values;
+ size_t rows = 0;
+ ASSERT_NO_FATAL_FAILURE(
+ read_range(content, "||", 0, split, count_only, &values, &rows, header_mode));
+ ASSERT_NO_FATAL_FAILURE(read_range(content, "||", split, content.size() - split,
+ count_only, &values, &rows, header_mode));
+ EXPECT_EQ(rows, 3);
+ if (!count_only) {
+ EXPECT_EQ(values, (std::vector<std::string> {"1", "|2", "3"}));
+ }
+ }
+ }
+}
+
+INSTANTIATE_TEST_SUITE_P(LegacyAndV2, PlainCsvSplitTest, testing::Bool(),
+ [](const testing::TestParamInfo<bool>& info) {
+ return info.param ? "V2" : "Legacy";
+ });
+
class CsvV2ReaderTest : public testing::Test {
public:
void SetUp() override {
diff --git a/be/test/format_v2/json/json_reader_test.cpp b/be/test/format_v2/json/json_reader_test.cpp
index e8d9aec40af..1d94e244f64 100644
--- a/be/test/format_v2/json/json_reader_test.cpp
+++ b/be/test/format_v2/json/json_reader_test.cpp
@@ -38,8 +38,10 @@
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
#include "core/data_type/data_type_struct.h"
+#include "exec/scan/scanner.h"
#include "exprs/vexpr.h"
#include "exprs/vexpr_context.h"
+#include "format/json/new_json_reader.h"
#include "format_v2/column_data.h"
#include "io/io_common.h"
#include "runtime/descriptors.h"
@@ -301,6 +303,179 @@ VExprContextSPtr prepared_conjunct(RuntimeState* state, const VExprSPtr& expr) {
} // namespace
+// Exercise both readers with the same physical splits and compare the complete record sequence,
+// since a row-count assertion alone can hide one lost record and one duplicated record.
+class JsonReaderSplitTest : public testing::TestWithParam<bool> {
+protected:
+ void read_range(const std::filesystem::path& path, const std::string& delimiter, int64_t start,
+ int64_t size, std::vector<int32_t>* ids) {
+ auto params = json_scan_params();
+ params.file_attributes.text_params.__set_line_delimiter(delimiter);
+ auto range = file_range(path);
+ range.__set_start_offset(start);
+ range.__set_size(size);
+ ObjectPool pool;
+ auto type = make_nullable(std::make_shared<DataTypeInt32>());
+ std::vector<SlotDescriptor*> slots {make_test_slot(&pool, 0, 0, type, "id")};
+ RuntimeProfile profile("json_split_test");
+ MockRuntimeState state;
+ state._batch_size = 2;
+
+ auto read_blocks = [&](auto&& next_block) {
+ bool eof = false;
+ while (!eof) {
+ Block block;
+ block.insert({type->create_column(), type, "id"});
+ size_t rows = 0;
+ auto status = next_block(&block, &rows, &eof);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_EQ(rows, block.rows());
+ const auto& nullable =
+ assert_cast<const ColumnNullable&>(*block.get_by_position(0).column);
+ const auto& column = assert_cast<const ColumnInt32&>(nullable.get_nested_column());
+ for (size_t row = 0; row < rows; ++row) {
+ ASSERT_FALSE(nullable.is_null_at(row));
+ ids->push_back(column.get_element(row));
+ }
+ }
+ };
+
+ if (GetParam()) {
+ auto properties = std::make_shared<io::FileSystemProperties>();
+ properties->system_type = TFileType::FILE_LOCAL;
+ auto desc = file_description(path.string());
+ desc->range_start_offset = start;
+ desc->range_size = size;
+ JsonReader reader(properties, desc, nullptr, &profile, ¶ms, range, slots);
+ ASSERT_TRUE(reader.init(&state).ok());
+ auto request = std::make_shared<FileScanRequest>();
+ request->local_positions.emplace(LocalColumnId(0), LocalIndex(0));
+ ASSERT_TRUE(reader.open(request).ok());
+ read_blocks([&](Block* block, size_t* rows, bool* eof) {
+ return reader.get_block(block, rows, eof);
+ });
+ } else {
+ ScannerCounter counter;
+ bool scanner_eof = false;
+ auto reader =
+ NewJsonReader::create_unique(&state, &profile, &counter, params, range, slots,
+ &scanner_eof, state.batch_size(), nullptr);
+ ASSERT_TRUE(reader->init_reader({}, true).ok());
+ read_blocks([&](Block* block, size_t* rows, bool* eof) {
+ return reader->get_next_block(block, rows, eof);
+ });
+ EXPECT_EQ(counter.num_rows_filtered, 0);
+ }
+ }
+};
+
+TEST_P(JsonReaderSplitTest, EveryByteBoundaryPreservesRecords) {
+ // Includes single-byte delimiters, CRLF, UTF-8 bytes, and a delimiter longer than a record
+ // to cover starts smaller than the amount of lookbehind. Test EOF with and without a delimiter.
+ for (const std::string delimiter : {"\n", "\r\n", "ABCDE", "\xE2\x98\x83", "ABCDEFGHIJKLM"}) {
+ for (bool trailing_delimiter : {false, true}) {
+ SCOPED_TRACE(testing::Message()
+ << "delimiter=" << delimiter << ", trailing=" << trailing_delimiter);
+ std::string content =
+ R"({"id":1})" + delimiter + R"({"id":2})" + delimiter + R"({"id":3})";
+ if (trailing_delimiter) {
+ content += delimiter;
+ }
+ const auto path = write_json_file("split_boundaries.json", content);
+ const auto file_size = static_cast<int64_t>(content.size());
+ const std::vector<int32_t> expected {1, 2, 3};
+ std::vector<int32_t> unsplit;
+ ASSERT_NO_FATAL_FAILURE(read_range(path, delimiter, 0, file_size, &unsplit));
+ ASSERT_EQ(unsplit, expected);
+ for (int64_t split = 1; split < file_size; ++split) {
+ SCOPED_TRACE(testing::Message() << "split=" << split);
+ std::vector<int32_t> ids;
+ ASSERT_NO_FATAL_FAILURE(read_range(path, delimiter, 0, split, &ids));
+ ASSERT_NO_FATAL_FAILURE(
+ read_range(path, delimiter, split, file_size - split, &ids));
+ ASSERT_EQ(ids, expected);
+ }
+ }
+ }
+}
+
+TEST_P(JsonReaderSplitTest, OverlappingDelimitersPreserveRecords) {
+ for (const std::string delimiter : {"\n\n", "\r\n\r\n", " \t "}) {
+ for (bool trailing_delimiter : {false, true}) {
+ SCOPED_TRACE(testing::Message()
+ << "delimiter=" << delimiter << ", trailing=" << trailing_delimiter);
+ // The delimiter followed by its prefix has overlapping matches. For "\n\n", a split
+ // at byte 11 previously returned {1, 2, 2, 3}: the two splits matched different pairs
+ // of newlines in the three-newline run before id=2.
+ std::string content = R"({"id":1})" + delimiter +
+ delimiter.substr(0, delimiter.size() / 2) + R"({"id":2})" +
+ delimiter + R"({"id":3})";
+ if (trailing_delimiter) {
+ content += delimiter;
+ }
+ const auto path = write_json_file("overlapping_delimiters.json", content);
+ const auto file_size = static_cast<int64_t>(content.size());
+ const std::vector<int32_t> expected {1, 2, 3};
+ std::vector<int32_t> unsplit;
+ ASSERT_NO_FATAL_FAILURE(read_range(path, delimiter, 0, file_size, &unsplit));
+ ASSERT_EQ(unsplit, expected);
+ for (int64_t split = 1; split < file_size; ++split) {
+ SCOPED_TRACE(testing::Message() << "split=" << split);
+ std::vector<int32_t> ids;
+ ASSERT_NO_FATAL_FAILURE(read_range(path, delimiter, 0, split, &ids));
+ ASSERT_NO_FATAL_FAILURE(
+ read_range(path, delimiter, split, file_size - split, &ids));
+ ASSERT_EQ(ids, expected);
+ }
+ std::vector<int32_t> ids;
+ for (int64_t start = 0; start < file_size; ++start) {
+ ASSERT_NO_FATAL_FAILURE(read_range(path, delimiter, start, 1, &ids));
+ }
+ EXPECT_EQ(ids, expected);
+ }
+ }
+}
+
+TEST_P(JsonReaderSplitTest, OverlappingDelimiterRunCrossesLookbehindBuffers) {
+ const std::string delimiter = "\n\n";
+ // An odd run longer than the alignment scratch buffer must be replayed from its true start.
+ const std::string prefix = R"({"id":1})" + std::string(64 * 1024 + 3, '\n');
+ const std::string content = prefix + R"({"id":2})" + delimiter + R"({"id":3})";
+ const auto path = write_json_file("long_overlapping_delimiters.json", content);
+ const auto file_size = static_cast<int64_t>(content.size());
+ const auto split = static_cast<int64_t>(prefix.size());
+ const std::vector<int32_t> expected {1, 2, 3};
+ std::vector<int32_t> ids;
+ ASSERT_NO_FATAL_FAILURE(read_range(path, delimiter, 0, split, &ids));
+ ASSERT_NO_FATAL_FAILURE(read_range(path, delimiter, split, file_size - split, &ids));
+ EXPECT_EQ(ids, expected);
+}
+
+TEST_P(JsonReaderSplitTest, FourRangesPreserveEveryRecord) {
+ const std::string delimiter = "ABCDE";
+ std::string content;
+ std::vector<int32_t> expected;
+ // Equal-width, distinct IDs retain deterministic byte boundaries while detecting duplicates.
+ for (int32_t id = 100; id < 503; ++id) {
+ content += "{\"id\":" + std::to_string(id) + "}" + delimiter;
+ expected.push_back(id);
+ }
+ const auto path = write_json_file("four_ranges.json", content);
+ const auto file_size = static_cast<int64_t>(content.size());
+ const int64_t bytes_per_range = file_size / 4 + 1;
+ std::vector<int32_t> ids;
+ for (int64_t start = 0; start < file_size; start += bytes_per_range) {
+ ASSERT_NO_FATAL_FAILURE(read_range(path, delimiter, start,
+ std::min(bytes_per_range, file_size - start), &ids));
+ }
+ EXPECT_EQ(ids, expected);
+}
+
+INSTANTIATE_TEST_SUITE_P(LegacyAndV2, JsonReaderSplitTest, testing::Bool(),
+ [](const testing::TestParamInfo<bool>& info) {
+ return info.param ? "V2" : "Legacy";
+ });
+
TEST(JsonReaderTest, ReadsRequestedColumnsInFileScanRequestOrder) {
ObjectPool pool;
auto slots = build_slots(&pool);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 25/35: branch-4.1: [opt](cloud) Add recycler rowset recycling benchmarks test #68043 (#68581)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ac940a48c121a34daa429ec4e8d9850c69e71107
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Wed Sep 30 22:03:16 2026 +0800
branch-4.1: [opt](cloud) Add recycler rowset recycling benchmarks test #68043 (#68581)
Cherry-picked from #68043
Co-authored-by: Yixuan Wang <wa...@selectdb.com>
---
cloud/src/recycler/recycler.cpp | 2 +-
cloud/test/CMakeLists.txt | 1 +
cloud/test/recycler_benchmark_test.cpp | 1069 ++++++++++++++++++++++++++++++++
3 files changed, 1071 insertions(+), 1 deletion(-)
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index 72a444f3da4..76a1f048b59 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -5651,7 +5651,7 @@ int InstanceRecycler::recycle_rowsets() {
LOG(INFO) << "delete the recycle rowset kv that has empty resource_id, key="
<< hex(k) << " value=" << proto_to_json(rowset);
rowset_keys.emplace_back(k);
- return -1;
+ return 0;
}
// decode rowset_id
auto k1 = k;
diff --git a/cloud/test/CMakeLists.txt b/cloud/test/CMakeLists.txt
index 4d768601044..057b0132356 100644
--- a/cloud/test/CMakeLists.txt
+++ b/cloud/test/CMakeLists.txt
@@ -41,6 +41,7 @@ set_target_properties(txn_kv_test PROPERTIES COMPILE_FLAGS "-fno-access-control"
add_executable(recycler_test
recycler_test.cpp
+ recycler_benchmark_test.cpp
recycler_operation_log_test.cpp
snapshot_data_size_calculator_test.cpp
recycle_versioned_keys_test.cpp
diff --git a/cloud/test/recycler_benchmark_test.cpp b/cloud/test/recycler_benchmark_test.cpp
new file mode 100644
index 00000000000..9ca31a04f5d
--- /dev/null
+++ b/cloud/test/recycler_benchmark_test.cpp
@@ -0,0 +1,1069 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <butil/guid.h>
+#include <fmt/core.h>
+#include <gen_cpp/olap_file.pb.h>
+#include <gtest/gtest-spi.h>
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+#include <limits>
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "common/bvars.h"
+#include "common/config.h"
+#include "common/defer.h"
+#include "common/simple_thread_pool.h"
+#include "cpp/sync_point.h"
+#include "meta-service/meta_service_schema.h"
+#include "meta-service/txn_lazy_committer.h"
+#include "meta-store/blob_message.h"
+#include "meta-store/keys.h"
+#include "meta-store/mem_txn_kv.h"
+#include "meta-store/txn_kv.h"
+#include "meta-store/txn_kv_error.h"
+#include "recycler/recycler.h"
+#include "recycler/s3_accessor.h"
+#include "recycler/util.h"
+
+namespace doris::cloud {
+namespace {
+
+const std::string kBenchmarkInstanceId = "recycler_benchmark_instance";
+const std::string kBenchmarkResourceId = "recycler_benchmark_resource";
+double recycler_benchmark_duration_tolerance_ratio = 0.8;
+
+std::string get_env(const char* name) {
+ const char* value = std::getenv(name);
+ return value == nullptr ? "" : value;
+}
+
+std::string get_env_with_fallback(const char* primary, const char* fallback) {
+ const char* value = std::getenv(primary);
+ return value == nullptr ? get_env(fallback) : std::string(value);
+}
+
+struct BenchmarkS3Config {
+ bool enabled = false;
+ std::string access_key;
+ std::string secret_key;
+ std::string role_arn;
+ std::string external_id;
+ std::string endpoint;
+ std::string provider;
+ std::string bucket;
+ std::string region;
+ std::string prefix;
+};
+
+BenchmarkS3Config load_benchmark_s3_config() {
+ BenchmarkS3Config config;
+ config.enabled = get_env("ENABLE_S3_CLIENT") == "1";
+ if (!config.enabled) {
+ return config;
+ }
+
+ config.access_key = get_env("S3_AK");
+ config.secret_key = get_env("S3_SK");
+ config.role_arn = get_env("AWS_ROLE_ARN");
+ config.external_id = get_env("AWS_EXTERNAL_ID");
+ config.endpoint = get_env_with_fallback("S3_ENDPOINT", "AWS_ENDPOINT");
+ config.provider = get_env("S3_PROVIDER");
+ config.bucket = get_env_with_fallback("S3_BUCKET", "AWS_BUCKET");
+ config.region = get_env_with_fallback("S3_REGION", "AWS_REGION");
+ config.prefix = get_env_with_fallback("S3_PREFIX", "AWS_PREFIX");
+ return config;
+}
+
+void set_obj_store_provider(const std::string& provider, ObjectStoreInfoPB* obj_info) {
+ if (provider == "AZURE") {
+ obj_info->set_provider(ObjectStoreInfoPB_Provider_AZURE);
+ } else if (provider == "GCS") {
+ obj_info->set_provider(ObjectStoreInfoPB_Provider_GCP);
+ } else {
+ obj_info->set_provider(ObjectStoreInfoPB_Provider_S3);
+ }
+}
+
+// Number of recyclable rowsets seeded per branch.
+constexpr int64_t kRowsetsPerBranch = 10000;
+// Commit the seeded recycle rowset KVs in batches to keep each txn small.
+constexpr int64_t kSeedCommitBatch = 2000;
+constexpr int64_t kRowsetsPerPackedFile = 2;
+constexpr int64_t kPackedFileCount = kRowsetsPerBranch / kRowsetsPerPackedFile;
+constexpr int kMaxPackedRecyclePasses = 10;
+static_assert(kRowsetsPerBranch % kRowsetsPerPackedFile == 0);
+static_assert(kSeedCommitBatch % kRowsetsPerPackedFile == 0);
+constexpr int64_t kBenchmarkIndexId = 20000;
+constexpr int32_t kBenchmarkSchemaVersion = 1;
+constexpr int64_t kBenchmarkDbId = 10000;
+
+struct RecycleRowsetConfigGuard {
+ RecycleRowsetConfigGuard()
+ : old_enable_mark(config::enable_mark_delete_rowset_before_recycle),
+ old_enable_abort(config::enable_abort_txn_and_job_for_delete_rowset_before_recycle) {
+ config::enable_mark_delete_rowset_before_recycle = true;
+ config::enable_abort_txn_and_job_for_delete_rowset_before_recycle = true;
+ }
+
+ ~RecycleRowsetConfigGuard() {
+ config::enable_mark_delete_rowset_before_recycle = old_enable_mark;
+ config::enable_abort_txn_and_job_for_delete_rowset_before_recycle = old_enable_abort;
+ }
+
+ bool old_enable_mark;
+ bool old_enable_abort;
+};
+
+doris::TabletSchemaCloudPB make_benchmark_schema() {
+ doris::TabletSchemaCloudPB schema;
+ schema.set_schema_version(kBenchmarkSchemaVersion);
+ schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1);
+ auto* index = schema.add_index();
+ index->set_index_id(1);
+ index->set_index_type(IndexType::INVERTED);
+ return schema;
+}
+
+int put_benchmark_schema(TxnKv* txn_kv, const std::string& instance_id) {
+ std::unique_ptr<Transaction> txn;
+ if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) {
+ return -1;
+ }
+ std::string schema_key;
+ meta_schema_key({instance_id, kBenchmarkIndexId, kBenchmarkSchemaVersion}, &schema_key);
+ auto schema = make_benchmark_schema();
+ MetaServiceCode code = MetaServiceCode::OK;
+ std::string msg;
+ put_schema_kv(code, msg, txn.get(), schema_key, schema);
+ if (code != MetaServiceCode::OK) {
+ return -1;
+ }
+ return txn->commit() == TxnErrorCode::TXN_OK ? 0 : -1;
+}
+
+int remove_benchmark_schema(TxnKv* txn_kv, const std::string& instance_id) {
+ std::unique_ptr<Transaction> txn;
+ if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) {
+ return -1;
+ }
+ std::string schema_key;
+ meta_schema_key({instance_id, kBenchmarkIndexId, kBenchmarkSchemaVersion}, &schema_key);
+ ValueBuf schema_value;
+ if (blob_get(txn.get(), schema_key, &schema_value) != TxnErrorCode::TXN_OK) {
+ return -1;
+ }
+ schema_value.remove(txn.get());
+ return txn->commit() == TxnErrorCode::TXN_OK ? 0 : -1;
+}
+
+// Each value maps to one recyclable branch inside
+// InstanceRecycler::recycle_rowsets()::handle_rowset_kv. Both mark and abort
+// flags are enabled by RecycleRowsetConfigGuard for every scenario.
+enum class RecycleRowsetBranch {
+ // Old-version RecycleRowsetPB (no `type`) whose resource_id is empty: the
+ // recycler removes the KV only, without touching object storage.
+ kLegacyEmptyResource,
+ // Old-version RecycleRowsetPB with a real resource_id: recycled through the
+ // single-rowset delete_rowset_data_by_prefix path.
+ kLegacyWithResource,
+ // Already-marked PREPARE rowset without a related txn/job: recycled directly.
+ kPrepareDirect,
+ // New PREPARE rowset queued to be marked as recycled first
+ // (enable_mark_delete_rowset_before_recycle = true).
+ kPrepareMark,
+ // New PREPARE rowset carrying a load_id that triggers the abort-txn/job path
+ // (enable_abort_txn_and_job_for_delete_rowset_before_recycle = true).
+ kPrepareAbort,
+ // New COMPACT/DROP rowset with segments: recycled via the batched
+ // delete_rowset_data path.
+ kCompactedWithData,
+ // New COMPACT/DROP rowset without segments: treated as an empty rowset and
+ // removed by KV delete only.
+ kCompactedEmpty,
+};
+
+// Build one RowsetMetaCloudPB for the given branch. Kept intentionally minimal
+// and self-contained so this benchmark does not depend on recycler_test.cpp.
+doris::RowsetMetaCloudPB make_rowset_meta(RecycleRowsetBranch branch, int64_t tablet_id,
+ const std::string& rowset_id) {
+ doris::RowsetMetaCloudPB meta;
+ meta.set_rowset_id(0); // deprecated but required
+ meta.set_rowset_id_v2(rowset_id);
+ meta.set_tablet_id(tablet_id);
+ meta.set_index_id(kBenchmarkIndexId);
+ meta.set_schema_version(kBenchmarkSchemaVersion);
+ meta.mutable_tablet_schema()->CopyFrom(make_benchmark_schema());
+ meta.set_start_version(2);
+ meta.set_end_version(2);
+ meta.set_data_disk_size(1024);
+ meta.set_index_disk_size(512);
+ meta.set_total_disk_size(1536);
+ if (branch == RecycleRowsetBranch::kPrepareDirect) {
+ meta.set_is_recycled(true);
+ }
+ switch (branch) {
+ case RecycleRowsetBranch::kCompactedEmpty:
+ meta.set_num_segments(0);
+ break;
+ default:
+ meta.set_num_segments(1);
+ break;
+ }
+ // Only the branches that reach delete_rowset_data need a resolvable resource.
+ if (branch != RecycleRowsetBranch::kLegacyEmptyResource) {
+ meta.set_resource_id(kBenchmarkResourceId);
+ }
+ if (branch == RecycleRowsetBranch::kPrepareAbort) {
+ // A load_id makes make_related_txn_or_job_abort_task emit a TXN abort
+ // task; end_version != 1 is required to enter the abort branch.
+ meta.mutable_load_id()->set_hi(tablet_id);
+ meta.mutable_load_id()->set_lo(1);
+ meta.set_txn_id(tablet_id);
+ }
+ return meta;
+}
+
+// Wrap a RowsetMetaCloudPB into a RecycleRowsetPB shaped for the branch.
+RecycleRowsetPB make_recycle_rowset(RecycleRowsetBranch branch,
+ const doris::RowsetMetaCloudPB& meta) {
+ RecycleRowsetPB pb;
+ pb.set_creation_time(1); // long expired once retention is 0 / immediate recycle
+ pb.set_expiration(1);
+ switch (branch) {
+ case RecycleRowsetBranch::kLegacyEmptyResource:
+ // Old-version layout: no `type`, resource_id left empty on purpose.
+ pb.set_tablet_id(meta.tablet_id());
+ pb.set_resource_id("");
+ break;
+ case RecycleRowsetBranch::kLegacyWithResource:
+ // Old-version layout: no `type`, resource_id populated.
+ pb.set_tablet_id(meta.tablet_id());
+ pb.set_resource_id(meta.resource_id());
+ break;
+ case RecycleRowsetBranch::kPrepareDirect:
+ case RecycleRowsetBranch::kPrepareMark:
+ case RecycleRowsetBranch::kPrepareAbort:
+ pb.set_type(RecycleRowsetPB::PREPARE);
+ pb.mutable_rowset_meta()->CopyFrom(meta);
+ break;
+ case RecycleRowsetBranch::kCompactedWithData:
+ case RecycleRowsetBranch::kCompactedEmpty:
+ pb.set_type(RecycleRowsetPB::COMPACT);
+ pb.mutable_rowset_meta()->CopyFrom(meta);
+ if (branch == RecycleRowsetBranch::kCompactedWithData) {
+ // Match production rowsets whose schema is stored separately in the schema KV.
+ pb.mutable_rowset_meta()->clear_tablet_schema();
+ }
+ break;
+ }
+ return pb;
+}
+
+enum class DeleteBitmapVersion { kNone, kV1, kV2 };
+
+void put_benchmark_delete_bitmap(Transaction* txn, const std::string& instance_id,
+ const doris::RowsetMetaCloudPB& meta,
+ DeleteBitmapVersion version) {
+ switch (version) {
+ case DeleteBitmapVersion::kNone:
+ break;
+ case DeleteBitmapVersion::kV1:
+ txn->put(meta_delete_bitmap_key({instance_id, meta.tablet_id(), meta.rowset_id_v2(), 0, 0}),
+ "delete_bitmap_data");
+ break;
+ case DeleteBitmapVersion::kV2: {
+ // As with segment/index files, metadata is enough to exercise deletion of
+ // the standalone .dbm file. Keep bitmap storage independent of data packing.
+ DeleteBitmapStoragePB storage;
+ storage.set_store_in_fdb(false);
+ blob_put(txn,
+ versioned::meta_delete_bitmap_key(
+ {instance_id, meta.tablet_id(), meta.rowset_id_v2()}),
+ storage, 0);
+ break;
+ }
+ }
+}
+
+// Seed `count` recycle rowset KVs for one branch. Every rowset gets a distinct
+// tablet_id so the per-tablet recycle batch limit never truncates the workload.
+int seed_recycle_rowsets(TxnKv* txn_kv, const std::string& instance_id, RecycleRowsetBranch branch,
+ int64_t count, int64_t tablet_id_base, bool write_schema_kv = true,
+ DeleteBitmapVersion bitmap_version = DeleteBitmapVersion::kNone) {
+ if (write_schema_kv && put_benchmark_schema(txn_kv, instance_id) != 0) {
+ return -1;
+ }
+ std::unique_ptr<Transaction> txn;
+ for (int64_t i = 0; i < count; ++i) {
+ if (i % kSeedCommitBatch == 0) {
+ if (txn) {
+ if (txn->commit() != TxnErrorCode::TXN_OK) {
+ return -1;
+ }
+ }
+ if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) {
+ return -1;
+ }
+ }
+ int64_t tablet_id = tablet_id_base + i;
+ std::string rowset_id = fmt::format("{:018d}", i);
+ auto meta = make_rowset_meta(branch, tablet_id, rowset_id);
+ auto pb = make_recycle_rowset(branch, meta);
+
+ std::string key;
+ recycle_rowset_key({instance_id, tablet_id, rowset_id}, &key);
+ std::string val;
+ pb.SerializeToString(&val);
+ txn->put(key, val);
+ put_benchmark_delete_bitmap(txn.get(), instance_id, meta, bitmap_version);
+
+ if (branch == RecycleRowsetBranch::kPrepareAbort) {
+ const auto txn_id = meta.txn_id();
+ const auto label = fmt::format("recycler_benchmark_txn_{}", txn_id);
+ TxnIndexPB index;
+ index.mutable_tablet_index()->set_db_id(kBenchmarkDbId);
+ index.mutable_tablet_index()->set_tablet_id(tablet_id);
+ TxnInfoPB info;
+ info.set_txn_id(txn_id);
+ info.set_db_id(kBenchmarkDbId);
+ info.set_label(label);
+ info.set_status(TxnStatusPB::TXN_STATUS_PREPARED);
+ info.set_prepare_time(1);
+ info.set_timeout_ms(60000);
+ TxnRunningPB running;
+ running.set_timeout_time(info.prepare_time() + info.timeout_ms());
+ TxnLabelPB txn_label;
+ txn_label.add_txn_ids(txn_id);
+ txn->put(txn_index_key({instance_id, txn_id}), index.SerializeAsString());
+ txn->put(txn_info_key({instance_id, kBenchmarkDbId, txn_id}), info.SerializeAsString());
+ txn->put(txn_running_key({instance_id, kBenchmarkDbId, txn_id}),
+ running.SerializeAsString());
+ txn->atomic_set_ver_value(txn_label_key({instance_id, kBenchmarkDbId, label}),
+ txn_label.SerializeAsString());
+ }
+ }
+ if (txn && txn->commit() != TxnErrorCode::TXN_OK) {
+ return -1;
+ }
+ return 0;
+}
+
+std::string benchmark_packed_file_path(int64_t file_id) {
+ return fmt::format("data/packed_file/recycler_benchmark_{}.pack", file_id);
+}
+
+// Seed a complete packed file and its rowsets in the same transaction.
+void put_packed_recycle_rowsets(Transaction* txn, const std::string& instance_id,
+ int64_t tablet_id_base, int64_t file_id,
+ DeleteBitmapVersion bitmap_version) {
+ const auto path = benchmark_packed_file_path(file_id);
+ PackedFileInfoPB packed_info;
+ packed_info.set_resource_id(kBenchmarkResourceId);
+ packed_info.set_state(PackedFileInfoPB::NORMAL);
+ int64_t offset = 0;
+ for (int64_t j = 0; j < kRowsetsPerPackedFile; ++j) {
+ // Spread references across scan batches so different recycler workers can
+ // contend on the same packed file KV, instead of handling them serially.
+ const auto rowset_number = file_id + j * kPackedFileCount;
+ const auto tablet_id = tablet_id_base + rowset_number;
+ const auto rowset_id = fmt::format("{:018d}", rowset_number);
+ auto meta = make_rowset_meta(RecycleRowsetBranch::kCompactedWithData, tablet_id, rowset_id);
+ const std::pair<std::string, int64_t> files[] = {
+ {segment_path(tablet_id, rowset_id, 0), meta.data_disk_size()},
+ {inverted_index_path_v1(tablet_id, rowset_id, 0, 1, ""), meta.index_disk_size()}};
+ for (const auto& [small_path, size] : files) {
+ auto& location = (*meta.mutable_packed_slice_locations())[small_path];
+ location.set_packed_file_path(path);
+ location.set_offset(offset);
+ location.set_size(size);
+ auto* slice = packed_info.add_slices();
+ slice->set_path(small_path);
+ slice->set_offset(offset);
+ slice->set_size(size);
+ slice->set_deleted(false);
+ slice->set_tablet_id(tablet_id);
+ slice->set_rowset_id(rowset_id);
+ offset += size;
+ }
+ auto rowset = make_recycle_rowset(RecycleRowsetBranch::kCompactedWithData, meta);
+ txn->put(recycle_rowset_key({instance_id, tablet_id, rowset_id}),
+ rowset.SerializeAsString());
+ put_benchmark_delete_bitmap(txn, instance_id, meta, bitmap_version);
+ }
+ packed_info.set_ref_cnt(packed_info.slices_size());
+ packed_info.set_total_slice_num(packed_info.slices_size());
+ packed_info.set_total_slice_bytes(offset);
+ packed_info.set_remaining_slice_bytes(offset);
+ txn->put(packed_file_key({instance_id, path}), packed_info.SerializeAsString());
+}
+
+class RecyclerBenchmarkTest : public ::testing::Test {
+protected:
+ struct RecycleMetrics {
+ int64_t num = 0;
+ int64_t bytes = 0;
+ };
+
+ struct TxnKvCounts {
+ int64_t get = 0;
+ int64_t put = 0;
+ int64_t del = 0;
+
+ int64_t total() const { return get + put + del; }
+
+ TxnKvCounts& operator+=(const TxnKvCounts& rhs) {
+ get += rhs.get;
+ put += rhs.put;
+ del += rhs.del;
+ return *this;
+ }
+ };
+
+ struct BenchmarkResult {
+ RecycleMetrics metrics;
+ TxnKvCounts txn_kv_counts;
+ double elapsed_ms = 0;
+ };
+
+ using RecycleFunction = int (InstanceRecycler::*)();
+
+ static const std::map<std::string, RecycleFunction>& recycle_functions() {
+ static const std::map<std::string, RecycleFunction> functions = {
+ {"recycle_cluster_snapshots", &InstanceRecycler::recycle_cluster_snapshots},
+ {"recycle_operation_logs", &InstanceRecycler::recycle_operation_logs},
+ {"recycle_indexes", &InstanceRecycler::recycle_indexes},
+ {"recycle_partitions", &InstanceRecycler::recycle_partitions},
+ {"recycle_tmp_rowsets", &InstanceRecycler::recycle_tmp_rowsets},
+ {"recycle_rowsets", &InstanceRecycler::recycle_rowsets},
+ {"recycle_packed_files", &InstanceRecycler::recycle_packed_files},
+ {"abort_timeout_txn", &InstanceRecycler::abort_timeout_txn},
+ {"recycle_expired_txn_label", &InstanceRecycler::recycle_expired_txn_label},
+ {"recycle_copy_jobs", &InstanceRecycler::recycle_copy_jobs},
+ {"recycle_stage", &InstanceRecycler::recycle_stage},
+ {"recycle_expired_stage_objects", &InstanceRecycler::recycle_expired_stage_objects},
+ {"recycle_versions", &InstanceRecycler::recycle_versions},
+ {"recycle_restore_jobs", &InstanceRecycler::recycle_restore_jobs},
+ };
+ return functions;
+ }
+
+ void SetUp() override {
+ old_force_immediate_recycle_ = config::force_immediate_recycle;
+ old_retention_seconds_ = config::retention_seconds;
+
+ const auto tolerance_env = get_env("recycler_benchmark_duration_tolerance_ratio");
+ size_t parsed_size = 0;
+ recycler_benchmark_duration_tolerance_ratio =
+ tolerance_env.empty() ? 0.8 : std::stod(tolerance_env, &parsed_size);
+ ASSERT_EQ(parsed_size, tolerance_env.size()) << "invalid recycler benchmark tolerance";
+ ASSERT_TRUE(std::isfinite(recycler_benchmark_duration_tolerance_ratio));
+ ASSERT_GE(recycler_benchmark_duration_tolerance_ratio, 0);
+
+ config::force_immediate_recycle = true;
+ config::retention_seconds = 0;
+
+ instance_.set_instance_id(std::string(kBenchmarkInstanceId));
+ auto* obj_info = instance_.add_obj_info();
+ obj_info->set_id(kBenchmarkResourceId);
+ const auto s3_config = load_benchmark_s3_config();
+ ASSERT_NO_FATAL_FAILURE(configure_obj_info(s3_config, obj_info));
+
+ s3_producer_pool_ = std::make_shared<SimpleThreadPool>(
+ config::recycle_pool_parallelism, "recycler_benchmark_s3_producer_pool");
+ recycle_tablet_pool_ = std::make_shared<SimpleThreadPool>(
+ config::recycle_pool_parallelism, "recycler_benchmark_recycle_tablet_pool");
+ group_recycle_function_pool_ = std::make_shared<SimpleThreadPool>(
+ config::recycle_pool_parallelism, "recycler_benchmark_group_recycle_function_pool");
+ ASSERT_EQ(s3_producer_pool_->start(), 0);
+ ASSERT_EQ(recycle_tablet_pool_->start(), 0);
+ ASSERT_EQ(group_recycle_function_pool_->start(), 0);
+
+ thread_pool_group_ = RecyclerThreadPoolGroup(s3_producer_pool_, recycle_tablet_pool_,
+ group_recycle_function_pool_);
+ ASSERT_NO_FATAL_FAILURE(init_recycler());
+ }
+
+ void init_recycler() {
+ recycler_.reset();
+ txn_lazy_committer_.reset();
+ config::fdb_cluster_file_path = "fdb.cluster";
+ txn_kv_ = std::make_shared<FdbTxnKv>();
+ ASSERT_EQ(txn_kv_->init(), 0);
+ txn_lazy_committer_ = std::make_shared<TxnLazyCommitter>(txn_kv_);
+ recycler_ = std::make_unique<InstanceRecycler>(txn_kv_, instance_, thread_pool_group_,
+ txn_lazy_committer_);
+
+ if (load_benchmark_s3_config().enabled) {
+ auto s3_conf = S3Conf::from_obj_store_info(instance_.obj_info(0));
+ ASSERT_TRUE(s3_conf.has_value());
+
+ std::shared_ptr<S3Accessor> accessor;
+ ASSERT_EQ(S3Accessor::create(std::move(*s3_conf), &accessor), 0);
+ s3_accessor_ = std::move(accessor);
+ recycler_->TEST_add_accessor(kBenchmarkResourceId, s3_accessor_);
+ }
+ ASSERT_EQ(recycler_->init(), 0);
+ }
+
+ template <typename Benchmark>
+ void run_benchmark(const std::string& branch, Benchmark benchmark) {
+ ::testing::TestPartResultArray failures;
+ {
+ ::testing::ScopedFakeTestPartResultReporter reporter(
+ ::testing::ScopedFakeTestPartResultReporter::INTERCEPT_ALL_THREADS, &failures);
+ SCOPED_TRACE(branch);
+ try {
+ // Fatal assertions return from this lambda or the scenario, not the whole test.
+ [&] { benchmark(); }();
+ } catch (const std::exception& e) {
+ ADD_FAILURE() << "benchmark threw: " << e.what();
+ } catch (...) {
+ ADD_FAILURE() << "benchmark threw an unknown exception";
+ }
+ }
+ for (int i = 0; i < failures.size(); ++i) {
+ const auto& failure = failures.GetTestPartResult(i);
+ if (failure.failed()) {
+ benchmark_failures_ +=
+ fmt::format("branch={}\n{}\n", branch, ::testing::PrintToString(failure));
+ }
+ }
+ }
+
+ static void configure_obj_info(const BenchmarkS3Config& s3_config,
+ ObjectStoreInfoPB* obj_info) {
+ if (!s3_config.enabled) {
+ obj_info->set_prefix(kBenchmarkResourceId);
+ return;
+ }
+ ASSERT_FALSE(s3_config.endpoint.empty());
+ ASSERT_FALSE(s3_config.region.empty());
+ ASSERT_FALSE(s3_config.bucket.empty());
+ ASSERT_FALSE(s3_config.prefix.empty());
+ ASSERT_TRUE((s3_config.access_key.empty() && s3_config.secret_key.empty()) ||
+ (!s3_config.access_key.empty() && !s3_config.secret_key.empty()));
+
+ obj_info->set_ak(s3_config.access_key);
+ obj_info->set_sk(s3_config.secret_key);
+ obj_info->set_endpoint(s3_config.endpoint);
+ obj_info->set_region(s3_config.region);
+ obj_info->set_bucket(s3_config.bucket);
+ obj_info->set_prefix(fmt::format("{}{}recycler_benchmark/{}", s3_config.prefix,
+ s3_config.prefix.ends_with('/') ? "" : "/",
+ butil::GenerateGUID()));
+ set_obj_store_provider(s3_config.provider, obj_info);
+ if (s3_config.access_key.empty()) {
+ obj_info->set_role_arn(s3_config.role_arn);
+ obj_info->set_external_id(s3_config.external_id);
+ obj_info->set_cred_provider_type(CredProviderTypePB::INSTANCE_PROFILE);
+ }
+ }
+
+ void TearDown() override {
+ recycler_.reset();
+ txn_lazy_committer_.reset();
+ if (s3_accessor_) {
+ // The accessor is rooted at this run's GUID directory, never the external prefix.
+ const int ret = s3_accessor_->delete_all();
+ if (ret != 0) {
+ benchmark_failures_ += fmt::format("branch=s3_cleanup ret={} prefix={}\n", ret,
+ instance_.obj_info(0).prefix());
+ }
+ s3_accessor_.reset();
+ }
+
+ std::string report;
+ for (const auto& [operation_type, branches] : benchmark_results_) {
+ size_t branch_width = 24;
+ for (const auto& entry : branches) {
+ branch_width = std::max(branch_width, entry.first.size());
+ }
+ report += fmt::format("recycler benchmark: operation={}\n", operation_type);
+ report += fmt::format(
+ " {:<{}} {:>12} {:>13} {:>16} {:>12} {:>12} {:>12}"
+ " {:>12}\n",
+ "branch", branch_width, "total_ms", "recycled_num", "recycled_bytes",
+ "get_keys", "put_keys", "del_keys", "total_keys");
+ double total_elapsed_ms = 0;
+ RecycleMetrics total_recycle_metrics;
+ TxnKvCounts total_txn_kv_counts;
+ for (const auto& [branch, result] : branches) {
+ report += fmt::format(
+ " {:<{}} {:>12.2f} {:>13} {:>16} {:>12} {:>12} {:>12} {:>12}\n",
+ branch, branch_width, result.elapsed_ms, result.metrics.num,
+ result.metrics.bytes, result.txn_kv_counts.get, result.txn_kv_counts.put,
+ result.txn_kv_counts.del, result.txn_kv_counts.total());
+ total_elapsed_ms += result.elapsed_ms;
+ total_recycle_metrics.num += result.metrics.num;
+ total_recycle_metrics.bytes += result.metrics.bytes;
+ total_txn_kv_counts += result.txn_kv_counts;
+ }
+ report += fmt::format(
+ " {:<{}} {:>12.2f} {:>13} {:>16} {:>12} {:>12} {:>12} {:>12}\n",
+ "total", branch_width, total_elapsed_ms, total_recycle_metrics.num,
+ total_recycle_metrics.bytes, total_txn_kv_counts.get, total_txn_kv_counts.put,
+ total_txn_kv_counts.del, total_txn_kv_counts.total());
+ }
+ std::cout << report << std::flush;
+ if (!benchmark_failures_.empty()) {
+ ADD_FAILURE() << "recycler benchmark failures:\n" << benchmark_failures_;
+ }
+
+ thread_pool_group_ = {};
+ if (group_recycle_function_pool_) {
+ ASSERT_EQ(group_recycle_function_pool_->stop(), 0);
+ }
+ if (recycle_tablet_pool_) {
+ ASSERT_EQ(recycle_tablet_pool_->stop(), 0);
+ }
+ if (s3_producer_pool_) {
+ ASSERT_EQ(s3_producer_pool_->stop(), 0);
+ }
+ group_recycle_function_pool_.reset();
+ recycle_tablet_pool_.reset();
+ s3_producer_pool_.reset();
+ txn_kv_.reset();
+
+ config::force_immediate_recycle = old_force_immediate_recycle_;
+ config::retention_seconds = old_retention_seconds_;
+ }
+
+ RecycleMetrics read_recycle_metrics(const std::string& operation_type) const {
+ return {.num = g_bvar_recycler_instance_recycle_total_num_since_started.get(
+ {kBenchmarkInstanceId, operation_type}),
+ .bytes = g_bvar_recycler_instance_recycle_total_bytes_since_started.get(
+ {kBenchmarkInstanceId, operation_type})};
+ }
+
+ TxnKvCounts read_txn_kv_counts() const {
+ return {.get = g_bvar_txn_kv_get_count_normalized.get_value(),
+ .put = g_bvar_txn_kv_put.count() + g_bvar_txn_kv_atomic_set_ver_key.count() +
+ g_bvar_txn_kv_atomic_set_ver_value.count() +
+ g_bvar_txn_kv_atomic_add.count(),
+ .del = g_bvar_txn_kv_remove.count() + g_bvar_txn_kv_range_remove.count()};
+ }
+
+ void check_elapsed_ms(const std::string& branch, double actual_ms, double baseline_ms) {
+ const double limit_ms = baseline_ms * (1 + recycler_benchmark_duration_tolerance_ratio);
+ if (actual_ms > limit_ms) {
+ benchmark_failures_ +=
+ fmt::format("branch={} actual_ms={:.2f} baseline_ms={:.2f} limit_ms={:.2f}\n",
+ branch, actual_ms, baseline_ms, limit_ms);
+ }
+ }
+
+ void check_txn_kv_counts(const std::string& branch, const TxnKvCounts& actual,
+ const TxnKvCounts& baseline) {
+ if (actual.get != baseline.get || actual.put != baseline.put ||
+ actual.del != baseline.del) {
+ benchmark_failures_ += fmt::format(
+ "branch={} txn_kv_counts actual=[get={}, put={}, del={}] "
+ "baseline=[get={}, put={}, del={}]\n",
+ branch, actual.get, actual.put, actual.del, baseline.get, baseline.put,
+ baseline.del);
+ }
+ }
+
+ // Time only the recycler call; seeding, assertions and metrics reads are excluded.
+ void measure(const std::string& operation_type, const std::string& branch,
+ const std::string& phase) {
+ SCOPED_TRACE(phase);
+ const auto function_it = recycle_functions().find(operation_type);
+ ASSERT_NE(function_it, recycle_functions().end())
+ << "unknown recycler operation: " << operation_type;
+
+ const auto recycle_metrics_before = read_recycle_metrics(operation_type);
+ const auto txn_kv_counts_before = read_txn_kv_counts();
+ const auto start = std::chrono::steady_clock::now();
+ const int ret = (recycler_.get()->*(function_it->second))();
+ const auto elapsed =
+ std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start);
+ const auto recycle_metrics_after = read_recycle_metrics(operation_type);
+ const auto txn_kv_counts_after = read_txn_kv_counts();
+ const RecycleMetrics metrics {
+ .num = recycle_metrics_after.num - recycle_metrics_before.num,
+ .bytes = recycle_metrics_after.bytes - recycle_metrics_before.bytes};
+ const int64_t get_keys = txn_kv_counts_after.get - txn_kv_counts_before.get;
+ const int64_t put_keys = txn_kv_counts_after.put - txn_kv_counts_before.put;
+ const int64_t del_keys = txn_kv_counts_after.del - txn_kv_counts_before.del;
+ const TxnKvCounts txn_kv_counts {.get = get_keys, .put = put_keys, .del = del_keys};
+ auto& result = benchmark_results_[operation_type][branch];
+ result.elapsed_ms += elapsed.count();
+ result.metrics.num += metrics.num;
+ result.metrics.bytes += metrics.bytes;
+ result.txn_kv_counts += txn_kv_counts;
+ ASSERT_EQ(ret, 0) << "recycler operation failed: " << operation_type;
+ }
+
+ void seed_packed_recycle_rowsets(int64_t tablet_id_base, DeleteBitmapVersion bitmap_version) {
+ ASSERT_EQ(put_benchmark_schema(txn_kv_.get(), kBenchmarkInstanceId), 0);
+ std::unique_ptr<Transaction> txn;
+ for (int64_t file_id = 0; file_id < kPackedFileCount; ++file_id) {
+ if (file_id % (kSeedCommitBatch / kRowsetsPerPackedFile) == 0) {
+ if (txn) {
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ }
+ ASSERT_EQ(txn_kv_->create_txn(&txn), TxnErrorCode::TXN_OK);
+ }
+ put_packed_recycle_rowsets(txn.get(), kBenchmarkInstanceId, tablet_id_base, file_id,
+ bitmap_version);
+ }
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ }
+
+ int64_t count_recycle_rowsets() {
+ std::string begin = recycle_rowset_key({kBenchmarkInstanceId, 0, ""});
+ const auto end =
+ recycle_rowset_key({kBenchmarkInstanceId, std::numeric_limits<int64_t>::max(), ""});
+ std::unique_ptr<Transaction> txn;
+ if (txn_kv_->create_txn(&txn) != TxnErrorCode::TXN_OK) {
+ return -1;
+ }
+ int64_t count = 0;
+ std::unique_ptr<RangeGetIterator> it;
+ do {
+ if (txn->get(begin, end, &it) != TxnErrorCode::TXN_OK) {
+ return -1;
+ }
+ count += it->size();
+ begin = it->next_begin_key();
+ } while (it->more());
+ return count;
+ }
+
+ void check_recycled_bitmap_range(int64_t tablet_id_base) {
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(txn_kv_->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::unique_ptr<RangeGetIterator> it;
+ ASSERT_EQ(txn->get(versioned::meta_delete_bitmap_key(
+ {kBenchmarkInstanceId, tablet_id_base, ""}),
+ versioned::meta_delete_bitmap_key(
+ {kBenchmarkInstanceId, tablet_id_base + kRowsetsPerBranch, ""}),
+ &it),
+ TxnErrorCode::TXN_OK);
+ ASSERT_FALSE(it->has_next());
+ }
+
+ std::shared_ptr<TxnKv> txn_kv_;
+ InstanceInfoPB instance_;
+ RecyclerThreadPoolGroup thread_pool_group_;
+ std::shared_ptr<SimpleThreadPool> s3_producer_pool_;
+ std::shared_ptr<SimpleThreadPool> recycle_tablet_pool_;
+ std::shared_ptr<SimpleThreadPool> group_recycle_function_pool_;
+ std::shared_ptr<TxnLazyCommitter> txn_lazy_committer_;
+ std::unique_ptr<InstanceRecycler> recycler_;
+ std::shared_ptr<S3Accessor> s3_accessor_;
+ std::map<std::string, std::map<std::string, BenchmarkResult>> benchmark_results_;
+ std::string benchmark_failures_;
+ bool reset_before_next_benchmark_ = false;
+ double test_elapsed_ms_ = 0;
+
+ bool old_force_immediate_recycle_ = false;
+ int64_t old_retention_seconds_ = 0;
+};
+
+// Non-overlapping tablet id ranges per branch so a single recycle_rowsets() run
+// can cover several branches at once without id collisions.
+constexpr int64_t kTabletBase = 1'000'000;
+constexpr int64_t kTabletStride = 1'000'000'000LL;
+
+int64_t tablet_base_for(RecycleRowsetBranch branch) {
+ return kTabletBase + static_cast<int64_t>(branch) * kTabletStride;
+}
+
+TEST_F(RecyclerBenchmarkTest, RecycleRowsets) {
+ RecycleRowsetConfigGuard config_guard;
+ const auto test_start = std::chrono::steady_clock::now();
+ DORIS_CLOUD_DEFER {
+ const auto elapsed = std::chrono::duration<double, std::milli>(
+ std::chrono::steady_clock::now() - test_start);
+ // Includes seeding and validation, unlike the sum of timed recycler calls.
+ test_elapsed_ms_ = elapsed.count();
+ };
+
+ run_benchmark("legacy_empty_resource", [&] {
+ ASSERT_EQ(seed_recycle_rowsets(txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kLegacyEmptyResource, kRowsetsPerBranch,
+ tablet_base_for(RecycleRowsetBranch::kLegacyEmptyResource)),
+ 0);
+ ASSERT_NO_FATAL_FAILURE(measure("recycle_rowsets", "legacy_empty_resource", "delete"));
+ ASSERT_EQ(count_recycle_rowsets(), 0);
+ });
+
+ run_benchmark("legacy_with_resource", [&] {
+ ASSERT_EQ(seed_recycle_rowsets(txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kLegacyWithResource, kRowsetsPerBranch,
+ tablet_base_for(RecycleRowsetBranch::kLegacyWithResource)),
+ 0);
+ ASSERT_NO_FATAL_FAILURE(measure("recycle_rowsets", "legacy_with_resource", "delete"));
+ ASSERT_EQ(count_recycle_rowsets(), 0);
+ });
+
+ run_benchmark("prepare_direct", [&] {
+ ASSERT_EQ(seed_recycle_rowsets(txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kPrepareDirect, kRowsetsPerBranch,
+ tablet_base_for(RecycleRowsetBranch::kPrepareDirect)),
+ 0);
+ ASSERT_NO_FATAL_FAILURE(measure("recycle_rowsets", "prepare_direct", "delete"));
+ ASSERT_EQ(count_recycle_rowsets(), 0);
+ });
+
+ run_benchmark("prepare_mark", [&] {
+ const auto tablet_id_base = tablet_base_for(RecycleRowsetBranch::kPrepareMark);
+ ASSERT_EQ(seed_recycle_rowsets(txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kPrepareMark, kRowsetsPerBranch,
+ tablet_id_base),
+ 0);
+ ASSERT_NO_FATAL_FAILURE(measure("recycle_rowsets", "prepare_mark", "mark"));
+ ASSERT_EQ(count_recycle_rowsets(), kRowsetsPerBranch);
+ ASSERT_NO_FATAL_FAILURE(measure("recycle_rowsets", "prepare_mark", "delete"));
+ ASSERT_EQ(count_recycle_rowsets(), 0);
+ });
+
+ run_benchmark("prepare_abort", [&] {
+ const auto txn_id_base = tablet_base_for(RecycleRowsetBranch::kPrepareAbort);
+ ASSERT_EQ(seed_recycle_rowsets(txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kPrepareAbort, kRowsetsPerBranch,
+ txn_id_base),
+ 0);
+ ASSERT_NO_FATAL_FAILURE(measure("recycle_rowsets", "prepare_abort", "mark"));
+ ASSERT_EQ(count_recycle_rowsets(), kRowsetsPerBranch);
+ ASSERT_NO_FATAL_FAILURE(measure("recycle_rowsets", "prepare_abort", "abort_and_delete"));
+ ASSERT_EQ(count_recycle_rowsets(), 0);
+ });
+
+ // Keep the missing-schema case separate from the bitmap ablation below.
+ run_benchmark("compacted_without_schema", [&] {
+ const auto tablet_id_base = tablet_base_for(RecycleRowsetBranch::kCompactedWithData);
+ ASSERT_EQ(seed_recycle_rowsets(txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kCompactedWithData, kRowsetsPerBranch,
+ tablet_id_base),
+ 0);
+ ASSERT_EQ(remove_benchmark_schema(txn_kv_.get(), kBenchmarkInstanceId), 0);
+ ASSERT_NO_FATAL_FAILURE(measure("recycle_rowsets", "compacted_without_schema", "delete"));
+ ASSERT_EQ(count_recycle_rowsets(), 0);
+ });
+
+ // Hold rowset count, schema and file sizes fixed; vary only data packing and
+ // bitmap version. V1 bitmap KVs survive rowset recycling, so isolate each case.
+ for (const auto& [bitmap_version, bitmap_name] :
+ {std::pair {DeleteBitmapVersion::kNone, "data_only"},
+ std::pair {DeleteBitmapVersion::kV1, "delete_bitmap_v1"},
+ std::pair {DeleteBitmapVersion::kV2, "delete_bitmap_v2"}}) {
+ SCOPED_TRACE(bitmap_name);
+ const auto tablet_id_base =
+ tablet_base_for(RecycleRowsetBranch::kCompactedWithData) +
+ (1 + 2 * static_cast<int64_t>(bitmap_version)) * kRowsetsPerBranch;
+ run_benchmark(fmt::format("compacted_with_data/{}", bitmap_name), [&] {
+ ASSERT_EQ(seed_recycle_rowsets(txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kCompactedWithData,
+ kRowsetsPerBranch, tablet_id_base, true, bitmap_version),
+ 0);
+ ASSERT_EQ(count_recycle_rowsets(), kRowsetsPerBranch);
+ ASSERT_NO_FATAL_FAILURE(measure("recycle_rowsets",
+ fmt::format("compacted_with_data/{}", bitmap_name),
+ "delete"));
+ ASSERT_EQ(count_recycle_rowsets(), 0);
+ ASSERT_NO_FATAL_FAILURE(check_recycled_bitmap_range(tablet_id_base));
+ });
+
+ run_benchmark(fmt::format("compacted_with_packed_data/{}", bitmap_name), [&] {
+ ASSERT_NO_FATAL_FAILURE(seed_packed_recycle_rowsets(tablet_id_base + kRowsetsPerBranch,
+ bitmap_version));
+
+ int64_t remaining = count_recycle_rowsets();
+ ASSERT_EQ(remaining, kRowsetsPerBranch);
+ for (int pass = 1; pass <= kMaxPackedRecyclePasses && remaining > 0; ++pass) {
+ const int64_t previous_remaining = remaining;
+ // A worker may exhaust packed-file transaction retries while recycle_rowsets()
+ // still returns success. Include every pass in the reported total elapsed time.
+ ASSERT_NO_FATAL_FAILURE(
+ measure("recycle_rowsets",
+ fmt::format("compacted_with_packed_data/{}", bitmap_name),
+ fmt::format("delete_pass_{}", pass)));
+ remaining = count_recycle_rowsets();
+ ASSERT_GE(remaining, 0);
+ ASSERT_LT(remaining, previous_remaining)
+ << "packed rowset recycling made no progress, pass=" << pass;
+ }
+ ASSERT_EQ(remaining, 0)
+ << "packed rowset recycling exceeded " << kMaxPackedRecyclePasses << " passes";
+
+ // Check outside the timed calls that every packed file reached zero references.
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(txn_kv_->create_txn(&txn), TxnErrorCode::TXN_OK);
+ for (int64_t file_id = 0; file_id < kPackedFileCount; ++file_id) {
+ std::string value;
+ ASSERT_EQ(txn->get(packed_file_key({kBenchmarkInstanceId,
+ benchmark_packed_file_path(file_id)}),
+ &value),
+ TxnErrorCode::TXN_KEY_NOT_FOUND);
+ }
+ ASSERT_NO_FATAL_FAILURE(
+ check_recycled_bitmap_range(tablet_id_base + kRowsetsPerBranch));
+ });
+ }
+
+ run_benchmark("compacted_empty", [&] {
+ ASSERT_EQ(seed_recycle_rowsets(txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kCompactedEmpty, kRowsetsPerBranch,
+ tablet_base_for(RecycleRowsetBranch::kCompactedEmpty)),
+ 0);
+ ASSERT_NO_FATAL_FAILURE(measure("recycle_rowsets", "compacted_empty", "delete"));
+ ASSERT_EQ(count_recycle_rowsets(), 0);
+ });
+
+ // Validate ordering outside the timed calls so callback KV reads do not skew timings.
+ run_benchmark("prepare_abort_before_delete", [&] {
+ const auto txn_id = tablet_base_for(RecycleRowsetBranch::kPrepareAbort) + kRowsetsPerBranch;
+ ASSERT_EQ(seed_recycle_rowsets(txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kPrepareAbort, 1, txn_id),
+ 0);
+ ASSERT_EQ(recycler_->recycle_rowsets(), 0);
+ ASSERT_EQ(count_recycle_rowsets(), 1);
+
+ auto* sp = SyncPoint::get_instance();
+ DORIS_CLOUD_DEFER {
+ sp->clear_all_call_backs();
+ sp->disable_processing();
+ };
+ sp->enable_processing();
+
+ ASSERT_EQ(recycler_->recycle_rowsets(), 0);
+ ASSERT_EQ(count_recycle_rowsets(), 0);
+ });
+
+ // Keep mixed-workload IDs separate from the transactions aborted above.
+ // Already-marked direct rowsets are deleted in the first pass.
+ run_benchmark("mixed", [&] {
+ constexpr int64_t mixed_tablet_offset = 7 * kTabletStride;
+ ASSERT_EQ(seed_recycle_rowsets(
+ txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kLegacyEmptyResource, kRowsetsPerBranch,
+ mixed_tablet_offset +
+ tablet_base_for(RecycleRowsetBranch::kLegacyEmptyResource)),
+ 0);
+ ASSERT_EQ(seed_recycle_rowsets(
+ txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kLegacyWithResource, kRowsetsPerBranch,
+ mixed_tablet_offset +
+ tablet_base_for(RecycleRowsetBranch::kLegacyWithResource)),
+ 0);
+ ASSERT_EQ(
+ seed_recycle_rowsets(
+ txn_kv_.get(), kBenchmarkInstanceId, RecycleRowsetBranch::kPrepareDirect,
+ kRowsetsPerBranch,
+ mixed_tablet_offset + tablet_base_for(RecycleRowsetBranch::kPrepareDirect)),
+ 0);
+ const auto mark_tablet_id_base =
+ mixed_tablet_offset + tablet_base_for(RecycleRowsetBranch::kPrepareMark);
+ ASSERT_EQ(seed_recycle_rowsets(txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kPrepareMark, kRowsetsPerBranch,
+ mark_tablet_id_base),
+ 0);
+ const auto txn_id_base =
+ mixed_tablet_offset + tablet_base_for(RecycleRowsetBranch::kPrepareAbort);
+ ASSERT_EQ(seed_recycle_rowsets(txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kPrepareAbort, kRowsetsPerBranch,
+ txn_id_base),
+ 0);
+ ASSERT_EQ(seed_recycle_rowsets(
+ txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kCompactedWithData, kRowsetsPerBranch,
+ mixed_tablet_offset +
+ tablet_base_for(RecycleRowsetBranch::kCompactedWithData)),
+ 0);
+ ASSERT_EQ(
+ seed_recycle_rowsets(txn_kv_.get(), kBenchmarkInstanceId,
+ RecycleRowsetBranch::kCompactedEmpty, kRowsetsPerBranch,
+ mixed_tablet_offset +
+ tablet_base_for(RecycleRowsetBranch::kCompactedEmpty)),
+ 0);
+ ASSERT_NO_FATAL_FAILURE(measure("recycle_rowsets", "mixed", "mark_and_delete_ready"));
+ ASSERT_EQ(count_recycle_rowsets(), 2 * kRowsetsPerBranch);
+ ASSERT_NO_FATAL_FAILURE(measure("recycle_rowsets", "mixed", "abort_and_delete_prepare"));
+ ASSERT_EQ(count_recycle_rowsets(), 0);
+ });
+
+ // Recorded with 10,000 rowsets per branch. Recalibrate if the workload changes.
+ static_assert(kRowsetsPerBranch == 10000);
+ const std::map<std::string, double> baseline_elapsed_ms = {
+ {"compacted_empty", 794.18},
+ {"compacted_with_data/data_only", 2285.80},
+ {"compacted_with_data/delete_bitmap_v1", 2461.84},
+ {"compacted_with_data/delete_bitmap_v2", 2726.75},
+ {"compacted_with_packed_data/data_only", 8583.32},
+ {"compacted_with_packed_data/delete_bitmap_v1", 9315.59},
+ {"compacted_with_packed_data/delete_bitmap_v2", 8948.54},
+ {"compacted_without_schema", 3924.04},
+ {"legacy_empty_resource", 945.75},
+ {"legacy_with_resource", 6718.37},
+ {"mixed", 35178.52},
+ {"prepare_abort", 16465.44},
+ {"prepare_direct", 5360.28},
+ {"prepare_mark", 7039.65},
+ };
+ const std::map<std::string, TxnKvCounts> baseline_txn_kv_counts = {
+ {"compacted_empty", {.get = 10000, .put = 0, .del = 10000}},
+ {"compacted_with_data/data_only", {.get = 10001, .put = 0, .del = 10000}},
+ {"compacted_with_data/delete_bitmap_v1", {.get = 10000, .put = 0, .del = 10000}},
+ {"compacted_with_data/delete_bitmap_v2", {.get = 20000, .put = 0, .del = 20000}},
+ {"compacted_with_packed_data/data_only", {.get = 25000, .put = 10000, .del = 15000}},
+ {"compacted_with_packed_data/delete_bitmap_v1",
+ {.get = 25000, .put = 10000, .del = 15000}},
+ {"compacted_with_packed_data/delete_bitmap_v2",
+ {.get = 35000, .put = 10000, .del = 25000}},
+ {"compacted_without_schema", {.get = 10000, .put = 0, .del = 10000}},
+ {"legacy_empty_resource", {.get = 10000, .put = 0, .del = 10000}},
+ {"legacy_with_resource", {.get = 20000, .put = 0, .del = 20000}},
+ {"mixed", {.get = 200000, .put = 40000, .del = 120000}},
+ {"prepare_abort", {.get = 90000, .put = 30000, .del = 30000}},
+ {"prepare_direct", {.get = 20000, .put = 0, .del = 20000}},
+ {"prepare_mark", {.get = 40000, .put = 10000, .del = 20000}},
+ };
+ const auto& results = benchmark_results_["recycle_rowsets"];
+ double total_elapsed_ms = 0;
+ for (const auto& [branch, baseline_ms] : baseline_elapsed_ms) {
+ const auto result = results.find(branch);
+ if (result == results.end()) {
+ benchmark_failures_ +=
+ fmt::format("branch={} did not produce a timing result\n", branch);
+ continue;
+ }
+ check_elapsed_ms(branch, result->second.elapsed_ms, baseline_ms);
+ check_txn_kv_counts(branch, result->second.txn_kv_counts,
+ baseline_txn_kv_counts.at(branch));
+ total_elapsed_ms += result->second.elapsed_ms;
+ }
+ check_elapsed_ms("total_elapsed_ms", total_elapsed_ms, 110748.07);
+}
+
+} // namespace
+} // namespace doris::cloud
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 05/35: [fix](arrow) Validate Flight result arrays before publishing batches (#68596)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ffa16d4b68de5cba085f13b039fa9425b6eb89e4
Author: Gabriel <li...@selectdb.com>
AuthorDate: Tue Sep 29 16:35:21 2026 +0800
[fix](arrow) Validate Flight result arrays before publishing batches (#68596)
### What problem does this PR solve?
Arrow Flight SQL can return a string array containing invalid UTF-8, for
example from `SELECT unhex(encoded) FROM input_table` when a row
contains the hex string `84`. Arrow string builders accept the bytes,
but clients such as PyArrow fail when decoding the result.
Validate each completed array in `ArrowFlightArrowBlockConvertor` before
publishing the batch. Reuse Arrow's recursive validation to cover nested
strings and per-value UTF-8 boundaries. Return an invalid-argument error
identifying the column ordinal, field name and Arrow validation failure.
Binary values retain their byte representation, and NULL payloads are
handled through the serialized validity bitmap.
Add six BE unit tests and a Flight SQL regression suite covering
malformed encodings, separate rows whose concatenation is valid UTF-8,
subsequent batches, nested arrays/structs/map keys and values, large
strings, valid Unicode, NULLs and binary results.
### Validation
- Reproduced four failing tests before the fix; the two valid-data/NULL
tests passed.
- ASAN: all 46 tests in `ArrowBlockConvertorTest` and
`DataTypeSerDeArrowTest` passed after the fix.
- clang-format 16 passed for all three affected C++ files.
- Updated the Flight regression suite to bypass constant folding for the
constant-input check, reuse the configured Flight connection, assert
valid results directly, and clean up in a finally block.
- Groovy 4.0.19: six mock JDBC execution checks passed, covering TLS
on/off, absent expected errors, unrelated errors, and cleanup. These
checks do not replace live Flight SQL execution; the updated regression
is pending CI.
- Re-ran the six focused Flight BE tests under ASAN after the
regression-only update: all passed.
The local BE unit-test build selected the relevant suites and their test
support files; the build configuration was restored and is not part of
this PR.
### Release note
Arrow Flight SQL now rejects invalid UTF-8 string results with a
descriptive server-side error instead of returning malformed Arrow data.
### Check List (For Author)
- Test
- [x] Regression test added (live execution pending CI)
- [x] Unit Test
- Behavior changed:
- [x] Yes. Invalid Arrow Flight string payloads fail on the server
before the affected batch is returned.
- Does this need documentation?
- [x] No.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
be/src/format/arrow/arrow_block_convertor.cpp | 19 +++
be/src/format/arrow/arrow_block_convertor.h | 10 +-
.../format/arrow/arrow_block_convertor_test.cpp | 133 +++++++++++++++++++++
.../test_flight_utf8_validation.groovy | 71 +++++++++++
4 files changed, 230 insertions(+), 3 deletions(-)
diff --git a/be/src/format/arrow/arrow_block_convertor.cpp b/be/src/format/arrow/arrow_block_convertor.cpp
index 0046ad3dacc..b62ca98cde9 100644
--- a/be/src/format/arrow/arrow_block_convertor.cpp
+++ b/be/src/format/arrow/arrow_block_convertor.cpp
@@ -341,6 +341,25 @@ Status ArrowBlockConvertor::init() {
return Status::OK();
}
+Status ArrowFlightArrowBlockConvertor::convert_to_arrow(const Block& block, arrow::MemoryPool* pool,
+ std::shared_ptr<arrow::RecordBatch>* result,
+ size_t start_row, size_t end_row) const {
+ std::shared_ptr<arrow::RecordBatch> batch;
+ RETURN_IF_ERROR(ArrowBlockConvertor::convert_to_arrow(block, pool, &batch, start_row, end_row));
+ // String builders accept arbitrary bytes, but Flight UTF-8 values must be valid per row,
+ // including nested children. Validate before publishing the batch to the reader.
+ for (int i = 0; i < batch->num_columns(); ++i) {
+ auto status = batch->column(i)->ValidateFull();
+ if (!status.ok()) {
+ return Status::InvalidArgument("Invalid Arrow Flight result in column {} ('{}'): {}",
+ i + 1, batch->schema()->field(i)->name(),
+ status.ToString());
+ }
+ }
+ *result = std::move(batch);
+ return Status::OK();
+}
+
Status DorisArrowBlockConvertor::init() {
if (_arrow_schema == nullptr) {
// cctz names fixed offsets as "Fixed/UTC+HH:MM:SS", which is not the Arrow
diff --git a/be/src/format/arrow/arrow_block_convertor.h b/be/src/format/arrow/arrow_block_convertor.h
index ec1083505a9..c923507786d 100644
--- a/be/src/format/arrow/arrow_block_convertor.h
+++ b/be/src/format/arrow/arrow_block_convertor.h
@@ -58,9 +58,9 @@ public:
virtual Status init();
const std::shared_ptr<arrow::Schema>& arrow_schema() const { return _arrow_schema; }
- Status convert_to_arrow(const Block& block, arrow::MemoryPool* pool,
- std::shared_ptr<arrow::RecordBatch>* result, size_t start_row = 0,
- size_t end_row = 0) const;
+ virtual Status convert_to_arrow(const Block& block, arrow::MemoryPool* pool,
+ std::shared_ptr<arrow::RecordBatch>* result,
+ size_t start_row = 0, size_t end_row = 0) const;
virtual Status convert_from_arrow(const std::shared_ptr<arrow::RecordBatch>& batch,
const DataTypes& types, Block* block) const;
@@ -115,6 +115,10 @@ private:
class ArrowFlightArrowBlockConvertor final : public DorisArrowBlockConvertor {
public:
using DorisArrowBlockConvertor::DorisArrowBlockConvertor;
+
+ Status convert_to_arrow(const Block& block, arrow::MemoryPool* pool,
+ std::shared_ptr<arrow::RecordBatch>* result, size_t start_row = 0,
+ size_t end_row = 0) const override;
};
class PythonArrowBlockConvertor final : public DorisArrowBlockConvertor {
diff --git a/be/test/format/arrow/arrow_block_convertor_test.cpp b/be/test/format/arrow/arrow_block_convertor_test.cpp
index f3db16d59de..b02b60cb7a7 100644
--- a/be/test/format/arrow/arrow_block_convertor_test.cpp
+++ b/be/test/format/arrow/arrow_block_convertor_test.cpp
@@ -23,6 +23,7 @@
#include <arrow/ipc/api.h>
#include <gtest/gtest.h>
+#include "core/column/column_nullable.h"
#include "core/column/column_vector.h"
#include "core/data_type/data_type_array.h"
#include "core/data_type/data_type_factory.hpp"
@@ -289,4 +290,136 @@ TEST_F(ArrowBlockConvertorTest, TableConvertersRejectMismatchedNestedSchemas) {
}
}
+TEST_F(ArrowBlockConvertorTest, FlightRejectsInvalidUtf8BeforeReturningBatch) {
+ auto type = DataTypeFactory::instance().create_data_type(TYPE_STRING, false);
+ for (const std::string& value :
+ {std::string("\x84"), std::string("\xc0\xaf"), std::string("\xed\xa0\x80"),
+ std::string("\xf4\x90\x80\x80")}) {
+ auto column = type->create_column();
+ column->insert(Field::create_field<TYPE_STRING>(value));
+ Block block {{std::move(column), type, "payload"}};
+ ArrowFlightArrowBlockConvertor flight(block, "UTC", cctz::utc_time_zone());
+ ASSERT_TRUE(flight.init().ok());
+ const ArrowBlockConvertor& converter = flight;
+ std::shared_ptr<arrow::RecordBatch> batch;
+ const auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()) << status;
+ EXPECT_NE(std::string::npos, status.to_string().find("payload"));
+ EXPECT_NE(std::string::npos, status.to_string().find("UTF8"));
+ EXPECT_EQ(nullptr, batch);
+
+ DorisArrowBlockConvertor ordinary(block, "UTC", cctz::utc_time_zone());
+ ASSERT_TRUE(ordinary.init().ok());
+ EXPECT_TRUE(ordinary.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
+ }
+}
+
+TEST_F(ArrowBlockConvertorTest, FlightChecksEachStringAndEveryBatch) {
+ auto type = DataTypeFactory::instance().create_data_type(TYPE_STRING, false);
+ auto column = type->create_column();
+ for (const std::string& value :
+ {std::string("valid"), std::string("\xc2"), std::string("\xa2")}) {
+ column->insert(Field::create_field<TYPE_STRING>(value));
+ }
+ Block block {{std::move(column), type, "payload"}};
+ ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone());
+ ASSERT_TRUE(converter.init().ok());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch, 0, 1).ok());
+ ASSERT_EQ(1, batch->num_rows());
+ batch.reset();
+ // Adjacent invalid strings form valid UTF-8 when concatenated; row boundaries matter.
+ EXPECT_FALSE(
+ converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch, 1, 3).ok());
+ EXPECT_EQ(nullptr, batch);
+}
+
+TEST_F(ArrowBlockConvertorTest, FlightPreservesValidTextNullsAndBinary) {
+ auto text = make_nullable(DataTypeFactory::instance().create_data_type(TYPE_STRING, false));
+ auto strings = text->create_column();
+ strings->insert(Field::create_field<TYPE_STRING>(std::string("\xe4\xb8\xad\xf0\x9f\x98\x80")));
+ strings->insert(Field::create_field<TYPE_STRING>(std::string()));
+ strings->insert(Field::create_field<TYPE_STRING>(std::string("a\0b", 3)));
+ strings->insert_default();
+ auto binary = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, false);
+ auto bytes = binary->create_column();
+ for (int i = 0; i < 4; ++i) {
+ bytes->insert_data("\x84\0\xff", 3);
+ }
+ Block block {{std::move(strings), text, "text"}, {std::move(bytes), binary, "binary"}};
+ ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone());
+ ASSERT_TRUE(converter.init().ok());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ const auto& values = static_cast<const arrow::StringArray&>(*batch->column(0));
+ EXPECT_EQ("\xe4\xb8\xad\xf0\x9f\x98\x80", values.GetString(0));
+ EXPECT_EQ("", values.GetString(1));
+ EXPECT_EQ(std::string("a\0b", 3), values.GetString(2));
+ EXPECT_TRUE(values.IsNull(3));
+ ASSERT_EQ(arrow::Type::BINARY, batch->column(1)->type_id());
+ const auto& binary_values = static_cast<const arrow::BinaryArray&>(*batch->column(1));
+ EXPECT_EQ(std::string("\x84\0\xff", 3), binary_values.GetString(0));
+}
+
+TEST_F(ArrowBlockConvertorTest, FlightRejectsNestedInvalidUtf8) {
+ auto text = make_nullable(DataTypeFactory::instance().create_data_type(TYPE_STRING, false));
+ const auto invalid = Field::create_field<TYPE_STRING>(std::string("\x84"));
+ const auto valid = Field::create_field<TYPE_STRING>(std::string("key"));
+ DataTypes types {std::make_shared<DataTypeArray>(text),
+ std::make_shared<DataTypeStruct>(DataTypes {text}, Strings {"child"}),
+ std::make_shared<DataTypeMap>(text, text),
+ std::make_shared<DataTypeMap>(text, text)};
+ FieldVector fields {
+ Field::create_field<TYPE_ARRAY>(Array {invalid}),
+ Field::create_field<TYPE_STRUCT>(Struct {invalid}),
+ Field::create_field<TYPE_MAP>(Map {Field::create_field<TYPE_ARRAY>(Array {valid}),
+ Field::create_field<TYPE_ARRAY>(Array {invalid})}),
+ Field::create_field<TYPE_MAP>(Map {Field::create_field<TYPE_ARRAY>(Array {invalid}),
+ Field::create_field<TYPE_ARRAY>(Array {valid})})};
+ for (size_t i = 0; i < types.size(); ++i) {
+ SCOPED_TRACE(types[i]->get_name());
+ auto column = types[i]->create_column();
+ column->insert(fields[i]);
+ Block block {{std::move(column), types[i], "nested"}};
+ ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone());
+ ASSERT_TRUE(converter.init().ok());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ const auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()) << status;
+ EXPECT_NE(std::string::npos, status.to_string().find("nested"));
+ EXPECT_EQ(nullptr, batch);
+ }
+}
+
+TEST_F(ArrowBlockConvertorTest, FlightRejectsInvalidLargeString) {
+ auto type = DataTypeFactory::instance().create_data_type(TYPE_STRING, false);
+ auto column = type->create_column();
+ column->insert(Field::create_field<TYPE_STRING>(std::string("\x84")));
+ Block block {{std::move(column), type, "large_text"}};
+ auto schema = arrow::schema({arrow::field("large_text", arrow::large_utf8(), false)});
+ ArrowFlightArrowBlockConvertor converter(schema, cctz::utc_time_zone());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ const auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()) << status;
+ EXPECT_NE(std::string::npos, status.to_string().find("large_text"));
+ EXPECT_EQ(nullptr, batch);
+}
+
+TEST_F(ArrowBlockConvertorTest, FlightIgnoresBytesMaskedByNull) {
+ auto text = DataTypeFactory::instance().create_data_type(TYPE_STRING, false);
+ auto values = text->create_column();
+ values->insert_data("\x84", 1);
+ auto nulls = ColumnUInt8::create();
+ nulls->insert_value(1);
+ Block block {{ColumnNullable::create(std::move(values), std::move(nulls)), make_nullable(text),
+ "nullable_text"}};
+ ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone());
+ ASSERT_TRUE(converter.init().ok());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ EXPECT_TRUE(batch->column(0)->IsNull(0));
+}
+
} // namespace doris
diff --git a/regression-test/suites/arrow_flight_sql_p0/test_flight_utf8_validation.groovy b/regression-test/suites/arrow_flight_sql_p0/test_flight_utf8_validation.groovy
new file mode 100644
index 00000000000..f19312b70d3
--- /dev/null
+++ b/regression-test/suites/arrow_flight_sql_p0/test_flight_utf8_validation.groovy
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_flight_utf8_validation", "arrow_flight_sql") {
+ // Reuse Flight credentials and avoid the TLS-dependent early return in connect().
+ def flight = context.getArrowFlightSqlConnection()
+ def input = "${context.dbName}.flight_utf8_input"
+ def expectInvalidUtf8 = { String query ->
+ try {
+ arrow_flight_sql(query)
+ assertTrue(false, "Expected an invalid UTF-8 Flight result error")
+ } catch (Exception error) {
+ assertTrue(error.toString().contains("Invalid UTF8"), error.toString())
+ }
+ }
+ arrow_flight_sql "DROP TABLE IF EXISTS ${input}"
+ arrow_flight_sql """CREATE TABLE ${input} (id INT, encoded STRING)
+ DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")"""
+ try {
+ arrow_flight_sql """INSERT INTO ${input} VALUES
+ (1, '616263'), (2, 'E4B8ADF09F9880'), (3, ''), (4, NULL),
+ (5, '84'), (6, 'C0AF'), (7, 'EDA080'), (8, 'F4908080'),
+ (9, 'C2'), (10, 'A2')"""
+ for (def id : [5, 6, 7, 8]) {
+ expectInvalidUtf8("SELECT unhex(encoded) AS payload FROM ${input} WHERE id = ${id}")
+ }
+ // Constant folding can replace invalid bytes with valid text before Flight conversion.
+ // Keep a table scan and skip folding so the constant is evaluated and validated on the BE.
+ expectInvalidUtf8("""SELECT /*+ SET_VAR(debug_skip_fold_constant=true) */
+ unhex('84') AS payload FROM ${input} WHERE id = 1""")
+ // These bytes are valid only when concatenated; each row must be valid independently.
+ expectInvalidUtf8("SELECT unhex(encoded) AS payload FROM ${input} WHERE id IN (9, 10) ORDER BY id")
+ for (def expression : ["array(unhex(encoded))",
+ "named_struct('child', unhex(encoded))",
+ "map('key', unhex(encoded))",
+ "map(unhex(encoded), 'value')"]) {
+ expectInvalidUtf8("SELECT ${expression} AS nested FROM ${input} WHERE id = 5")
+ }
+ // Reaching an invalid value after valid rows must fail rather than publish malformed text.
+ expectInvalidUtf8("SELECT unhex(encoded) AS payload FROM ${input} ORDER BY id")
+ assertEquals([['abc'], ['中😀'], [''], [null]],
+ arrow_flight_sql("SELECT unhex(encoded) FROM ${input} WHERE id <= 4 ORDER BY id"))
+ // Binary payloads may contain arbitrary bytes and must keep their binary Arrow type.
+ flight.createStatement().withCloseable { statement ->
+ statement.executeQuery("SELECT to_binary(encoded) AS payload FROM ${input} WHERE id = 5")
+ .withCloseable { rows ->
+ assertTrue(rows.next())
+ assertTrue(rows.getObject(1) instanceof byte[])
+ assertEquals([0x84], rows.getBytes(1).collect { it & 0xff })
+ assertFalse(rows.next())
+ }
+ }
+ } finally {
+ arrow_flight_sql "DROP TABLE IF EXISTS ${input}"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 06/35: branch-4.1: [fix](function) Keep the leading NULLs of every row in array_cum_sum #68554 (#68605)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 69e3bffa030683418c3feff5d014bbbae29e4e44
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Sep 29 17:19:07 2026 +0800
branch-4.1: [fix](function) Keep the leading NULLs of every row in array_cum_sum #68554 (#68605)
Cherry-picked from #68554
Co-authored-by: Chenyang Sun <su...@selectdb.com>
Co-authored-by: Claude Opus 5.5 (1M context) <no...@anthropic.com>
---
.../function/array/function_array_cum_sum.cpp | 14 +--
.../exprs/function/function_array_cum_sum_test.cpp | 100 +++++++++++++++++++++
.../test_array_cum_sum_leading_null.out | 14 +++
.../test_array_cum_sum_leading_null.groovy | 51 +++++++++++
4 files changed, 174 insertions(+), 5 deletions(-)
diff --git a/be/src/exprs/function/array/function_array_cum_sum.cpp b/be/src/exprs/function/array/function_array_cum_sum.cpp
index d23d54f0e03..653ebfc09ac 100644
--- a/be/src/exprs/function/array/function_array_cum_sum.cpp
+++ b/be/src/exprs/function/array/function_array_cum_sum.cpp
@@ -253,11 +253,15 @@ private:
// handle null value in res_datas for first null value
auto res_null_map_col = ColumnUInt8::create(size, 0);
- size_t first_not_null_pos =
- VectorizedUtils::find_first_valid_simd(src_null_map, 0, size);
- VLOG_DEBUG << "first_not_null_pos: " << std::to_string(first_not_null_pos);
- VectorizedUtils::range_set_nullmap_to_true_simd(res_null_map_col->get_data(), 0,
- first_not_null_pos);
+ size_t prev_offset = 0;
+ for (auto cur_offset : src_offsets) {
+ // The NULLs before the first non-NULL element of each array stay NULL.
+ size_t first_not_null_pos = VectorizedUtils::find_first_valid_simd(
+ src_null_map, prev_offset, cur_offset);
+ VectorizedUtils::range_set_nullmap_to_true_simd(res_null_map_col->get_data(),
+ prev_offset, first_not_null_pos);
+ prev_offset = cur_offset;
+ }
res_nested_ptr = ColumnNullable::create(std::move(res_nested_mut_ptr),
std::move(res_null_map_col));
diff --git a/be/test/exprs/function/function_array_cum_sum_test.cpp b/be/test/exprs/function/function_array_cum_sum_test.cpp
new file mode 100644
index 00000000000..d990528d93e
--- /dev/null
+++ b/be/test/exprs/function/function_array_cum_sum_test.cpp
@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "core/block/block.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "exprs/function/function_test_util.h"
+#include "exprs/function/simple_function_factory.h"
+
+namespace doris {
+
+// Runs array_cum_sum on one block with one row per element of arrays, and returns each result
+// row as a string.
+static std::vector<std::string> run_array_cum_sum(const std::vector<TestArray>& arrays) {
+ auto array_type =
+ std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeInt32>()));
+ auto return_type =
+ std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeInt64>()));
+ const size_t row_size = arrays.size();
+ // Empty rows make a failed call fail the checks instead of reading past the end.
+ std::vector<std::string> results(row_size);
+
+ MutableColumnPtr array_column = array_type->create_column();
+ for (const auto& array : arrays) {
+ EXPECT_TRUE(insert_cell(array_column, array_type, array));
+ }
+ Block block;
+ block.insert({std::move(array_column), array_type, "array"});
+
+ DataTypePtr result_type = return_type;
+ FunctionBasePtr func = SimpleFunctionFactory::instance().get_function(
+ "array_cum_sum", block.get_columns_with_type_and_name(), result_type);
+ EXPECT_NE(func, nullptr);
+
+ FunctionUtils fn_utils(result_type, {array_type}, false);
+ auto* fn_ctx = fn_utils.get_fn_ctx();
+ fn_ctx->set_constant_cols({nullptr});
+ EXPECT_TRUE(func->open(fn_ctx, FunctionContext::FRAGMENT_LOCAL).ok());
+ EXPECT_TRUE(func->open(fn_ctx, FunctionContext::THREAD_LOCAL).ok());
+
+ block.insert({nullptr, result_type, "result"});
+ auto result_idx = block.columns() - 1;
+ auto st = func->execute(fn_ctx, block, {0}, result_idx, row_size);
+ EXPECT_TRUE(st.ok()) << st;
+ static_cast<void>(func->close(fn_ctx, FunctionContext::THREAD_LOCAL));
+ static_cast<void>(func->close(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
+ if (!st.ok()) {
+ return results;
+ }
+
+ const auto& result_column = *block.get_by_position(result_idx).column;
+ for (size_t i = 0; i < row_size; ++i) {
+ results[i] = result_type->to_string(result_column, i);
+ }
+ return results;
+}
+
+// The NULLs before the first non-NULL element of each array stay NULL, and a later NULL keeps the
+// running sum. This holds for every row of a block, so each row gives the same result in a block
+// with other rows as alone.
+TEST(function_array_cum_sum_test, leading_null_per_row) {
+ const std::vector<TestArray> arrays = {
+ {Int32(1), Int32(2)},
+ {Null(), Int32(2)},
+ {Null(), Null(), Int32(3)},
+ {Int32(5), Null(), Int32(1)},
+ {Null(), Null()},
+ {},
+ {Null(), Int32(1), Null(), Int32(2), Int32(3)},
+ };
+ auto results = run_array_cum_sum(arrays);
+ for (size_t i = 0; i < arrays.size(); ++i) {
+ EXPECT_EQ(results[i], run_array_cum_sum({arrays[i]})[0]) << "row " << i;
+ }
+ EXPECT_EQ(results[1], "[null, 2]");
+ EXPECT_EQ(results[6], "[null, 1, 1, 3, 6]");
+}
+
+} // namespace doris
diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_cum_sum_leading_null.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_cum_sum_leading_null.out
new file mode 100644
index 00000000000..a7419dbc53e
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_cum_sum_leading_null.out
@@ -0,0 +1,14 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !column --
+1 [1, 2] [1, 3]
+2 [null, 2] [null, 2]
+3 [null, null, 3] [null, null, 3]
+4 [5, null, 1] [5, 5, 6]
+5 [null, null] [null, null]
+6 [null, 1, null, 2, 3] [null, 1, 1, 3, 6]
+7 [] []
+8 \N \N
+
+-- !literal --
+[1, 3] [null, 2] [null, null, 3] [5, 5, 6] [null, null] [null, 1, 1, 3, 6]
+
diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_cum_sum_leading_null.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_cum_sum_leading_null.groovy
new file mode 100644
index 00000000000..36e5c5596e6
--- /dev/null
+++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_cum_sum_leading_null.groovy
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_array_cum_sum_leading_null") {
+ sql "DROP TABLE IF EXISTS test_array_cum_sum_leading_null"
+ sql """
+ CREATE TABLE test_array_cum_sum_leading_null (
+ id INT,
+ a ARRAY<INT> NULL
+ ) DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ // The rows are loaded together and read in one block, so every row after the first one checks
+ // that the NULLs at the start of its own array stay NULL.
+ sql """
+ INSERT INTO test_array_cum_sum_leading_null VALUES
+ (1, [1, 2]),
+ (2, [NULL, 2]),
+ (3, [NULL, NULL, 3]),
+ (4, [5, NULL, 1]),
+ (5, [NULL, NULL]),
+ (6, [NULL, 1, NULL, 2, 3]),
+ (7, []),
+ (8, NULL)
+ """
+
+ // The NULLs before the first non-NULL element of each array stay NULL, and a later NULL keeps
+ // the running sum.
+ order_qt_column "SELECT id, a, array_cum_sum(a) FROM test_array_cum_sum_leading_null"
+ // Each row gives the same result as the same array written as a literal.
+ order_qt_literal """
+ SELECT array_cum_sum([1, 2]), array_cum_sum([NULL, 2]), array_cum_sum([NULL, NULL, 3]),
+ array_cum_sum([5, NULL, 1]), array_cum_sum([NULL, NULL]),
+ array_cum_sum([NULL, 1, NULL, 2, 3])
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 11/35: branch-4.1: [fix](fe) Apply the same checks to HTTP Basic and cookie requests on /rest/v1 (#68635)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8cbfea08b3e80e6b2886a6582d3a37b81f86cbd5
Author: Calvin Kirs <gu...@selectdb.com>
AuthorDate: Tue Sep 29 21:46:39 2026 +0800
branch-4.1: [fix](fe) Apply the same checks to HTTP Basic and cookie requests on /rest/v1 (#68635)
https://github.com/apache/doris/pull/68549
---
.../doris/httpv2/controller/BaseController.java | 9 +-
.../doris/httpv2/controller/LoginController.java | 11 +-
.../controller/BaseControllerBasicAuthTest.java | 188 +++++++++++++++++++++
.../suites/auth_p0/test_http_rest_v1_auth.groovy | 90 ++++++++++
4 files changed, 295 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/BaseController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/BaseController.java
index 5a2b97e44d8..7e09ea0eaf4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/BaseController.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/BaseController.java
@@ -77,8 +77,13 @@ public class BaseController {
ActionAuthorizationInfo authInfo = getAuthorizationInfo(request);
UserIdentity currentUser = checkPassword(authInfo, request);
- if (Config.isCloudMode() && checkAuth) {
- checkInstanceOverdue(currentUser);
+ // The cookie branch below requires ADMIN_OR_NODE whenever checkAuth is set, in every deployment
+ // mode, so this branch must as well: which of the two ways a caller authenticates must not change
+ // what it is allowed to reach. Only the overdue fence is specific to cloud mode.
+ if (checkAuth) {
+ if (Config.isCloudMode()) {
+ checkInstanceOverdue(currentUser);
+ }
checkGlobalAuth(currentUser, PrivPredicate.ADMIN_OR_NODE);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LoginController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LoginController.java
index fbf0a16b02b..9d7ad312596 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LoginController.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LoginController.java
@@ -17,6 +17,9 @@
package org.apache.doris.httpv2.controller;
+import org.apache.doris.common.Config;
+import org.apache.doris.qe.ConnectContext;
+
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.web.bind.annotation.RequestMapping;
@@ -32,7 +35,13 @@ public class LoginController extends BaseController {
@RequestMapping(path = "/login", method = RequestMethod.POST)
public Object login(HttpServletRequest request, HttpServletResponse response) {
- checkAuthWithCookie(request, response);
+ // Login only establishes who the caller is. What the account may reach is decided on each /rest/v1 request
+ // that follows, where the session issued here is checked for the required privilege; this lets the UI
+ // tell an account that lacks it apart from a failed sign-in.
+ checkWithCookie(request, response, false);
+ if (Config.isCloudMode()) {
+ checkInstanceOverdue(ConnectContext.get().getCurrentUserIdentity());
+ }
Map<String, Object> msg = new HashMap<>();
msg.put("code", 200);
msg.put("msg", "Login success!");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/httpv2/controller/BaseControllerBasicAuthTest.java b/fe/fe-core/src/test/java/org/apache/doris/httpv2/controller/BaseControllerBasicAuthTest.java
new file mode 100644
index 00000000000..7febfcff270
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/httpv2/controller/BaseControllerBasicAuthTest.java
@@ -0,0 +1,188 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.httpv2.controller;
+
+import org.apache.doris.analysis.UserIdentity;
+import org.apache.doris.common.Config;
+import org.apache.doris.httpv2.HttpAuthManager.SessionValue;
+import org.apache.doris.httpv2.controller.BaseController.ActionAuthorizationInfo;
+import org.apache.doris.httpv2.exception.UnauthorizedException;
+import org.apache.doris.httpv2.interceptor.AuthInterceptor;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.qe.ConnectContext;
+
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.Proxy;
+import java.util.Map;
+
+/**
+ * HTTP Basic requests to the /rest/v1 surface, outside cloud mode.
+ */
+class BaseControllerBasicAuthTest {
+ private final UserIdentity analyst = UserIdentity.createAnalyzedUserIdentWithIp("analyst", "%");
+
+ private PrivPredicate checkedPredicate;
+ private SessionValue issuedSession;
+
+ @BeforeEach
+ void setUp() {
+ Assertions.assertFalse(Config.isCloudMode());
+ checkedPredicate = null;
+ issuedSession = null;
+ }
+
+ @AfterEach
+ void tearDown() {
+ ConnectContext.remove();
+ }
+
+ @Test
+ void rejectsBasicAuthenticationWithoutPrivilege() {
+ AuthInterceptor interceptor = interceptor(false);
+
+ Assertions.assertThrows(UnauthorizedException.class,
+ () -> interceptor.preHandle(request("/rest/v1/system"), response(), new Object()));
+ Assertions.assertEquals(PrivPredicate.ADMIN_OR_NODE, checkedPredicate);
+ Assertions.assertNull(issuedSession);
+ }
+
+ @Test
+ void acceptsBasicAuthenticationWithPrivilege() {
+ AuthInterceptor interceptor = interceptor(true);
+
+ Assertions.assertTrue(interceptor.preHandle(request("/rest/v1/system"), response(), new Object()));
+ Assertions.assertEquals(PrivPredicate.ADMIN_OR_NODE, checkedPredicate);
+ Assertions.assertEquals(analyst, issuedSession.currentUser);
+ }
+
+ @Test
+ void loginAuthenticatesAnAccountWithoutPrivilege() {
+ // The UI tells an account that lacks the privilege apart from a failed sign-in, so login itself only
+ // authenticates; the privilege is checked on the requests that follow.
+ LoginController controller = new LoginController() {
+ @Override
+ public ActionAuthorizationInfo getAuthorizationInfo(HttpServletRequest request) {
+ return authorizationInfo();
+ }
+
+ @Override
+ protected UserIdentity checkPassword(ActionAuthorizationInfo authInfo, HttpServletRequest request) {
+ return analyst;
+ }
+
+ @Override
+ protected void checkGlobalAuth(UserIdentity currentUser, PrivPredicate predicate) {
+ checkedPredicate = predicate;
+ throw new UnauthorizedException("Access denied");
+ }
+
+ @Override
+ protected void addSession(HttpServletRequest request, HttpServletResponse response,
+ SessionValue value) {
+ issuedSession = value;
+ }
+ };
+
+ @SuppressWarnings("unchecked")
+ Map<String, Object> result = (Map<String, Object>) controller.login(request("/rest/v1/login"), response());
+ Assertions.assertEquals(200, result.get("code"));
+ Assertions.assertNull(checkedPredicate);
+ Assertions.assertEquals(analyst, issuedSession.currentUser);
+ }
+
+ private AuthInterceptor interceptor(boolean privileged) {
+ return new AuthInterceptor() {
+ @Override
+ public ActionAuthorizationInfo getAuthorizationInfo(HttpServletRequest request) {
+ return authorizationInfo();
+ }
+
+ @Override
+ protected UserIdentity checkPassword(ActionAuthorizationInfo authInfo, HttpServletRequest request) {
+ return analyst;
+ }
+
+ @Override
+ protected void checkGlobalAuth(UserIdentity currentUser, PrivPredicate predicate) {
+ checkedPredicate = predicate;
+ Assertions.assertEquals(analyst, currentUser);
+ if (!privileged) {
+ throw new UnauthorizedException("Access denied");
+ }
+ }
+
+ @Override
+ protected void addSession(HttpServletRequest request, HttpServletResponse response,
+ SessionValue value) {
+ issuedSession = value;
+ }
+ };
+ }
+
+ private ActionAuthorizationInfo authorizationInfo() {
+ ActionAuthorizationInfo authInfo = new ActionAuthorizationInfo();
+ authInfo.fullUserName = analyst.getQualifiedUser();
+ authInfo.password = "secret";
+ authInfo.remoteIp = "127.0.0.1";
+ return authInfo;
+ }
+
+ private HttpServletRequest request(String requestUri) {
+ return (HttpServletRequest) Proxy.newProxyInstance(
+ HttpServletRequest.class.getClassLoader(),
+ new Class<?>[] {HttpServletRequest.class},
+ (proxy, method, args) -> {
+ switch (method.getName()) {
+ case "getMethod":
+ return "GET";
+ case "getRequestURI":
+ return requestUri;
+ case "getHeader":
+ return "Authorization".equals(args[0]) ? "Basic ignored-by-test" : null;
+ default:
+ return defaultValue(method.getReturnType());
+ }
+ });
+ }
+
+ private HttpServletResponse response() {
+ return (HttpServletResponse) Proxy.newProxyInstance(
+ HttpServletResponse.class.getClassLoader(),
+ new Class<?>[] {HttpServletResponse.class},
+ (proxy, method, args) -> defaultValue(method.getReturnType()));
+ }
+
+ private Object defaultValue(Class<?> returnType) {
+ if (!returnType.isPrimitive()) {
+ return null;
+ }
+ if (returnType == boolean.class) {
+ return false;
+ }
+ if (returnType == char.class) {
+ return '\0';
+ }
+ return 0;
+ }
+}
diff --git a/regression-test/suites/auth_p0/test_http_rest_v1_auth.groovy b/regression-test/suites/auth_p0/test_http_rest_v1_auth.groovy
new file mode 100644
index 00000000000..f6cdd0444c8
--- /dev/null
+++ b/regression-test/suites/auth_p0/test_http_rest_v1_auth.groovy
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// The /rest/v1 endpoints behind the Web UI require ADMIN_OR_NODE. HTTP Basic requests are checked for it
+// just as session-cookie requests are, in every deployment mode. /rest/v1/login itself only authenticates,
+// so the UI can tell an account without the privilege apart from a failed sign-in.
+suite("test_http_rest_v1_auth", "p0,auth") {
+ String user = "test_http_rest_v1_auth_user"
+ String pwd = 'C123_567p'
+ try_sql("DROP USER ${user}")
+ sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
+
+ def uris = [
+ "/rest/v1/system?path=/",
+ "/rest/v1/config/fe",
+ "/rest/v1/session",
+ "/rest/v1/query_profile",
+ "/rest/v1/log",
+ "/rest/v1/ha"
+ ]
+
+ def getRestV1 = { uriPath, checkFunc ->
+ httpTest {
+ basicAuthorization "${user}", "${pwd}"
+ endpoint "${context.config.feHttpAddress}"
+ uri uriPath
+ op "get"
+ check checkFunc
+ }
+ }
+
+ def login = { checkFunc ->
+ httpTest {
+ basicAuthorization "${user}", "${pwd}"
+ endpoint "${context.config.feHttpAddress}"
+ uri "/rest/v1/login"
+ op "post"
+ body "{}"
+ check checkFunc
+ }
+ }
+
+ uris.each { uriPath ->
+ getRestV1.call(uriPath) {
+ respCode, body ->
+ log.info("${uriPath} (no privilege) body:${body}")
+ assertEquals(200, respCode)
+ assertEquals(401, parseJson(body).code)
+ }
+ }
+
+ login.call {
+ respCode, body ->
+ log.info("login (no privilege) body:${body}")
+ assertEquals(200, respCode)
+ assertEquals(200, parseJson(body).code)
+ }
+
+ sql """GRANT 'admin' TO '${user}'"""
+
+ uris.each { uriPath ->
+ getRestV1.call(uriPath) {
+ respCode, body ->
+ log.info("${uriPath} (admin) body:${body}")
+ assertEquals(200, respCode)
+ assertEquals(0, parseJson(body).code)
+ }
+ }
+
+ login.call {
+ respCode, body ->
+ log.info("login (admin) body:${body}")
+ assertEquals(200, respCode)
+ assertEquals(200, parseJson(body).code)
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/35: [feature](lance) Support time travel, tags, branches and REST Namespace managed versioning (#68453)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c5ebba21dfdf1c8b49913cd682eef93280384e44
Author: zy-kkk <zh...@gmail.com>
AuthorDate: Tue Sep 29 14:53:39 2026 +0800
[feature](lance) Support time travel, tags, branches and REST Namespace managed versioning (#68453)
### What problem does this PR solve?
Issue Number: close #66491
Related PR: #65730 (baseline Lance read integration, which already
carried `FOR VERSION AS OF` / `FOR TIME AS OF` on the FE without tests
or documentation)
Problem Summary:
Lance catalogs could select a dataset version on the FE, but nothing
proved it end to end, the error messages leaked SDK internals
(`NumberFormatException: For input string`, storage paths and Rust
source locations), and tables whose versions are managed by a REST
Namespace (`DescribeTable` returns `managed_versioning = true`) were
rejected outright with "not supported by the current BE reader".
This PR completes the four scope items of the issue:
1. **Time travel** – `FOR VERSION AS OF <n>` and `FOR TIME AS OF
'<timestamp>'` are covered by regression suites on a filesystem catalog
and a REST catalog, with a committed three-version fixture
(`time_travel.lance`) and the existing `multi_frag.lance` (three
appends, then a delete) so that reading an older version across a
deletion file is exercised on the BE.
The semantics follow the Iceberg and Paimon integrations: the selected
version is the one committed last at or before the timestamp (of
versions committed at the same instant, the newer one), with the
timestamp parsed in the session time zone with second or millisecond
precision and compared with the full commit time, which Lance records
below a millisecond, so a commit later within the requested millisecond
is not selected; commit times are compared as recorded, without assuming
they grow with version numbers. As Iceberg drops its snapshot log before
a removed snapshot, only the versions newer than the most recent one
removed by cleanup are candidates, so a timestamp earlier than all of
them (before the first version, or in removed history) is an error.
2. **REST Namespace managed versioning** – the namespace decides which
versions of such a table exist and which is the newest: the latest
version comes from `ListTableVersions`, an explicit version is checked
with `DescribeTableVersion`, and `FOR TIME AS OF` selects among the
versions the namespace lists. The FE then reads the chosen version the
way it reads any Lance dataset, and the way the BE does through lance-c:
by the location and storage options of the table's `DescribeTable` and
the version number. No BE or thrift change is needed. `EXPLAIN` shows
`lanceManagedVersioning=true` for these tables.
3. **Selected version fixed for the whole statement** – schema binding,
fragment planning, predicate pushdown, metadata COUNT and the BE scan
all use the selected version; two references to the same table in one
statement can select different versions (self join / union tests).
4. **Tags and branches** – `tbl@tag(name)` resolves the tag to the
branch and version it points at, read from the dataset's `_refs/tags/`
for both storage-versioned and managed tables, and only that tag's file
is read (Lance's namespace tag APIs expose the same refs; the selected
version of a managed table is then resolved through the namespace); a
tag created on a branch selects that branch's version, not the same
number on `main`. As for Paimon, and for tag names in Iceberg, a
non-numeric `FOR VERSION AS OF 'v2'` names a tag; a tag name is never
combined with `@branch`, `@branch(main)` included, because the tag
already determines its branch. `@branch(main)` is `main`; `@tag` /
`@branch` take exactly one name and reject extra parameters. Inside a
branch, versions and times select among the branch's own versions, which
start at the version it was created from. `tbl@branch(name)` opens the
branch under `<table>/tree/<name>/`, optionally with `FOR VERSION AS OF`
/ `FOR TIME AS OF` inside the branch; the BE receives the branch
directory as the dataset URI and needs no change (a branch is a shallow
clone with its own `_versions/`). For managed tables Doris sends the
branch in the `branch` field of the version APIs. Previously `@tag` /
`@branch` were silently ignored and the query read the latest version of
`main`.
5. **Errors** – a missing version (never committed, or removed by
`cleanup_old_versions`) is reported as `Lance version N of db.tbl[ (tag
'x')] was not found[ in the namespace]` without the storage path; a
missing tag or branch, an out-of-range or non-positive version (a signed
number is a version, not a tag name), a timestamp before the first
version and an unparsable timestamp have their own messages; a namespace
that lists no versions is reported as such; a table that is only
declared in the namespace fails with "declared in the namespace but has
no data yet"; for a managed table, a namespace `location` that is empty
or names another place than `table_uri` (a query string aside), or an
`s3+ddb` URI, whose DynamoDB commit handler would record and finalize
versions on read, fails the query before anything is read; every other
failure keeps the sanitized provider message.
Design decisions worth calling out:
- For a managed table the namespace is the source of truth. A version
whose manifest is still in storage but which the namespace does not
record is reported as not found, and the table's latest version is the
namespace's latest even when storage already holds a newer manifest.
Reading unrecorded manifests would expose staged or rolled-back commits.
- `FOR TIME AS OF` uses the commit times the manifests record, for
storage-versioned and managed tables alike, as Lance's `asof` does; a
managed table only selects among the versions its namespace lists. The
namespace's `timestamp_millis` is not used: it is optional in the spec
and its meaning varies (Lance's Directory namespace reports the manifest
object's modification time, second-granular on S3 and reset when a
dataset is copied), so the same dataset would otherwise resolve
differently depending on how it is catalogued. A removed version takes
its commit time with it, which is why history before it is not used. On
a managed chain, a version number the namespace no longer records
between versions it records is a removed version too (Lance numbers a
chain's commits consecutively, and Lance's Directory namespace only
accepts the next version), and so is a recorded version whose manifest
is not in storage.
- For a managed table every read of the latest version (a plain query,
the search functions, a branch head) is pinned to the newest version the
namespace records, asked for the way Lance's own reader asks
(`ListTableVersions`, newest first, one entry); an empty history is an
error. Opening the storage latest would expose a version the namespace
never published.
- The FE and the BE read a managed table the same way, by URI and
version number, rather than the FE opening it through the SDK's
namespace client (`Dataset.open().namespaceClient(..)`). The BE's
lance-c can only open by URI and version, and with the FE in namespace
mode the two readers resolved the store, the credentials and the
manifest differently; earlier rounds of this PR kept adding code to
align them (a wrapper around the namespace client, store identities,
reconciliation of a second describe). Reading both sides by URI removes
that class of mismatch. The namespace still decides the versions: Doris
asks it directly for every managed read, and a managed table's access is
never cached, so each read uses the location and options of its own
`DescribeTable`, which the BE receives unchanged. The table describe and
the version requests run outside the catalog's namespace lock, which
keeps only the namespace and table listings: the REST and Directory
namespaces are safe to call concurrently, a managed table makes these
requests on every read, and Lance's REST client sets no request timeout,
so a stalled request must not hold up every other table.
- The BE reads a version by number at its canonical manifest, `<table
root>[/tree/<branch>]/_versions/<u64::MAX - v>.manifest` (or the V1
`<v>.manifest`). Doris checks that the manifest the namespace records
for the selected version is that path, or a staged manifest beside it
(`<canonical>-<id>`, which a namespace that records the path Lance's
client sends in `CreateTableVersion` keeps recording after the commit is
finalized, since Lance's namespace store cannot update a record). Any
other path means the namespace names a manifest neither reader would
open (a manifest recorded elsewhere, or a table moved between the
describe and the version lookup), and the read fails with both paths in
the message and a hint to retry. `FOR TIME AS OF` checks every version
the namespace records on the chain, since the commit times of those
manifests decide the selection. The table root is derived from the
location as lance-io derives it.
- A version the namespace records only at a staged manifest whose
canonical manifest does not exist (a commit that was never finalized, or
whose manifest cleanup removed) cannot be read: the query fails and says
so, instead of falling back to an older version, and Doris writes
nothing on read. This is how Doris's Iceberg and Paimon connectors treat
a snapshot their metadata names but they cannot load. Lance's own
namespaces finalize a commit inside `CreateTableVersion`, before they
list it, so they do not produce this state for committed versions.
- `FOR VERSION AS OF` on a dataset whose directory is missing altogether
reports "version N was not found", because the SDK probes the version's
manifest path first; the provider's own message is kept in the exception
cause, handled the same way as every other Lance error.
- `org.lance:lance-core` is bumped from 11.0.0 to 12.0.0 (its
`lance-namespace` client from 0.7.7 to 0.11.1). The bump is for the
namespace client, whose `branch` field on the version requests is what
managed-table branches need; `Dataset.checkout(Ref)` already existed.
The FE code compiled unchanged and every Lance unit test class and
regression suite passes on it. FE packaging replaces the Linux x86_64
JNI library in that JAR with a glibc 2.17 rebuild, so the pins in
`docker/thirdparties/lance-jni-helpers.sh` move to the 12.0.0 rebuild
published as
[lance-jni-12.0.0-glibc2.17-r1](https://github.com/apache/doris-thirdparty/releases/tag/lance-jni-12.0.0-glibc2.17-r1)
(built from unmodified Lance v12.0.0 with the procedure of the 11.0.0
asset; its 288 JNI exports match the upstream library, and the Lance v12
Java test suite passes with it on CentOS 7). The BE stays on lance-c
0.1.9 (Lance 11.0.0): the FE only reads metadata and never writes a
dataset, so the newer FE release does not change what the BE can read.
- On a storage-versioned table, an expired version (removed by cleanup)
is indistinguishable from one that never existed, so both get the same
message. On a managed table the namespace's record tells them apart: a
version it does not record is "not found in the namespace", one it
records at its canonical path but storage lacks is "not found", and one
recorded only at a staged manifest "cannot be read".
- A managed table's branch is read from its own directory
(`<table>/tree/<branch>`, joined before any query string as Lance joins
it) by URI and version, as the BE reads it, so it needs neither a
recorded `main` version nor a readable `main` manifest. Every other
selector except an explicit version on `main` and the latest `main` is a
checkout from one open of the latest `main` in storage; for a managed
table that open only supplies the tag files and the manifest listing,
and the version finally read is always one the namespace records,
checked as above. The namespace version list is fetched completely, once
per read, without a page size: Lance's Directory namespace applies a
limit without returning a page token, and neither the order nor
monotonic commit times can be relied on to stop early.
- Known limitations: a version recorded at a staged manifest is read
from its canonical manifest when one exists; a namespace that lets one
version number be committed twice (deleting records while the earlier
commit's canonical manifest stays in storage) is not detected. While the
newest version of a managed table is not finalized, reading the table
without `FOR VERSION AS OF` fails, and so does loading its schema. Two
references to the same branch in one statement resolve the branch head
independently, and the optimizer's row count is that of the latest
`main` (both shared with the Iceberg integration). `vector_search()`,
`full_text_search()` and index inspection use the latest `main` and
reject a version, tag or branch in their `table` argument. Index paths
open by URI and are only reachable for filesystem catalogs, whose
Directory namespace never manages versions; an assertion guards that. On
a storage-versioned table, a branch whose metadata was deleted, or whose
creation stopped partway, stays readable through `@branch` until its
directory is cleaned up, because the Lance SDK checks a branch out by
its directory; on a managed table the namespace decides whether the
branch exists (a missing table reported by Lance's Directory namespace
for a branch request counts as the missing branch).
- A Lance branch is a shallow clone whose manifests record the parent's
location as an absolute URI (`Manifest.base_paths`), so a branch is
readable only where it was created. The committed `time_travel.lance`
branch was therefore created against the fixture's final location
`s3://warehouse/lance/time_travel.lance` (`lance_build_time_travel.py
--create-branch`) and synced back; the fixture's self-check verifies
that reference.
Examples (mysql client against a 4.1 FE; `time_travel.lance` has
versions 1..3 committed at 13:06:07.597 / 13:06:09.113 / 13:06:10.621
UTC with rows 1..3 / 4..6 / 7..9):
```sql
mysql> SET enable_file_scanner_v2 = true;
mysql> SET time_zone = 'UTC';
mysql> CREATE CATALOG lance_demo_fs PROPERTIES (
-> "type" = "lance",
-> "lance.catalog.type" = "filesystem",
-> "warehouse" = "s3://warehouse/lance",
-> "s3.endpoint" = "http://127.0.0.1:19000",
-> "s3.access_key" = "admin",
-> "s3.secret_key" = "password",
-> "s3.region" = "us-east-1",
-> "use_path_style" = "true"
-> );
mysql> SELECT * FROM lance_demo_fs.`default`.time_travel FOR VERSION AS OF 1 ORDER BY row_id;
+--------+------+
| row_id | tag |
+--------+------+
| 1 | v1 |
| 2 | v1 |
| 3 | v1 |
+--------+------+
mysql> SELECT count(*), max(row_id) FROM lance_demo_fs.`default`.time_travel FOR TIME AS OF '2026-09-19 13:06:10';
+----------+-------------+
| count(*) | max(row_id) |
+----------+-------------+
| 6 | 6 |
+----------+-------------+
mysql> SELECT a.row_id, a.tag AS tag_v1, b.tag AS tag_v3
-> FROM lance_demo_fs.`default`.time_travel FOR VERSION AS OF 1 a
-> JOIN lance_demo_fs.`default`.time_travel FOR VERSION AS OF 3 b ON a.row_id = b.row_id
-> ORDER BY a.row_id;
+--------+--------+--------+
| row_id | tag_v1 | tag_v3 |
+--------+--------+--------+
| 1 | v1 | v1 |
| 2 | v1 | v1 |
| 3 | v1 | v1 |
+--------+--------+--------+
mysql> SELECT count(*) FROM lance_demo_fs.`default`.time_travel FOR VERSION AS OF 99;
ERROR 1105 (HY000): errCode = 2, detailMessage = Lance version 99 of default.time_travel was not found
mysql> SELECT count(*) FROM lance_demo_fs.`default`.time_travel FOR TIME AS OF '2026-09-19 13:06:07';
ERROR 1105 (HY000): errCode = 2, detailMessage = Lance table default.time_travel has no version at or before '2026-09-19 13:06:07'
```
A REST Namespace with managed versioning. `time_travel_managed` records
versions 1..3, `time_travel_managed_lagging` records only 1..2 while
storage already holds version 3, `time_travel_managed_partial` records 1
and 3, and `time_travel_managed_pending` records a fourth version only
at a staged manifest that was never finalized:
```sql
mysql> CREATE CATALOG lance_demo_rest PROPERTIES (
-> "type" = "lance",
-> "lance.catalog.type" = "rest",
-> "lance.rest.uri" = "http://127.0.0.1:19102",
-> "lance.rest.security.type" = "bearer",
-> "lance.rest.bearer-token" = "doris-lance-rest-test-token",
-> "lance.namespace.root_database" = "default",
-> "s3.endpoint" = "http://127.0.0.1:19000",
-> "s3.region" = "us-east-1",
-> "use_path_style" = "true"
-> );
mysql> SELECT count(*), max(row_id) FROM lance_demo_rest.`default`.time_travel_managed FOR VERSION AS OF 2;
+----------+-------------+
| count(*) | max(row_id) |
+----------+-------------+
| 6 | 6 |
+----------+-------------+
mysql> SELECT count(*), max(row_id) FROM lance_demo_rest.`default`.time_travel_managed_lagging;
+----------+-------------+
| count(*) | max(row_id) |
+----------+-------------+
| 6 | 6 |
+----------+-------------+
mysql> SELECT count(*) FROM lance_demo_rest.`default`.time_travel_managed_lagging FOR VERSION AS OF 3;
ERROR 1105 (HY000): errCode = 2, detailMessage = Lance version 3 of default.time_travel_managed_lagging was not found in the namespace
mysql> SELECT count(*) FROM lance_demo_rest.`default`.time_travel_managed_partial FOR TIME AS OF '2026-09-19 13:06:10';
ERROR 1105 (HY000): errCode = 2, detailMessage = Lance cannot resolve FOR TIME AS OF '2026-09-19 13:06:10' on default.time_travel_managed_partial: version 2, which may hold the state at that time, no longer exists
mysql> SELECT count(*) FROM lance_demo_rest.`default`.time_travel_managed_pending;
ERROR 1105 (HY000): errCode = 2, detailMessage = Lance version 4 of default.time_travel_managed_pending cannot be read: the namespace records it at a staged manifest, and its canonical manifest, which Doris reads, does not exist (its commit was not finalized, or cleanup removed it)
mysql> SELECT count(*), max(row_id) FROM lance_demo_rest.`default`.time_travel_managed_pending FOR VERSION AS OF 3;
+----------+-------------+
| count(*) | max(row_id) |
+----------+-------------+
| 9 | 9 |
+----------+-------------+
mysql> EXPLAIN SELECT row_id FROM lance_demo_rest.`default`.time_travel_managed_lagging;
...
0:VLANCE_SCAN_NODE(44)
table: lance_demo_rest.default.time_travel_managed_lagging
lanceCatalogType=rest
lanceVersion=2
lanceManagedVersioning=true
lanceFragments=2
lanceFragmentGrouping=FRAGMENT
```
Tags and branches (`time_travel.lance` carries tags `v1`..`v3` and a
branch `dev` forked from version 2 with one extra row; on
`time_travel_managed_partial` the tag `v2` points at the version the
namespace no longer records):
```sql
mysql> SELECT * FROM lance_demo_fs.`default`.time_travel@tag(v2) ORDER BY row_id;
+--------+------+
| row_id | tag |
+--------+------+
| 1 | v1 |
| 2 | v1 |
| 3 | v1 |
| 4 | v2 |
| 5 | v2 |
| 6 | v2 |
+--------+------+
mysql> SELECT * FROM lance_demo_fs.`default`.time_travel@branch(dev) ORDER BY row_id;
+--------+------+
| row_id | tag |
+--------+------+
| 1 | v1 |
| 2 | v1 |
| 3 | v1 |
| 4 | v2 |
| 5 | v2 |
| 6 | v2 |
| 100 | dev |
+--------+------+
mysql> SELECT count(*), max(row_id) FROM lance_demo_fs.`default`.time_travel@branch(dev) FOR VERSION AS OF 2;
+----------+-------------+
| count(*) | max(row_id) |
+----------+-------------+
| 6 | 6 |
+----------+-------------+
mysql> SELECT count(*) FROM lance_demo_fs.`default`.time_travel@tag(no_such_tag);
ERROR 1105 (HY000): errCode = 2, detailMessage = Lance tag 'no_such_tag' of default.time_travel was not found
mysql> SELECT count(*) FROM lance_demo_fs.`default`.time_travel@branch(nope);
ERROR 1105 (HY000): errCode = 2, detailMessage = Lance branch 'nope' of default.time_travel was not found
mysql> SELECT count(*), max(row_id) FROM lance_demo_rest.`default`.time_travel_managed@branch(dev);
+----------+-------------+
| count(*) | max(row_id) |
+----------+-------------+
| 7 | 100 |
+----------+-------------+
mysql> SELECT count(*), max(row_id) FROM lance_demo_rest.`default`.time_travel_managed@tag(v1);
+----------+-------------+
| count(*) | max(row_id) |
+----------+-------------+
| 3 | 3 |
+----------+-------------+
mysql> SELECT count(*) FROM lance_demo_rest.`default`.time_travel_managed_partial@tag(v2);
ERROR 1105 (HY000): errCode = 2, detailMessage = Lance version 2 of default.time_travel_managed_partial (tag 'v2') was not found in the namespace
mysql> EXPLAIN SELECT row_id FROM lance_demo_fs.`default`.time_travel@branch(dev);
...
0:VLANCE_SCAN_NODE(44)
table: lance_demo_fs.default.time_travel
lanceCatalogType=filesystem
lanceVersion=3
lanceManagedVersioning=false
lanceBranch=dev
lanceFragments=3
lanceFragmentGrouping=FRAGMENT
```
Test environment changes: the Lance REST fixture
(`lance_rest_server.py`) gains `ListTableVersions` /
`DescribeTableVersion` for tables listed in
`LANCE_REST_MANAGED_TABLES_JSON`; `time_travel.lance` is a committed
three-version dataset with tags and a branch, carried over as-is by
`lance_build_preinstalled_catalog.py` because the suites hard-code its
commit times (`lance_build_time_travel.py` regenerates it and prints the
times to update; `--create-branch` forks the branch at the uploaded
location). The stub additionally serves the `branch` parameter of the
version APIs, and can record a version at a staged manifest (`"staged":
true`) or at any path (`"manifest_path"`), which the `_pending`,
`_staged` and `_misrecorded` fixture tables use.
### Release note
Lance catalogs support `FOR VERSION AS OF`, `FOR TIME AS OF`,
`@tag(name)` and `@branch(name)`, including tables whose versions are
managed by a REST Namespace (`managed_versioning = true`). Missing
versions, tags and branches and invalid selectors now report clear
errors.
### Check List (For Author)
- Test: Regression test / Unit Test
- `external_table_p0/lance/test_lance_time_travel`,
`test_lance_rest_time_travel`, `test_lance_rest_catalog` (Lance REST
fixture + MinIO)
- `LanceManagedVersioningTest` (real SDK dataset behind a REST stub:
managed / partial / lagging / untimed / growing / server error /
location checks / declared-only / cleaned-up version / tags / branches /
paging / empty version list / relocated table / table moved after it was
described / version recorded at a staged manifest, finalized or not, on
`main` and on a branch, newest or older / finalized manifest outside its
canonical path / the BE receiving the options of the read's describe /
history gaps / namespace times that differ from manifest times), plus
`test_lance_vector_search` and `test_lance_full_text_search` re-run on
lance-core 12.0.0, `LanceSnapshotTest` (including sub-millisecond commit
times, and a check of time selection against its rule on random
histories with microsecond commit times and versions a namespace records
but storage lacks), `LanceManifestPathsTest` (canonical and staged paths
in both naming schemes, branches, rejected paths, a non-ASCII-digit
locale, location paths as lance-io derives them),
`LanceTableAccessCacheTest` (including a managed table described on
every read), `LanceRestCatalogTest`, `LanceScanNodeTest`,
`LanceMetadataLoaderTest`, `LanceCatalogLifecycleTest`
- Behavior changed: Yes. Tables with `managed_versioning = true` were
rejected and are now readable; error messages for invalid or missing
versions changed to the user-facing form above; `@tag` / `@branch` on a
Lance table now select the tag or branch (including a tag that points
into a branch) instead of silently reading the latest version of `main`;
FE dependency `lance-core` 11.0.0 → 12.0.0.
- Does this need documentation: Yes (apache/doris-website#4172: feature
table, REST catalog caution, new "Time Travel" section with the
managed-versioning rules and limitations)
---
.../docker-compose/iceberg/iceberg.yaml.tpl | 24 +-
.../scripts/lance_build_preinstalled_catalog.py | 16 +-
.../iceberg/scripts/lance_build_time_travel.py | 178 +++
.../iceberg/scripts/lance_fixture_requirements.txt | 7 +-
.../iceberg/scripts/lance_rest_server.py | 198 ++-
.../time_travel.lance/_refs/branches/dev.json | 15 +
.../lance/time_travel.lance/_refs/tags/rel.json | 8 +
.../lance/time_travel.lance/_refs/tags/v1.json | 8 +
.../lance/time_travel.lance/_refs/tags/v2.json | 8 +
.../lance/time_travel.lance/_refs/tags/v3.json | 8 +
.../0-c7a3192f-4b4a-4e8b-b3a3-b736534c10f3.txn | Bin 0 -> 185 bytes
.../1-79502d42-ca15-43d2-99e3-8c734698ae2b.txn | Bin 0 -> 122 bytes
.../2-e678f5d6-e017-494d-adaf-fd1c0e245182.txn | Bin 0 -> 122 bytes
.../_versions/18446744073709551612.manifest | Bin 0 -> 546 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 464 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 447 bytes
.../_versions/latest_version_hint.json | 1 +
...11001001000111077951d4cb09c0a747e1eddc25e.lance | Bin 0 -> 557 bytes
...010000110100100bcc907454fa3151c2176cb1bc4.lance | Bin 0 -> 557 bytes
...1100100111000010ab68a492fa5d2c0fa972154e5.lance | Bin 0 -> 557 bytes
.../2-ca6009ab-1db7-4777-9e3e-a9374ed947b9.txn | Bin 0 -> 122 bytes
.../2-d76280f6-7616-446b-9575-5b8ec5ea2df1.txn | 1 +
.../dev/_versions/18446744073709551612.manifest | Bin 0 -> 605 bytes
.../dev/_versions/18446744073709551613.manifest | Bin 0 -> 494 bytes
...0110110000010015e5eaf41b5892dff0697f4cbcd.lance | Bin 0 -> 414 bytes
docker/thirdparties/lance-jni-helpers.sh | 10 +-
docs/lance-jni-packaging.md | 6 +-
.../doris/datasource/lance/LanceCatalogClient.java | 556 +++++++-
.../datasource/lance/LanceExternalCatalog.java | 7 +-
.../doris/datasource/lance/LanceExternalTable.java | 75 +-
.../doris/datasource/lance/LanceManifestPaths.java | 141 ++
.../datasource/lance/LanceNamespaceClient.java | 154 ++-
.../lance/metadata/LanceRefSelector.java | 75 +
.../lance/metadata/LanceSnapshotResolver.java | 141 +-
.../lance/metadata/LanceTableAccess.java | 60 +-
.../lance/metadata/LanceTableMetadata.java | 14 +
.../datasource/lance/source/LanceScanNode.java | 4 +
.../LanceExternalSearchTableValuedFunction.java | 20 +-
.../datasource/lance/LanceManagedAccessTest.java | 168 +++
.../lance/LanceManagedVersioningTest.java | 1460 ++++++++++++++++++++
.../datasource/lance/LanceManifestPathsTest.java | 120 ++
.../datasource/lance/LanceSharedSessionTest.java | 11 +-
.../lance/LanceTableAccessCacheTest.java | 17 +
.../lance/metadata/LanceSnapshotTest.java | 211 +++
.../VectorSearchTableValuedFunctionTest.java | 18 +
fe/pom.xml | 2 +-
.../lance/test_lance_rest_catalog.out | 10 +
.../lance/test_lance_rest_time_travel.out | 134 ++
.../lance/test_lance_time_travel.out | 141 ++
.../lance/test_lance_rest_catalog.groovy | 4 +-
.../lance/test_lance_rest_time_travel.groovy | 294 ++++
.../lance/test_lance_time_travel.groovy | 267 ++++
52 files changed, 4495 insertions(+), 97 deletions(-)
diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
index 066d45ff5a0..098a1cf4eed 100644
--- a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
+++ b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
@@ -118,10 +118,30 @@ services:
- ./scripts/lance_rest_server.py:/opt/lance-rest/server.py:ro
environment:
LANCE_REST_BEARER_TOKEN: doris-lance-rest-test-token
- LANCE_REST_TABLES_JSON: '{"all_types":"s3://warehouse/lance/all_types.lance","all_types_unprefixed":"s3://warehouse/lance/all_types.lance"}'
+ LANCE_REST_TABLES_JSON: '{"all_types":"s3://warehouse/lance/all_types.lance","all_types_unprefixed":"s3://warehouse/lance/all_types.lance","time_travel":"s3://warehouse/lance/time_travel.lance"}'
# all_types_unprefixed serves the same dataset but vends its credentials under the
# unprefixed object-store spelling, which is what real namespace servers emit.
- LANCE_REST_UNPREFIXED_TABLES_JSON: '["all_types_unprefixed"]'
+ LANCE_REST_UNPREFIXED_TABLES_JSON: '["all_types_unprefixed","time_travel_managed_unprefixed"]'
+ # All managed tables are the same three-version dataset; they differ in what the namespace
+ # records. A reader that resolves versions through the namespace cannot see a version
+ # missing there even though its manifest is still in storage, and its latest version is
+ # the namespace's latest, not storage's:
+ # time_travel_managed every version, with commit times, and branch dev
+ # (versions 2 and 3 under tree/dev/)
+ # time_travel_managed_partial versions 1 and 3
+ # time_travel_managed_lagging versions 1 and 2, storage already has 3
+ # time_travel_managed_untimed every version, no commit times reported
+ # time_travel_managed_unprefixed every version, credentials vended unprefixed
+ # time_travel_managed_pending every version, and version 4 (main and dev) recorded at a
+ # staged manifest whose commit never finalized
+ # time_travel_managed_staged every version, version 3 recorded at its staged manifest
+ # although its canonical one exists
+ # time_travel_managed_misrecorded every version, version 3 recorded at a manifest Doris
+ # does not read
+ # Versions and commit times (epoch millis, UTC) match the committed time_travel.lance,
+ # see lance_build_time_travel.py. Doris resolves FOR TIME AS OF from the manifests' commit
+ # times, so the reported ones are not what the time-travel results depend on.
+ LANCE_REST_MANAGED_TABLES_JSON: '{"time_travel_managed":{"uri":"s3://warehouse/lance/time_travel.lance","versions":[{"version":1,"timestamp_millis":1789823167597},{"version":2,"timestamp_millis":1789823169113},{"version":3,"timestamp_millis":1789823170621}],"branches":{"dev":{"versions":[2,3]}}},"time_travel_managed_partial":{"uri":"s3://warehouse/lance/time_travel.lance","versions":[{"version":1,"timestamp_millis":1789823167597},{"version":3,"timestamp_millis":1789823170621}]},"ti [...]
LANCE_S3_ACCESS_KEY: admin
LANCE_S3_SECRET_KEY: password
LANCE_S3_REGION: us-east-1
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py
index b034b79212e..54560857370 100644
--- a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py
@@ -29,6 +29,9 @@ The generated catalog contains:
- __manifest Directory Namespace V2 manifest table (with its scalar indexes).
- all_types.lance The pre-existing compatibility-mode root table, re-registered as-is.
- nested_null.lance Nullable Null leaves inside lists, structs, and maps.
+ - time_travel.lance Three uncompacted versions for FOR VERSION / TIME AS OF; carried
+ over as-is because the suites hard-code its commit times
+ (see lance_build_time_travel.py).
- The `doris` namespace with two full-text-search fixtures, one indexed vector table per cell of the
algorithm x element type x metric matrix (hash-prefixed directories), listed in
VECTOR_TABLES below; BREADTH_TABLE, one table carrying the remaining cells at plan
@@ -88,6 +91,7 @@ import pyarrow as pa
import pyarrow.ipc as ipc
from lance_build_multivector import build as build_multivector, check as check_multivector
from lance_build_nested_null import build as build_nested_null, check as check_nested_null
+from lance_build_time_travel import check as check_time_travel
from lance_namespace_urllib3_client.models import (
CreateNamespaceRequest,
CreateTableRequest,
@@ -103,6 +107,7 @@ NUM_PARTITIONS = 4
NAMESPACE = "doris"
ALL_TYPES_DIR = "all_types.lance"
NESTED_NULL_DIR = "nested_null.lance"
+TIME_TRAVEL_DIR = "time_travel.lance"
MANIFEST_DIR = "__manifest"
# 4-bit PQ keeps codebook training comfortable on 1024 rows. This only serves fixture
@@ -931,8 +936,10 @@ def build_multi_frag(root: Path) -> None:
lance.dataset(location).delete(f"row_id in ({deleted})")
-def build(root: Path, all_types_source: Path) -> None:
+def build(root: Path, all_types_source: Path, time_travel_source: Path) -> None:
shutil.copytree(all_types_source, root / ALL_TYPES_DIR)
+ # Not rebuilt: its commit times are hard-coded in the time-travel suites.
+ shutil.copytree(time_travel_source, root / TIME_TRAVEL_DIR)
build_multi_frag(root)
# Recreate this fixture in staging because promotion replaces the entire catalog tree.
build_nested_null(root / NESTED_NULL_DIR)
@@ -1726,6 +1733,7 @@ def check_catalog(root: Path) -> None:
check_nested_dataset(nested.location)
check_multi_frag(root)
check_nested_null(root / NESTED_NULL_DIR)
+ check_time_travel(root / TIME_TRAVEL_DIR)
check_multivector(root / "multivector.lance")
full_fts = namespace.describe_table(DescribeTableRequest(id=[NAMESPACE, FTS_TABLE]))
@@ -1793,11 +1801,15 @@ def main() -> int:
if not all_types_source.is_dir():
print(f"missing all_types source: {all_types_source}", file=sys.stderr)
return 1
+ time_travel_source = output / TIME_TRAVEL_DIR
+ if not time_travel_source.is_dir():
+ print(f"missing time_travel source: {time_travel_source}", file=sys.stderr)
+ return 1
with tempfile.TemporaryDirectory(prefix="lance_fixture_") as staging_name:
staging = Path(staging_name) / "lance"
staging.mkdir()
- build(staging, all_types_source)
+ build(staging, all_types_source, time_travel_source)
check_catalog(staging)
backup = output.with_name(output.name + ".old")
if backup.exists():
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_time_travel.py b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_time_travel.py
new file mode 100644
index 00000000000..305b2dc319d
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_time_travel.py
@@ -0,0 +1,178 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Generate the Lance time-travel regression fixture.
+
+time_travel.lance is a root table of the preinstalled Directory catalog next to
+all_types.lance. Unlike the other fixtures, most of which are compacted to a single version,
+it keeps a history for FOR TIME AS OF, tags and branches: three commits, none of them cleaned up.
+
+ version 1 create row_id 1..3 tag column "v1" Lance tag v1
+ version 2 append row_id 4..6 tag column "v2" Lance tag v2
+ version 3 append row_id 7..9 tag column "v3" Lance tag v3
+
+Each version also carries a Lance tag of the same name (stored under _refs/tags/), which the
+suites select with tbl@tag(v2). A branch "dev" (stored under tree/dev/, metadata in
+_refs/branches/) is forked from version 2 and carries one extra commit:
+
+ dev version 3 append row_id 100 tag column "dev"
+
+so the branch's version 3 has rows 1..6 and 100, while main's version 3 has rows 1..9; the
+suites select it with tbl@branch(dev). The main chain is not touched by the branch. A tag "rel"
+points at the branch's version 3, so tbl@tag(rel) must read the branch, not main's version 3.
+
+A Lance branch is a shallow clone: its manifests record the parent's location as an absolute
+URI (Manifest.base_paths), so a branch only reads where it was created. The committed
+branch was therefore created against the fixture's final location, s3://warehouse/lance/
+time_travel.lance, and synced back into this directory; build() writes the main chain and
+the tags locally, and --create-branch <uri> creates the branch at the uploaded location:
+
+ python3 lance_build_time_travel.py --create-branch s3://warehouse/lance/time_travel.lance \
+ --storage-option endpoint=http://127.0.0.1:19000 --storage-option access_key_id=admin \
+ --storage-option secret_access_key=password --storage-option region=us-east-1 \
+ --storage-option allow_http=true
+ # then sync tree/, _refs/branches/ and _refs/tags/rel.json from that location into
+ # preinstalled_data/lance/time_travel.lance/
+
+check() verifies the branch files and that they point at that URI without opening the branch,
+which is not possible offline.
+
+The commits are spaced apart so that FOR TIME AS OF can land between two of them. Lance
+stores the commit time of each version in its manifest, and that time is whatever wall
+clock this script ran at. Doris has no SQL to read those times back, so the regression
+suites hard-code them (test_lance_time_travel and test_lance_rest_time_travel, together
+with their .out files). Regenerating this dataset therefore means updating those suites
+from the times this script prints; that is why lance_build_preinstalled_catalog.py carries
+the committed directory over as-is, like all_types.lance, instead of rebuilding it, and only
+runs check() on it.
+
+The same directory serves three catalogs in the regression suites: the filesystem catalog,
+the REST catalog with storage-native versions, and the REST catalog with namespace-managed
+versions, where lance_rest_server.py answers the version endpoints from a static list
+matching the versions written here.
+
+Run it with the writer pinned in lance_fixture_requirements.txt; the main script's
+check_pinned_writer() enforces the pin for the whole catalog, this script alone does not.
+
+Usage:
+ python3 lance_build_time_travel.py preinstalled_data/lance/time_travel.lance
+ python3 lance_build_time_travel.py --check preinstalled_data/lance/time_travel.lance
+"""
+import argparse
+import shutil
+import time
+from pathlib import Path
+
+import lance
+import pyarrow as pa
+
+COMMITS = (("create", 1, 3, "v1"), ("append", 4, 6, "v2"), ("append", 7, 9, "v3"))
+BRANCH = "dev"
+BRANCH_ROW_ID = 100
+BRANCH_TAG = "rel"
+BRANCH_PARENT_URI = "s3://warehouse/lance/time_travel.lance"
+COMMIT_GAP_SECONDS = 1.5
+
+
+def rows_of(low: int, high: int, tag: str) -> pa.Table:
+ return pa.table({
+ "row_id": pa.array(range(low, high + 1), pa.int32()),
+ "tag": pa.array([tag] * (high - low + 1), pa.string()),
+ })
+
+
+def build(output: Path) -> None:
+ if output.exists():
+ shutil.rmtree(output)
+ for index, (mode, low, high, tag) in enumerate(COMMITS):
+ if index > 0:
+ time.sleep(COMMIT_GAP_SECONDS)
+ # Match all_types.lance (data storage version 2.2) so every committed Lance data file
+ # shares one on-disk format with the rest of the fixture.
+ lance.write_dataset(rows_of(low, high, tag), str(output), mode=mode,
+ data_storage_version="2.2")
+ dataset = lance.dataset(str(output))
+ for version, (_, _, _, tag) in zip((1, 2, 3), COMMITS):
+ dataset.tags.create(tag, version)
+ print(f"main chain and tags written; create the branch with --create-branch {BRANCH_PARENT_URI}")
+
+
+def create_branch(uri: str, storage_options: dict) -> None:
+ """Forks the branch at the dataset's final location and appends its extra row there."""
+ dataset = lance.dataset(uri, storage_options=storage_options)
+ assert [v["version"] for v in dataset.versions()] == [1, 2, 3], "upload the main chain first"
+ dataset.create_branch(BRANCH, 2)
+ lance.write_dataset(rows_of(BRANCH_ROW_ID, BRANCH_ROW_ID, BRANCH), f"{uri}/tree/{BRANCH}",
+ mode="append", data_storage_version="2.2", storage_options=storage_options)
+ branch = lance.dataset(f"{uri}/tree/{BRANCH}", storage_options=storage_options)
+ assert branch.version == 3 and sorted(branch.to_table()["row_id"].to_pylist()) == [1, 2, 3, 4, 5, 6, BRANCH_ROW_ID]
+ dataset.tags.create(BRANCH_TAG, (BRANCH, 3))
+ print(f"branch {BRANCH} and tag {BRANCH_TAG} created at {uri}; sync tree/, _refs/branches/ and _refs/tags/ back")
+
+
+def check(output: Path) -> None:
+ dataset = lance.dataset(str(output))
+ versions = dataset.versions()
+ assert [v["version"] for v in versions] == [1, 2, 3], (
+ f"time-travel fixture must keep exactly versions 1..3: {versions}")
+ timestamps = [v["timestamp"] for v in versions]
+ assert timestamps == sorted(timestamps) and len(set(timestamps)) == 3, (
+ f"time-travel fixture commit times must be distinct and increasing: {timestamps}")
+ assert all((b - a).total_seconds() >= 1 for a, b in zip(timestamps, timestamps[1:])), (
+ f"time-travel fixture commits must be at least one second apart: {timestamps}")
+ for version, (_, _, high, tag) in zip((1, 2, 3), COMMITS):
+ table = dataset.checkout_version(version).to_table().sort_by("row_id")
+ assert table["row_id"].to_pylist() == list(range(1, high + 1)), (
+ f"version {version} rows differ from expected: {table}")
+ assert table["tag"].to_pylist()[-1] == tag, f"version {version} tag differs: {table}"
+ tags = {name: (ref["branch"], ref["version"]) for name, ref in dataset.tags.list().items()}
+ assert tags == {"v1": (None, 1), "v2": (None, 2), "v3": (None, 3), BRANCH_TAG: (BRANCH, 3)}, (
+ f"time-travel fixture tags differ: {tags}")
+ assert list(dataset.branches.list()) == [BRANCH], f"time-travel fixture branches differ: {dataset.branches.list()}"
+ branch_manifests = sorted((output / "tree" / BRANCH / "_versions").glob("*.manifest"))
+ assert len(branch_manifests) == 2, f"branch {BRANCH} must carry versions 2 and 3: {branch_manifests}"
+ for manifest in branch_manifests:
+ assert BRANCH_PARENT_URI.encode() in manifest.read_bytes(), (
+ f"{manifest} must reference the parent at {BRANCH_PARENT_URI}; a branch created elsewhere is unreadable there")
+ for version in versions:
+ print(f"time_travel.lance version {version['version']} committed at "
+ f"{version['timestamp'].isoformat()}")
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("output", type=Path, nargs="?",
+ help="path of time_travel.lance (not used with --create-branch)")
+ parser.add_argument("--check", action="store_true", help="verify the existing fixture")
+ parser.add_argument("--create-branch", metavar="URI",
+ help="create the dev branch at the uploaded dataset URI instead of building")
+ parser.add_argument("--storage-option", action="append", default=[], metavar="KEY=VALUE",
+ help="Lance storage option for --create-branch (repeatable)")
+ args = parser.parse_args()
+ if not args.create_branch and args.output is None:
+ parser.error("the output path is required unless --create-branch is given")
+ if args.create_branch:
+ create_branch(args.create_branch, dict(item.split("=", 1) for item in args.storage_option))
+ elif args.check:
+ check(args.output)
+ else:
+ build(args.output)
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/lance_fixture_requirements.txt b/docker/thirdparties/docker-compose/iceberg/scripts/lance_fixture_requirements.txt
index 993ad4849cc..1af5d1305b2 100644
--- a/docker/thirdparties/docker-compose/iceberg/scripts/lance_fixture_requirements.txt
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/lance_fixture_requirements.txt
@@ -19,13 +19,14 @@
#
# The fixture has several readers and they do not all run the same Lance version:
#
-# lance-c v0.1.6 (lance-rs 7.0.0-beta.7) BE, both from source and from the prebuilt
+# lance-c v0.1.9 (lance-rs 11.0.0, patched) BE, both from source and from the prebuilt
# doris-thirdparty package - thirdparty/vars.sh
# lance-java 4.0.0 Spark, via lance-spark-bundle 0.4.0; it
# registers runtime tables into the same __manifest
-# lance-java 9.1.0-beta.3 Doris FE Directory Namespace client - fe/pom.xml
+# lance-java 12.0.0 Doris FE Directory Namespace client - fe/pom.xml
#
-# The writer is pinned to the Lance generation the BE actually reads. It used to be pinned
+# The writer stays on Lance 7, the generation the BE read when this pin was set; the BE's
+# lance-rs 11 reads what it writes, which the regression run verifies. It used to be pinned
# to the oldest reader instead, on the theory that an older writer is readable by every
# reader above; that stopped being possible once the fixture had to carry non-Float32
# indexes. Measured on pylance 4.0.1 (Lance 4): IVF_HNSW_FLAT + float64 and
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/lance_rest_server.py b/docker/thirdparties/docker-compose/iceberg/scripts/lance_rest_server.py
index b8436fa686b..0572a77f2ce 100644
--- a/docker/thirdparties/docker-compose/iceberg/scripts/lance_rest_server.py
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/lance_rest_server.py
@@ -25,13 +25,37 @@ LANCE_REST_TABLES_JSON. Keys are Lance identifiers joined with "$", for example:
{"all_types": "s3://warehouse/lance/all_types.lance",
"doris$items": "s3://warehouse/lance/doris/items.lance"}
+
+Tables listed in LANCE_REST_MANAGED_TABLES_JSON are described with
+managed_versioning=true and answer the ListTableVersions / DescribeTableVersion
+endpoints from a static version list, the way a namespace that owns the table's
+manifests would. Only the versions listed are visible, so a version that exists
+in storage but is absent here behaves like one the namespace has dropped:
+
+ {"time_travel_managed": {"uri": "s3://warehouse/lance/time_travel.lance",
+ "versions": [{"version": 1, "timestamp_millis": 1789823167597},
+ 2, 3],
+ "branches": {"dev": {"versions": [2, 3]}}}}
+
+A version given as an object also reports its commit time, as a real namespace does; a bare
+integer reports none. Doris resolves FOR TIME AS OF from the manifests' commit times either way. Tags are not served:
+they live in the dataset's _refs/tags/ for managed tables too. "branches" lists the versions
+recorded on each branch (manifests under <table>/tree/<branch>/_versions/), answering the
+version endpoints when a request carries a branch; its versions take the same forms.
+
+A version is recorded at its canonical V2 path,
+``<chain>/_versions/<u64::MAX - version>.manifest``, which is where pylance
+writes it. An object may record it elsewhere: ``"staged": true`` records the
+staged manifest beside the canonical path (``<canonical>-<id>``), as a Lance
+namespace does for a commit it has not finalized, and ``"manifest_path"``
+records the given path as is.
"""
import json
import os
import re
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from urllib.parse import unquote, urlparse
+from urllib.parse import parse_qs, unquote, urlparse
HOST = os.environ.get("LANCE_REST_HOST", "0.0.0.0")
@@ -83,6 +107,108 @@ def _load_unprefixed_tables() -> set[tuple[str, ...]]:
UNPREFIXED_TABLES = _load_unprefixed_tables()
+U64_MAX = 2**64 - 1
+
+# The id a staged manifest carries after the canonical name; any id Lance generates looks like this.
+STAGED_ID = "3c9d0e1f-2a4b-4c6d-8e0f-1a2b3c4d5e6f"
+
+
+def _load_versions(entries) -> dict[int, dict]:
+ """Parses recorded versions into {version: {"timestamp_millis", "staged", "manifest_path"}}.
+
+ A version may be given as a bare integer, in which case no commit time is reported, or as
+ {"version": n, "timestamp_millis": ms}, which is what a real namespace returns, optionally
+ with "staged" or "manifest_path" to record the version elsewhere than its canonical path.
+ """
+ if not isinstance(entries, list):
+ raise ValueError("Managed Lance versions must be a list")
+ recorded: dict[int, dict] = {}
+ for entry in entries:
+ spec = entry if isinstance(entry, dict) else {"version": entry}
+ version = spec.get("version")
+ timestamp = spec.get("timestamp_millis")
+ staged = spec.get("staged", False)
+ manifest_path = spec.get("manifest_path")
+ if not isinstance(version, int) or isinstance(version, bool) or version <= 0:
+ raise ValueError("Managed Lance versions must be positive integers")
+ if timestamp is not None and not isinstance(timestamp, int):
+ raise ValueError("Managed Lance timestamp_millis must be an integer")
+ if not isinstance(staged, bool) or (manifest_path is not None and not isinstance(manifest_path, str)):
+ raise ValueError("Managed Lance 'staged' must be a boolean and 'manifest_path' a string")
+ recorded[version] = {
+ "timestamp_millis": timestamp,
+ "staged": staged,
+ "manifest_path": manifest_path,
+ }
+ return dict(sorted(recorded.items()))
+
+
+def _load_managed_tables() -> dict[tuple[str, ...], dict]:
+ """Tables whose versions this namespace manages.
+
+ Each entry maps to {"versions": {version: record}, "branches": {name: {version: record}}},
+ with records as _load_versions parses them.
+ """
+ raw = os.environ.get("LANCE_REST_MANAGED_TABLES_JSON", "{}")
+ managed = json.loads(raw)
+ if not isinstance(managed, dict):
+ raise ValueError("LANCE_REST_MANAGED_TABLES_JSON must be a JSON object")
+ result = {}
+ for identifier, spec in managed.items():
+ parts = tuple(part for part in identifier.split(DELIMITER) if part)
+ if not parts or not isinstance(spec, dict):
+ raise ValueError("A managed Lance table needs an identifier and a spec object")
+ uri = spec.get("uri")
+ if not isinstance(uri, str) or "versions" not in spec:
+ raise ValueError("A managed Lance table spec needs 'uri' and 'versions'")
+ branches = spec.get("branches", {})
+ if not isinstance(branches, dict) or any(
+ not isinstance(name, str) or not isinstance(b, dict) for name, b in branches.items()):
+ raise ValueError("Managed Lance branches must map branch names to {'versions': [...]}")
+ TABLES[parts] = uri
+ result[parts] = {
+ "versions": _load_versions(spec["versions"]),
+ "branches": {name: _load_versions(b.get("versions")) for name, b in branches.items()},
+ }
+ return result
+
+
+MANAGED_TABLES = _load_managed_tables()
+
+
+def _object_store_path(table_uri: str) -> str:
+ """The table root as Lance's object store sees it: bucket-relative for s3, no leading slash."""
+ parsed = urlparse(table_uri)
+ if parsed.scheme in ("s3", "s3a", "oss"):
+ return parsed.path.lstrip("/")
+ if parsed.scheme in ("", "file"):
+ return parsed.path.lstrip("/")
+ raise ValueError(f"unsupported Lance table URI scheme: {table_uri}")
+
+
+def _table_version(identifier: tuple[str, ...], version: int, branch: str | None = None) -> dict:
+ # A branch is its own manifest chain under <table>/tree/<branch>/.
+ chain = _object_store_path(TABLES[identifier])
+ if branch is not None:
+ chain = f"{chain}/tree/{branch}"
+ record = _branch_versions(MANAGED_TABLES[identifier], branch)[version]
+ manifest_path = record["manifest_path"]
+ if manifest_path is None:
+ manifest_path = f"{chain}/_versions/{U64_MAX - version:020d}.manifest"
+ if record["staged"]:
+ manifest_path = f"{manifest_path}-{STAGED_ID}"
+ result = {"version": version, "manifest_path": manifest_path}
+ if record["timestamp_millis"] is not None:
+ result["timestamp_millis"] = record["timestamp_millis"]
+ return result
+
+
+def _branch_versions(managed: dict, branch: str | None):
+ """Versions recorded on the main chain (branch None) or on a branch; None if unknown."""
+ if branch is None:
+ return managed["versions"]
+ return managed["branches"].get(branch)
+
def _storage_options(identifier: tuple[str, ...]) -> dict[str, str]:
access_key = os.environ.get("LANCE_S3_ACCESS_KEY", "admin")
@@ -169,7 +295,71 @@ class LanceRestHandler(BaseHTTPRequestHandler):
path = urlparse(self.path).path.rstrip("/")
if not self._authorized():
return
- self._read_body()
+ raw_body = self._read_body()
+ try:
+ body = json.loads(raw_body) if raw_body else {}
+ except ValueError:
+ self._write_json(400, {"error": "request body is not JSON", "code": 13})
+ return
+
+ # Version endpoints first: the generic describe pattern would otherwise swallow
+ # "<id>/version/describe" as a table id.
+ version_list_match = re.fullmatch(r"/v1/table/(.+)/version/list", path)
+ if version_list_match:
+ identifier = _decode_identifier(version_list_match.group(1))
+ managed = MANAGED_TABLES.get(identifier)
+ if managed is None:
+ if identifier not in TABLES:
+ self._write_json(404, {"error": "table not found", "code": 4})
+ else:
+ # 13 = InvalidInput: the table exists but its versions are not managed here.
+ self._write_json(400, {"error": "table versions are not managed", "code": 13})
+ return
+ query = parse_qs(urlparse(self.path).query)
+ branch = query.get("branch", [None])[0]
+ versions = _branch_versions(managed, branch)
+ if versions is None:
+ # 22 = TableBranchNotFound
+ self._write_json(404, {"error": f"table branch {branch} not found", "code": 22})
+ return
+ ordered = list(versions)
+ if query.get("descending", ["false"])[0].lower() == "true":
+ ordered.reverse()
+ limit = query.get("limit", [None])[0]
+ if limit is not None:
+ if not limit.isdigit():
+ self._write_json(400, {"error": "limit must be a non-negative integer", "code": 13})
+ return
+ ordered = ordered[: int(limit)]
+ self._write_json(
+ 200,
+ {"versions": [_table_version(identifier, v, branch) for v in ordered]},
+ )
+ return
+
+ version_describe_match = re.fullmatch(r"/v1/table/(.+)/version/describe", path)
+ if version_describe_match:
+ identifier = _decode_identifier(version_describe_match.group(1))
+ managed = MANAGED_TABLES.get(identifier)
+ if managed is None:
+ if identifier not in TABLES:
+ self._write_json(404, {"error": "table not found", "code": 4})
+ else:
+ self._write_json(400, {"error": "table versions are not managed", "code": 13})
+ return
+ branch = body.get("branch") if isinstance(body, dict) else None
+ versions = _branch_versions(managed, branch)
+ if versions is None:
+ self._write_json(404, {"error": f"table branch {branch} not found", "code": 22})
+ return
+ version = body.get("version") if isinstance(body, dict) else None
+ if not isinstance(version, int) or version not in versions:
+ self._write_json(
+ 404, {"error": f"table version {version} not found", "code": 11}
+ )
+ return
+ self._write_json(200, {"version": _table_version(identifier, version, branch)})
+ return
describe_match = re.fullmatch(r"/v1/table/(.+)/describe", path)
if describe_match:
@@ -186,7 +376,7 @@ class LanceRestHandler(BaseHTTPRequestHandler):
"location": table_uri,
"table_uri": table_uri,
"storage_options": _storage_options(identifier),
- "managed_versioning": False,
+ "managed_versioning": identifier in MANAGED_TABLES,
"is_only_declared": False,
},
)
@@ -237,7 +427,7 @@ class LanceRestHandler(BaseHTTPRequestHandler):
if __name__ == "__main__":
print(
f"Starting Lance REST Namespace fixture on {HOST}:{PORT} "
- f"with {len(TABLES)} table(s)",
+ f"with {len(TABLES)} table(s), {len(MANAGED_TABLES)} managed",
flush=True,
)
ThreadingHTTPServer((HOST, PORT), LanceRestHandler).serve_forever()
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/branches/dev.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/branches/dev.json
new file mode 100644
index 00000000000..4b7457c1c9f
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/branches/dev.json
@@ -0,0 +1,15 @@
+{
+ "parentBranch": null,
+ "identifier": {
+ "version_mapping": [
+ [
+ 2,
+ "7c0a6c057c0d4da8bd80f539cec59d1d"
+ ]
+ ]
+ },
+ "parentVersion": 2,
+ "createAt": 1790087074,
+ "manifestSize": 464,
+ "metadata": {}
+}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/rel.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/rel.json
new file mode 100644
index 00000000000..56368a416fe
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/rel.json
@@ -0,0 +1,8 @@
+{
+ "branch": "dev",
+ "version": 3,
+ "createdAt": "2026-09-22T15:20:09.563659Z",
+ "updatedAt": "2026-09-22T15:20:09.563659Z",
+ "manifestSize": 605,
+ "metadata": {}
+}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/v1.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/v1.json
new file mode 100644
index 00000000000..0388f6dce41
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/v1.json
@@ -0,0 +1,8 @@
+{
+ "branch": null,
+ "version": 1,
+ "createdAt": "2026-09-22T13:40:15.790905Z",
+ "updatedAt": "2026-09-22T13:40:15.790905Z",
+ "manifestSize": 447,
+ "metadata": {}
+}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/v2.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/v2.json
new file mode 100644
index 00000000000..3d86f40bbd6
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/v2.json
@@ -0,0 +1,8 @@
+{
+ "branch": null,
+ "version": 2,
+ "createdAt": "2026-09-22T13:40:15.794612Z",
+ "updatedAt": "2026-09-22T13:40:15.794612Z",
+ "manifestSize": 464,
+ "metadata": {}
+}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/v3.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/v3.json
new file mode 100644
index 00000000000..7c14804f301
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_refs/tags/v3.json
@@ -0,0 +1,8 @@
+{
+ "branch": null,
+ "version": 3,
+ "createdAt": "2026-09-22T13:40:15.794888Z",
+ "updatedAt": "2026-09-22T13:40:15.794888Z",
+ "manifestSize": 546,
+ "metadata": {}
+}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_transactions/0-c7a3192f-4b4a-4e8b-b3a3-b736534c10f3.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_transactions/0-c7a3192f-4b4a-4e8b-b3a3-b736534c10f3.txn
new file mode 100644
index 00000000000..8d14802b964
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_transactions/0-c7a3192f-4b4a-4e8b-b3a3-b736534c10f3.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_transactions/1-79502d42-ca15-43d2-99e3-8c734698ae2b.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_transactions/1-79502d42-ca15-43d2-99e3-8c734698ae2b.txn
new file mode 100644
index 00000000000..ddc9bf0c8e4
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_transactions/1-79502d42-ca15-43d2-99e3-8c734698ae2b.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_transactions/2-e678f5d6-e017-494d-adaf-fd1c0e245182.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_transactions/2-e678f5d6-e017-494d-adaf-fd1c0e245182.txn
new file mode 100644
index 00000000000..816d64c8821
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_transactions/2-e678f5d6-e017-494d-adaf-fd1c0e245182.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/18446744073709551612.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/18446744073709551612.manifest
new file mode 100644
index 00000000000..5a1c19b96b4
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/18446744073709551612.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/18446744073709551613.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/18446744073709551613.manifest
new file mode 100644
index 00000000000..35d346066f7
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/18446744073709551613.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/18446744073709551614.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/18446744073709551614.manifest
new file mode 100644
index 00000000000..65f80dddcad
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/18446744073709551614.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/latest_version_hint.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/latest_version_hint.json
new file mode 100644
index 00000000000..f93d3984472
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/_versions/latest_version_hint.json
@@ -0,0 +1 @@
+{"version":3}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/data/01001111111001001000111077951d4cb09c0a747e1eddc25e.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/data/01001111111001001000111077951d4cb09c0a747e1eddc25e.lance
new file mode 100644
index 00000000000..ffea364e8c4
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/data/01001111111001001000111077951d4cb09c0a747e1eddc25e.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/data/111001110010000110100100bcc907454fa3151c2176cb1bc4.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/data/111001110010000110100100bcc907454fa3151c2176cb1bc4.lance
new file mode 100644
index 00000000000..11016220947
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/data/111001110010000110100100bcc907454fa3151c2176cb1bc4.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/data/1111000011100100111000010ab68a492fa5d2c0fa972154e5.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/data/1111000011100100111000010ab68a492fa5d2c0fa972154e5.lance
new file mode 100644
index 00000000000..66d6b9b483f
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/data/1111000011100100111000010ab68a492fa5d2c0fa972154e5.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_transactions/2-ca6009ab-1db7-4777-9e3e-a9374ed947b9.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_transactions/2-ca6009ab-1db7-4777-9e3e-a9374ed947b9.txn
new file mode 100644
index 00000000000..bc3ddecad8e
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_transactions/2-ca6009ab-1db7-4777-9e3e-a9374ed947b9.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_transactions/2-d76280f6-7616-446b-9575-5b8ec5ea2df1.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_transactions/2-d76280f6-7616-446b-9575-5b8ec5ea2df1.txn
new file mode 100644
index 00000000000..c8d1c7dcd47
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_transactions/2-d76280f6-7616-446b-9575-5b8ec5ea2df1.txn
@@ -0,0 +1 @@
+$d76280f6-7616-446b-9575-5b8ec5ea2df1�1"&s3://warehouse/lance/time_travel.lance*dev
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_versions/18446744073709551612.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_versions/18446744073709551612.manifest
new file mode 100644
index 00000000000..5aeb004b54f
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_versions/18446744073709551612.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_versions/18446744073709551613.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_versions/18446744073709551613.manifest
new file mode 100644
index 00000000000..23bd893fa98
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/_versions/18446744073709551613.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/data/0010100010110110000010015e5eaf41b5892dff0697f4cbcd.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/data/0010100010110110000010015e5eaf41b5892dff0697f4cbcd.lance
new file mode 100644
index 00000000000..e074fb430a9
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/time_travel.lance/tree/dev/data/0010100010110110000010015e5eaf41b5892dff0697f4cbcd.lance differ
diff --git a/docker/thirdparties/lance-jni-helpers.sh b/docker/thirdparties/lance-jni-helpers.sh
index b48bc525fab..021a211cb7f 100644
--- a/docker/thirdparties/lance-jni-helpers.sh
+++ b/docker/thirdparties/lance-jni-helpers.sh
@@ -17,11 +17,11 @@
# under the License.
# Keep these pins in sync with lance.version in fe/pom.xml.
-LANCE_JNI_VERSION="11.0.0"
-LANCE_JNI_ASSET="liblance_jni-11.0.0-linux-x86_64-glibc2.17-r1.so.gz"
-LANCE_JNI_URL="https://github.com/apache/doris-thirdparty/releases/download/lance-jni-11.0.0-glibc2.17-r1/${LANCE_JNI_ASSET}"
-LANCE_JNI_ARCHIVE_SHA256="f9dc713269632e26c06ea2ea20637ca16ec9d76772dc20db0bead11e5cd34a6b"
-LANCE_JNI_LIBRARY_SHA256="b6540edd3bdcd76b04f96a9e78fe804a2f1bffbd75eb35abe84de9091ba432ac"
+LANCE_JNI_VERSION="12.0.0"
+LANCE_JNI_ASSET="liblance_jni-12.0.0-linux-x86_64-glibc2.17-r1.so.gz"
+LANCE_JNI_URL="https://github.com/apache/doris-thirdparty/releases/download/lance-jni-12.0.0-glibc2.17-r1/${LANCE_JNI_ASSET}"
+LANCE_JNI_ARCHIVE_SHA256="678d9454ad1a24ff07a504d8c318479f6b01d095f04945248dc170eecb303801"
+LANCE_JNI_LIBRARY_SHA256="72f26651b5500bf1483da1cd35130887274a42c2e8505fcfdcd846a521fc1549"
lance_jni_download() (
set -eo pipefail
diff --git a/docs/lance-jni-packaging.md b/docs/lance-jni-packaging.md
index b6941bdff2e..73f164eeb0c 100644
--- a/docs/lance-jni-packaging.md
+++ b/docs/lance-jni-packaging.md
@@ -1,17 +1,17 @@
# Lance JNI packaging for Linux x86_64
`bash build.sh --fe` automatically replaces the Linux x86_64 JNI entry in
-`lance-core-11.0.0.jar` with the glibc 2.17 rebuild. No local Lance checkout,
+`lance-core-12.0.0.jar` with the glibc 2.17 rebuild. No local Lance checkout,
Rust toolchain, or `LANCE_JNI_SO` variable is required on the Doris build host.
The artifact is pinned in `docker/thirdparties/lance-jni-helpers.sh` and published
-in [apache/doris-thirdparty](https://github.com/apache/doris-thirdparty/releases/tag/lance-jni-11.0.0-glibc2.17-r1).
+in [apache/doris-thirdparty](https://github.com/apache/doris-thirdparty/releases/tag/lance-jni-12.0.0-glibc2.17-r1).
Both the compressed archive and the extracted library have fixed SHA256 values.
The archive is cached under:
```text
-thirdparty/installed/lance-jni/liblance_jni-11.0.0-linux-x86_64-glibc2.17-r1.so.gz
+thirdparty/installed/lance-jni/liblance_jni-12.0.0-linux-x86_64-glibc2.17-r1.so.gz
```
With a custom `DORIS_THIRDPARTY`, the cache is under that directory's
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java
index f7cd271389e..8b745a868a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java
@@ -28,6 +28,7 @@ import org.apache.doris.datasource.lance.index.LanceShowIndexInfo;
import org.apache.doris.datasource.lance.job.LanceIndexDatasetLocator;
import org.apache.doris.datasource.lance.metadata.LanceMetadataLoader;
import org.apache.doris.datasource.lance.metadata.LanceReadOptions;
+import org.apache.doris.datasource.lance.metadata.LanceRefSelector;
import org.apache.doris.datasource.lance.metadata.LanceSnapshotResolver;
import org.apache.doris.datasource.lance.metadata.LanceTableAccess;
import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
@@ -40,19 +41,29 @@ import com.github.benmanes.caffeine.cache.Ticker;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.lance.Dataset;
+import org.lance.Ref;
import org.lance.Session;
import org.lance.namespace.LanceNamespace;
+import org.lance.namespace.errors.TableBranchNotFoundException;
+import org.lance.namespace.errors.TableNotFoundException;
+import org.lance.namespace.errors.TableVersionNotFoundException;
+import org.lance.namespace.model.TableVersion;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
+import java.util.NavigableSet;
import java.util.Optional;
import java.util.OptionalLong;
+import java.util.TreeSet;
import java.util.function.BiFunction;
/**
@@ -62,6 +73,8 @@ import java.util.function.BiFunction;
final class LanceCatalogClient implements AutoCloseable {
private static final Logger LOG = LogManager.getLogger(LanceCatalogClient.class);
+ /** Lance's name for the main chain; {@code @branch(main)} and tags on it mean the main chain. */
+ static final String MAIN_BRANCH = "main";
private static final long METADATA_CACHE_SIZE_BYTES = 64L * 1024 * 1024;
private static final long INDEX_CACHE_SIZE_BYTES = 128L * 1024 * 1024;
@@ -235,64 +248,545 @@ final class LanceCatalogClient implements AutoCloseable {
}
public Schema loadTableSchema(String dbName, String tableName) {
- return readTableSnapshot(dbName, tableName, Optional.empty(),
+ return readTableSnapshot(dbName, tableName, LanceRefSelector.latest(),
(dataset, access, metrics) -> metrics.measure(Stage.SCHEMA, dataset::getSchema));
}
public LanceTableMetadata loadTableMetadata(String dbName, String tableName,
Optional<TableSnapshot> tableSnapshot) {
- return loadQueryMetadata(dbName, tableName, tableSnapshot, LanceMetadataLoader.MetadataScope.WITH_INDEXES);
+ return loadTableMetadata(dbName, tableName, LanceRefSelector.snapshot(tableSnapshot));
+ }
+
+ public LanceTableMetadata loadTableMetadata(String dbName, String tableName, LanceRefSelector selector) {
+ return loadQueryMetadata(dbName, tableName, selector, LanceMetadataLoader.MetadataScope.WITH_INDEXES);
}
private LanceTableMetadata loadQueryMetadata(String dbName, String tableName,
Optional<TableSnapshot> tableSnapshot, LanceMetadataLoader.MetadataScope mode) {
- return readTableSnapshot(dbName, tableName, tableSnapshot,
+ return loadQueryMetadata(dbName, tableName, LanceRefSelector.snapshot(tableSnapshot), mode);
+ }
+
+ private LanceTableMetadata loadQueryMetadata(String dbName, String tableName,
+ LanceRefSelector selector, LanceMetadataLoader.MetadataScope mode) {
+ return readTableSnapshot(dbName, tableName, selector,
(dataset, access, metrics) -> LanceMetadataLoader.read(dataset, access, mode, metrics));
}
- /** Pins one resource generation, resolved table access, and the Dataset version for the whole read. */
- private <T> T readTableSnapshot(String dbName, String tableName, Optional<TableSnapshot> tableSnapshot,
+ /**
+ * Pins one resource generation, resolved table access, and the Dataset version for the whole read.
+ *
+ * <p>Every dataset is read by its URI and a version, as the BE reads it. The latest version of
+ * the main chain in storage is opened once as a handle, and the other selectors are checkouts
+ * from it, except an explicit version on the main chain and a managed table's branch (below).
+ * A tag is resolved first to the chain and version it points at, so a tag created on a branch
+ * selects that branch.
+ *
+ * <p>For a managed table the namespace decides which versions exist. "Latest" is the newest
+ * version it records, never the newest manifest in storage, and every version a read selects
+ * must be one it records, at the manifest path Doris reads ({@link LanceManifestPaths}). A
+ * branch is read from its own directory, as the BE reads it, so it does not depend on the main
+ * chain; the main handle only supplies tag files and the manifest listing that FOR TIME AS OF
+ * on main takes commit times from.
+ */
+ private <T> T readTableSnapshot(String dbName, String tableName, LanceRefSelector selector,
SnapshotReader<T> reader) {
- LanceTableAccess tableAccess = null;
+ ReadState state = new ReadState(selector, dbName + "." + tableName);
LanceMetadataMetrics metrics = LanceMetadataMetrics.startMetadataRead();
try {
T result;
try (BufferAllocator allocator = namespaceAllocator.newChildAllocator(
"lance-metadata-read", 0, namespaceAllocator.getLimit())) {
- tableAccess = metrics.measure(Stage.TABLE_ACCESS,
+ state.access = metrics.measure(Stage.TABLE_ACCESS,
() -> namespaceClient.resolveTableAccess(dbName, tableName));
- OptionalLong version = OptionalLong.empty();
- if (tableSnapshot.isPresent()) {
- TableSnapshot snapshot = tableSnapshot.get();
- if (snapshot.getType() == TableSnapshot.VersionType.VERSION) {
- version = OptionalLong.of(LanceSnapshotResolver.parseVersion(snapshot.getValue()));
- } else {
- long timestamp = TimeUtils.timeStringToLong(snapshot.getValue(), TimeUtils.getTimeZone());
- if (timestamp < 0) {
- throw new IllegalArgumentException(
- "Cannot parse Lance FOR TIME AS OF value '" + snapshot.getValue() + "'");
- }
- try (Dataset latest = openDataset(allocator, tableAccess, OptionalLong.empty(), metrics)) {
- version = OptionalLong.of(metrics.measure(Stage.VERSION_RESOLVE,
- () -> LanceSnapshotResolver.getVersionAtOrBefore(latest, timestamp)));
- }
+ OptionalLong direct = directMainVersion(state, metrics);
+ if (state.access.isManagedVersioning() && state.branch.isPresent() && !selector.getTag().isPresent()) {
+ result = readManagedBranch(allocator, state, reader, metrics);
+ } else if (direct.isPresent() || isLatestMain(selector)) {
+ state.version = direct;
+ try (Dataset dataset = openDataset(allocator, state.access, direct, metrics)) {
+ result = reader.read(dataset, state.access, metrics);
+ }
+ } else {
+ try (Dataset main = openDataset(allocator, state.access, OptionalLong.empty(), metrics)) {
+ result = readFromLatest(main, state, reader, metrics);
}
- }
- try (Dataset dataset = openDataset(allocator, tableAccess, version, metrics)) {
- result = reader.read(dataset, tableAccess, metrics);
}
}
metrics.succeeded();
return result;
+ } catch (LanceUserFacingException e) {
+ throw new RuntimeException(e.getMessage(), e);
} catch (Exception e) {
- throw LanceErrorMessages.failure("Failed to load Lance table metadata for " + dbName + "." + tableName, e,
- tableAccess == null ? null : tableAccess.getDatasetUri(),
- tableAccess == null ? namespaceStorageOptions : tableAccess.getStorageOptions(), catalogSecrets);
+ LanceTableAccess access = state.access;
+ String uri = access == null ? null : access.getDatasetUri();
+ Map<String, String> options = access == null ? namespaceStorageOptions : access.getStorageOptions();
+ String what = state.displayName();
+ // Lance's Directory namespace reports a branch it lacks as a missing table. The table
+ // was described in this read, so a missing table from a branch's version request
+ // means the branch.
+ boolean namespaceLacksBranch = access != null && access.isManagedVersioning()
+ && ExceptionUtils.indexOfType(e, TableNotFoundException.class) >= 0;
+ if (state.branch.isPresent() && !state.branchExists
+ && (namespaceLacksBranch || isBranchNotFound(e, state.branch.get()))) {
+ throw new RuntimeException("Lance branch '" + state.branch.get() + "' of " + state.tableName
+ + state.selector.getTag().map(tag -> " (tag '" + tag + "')").orElse("")
+ + " was not found" + (namespaceLacksBranch || isNamespaceMiss(e) ? " in the namespace" : ""),
+ sanitizedCause(e, uri, options));
+ }
+ if (isVersionNotFound(e) && state.pinned != null
+ && state.pinned.manifest == LanceManifestPaths.Recorded.STAGED) {
+ throw new RuntimeException(unreadableStaged(state.pinned, state), sanitizedCause(e, uri, options));
+ }
+ if (state.version.isPresent() && isVersionNotFound(e)) {
+ throw new RuntimeException("Lance version " + state.version.getAsLong() + " of " + what
+ + state.selector.getTag().map(tag -> " (tag '" + tag + "')").orElse("")
+ + " was not found" + (isNamespaceMiss(e) ? " in the namespace" : ""),
+ sanitizedCause(e, uri, options));
+ }
+ throw LanceErrorMessages.failure("Failed to load Lance table metadata for " + what, e, uri, options,
+ catalogSecrets);
} finally {
metrics.close();
}
}
+ /** What a read has resolved so far; the catch block reports errors against it. */
+ private static final class ReadState {
+ private final LanceRefSelector selector;
+ private final String tableName;
+ /** The table's access; a branch's access is derived from it with {@code onBranch}. */
+ private LanceTableAccess access;
+ private Optional<String> branch;
+ /**
+ * Set once the branch is known to exist: the namespace recorded versions for it, or its
+ * latest version was checked out. Later failures are not reported as a missing branch.
+ */
+ private boolean branchExists;
+ private OptionalLong version = OptionalLong.empty();
+ /** The managed version this read opens next, as the namespace records it. */
+ private Recorded pinned;
+ /** The namespace's version list of the chain a FOR TIME AS OF reads ("" is main), fetched once. */
+ private final Map<String, List<TableVersion>> namespaceVersions = new HashMap<>();
+
+ private ReadState(LanceRefSelector selector, String tableName) {
+ this.selector = selector;
+ this.tableName = tableName;
+ this.branch = selector.getBranch();
+ }
+
+ private String displayName() {
+ return tableName + branch.map(name -> "@" + name).orElse("");
+ }
+ }
+
+ /** A version of a managed chain the namespace records, and how it records its manifest. */
+ private static final class Recorded {
+ private final Optional<String> branch;
+ private final long version;
+ private final LanceManifestPaths.Recorded manifest;
+
+ private Recorded(Optional<String> branch, long version, LanceManifestPaths.Recorded manifest) {
+ this.branch = branch;
+ this.version = version;
+ this.manifest = manifest;
+ }
+ }
+
+ private static boolean isLatestMain(LanceRefSelector selector) {
+ return !selector.getTag().isPresent() && !selector.getBranch().isPresent()
+ && !selector.getSnapshot().isPresent();
+ }
+
+ /**
+ * The main-chain version a selector names without looking at the latest manifest: an explicit
+ * version, or the latest version of a managed table, which the namespace records.
+ */
+ private OptionalLong directMainVersion(ReadState state, LanceMetadataMetrics metrics) {
+ LanceRefSelector selector = state.selector;
+ if (selector.getTag().isPresent() || selector.getBranch().isPresent()) {
+ return OptionalLong.empty();
+ }
+ if (!selector.getSnapshot().isPresent()) {
+ return state.access.isManagedVersioning()
+ ? OptionalLong.of(recordedHead(state, Optional.empty(), metrics))
+ : OptionalLong.empty();
+ }
+ TableSnapshot snapshot = selector.getSnapshot().get();
+ if (snapshot.getType() != TableSnapshot.VersionType.VERSION) {
+ return OptionalLong.empty();
+ }
+ state.version = OptionalLong.of(LanceSnapshotResolver.parseVersion(snapshot.getValue()));
+ requireRecorded(state, Optional.empty(), state.version.getAsLong(), metrics);
+ return state.version;
+ }
+
+ /** Resolves the selector against the open latest main chain and reads the selected snapshot. */
+ private <T> T readFromLatest(Dataset main, ReadState state, SnapshotReader<T> reader, LanceMetadataMetrics metrics)
+ throws Exception {
+ LanceRefSelector selector = state.selector;
+ if (selector.getTag().isPresent()) {
+ // Only this tag's file is read, however many tags the table has. The SDK checks the tag
+ // out on the branch of the version it points at.
+ String tag = selector.getTag().get();
+ state.version = OptionalLong.of(metrics.measure(Stage.VERSION_RESOLVE, () -> tagVersion(main, tag, state)));
+ try (Dataset target = checkout(main, Ref.ofTag(tag), metrics)) {
+ // The checkout reads the tag file again; the version it read is the one to check.
+ state.version = OptionalLong.of(target.version());
+ state.branch = branchOf(target.uri(), state.access.getDatasetUri());
+ requireRecorded(state, state.branch, target.version(), metrics);
+ return reader.read(target, accessOf(target, state), metrics);
+ }
+ }
+ if (state.branch.isPresent()) {
+ String branch = state.branch.get();
+ // Check out the branch's latest version first even when a version is already known, so
+ // a missing branch and a missing version inside an existing branch are told apart.
+ try (Dataset latest = checkout(main, Ref.ofBranch(branch), metrics)) {
+ state.branchExists = true;
+ LanceTableAccess branchAccess = accessOf(latest, state);
+ if (!state.version.isPresent() && selector.getSnapshot().isPresent()) {
+ state.version = resolveSnapshotVersion(latest, branchAccess, selector.getSnapshot().get(), state,
+ metrics);
+ }
+ if (!state.version.isPresent()) {
+ return reader.read(latest, branchAccess, metrics);
+ }
+ try (Dataset dataset = checkout(latest, Ref.ofBranch(branch, state.version.getAsLong()), metrics)) {
+ return reader.read(dataset, branchAccess, metrics);
+ }
+ }
+ }
+ // FOR TIME AS OF on the main chain, resolved from manifest commit times.
+ state.version = resolveSnapshotVersion(main, state.access, selector.getSnapshot().get(), state, metrics);
+ try (Dataset dataset = checkout(main, Ref.ofMain(state.version.getAsLong()), metrics)) {
+ return reader.read(dataset, state.access, metrics);
+ }
+ }
+
+ /**
+ * Reads a branch of a managed table from the branch's directory, by URI and version as the BE
+ * reads it. The namespace answers for the branch first, so a branch it lacks is reported as
+ * missing, and the main chain need not be readable. FOR TIME AS OF lists the branch's
+ * manifests from its newest version in storage and, as on main, selects among the versions the
+ * namespace records.
+ */
+ private <T> T readManagedBranch(BufferAllocator allocator, ReadState state, SnapshotReader<T> reader,
+ LanceMetadataMetrics metrics) throws Exception {
+ String branch = state.branch.get();
+ LanceTableAccess branchAccess = state.access.onBranch(branch,
+ branchUri(state.access.getDatasetUri(), branch));
+ Optional<TableSnapshot> snapshot = state.selector.getSnapshot();
+ if (!snapshot.isPresent()) {
+ state.version = OptionalLong.of(recordedHead(state, state.branch, metrics));
+ } else if (snapshot.get().getType() == TableSnapshot.VersionType.VERSION) {
+ state.version = OptionalLong.of(LanceSnapshotResolver.parseVersion(snapshot.get().getValue()));
+ requireRecorded(state, state.branch, state.version.getAsLong(), metrics);
+ } else {
+ namespaceVersions(state, state.branch, metrics);
+ state.branchExists = true;
+ try (Dataset latest = openDataset(allocator, branchAccess, OptionalLong.empty(), metrics)) {
+ state.version = resolveSnapshotVersion(latest, branchAccess, snapshot.get(), state, metrics);
+ }
+ }
+ state.branchExists = true;
+ try (Dataset dataset = openDataset(allocator, branchAccess, state.version, metrics)) {
+ return reader.read(dataset, branchAccess, metrics);
+ }
+ }
+
+ /**
+ * The URI of a branch's directory, joined as Lance's {@code BranchLocation} joins it:
+ * {@code tree/<branch>} under the table root, before the URI's query string.
+ */
+ static String branchUri(String tableUri, String branch) {
+ int query = tableUri.indexOf('?');
+ String path = query < 0 ? tableUri : tableUri.substring(0, query);
+ String joined = path + (path.endsWith("/") ? "" : "/") + "tree/" + StringUtils.stripStart(branch, "/");
+ return query < 0 ? joined : joined + tableUri.substring(query);
+ }
+
+ private static long tagVersion(Dataset main, String tag, ReadState state) {
+ try {
+ return main.tags().getVersion(tag);
+ } catch (RuntimeException e) {
+ String rootMessage = ExceptionUtils.getRootCauseMessage(e);
+ if (rootMessage != null && rootMessage.contains("tag " + tag + " does not exist")) {
+ throw new LanceUserFacingException("Lance tag '" + tag + "' of " + state.tableName + " was not found");
+ }
+ throw e;
+ }
+ }
+
+ /**
+ * A dataset URI without its query and trailing slash. The query may carry credentials, which a
+ * namespace can vend anew on every describe.
+ */
+ private static String location(String uri) {
+ return StringUtils.removeEnd(StringUtils.substringBefore(uri, "?"), "/");
+ }
+
+ /**
+ * The branch a dataset checked out from the table root is on, from its root directory: the
+ * table root for main, {@code <root>/tree/<branch>} otherwise. Lance inserts the branch path
+ * before a URI's query string, so the query is compared apart. A URI that is neither is an
+ * error rather than main, which would hand the BE the wrong chain.
+ */
+ static Optional<String> branchOf(String checkedOutUri, String tableUri) {
+ String root = location(tableUri);
+ String uri = location(checkedOutUri);
+ if (uri.equals(root)) {
+ return Optional.empty();
+ }
+ String branchRoot = root + "/tree/";
+ if (!uri.startsWith(branchRoot) || uri.length() == branchRoot.length()) {
+ // The URIs may carry credentials in their query, so they stay out of the message.
+ throw new IllegalStateException("Cannot tell which branch a Lance tag was checked out on");
+ }
+ return Optional.of(uri.substring(branchRoot.length()));
+ }
+
+ /**
+ * The access for a dataset checked out from the table: the main chain keeps the table access,
+ * and a branch takes the directory the SDK checked out, which is what the BE opens by URI.
+ */
+ private static LanceTableAccess accessOf(Dataset dataset, ReadState state) {
+ return state.branch.isPresent() ? state.access.onBranch(state.branch.get(), dataset.uri()) : state.access;
+ }
+
+ /** A selector error whose message is user-facing as is, such as a tag that does not exist. */
+ private static final class LanceUserFacingException extends RuntimeException {
+ private LanceUserFacingException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * The newest version the namespace records for a managed chain, which the read then opens.
+ * Doris asks for it itself: opening "latest" by URI would read the newest manifest in storage,
+ * which the namespace may not have published.
+ */
+ private long recordedHead(ReadState state, Optional<String> branch, LanceMetadataMetrics metrics) {
+ state.pinned = null;
+ Optional<TableVersion> head = metrics.measure(Stage.VERSION_RESOLVE,
+ () -> namespaceClient.latestManagedVersion(state.access, branch));
+ if (!head.isPresent()) {
+ throw new LanceUserFacingException("Lance namespace lists no versions for " + state.tableName
+ + branch.map(name -> "@" + name).orElse(""));
+ }
+ long version = head.get().getVersion();
+ state.pinned = new Recorded(branch, version, LanceManifestPaths.check(state.access.getDatasetUri(), branch,
+ version, head.get().getManifestPath(), state.tableName));
+ return version;
+ }
+
+ /**
+ * Requires the namespace of a managed table to record {@code version} of the chain on
+ * {@code branch}, at the manifest path Doris reads; nothing for a storage-versioned table.
+ */
+ private void requireRecorded(ReadState state, Optional<String> branch, long version,
+ LanceMetadataMetrics metrics) {
+ if (!state.access.isManagedVersioning()) {
+ return;
+ }
+ state.pinned = null;
+ TableVersion recorded = metrics.measure(Stage.VERSION_RESOLVE,
+ () -> namespaceClient.describeManagedVersion(state.access, branch, version));
+ state.pinned = new Recorded(branch, version, LanceManifestPaths.check(state.access.getDatasetUri(), branch,
+ version, recorded.getManifestPath(), state.tableName));
+ }
+
+ /**
+ * The error for a version the namespace records at a staged manifest while its canonical
+ * manifest, which Doris reads, does not exist. Either the commit reserved the version and was
+ * not finalized, which a reader that uses the namespace would finish and Doris does not, or
+ * the version was finalized and cleanup later removed it; the namespace's record does not tell
+ * the two apart.
+ */
+ private static String unreadableStaged(Recorded pinned, ReadState state) {
+ return "Lance version " + pinned.version + " of " + state.tableName
+ + pinned.branch.map(name -> "@" + name).orElse("") + " cannot be read: " + stagedOnly();
+ }
+
+ private static String stagedOnly() {
+ return "the namespace records it at a staged manifest, and its canonical manifest, which Doris reads,"
+ + " does not exist (its commit was not finalized, or cleanup removed it)";
+ }
+
+ private RuntimeException sanitizedCause(Throwable error, String uri, Map<String, String> options) {
+ return new RuntimeException(LanceErrorMessages.sanitize(error, uri, options, catalogSecrets));
+ }
+
+ /** Checks out a ref of an already open dataset; the SDK resolves the ref from the dataset directory. */
+ private static Dataset checkout(Dataset dataset, Ref ref, LanceMetadataMetrics metrics) {
+ return metrics.measure(Stage.VERSION_RESOLVE, () -> dataset.checkout(ref));
+ }
+
+ /**
+ * Resolves a {@code FOR VERSION AS OF} / {@code FOR TIME AS OF} snapshot against the chain
+ * {@code latest} is checked out on: the main chain, or a branch when {@code access} is a
+ * branch access.
+ */
+ private OptionalLong resolveSnapshotVersion(Dataset latest, LanceTableAccess access, TableSnapshot snapshot,
+ ReadState state, LanceMetadataMetrics metrics) {
+ if (snapshot.getType() == TableSnapshot.VersionType.VERSION) {
+ state.version = OptionalLong.of(LanceSnapshotResolver.parseVersion(snapshot.getValue()));
+ requireRecorded(state, access.getBranch(), state.version.getAsLong(), metrics);
+ return state.version;
+ }
+ long timestamp = parseTimeTravelTimestamp(snapshot.getValue());
+ try {
+ return OptionalLong.of(resolveVersionAtOrBefore(latest, access, timestamp, snapshot.getValue(), state,
+ metrics));
+ } catch (LanceSnapshotResolver.NoVersionAtOrBeforeException e) {
+ if (!access.getBranch().isPresent()) {
+ throw new LanceUserFacingException("Lance table " + state.tableName + " has no version at or before '"
+ + snapshot.getValue() + "'");
+ }
+ // A branch's chain starts at the version it was created from and carries its own
+ // commit times, so an earlier timestamp has nothing to select on the branch.
+ throw new LanceUserFacingException("Lance branch '" + access.getBranch().get() + "' of "
+ + state.tableName + " has no version at or before '" + snapshot.getValue()
+ + "'; a branch only holds the versions from its creation on");
+ }
+ }
+
+ /**
+ * Whether a failure means the branch does not exist. A checkout reports "branch <name> does
+ * not exist", or a missing manifest under the branch directory when nothing was ever
+ * committed there; a namespace that reports the branch itself throws
+ * {@link TableBranchNotFoundException}.
+ */
+ private static boolean isBranchNotFound(Throwable throwable, String branch) {
+ if (ExceptionUtils.indexOfType(throwable, TableBranchNotFoundException.class) >= 0) {
+ return true;
+ }
+ String rootMessage = ExceptionUtils.getRootCauseMessage(throwable);
+ if (rootMessage == null) {
+ return false;
+ }
+ String lower = rootMessage.toLowerCase(Locale.ROOT);
+ String name = branch.toLowerCase(Locale.ROOT);
+ return lower.contains("branch " + name + " does not exist")
+ || (lower.contains("not found") && lower.contains("tree/" + name + "/"));
+ }
+
+ /** Whether a not-found came from the namespace client rather than storage. */
+ private static boolean isNamespaceMiss(Throwable throwable) {
+ return ExceptionUtils.indexOfType(throwable, TableVersionNotFoundException.class) >= 0
+ || ExceptionUtils.indexOfType(throwable, TableBranchNotFoundException.class) >= 0;
+ }
+
+ /**
+ * Every version the namespace records for the chain on {@code branch}, listed once per read.
+ * The whole list is needed: FOR TIME AS OF selects among it, and neither the order a namespace
+ * returns nor monotonic commit times can be relied on to stop early.
+ */
+ private List<TableVersion> namespaceVersions(ReadState state, Optional<String> branch,
+ LanceMetadataMetrics metrics) {
+ return state.namespaceVersions.computeIfAbsent(branch.orElse(""), chain -> {
+ List<TableVersion> versions = metrics.measure(Stage.VERSION_RESOLVE,
+ () -> namespaceClient.listManagedVersions(state.access, branch));
+ if (versions.isEmpty()) {
+ throw new LanceUserFacingException("Lance namespace lists no versions for "
+ + state.tableName + (chain.isEmpty() ? "" : "@" + chain));
+ }
+ return versions;
+ });
+ }
+
+ /**
+ * Resolves {@code FOR TIME AS OF} to a version on the chain {@code latest} is checked out on,
+ * from the commit times the manifests in storage record, over the history
+ * {@link LanceSnapshotResolver} describes. A managed table only selects among the versions its
+ * namespace records. A version it no longer records between recorded ones cuts the history
+ * like a removed one, and so does a recorded version storage lacks, since its commit time is
+ * unknown.
+ */
+ private long resolveVersionAtOrBefore(Dataset latest, LanceTableAccess access, long timestamp,
+ String requestedText, ReadState state, LanceMetadataMetrics metrics) {
+ state.pinned = null;
+ Map<Long, TableVersion> records = null;
+ if (access.isManagedVersioning()) {
+ records = new HashMap<>();
+ for (TableVersion recorded : namespaceVersions(state, access.getBranch(), metrics)) {
+ if (recorded.getVersion() != null) {
+ // Selection compares the commit times of the manifests Doris reads, so each
+ // recorded version must be at one of them, not only the selected one.
+ LanceManifestPaths.check(state.access.getDatasetUri(), access.getBranch(), recorded.getVersion(),
+ recorded.getManifestPath(), state.tableName);
+ records.put(recorded.getVersion(), recorded);
+ }
+ }
+ }
+ Map<Long, TableVersion> recordedById = records;
+ NavigableSet<Long> recorded = records == null ? null : new TreeSet<>(records.keySet());
+ long version = metrics.measure(Stage.VERSION_RESOLVE, () -> {
+ try {
+ return LanceSnapshotResolver.versionAtOrBefore(latest.listVersions(), recorded, timestamp,
+ requestedText);
+ } catch (LanceSnapshotResolver.HistoryRemovedException e) {
+ // A removed version the namespace still records may only be staged; one it no
+ // longer records is gone.
+ TableVersion removed = recordedById == null ? null : recordedById.get(e.getVersion());
+ boolean staged = removed != null && LanceManifestPaths.check(state.access.getDatasetUri(),
+ access.getBranch(), e.getVersion(), removed.getManifestPath(), state.tableName)
+ == LanceManifestPaths.Recorded.STAGED;
+ throw historyRemoved(e.getVersion(), staged, requestedText, state);
+ }
+ });
+ if (records != null) {
+ state.pinned = new Recorded(access.getBranch(), version, LanceManifestPaths.check(
+ state.access.getDatasetUri(), access.getBranch(), version, records.get(version).getManifestPath(),
+ state.tableName));
+ }
+ LOG.debug("Resolved Lance FOR TIME AS OF '{}' to version {} from manifest commit times", requestedText,
+ version);
+ return version;
+ }
+
+ private static LanceUserFacingException historyRemoved(long version, boolean staged, String requestedText,
+ ReadState state) {
+ return new LanceUserFacingException("Lance cannot resolve FOR TIME AS OF '" + requestedText + "' on "
+ + state.displayName() + ": version " + version + ", which may hold the state at that time, "
+ + (staged ? "cannot be read: " + stagedOnly() : "no longer exists"));
+ }
+
+ /**
+ * Parses a {@code FOR TIME AS OF} value in the session time zone. Second and millisecond
+ * precision are accepted. Commit times are compared in full, so a commit later within the
+ * requested millisecond is not selected.
+ */
+ private static long parseTimeTravelTimestamp(String value) {
+ long timestamp = TimeUtils.timeStringToLong(value, TimeUtils.getTimeZone());
+ if (timestamp < 0) {
+ timestamp = TimeUtils.msTimeStringToLong(value, TimeUtils.getTimeZone());
+ }
+ if (timestamp < 0) {
+ throw new IllegalArgumentException("Cannot parse Lance FOR TIME AS OF value '" + value
+ + "', expected 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd HH:mm:ss.SSS'");
+ }
+ return timestamp;
+ }
+
+ /**
+ * Whether a failed open of an explicitly requested version means that version does not exist.
+ * A namespace reports it through {@link TableVersionNotFoundException}. The storage reader
+ * reports it as a missing manifest under {@code _versions/} or as Lance's own version-not-found
+ * error; a missing dataset or an unreachable store fails differently and keeps its message.
+ */
+ private static boolean isVersionNotFound(Throwable throwable) {
+ if (ExceptionUtils.indexOfType(throwable, TableVersionNotFoundException.class) >= 0) {
+ return true;
+ }
+ String rootMessage = ExceptionUtils.getRootCauseMessage(throwable);
+ if (rootMessage == null) {
+ return false;
+ }
+ String lower = rootMessage.toLowerCase(Locale.ROOT);
+ return lower.contains("version not found")
+ || (lower.contains("not found") && lower.contains("_versions/"));
+ }
+
private Dataset openDataset(BufferAllocator allocator, LanceTableAccess access, OptionalLong version,
LanceMetadataMetrics metrics) {
return metrics.measure(Stage.DATASET_OPEN, () -> Dataset.open().allocator(allocator).uri(access.getDatasetUri())
@@ -331,6 +825,12 @@ final class LanceCatalogClient implements AutoCloseable {
// The worker owns its Dataset and Session even if the caller releases its lease on timeout.
// Index admission must verify the current target even while query access is cached.
tableAccess = namespaceClient.resolveTableAccessUncached(dbName, tableName);
+ // Index paths open the dataset by URI at storage-latest. They are only reachable for
+ // filesystem catalogs, whose Directory namespace never manages versions; a managed
+ // table here would bypass the namespace.
+ if (tableAccess.isManagedVersioning()) {
+ throw new IllegalStateException("Lance index inspection does not support namespace-managed tables");
+ }
LanceTableAccess access = tableAccess;
return LanceIndexInspectionExecutor.execute(() -> {
// The caller can time out while JNI is running; the worker must own resource cleanup.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java
index fd9eeb7011d..179697d436f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java
@@ -28,6 +28,7 @@ import org.apache.doris.datasource.lance.index.LancePhysicalIndexEntry;
import org.apache.doris.datasource.lance.index.LanceShowIndexInfo;
import org.apache.doris.datasource.lance.job.LanceIndexDatasetLocator;
import org.apache.doris.datasource.lance.metadata.LanceMetadataLoader;
+import org.apache.doris.datasource.lance.metadata.LanceRefSelector;
import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
import org.apache.doris.datasource.lance.storage.LanceStorageOptions;
import org.apache.doris.datasource.property.metastore.AbstractLanceProperties;
@@ -214,7 +215,11 @@ public class LanceExternalCatalog extends ExternalCatalog {
}
public LanceTableMetadata loadTableMetadata(String dbName, String tableName, Optional<TableSnapshot> snapshot) {
- return withClient(current -> current.loadTableMetadata(dbName, tableName, snapshot));
+ return loadTableMetadata(dbName, tableName, LanceRefSelector.snapshot(snapshot));
+ }
+
+ public LanceTableMetadata loadTableMetadata(String dbName, String tableName, LanceRefSelector selector) {
+ return withClient(current -> current.loadTableMetadata(dbName, tableName, selector));
}
public LanceTableMetadata loadTableMetadataForSearch(String dbName, String tableName) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java
index 2169beddfce..916976cdfaf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java
@@ -26,7 +26,9 @@ import org.apache.doris.datasource.SchemaCacheValue;
import org.apache.doris.datasource.lance.index.LancePhysicalIndexEntry;
import org.apache.doris.datasource.lance.index.LanceShowIndexInfo;
import org.apache.doris.datasource.lance.metadata.LanceMvccSnapshot;
+import org.apache.doris.datasource.lance.metadata.LanceRefSelector;
import org.apache.doris.datasource.lance.metadata.LanceSchemaHelper;
+import org.apache.doris.datasource.lance.metadata.LanceSnapshotResolver;
import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
import org.apache.doris.datasource.mvcc.MvccSnapshot;
import org.apache.doris.datasource.mvcc.MvccTable;
@@ -39,9 +41,11 @@ import org.apache.doris.thrift.TTableDescriptor;
import org.apache.doris.thrift.TTableType;
import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.commons.lang3.StringUtils;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
public class LanceExternalTable extends ExternalTable implements MvccTable {
@@ -79,11 +83,6 @@ public class LanceExternalTable extends ExternalTable implements MvccTable {
db.getRemoteName(), remoteName);
}
- private LanceTableMetadata loadMetadata(Optional<TableSnapshot> tableSnapshot) {
- return ((LanceExternalCatalog) catalog).loadTableMetadata(
- db.getRemoteName(), remoteName, tableSnapshot);
- }
-
public LanceTableMetadata getMetadata(Optional<MvccSnapshot> snapshot) {
if (snapshot.isPresent()) {
return ((LanceMvccSnapshot) snapshot.get()).getMetadata();
@@ -94,7 +93,71 @@ public class LanceExternalTable extends ExternalTable implements MvccTable {
@Override
public MvccSnapshot loadSnapshot(Optional<TableSnapshot> tableSnapshot,
Optional<TableScanParams> scanParams) {
- return new LanceMvccSnapshot(loadMetadata(tableSnapshot));
+ // As for Paimon tables, and for tag names in Iceberg, a non-numeric FOR VERSION AS OF names a tag.
+ boolean versionIsTag = tableSnapshot.isPresent()
+ && tableSnapshot.get().getType() == TableSnapshot.VersionType.VERSION
+ && !LanceSnapshotResolver.isVersionNumber(tableSnapshot.get().getValue());
+ LanceRefSelector selector = versionIsTag
+ ? LanceRefSelector.tag(tableSnapshot.get().getValue()) : LanceRefSelector.snapshot(tableSnapshot);
+ if (scanParams.isPresent()) {
+ TableScanParams params = scanParams.get();
+ if (params.isBranch()) {
+ String branch = refName(params);
+ // A tag already determines its branch, so naming one as well is redundant at best
+ // and contradictory when the tag points into another branch; @branch(main) included.
+ // Iceberg rejects any version with @branch; numeric versions and times stay allowed
+ // here because Lance numbers versions per branch.
+ if (versionIsTag) {
+ throw new IllegalArgumentException("Lance table " + getName() + ": FOR VERSION AS OF '"
+ + tableSnapshot.get().getValue() + "' names a tag, which cannot be combined with @branch;"
+ + " use @tag(...) or a numeric version");
+ }
+ // Lance calls the main chain "main"; it lives at the table root, not under tree/.
+ if (!LanceCatalogClient.MAIN_BRANCH.equals(branch)) {
+ selector = LanceRefSelector.branch(branch, tableSnapshot);
+ }
+ } else if (params.isTag()) {
+ if (tableSnapshot.isPresent()) {
+ throw new IllegalArgumentException("Lance table " + getName()
+ + ": @tag cannot be combined with FOR VERSION AS OF or FOR TIME AS OF");
+ }
+ selector = LanceRefSelector.tag(refName(params));
+ } else {
+ // Silently reading the latest version instead would return wrong data.
+ throw new IllegalArgumentException("Lance table " + getName() + " does not support @"
+ + params.getParamType() + "; use @branch, @tag, FOR VERSION AS OF or FOR TIME AS OF");
+ }
+ }
+ return new LanceMvccSnapshot(((LanceExternalCatalog) catalog).loadTableMetadata(
+ db.getRemoteName(), remoteName, selector));
+ }
+
+ /**
+ * {@code tbl@tag(name)} arrives as a list parameter, {@code tbl@tag('name'='x')} as a map;
+ * anything else, such as extra keys or arguments, is rejected rather than ignored.
+ */
+ private static String refName(TableScanParams params) {
+ String usage = "Lance @" + params.getParamType() + " takes exactly one name, as @"
+ + params.getParamType() + "(x) or @" + params.getParamType() + "('" + TableScanParams.PARAMS_NAME
+ + "'='x')";
+ Map<String, String> map = params.getMapParams();
+ List<String> list = params.getListParams();
+ String name;
+ if (!map.isEmpty()) {
+ if (!list.isEmpty() || map.size() != 1 || !map.containsKey(TableScanParams.PARAMS_NAME)) {
+ throw new IllegalArgumentException(usage);
+ }
+ name = map.get(TableScanParams.PARAMS_NAME);
+ } else {
+ if (list.size() != 1) {
+ throw new IllegalArgumentException(usage);
+ }
+ name = list.get(0);
+ }
+ if (StringUtils.isBlank(name)) {
+ throw new IllegalArgumentException(usage);
+ }
+ return name;
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceManifestPaths.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceManifestPaths.java
new file mode 100644
index 00000000000..993a9a44232
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceManifestPaths.java
@@ -0,0 +1,141 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import java.util.regex.Pattern;
+
+/**
+ * Where a namespace-managed version's manifest must be for Doris to read that version.
+ *
+ * <p>Doris reads a managed version as it reads any other, by the dataset URI and the version
+ * number, and so does the BE: Lance then opens {@code <chain>/_versions/<u64::MAX - v>.manifest},
+ * or {@code <v>.manifest} in the V1 naming scheme, where the chain is the table root or
+ * {@code <root>/tree/<branch>}. The namespace records a manifest path for each version. Lance's
+ * own namespaces finalize a commit in CreateTableVersion and record that canonical path. A
+ * namespace that records the path Lance's client sends records the staged manifest beside it,
+ * named {@code <canonical>-<id>} ({@code make_staging_manifest_path}), and keeps it after the
+ * commit is finalized, since Lance's namespace store cannot update a record. A recorded path
+ * anywhere else names a manifest Doris would not read. It is also what a namespace answers once
+ * it has moved the table away from the location this read described, if the move changed the
+ * path inside the bucket; a path is relative to its bucket or container, so a move to another one
+ * under the same path is not seen here.
+ */
+final class LanceManifestPaths {
+
+ /** How the namespace records a version whose manifest is where Doris reads it. */
+ enum Recorded {
+ /** At its canonical path. */
+ CANONICAL,
+ /**
+ * At a staged manifest beside the canonical path. The version may not have been finalized
+ * yet, or was finalized after the namespace recorded the staged path.
+ */
+ STAGED
+ }
+
+ private static final String MANIFEST_EXTENSION = ".manifest";
+
+ private static final BigInteger U64_MAX = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE);
+
+ /** A URL scheme; lance-io takes a single letter before the colon for a Windows drive instead. */
+ private static final Pattern URL_SCHEME = Pattern.compile("^[A-Za-z][A-Za-z0-9+.-]+:");
+
+ private LanceManifestPaths() {
+ }
+
+ /**
+ * How the namespace records version {@code version} of the chain of {@code tableUri} (the
+ * table root) on {@code branch}.
+ *
+ * @throws IllegalStateException if the recorded path is neither the canonical path nor a
+ * staged manifest beside it
+ */
+ static Recorded check(String tableUri, Optional<String> branch, long version, String manifestPath,
+ String tableName) {
+ String chain = objectStorePath(tableUri);
+ if (branch.isPresent()) {
+ chain = (chain.isEmpty() ? "" : chain + "/") + "tree/" + branch.get();
+ }
+ String versions = (chain.isEmpty() ? "" : chain + "/") + "_versions/";
+ // Padded by hand: String.format would use the FE's locale digits, and Lance writes ASCII.
+ String canonical = versions + StringUtils.leftPad(U64_MAX.subtract(BigInteger.valueOf(version)).toString(),
+ 20, '0') + MANIFEST_EXTENSION;
+ // Lance parses the recorded path first, which drops surrounding slashes.
+ String recorded = manifestPath == null ? "" : StringUtils.strip(manifestPath, "/");
+ for (String name : new String[] {canonical, versions + version + MANIFEST_EXTENSION}) {
+ if (recorded.equals(name)) {
+ return Recorded.CANONICAL;
+ }
+ if (recorded.startsWith(name + "-") && recorded.indexOf('/', name.length()) < 0) {
+ return Recorded.STAGED;
+ }
+ }
+ throw new IllegalStateException("Lance namespace records version " + version + " of " + tableName
+ + branch.map(name -> "@" + name).orElse("") + " at manifest '" + manifestPath
+ + "', but Doris reads that version from '" + canonical
+ + "'; if the table was moved during the query, retry it");
+ }
+
+ /**
+ * A dataset location as Lance's object store addresses it, derived as lance-io derives it: the
+ * path of a URL after its authority (the bucket, container or host), percent-decoded
+ * ({@code Path::from_url_path}), and a location without a scheme as is. Surrounding slashes
+ * are dropped, so a table at the root of a bucket has an empty path.
+ */
+ static String objectStorePath(String location) {
+ if (location == null) {
+ throw new IllegalArgumentException("Lance namespace returned no table location");
+ }
+ if (!URL_SCHEME.matcher(location).find()) {
+ return StringUtils.strip(location, "/");
+ }
+ String path = StringUtils.substringBefore(StringUtils.substringBefore(location, "?"), "#");
+ path = path.substring(path.indexOf(':') + 1);
+ if (path.startsWith("//")) {
+ int slash = path.indexOf('/', 2);
+ path = slash < 0 ? "" : path.substring(slash);
+ }
+ return StringUtils.strip(percentDecode(path), "/");
+ }
+
+ /** Decodes {@code %XX} escapes as UTF-8 bytes and keeps anything else, as Rust's percent_decode does. */
+ private static String percentDecode(String text) {
+ if (text.indexOf('%') < 0) {
+ return text;
+ }
+ byte[] raw = text.getBytes(StandardCharsets.UTF_8);
+ ByteArrayOutputStream decoded = new ByteArrayOutputStream(raw.length);
+ for (int i = 0; i < raw.length; i++) {
+ int high = i + 2 < raw.length && raw[i] == '%' ? Character.digit(raw[i + 1], 16) : -1;
+ int low = high < 0 ? -1 : Character.digit(raw[i + 2], 16);
+ if (low < 0) {
+ decoded.write(raw[i]);
+ } else {
+ decoded.write(high * 16 + low);
+ i += 2;
+ }
+ }
+ return new String(decoded.toByteArray(), StandardCharsets.UTF_8);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceNamespaceClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceNamespaceClient.java
index 79f47210b8b..40e0354d949 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceNamespaceClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceNamespaceClient.java
@@ -31,23 +31,31 @@ import org.apache.commons.lang3.StringUtils;
import org.lance.namespace.LanceNamespace;
import org.lance.namespace.errors.NamespaceNotFoundException;
import org.lance.namespace.errors.TableNotFoundException;
+import org.lance.namespace.errors.TableVersionNotFoundException;
import org.lance.namespace.model.DescribeTableRequest;
import org.lance.namespace.model.DescribeTableResponse;
+import org.lance.namespace.model.DescribeTableVersionRequest;
+import org.lance.namespace.model.DescribeTableVersionResponse;
import org.lance.namespace.model.ListNamespacesRequest;
import org.lance.namespace.model.ListNamespacesResponse;
+import org.lance.namespace.model.ListTableVersionsRequest;
+import org.lance.namespace.model.ListTableVersionsResponse;
import org.lance.namespace.model.ListTablesRequest;
import org.lance.namespace.model.ListTablesResponse;
import org.lance.namespace.model.TableExistsRequest;
+import org.lance.namespace.model.TableVersion;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -66,6 +74,13 @@ final class LanceNamespaceClient {
private final String rootDatabase;
private final List<String> parentNamespace;
private final List<StorageProperties> storageProperties;
+ /**
+ * Serializes the namespace and table listings and table existence checks. Table describes and
+ * version requests, which a read makes whenever the table's access is not cached (always for a
+ * managed table), do not take it: the REST and Directory namespaces are safe to call
+ * concurrently (lance-jni calls them through a shared reference on its multi-threaded runtime),
+ * and a stalled describe must not hold up every other table of the catalog.
+ */
private final Object namespaceLock = new Object();
private final long tableAccessTtlNanos;
private final Ticker ticker;
@@ -227,22 +242,137 @@ final class LanceNamespaceClient {
private CachedTableAccess loadTableAccess(List<String> tableId) {
DescribeTableResponse table = describeTable(tableId);
- if (Boolean.TRUE.equals(table.getManagedVersioning())) {
- throw new UnsupportedOperationException(
- "Lance managed versioning is not supported by the current BE reader");
+ if (Boolean.TRUE.equals(table.getIsOnlyDeclared())) {
+ throw new RuntimeException("Lance table is declared in the namespace but has no data yet");
}
String datasetUri = StringUtils.firstNonBlank(table.getTableUri(), table.getLocation());
if (datasetUri == null) {
throw new RuntimeException("Lance namespace returned no table URI for " + tableId);
}
- // One option map serves both readers: the FE opens the dataset through the Lance Java SDK
- // and the BE through lance-c, so neither can end up with credentials the other lacks. The
- // dataset URL picks the option vocabulary, the same way Lance picks a provider from it.
- Map<String, String> storageOptions = LanceStorageOptions.fromDorisAndVendedStorageOptions(datasetUri,
- storageProperties, table.getStorageOptions());
- return new CachedTableAccess(new LanceTableAccess(datasetUri, storageOptions),
- tableAccessTtlNanos(datasetUri, table.getStorageOptions()));
+ LanceTableAccess access;
+ boolean managed = Boolean.TRUE.equals(table.getManagedVersioning());
+ if (managed) {
+ // The namespace decides which versions exist; the FE and the BE both read one of them
+ // by URI, which is all lance-c supports. The manifest paths the namespace records are
+ // object-store paths under `location`, so `table_uri` must name the same place; it may
+ // add a query, such as presigned credentials, which Doris then opens it with.
+ if (StringUtils.isBlank(table.getLocation())) {
+ throw new RuntimeException("Lance namespace returned no location for managed table " + tableId);
+ }
+ // An s3+ddb URI commits through DynamoDB, whose handler records and finalizes versions
+ // itself, even on read; the namespace already decides the versions of a managed table.
+ if (StringUtils.startsWithIgnoreCase(datasetUri, "s3+ddb:")) {
+ throw new RuntimeException("Lance namespace returned an s3+ddb URI for managed table " + tableId
+ + ", whose versions the namespace manages");
+ }
+ if (StringUtils.isNotBlank(table.getTableUri())
+ && !withoutQuery(table.getTableUri()).equals(withoutQuery(table.getLocation()))) {
+ throw new RuntimeException("Lance namespace returned a table_uri that differs from location for "
+ + "managed table " + tableId);
+ }
+ access = LanceTableAccess.managedByNamespace(datasetUri,
+ storageOptions(datasetUri, table.getStorageOptions()), tableId);
+ } else {
+ access = new LanceTableAccess(datasetUri, storageOptions(datasetUri, table.getStorageOptions()));
+ }
+ // A managed access is not cached: the version list the read asks for next is the
+ // namespace's current one, and must be checked against the location the namespace
+ // reports now, not against one it reported before moving the table.
+ return new CachedTableAccess(access, managed ? 0 : tableAccessTtlNanos(datasetUri, table.getStorageOptions()));
+ }
+
+ /**
+ * A location without its query and trailing slash. A fragment is kept: Lance ignores it, but
+ * joins a branch directory after it, so it would move the branch onto the table root.
+ */
+ private static String withoutQuery(String uri) {
+ return StringUtils.removeEnd(StringUtils.substringBefore(uri, "?"), "/");
+ }
+
+ /**
+ * One option map serves both readers: the FE opens the dataset through the Lance Java SDK and
+ * the BE through lance-c, so neither can end up with credentials the other lacks. The dataset
+ * URL picks the option vocabulary, the same way Lance picks a provider from it.
+ */
+ private Map<String, String> storageOptions(String datasetUri, Map<String, String> vendedOptions) {
+ return LanceStorageOptions.fromDorisAndVendedStorageOptions(datasetUri, storageProperties, vendedOptions);
+ }
+
+ /**
+ * Every version the namespace records for a managed chain. No page size is requested: Lance's
+ * Directory namespace applies a limit without returning a page token, which would silently
+ * truncate the history, while a namespace that pages on its own still returns one.
+ */
+ List<TableVersion> listManagedVersions(LanceTableAccess access, Optional<String> branch) {
+ List<TableVersion> result = new ArrayList<>();
+ String pageToken = null;
+ Set<String> consumedTokens = new HashSet<>();
+ do {
+ ListTableVersionsRequest request = new ListTableVersionsRequest().id(access.getNamespaceTableId());
+ branch.ifPresent(request::branch);
+ if (pageToken != null) {
+ request.pageToken(pageToken);
+ }
+ ListTableVersionsResponse response = namespace.listTableVersions(request);
+ if (response.getVersions() != null) {
+ result.addAll(response.getVersions());
+ }
+ pageToken = response.getPageToken();
+ if (StringUtils.isNotEmpty(pageToken) && !consumedTokens.add(pageToken)) {
+ throw new IllegalStateException("Lance namespace repeated a pagination token");
+ }
+ } while (StringUtils.isNotEmpty(pageToken));
+ return result;
+ }
+
+ /**
+ * The newest version the namespace records for a managed chain, asked for the way the Lance
+ * SDK asks when it opens the latest version: newest first, one entry. Empty when the chain
+ * records no version.
+ */
+ Optional<TableVersion> latestManagedVersion(LanceTableAccess access, Optional<String> branch) {
+ String pageToken = null;
+ Set<String> consumedTokens = new HashSet<>();
+ do {
+ ListTableVersionsRequest request = new ListTableVersionsRequest().id(access.getNamespaceTableId())
+ .descending(true).limit(1);
+ branch.ifPresent(request::branch);
+ if (pageToken != null) {
+ request.pageToken(pageToken);
+ }
+ ListTableVersionsResponse response = namespace.listTableVersions(request);
+ // The maximum rather than the first entry, in case a namespace ignores the limit. A
+ // limit is an upper bound, so a page may be empty and the version on a later one.
+ Optional<TableVersion> newest = response.getVersions() == null ? Optional.empty()
+ : response.getVersions().stream().filter(version -> version.getVersion() != null)
+ .max(Comparator.comparingLong(TableVersion::getVersion));
+ if (newest.isPresent()) {
+ return newest;
+ }
+ pageToken = response.getPageToken();
+ if (StringUtils.isNotEmpty(pageToken) && !consumedTokens.add(pageToken)) {
+ throw new IllegalStateException("Lance namespace repeated a pagination token");
+ }
+ } while (StringUtils.isNotEmpty(pageToken));
+ return Optional.empty();
+ }
+
+ /**
+ * What the namespace records for one version of a managed chain.
+ *
+ * @throws TableVersionNotFoundException if the namespace does not record it
+ */
+ TableVersion describeManagedVersion(LanceTableAccess access, Optional<String> branch, long version) {
+ DescribeTableVersionRequest request = new DescribeTableVersionRequest().id(access.getNamespaceTableId())
+ .version(version);
+ branch.ifPresent(request::branch);
+ DescribeTableVersionResponse response = namespace.describeTableVersion(request);
+ if (response.getVersion() == null || !Long.valueOf(version).equals(response.getVersion().getVersion())) {
+ throw new IllegalStateException("Lance namespace described another version than " + version
+ + " of table " + access.getNamespaceTableId());
+ }
+ return response.getVersion();
}
private long tableAccessTtlNanos(String datasetUri, Map<String, String> vendedOptions) {
@@ -302,9 +432,7 @@ final class LanceNamespaceClient {
private DescribeTableResponse describeTable(List<String> tableId) {
DescribeTableRequest request = new DescribeTableRequest().id(tableId).withTableUri(true)
.vendCredentials(LANCE_REST.equals(catalogType));
- synchronized (namespaceLock) {
- return namespace.describeTable(request);
- }
+ return namespace.describeTable(request);
}
private List<String> buildNamespaceId(String dbName) throws DdlException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceRefSelector.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceRefSelector.java
new file mode 100644
index 00000000000..2ef80565be7
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceRefSelector.java
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.metadata;
+
+import org.apache.doris.analysis.TableSnapshot;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * What a query selects on a Lance table: the latest version of the main chain, an explicit
+ * {@code FOR VERSION AS OF} / {@code FOR TIME AS OF} snapshot, a tag, or a branch (optionally
+ * with a snapshot inside that branch). A tag and a snapshot exclude each other.
+ */
+public final class LanceRefSelector {
+ private static final LanceRefSelector LATEST = new LanceRefSelector(Optional.empty(), Optional.empty(),
+ Optional.empty());
+
+ private final Optional<TableSnapshot> snapshot;
+ private final Optional<String> tag;
+ private final Optional<String> branch;
+
+ private LanceRefSelector(Optional<TableSnapshot> snapshot, Optional<String> tag, Optional<String> branch) {
+ this.snapshot = Objects.requireNonNull(snapshot, "snapshot");
+ this.tag = Objects.requireNonNull(tag, "tag");
+ this.branch = Objects.requireNonNull(branch, "branch");
+ }
+
+ /** The latest version of the main chain. */
+ public static LanceRefSelector latest() {
+ return LATEST;
+ }
+
+ /** A {@code FOR VERSION AS OF} / {@code FOR TIME AS OF} snapshot on the main chain. */
+ public static LanceRefSelector snapshot(Optional<TableSnapshot> snapshot) {
+ return snapshot.isPresent() ? new LanceRefSelector(snapshot, Optional.empty(), Optional.empty()) : LATEST;
+ }
+
+ /** The version a tag points at. */
+ public static LanceRefSelector tag(String tag) {
+ return new LanceRefSelector(Optional.empty(), Optional.of(tag), Optional.empty());
+ }
+
+ /** A branch, at its latest version or at the snapshot selected inside it. */
+ public static LanceRefSelector branch(String branch, Optional<TableSnapshot> snapshot) {
+ return new LanceRefSelector(snapshot, Optional.empty(), Optional.of(branch));
+ }
+
+ public Optional<TableSnapshot> getSnapshot() {
+ return snapshot;
+ }
+
+ public Optional<String> getTag() {
+ return tag;
+ }
+
+ public Optional<String> getBranch() {
+ return branch;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceSnapshotResolver.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceSnapshotResolver.java
index 14a2add436e..7483e4892ad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceSnapshotResolver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceSnapshotResolver.java
@@ -17,14 +17,17 @@
package org.apache.doris.datasource.lance.metadata;
-import org.apache.doris.datasource.lance.LanceExternalCatalog;
-
-import org.lance.Dataset;
import org.lance.Version;
import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.Comparator;
import java.util.List;
+import java.util.NavigableMap;
+import java.util.NavigableSet;
+import java.util.TreeMap;
+import java.util.regex.Pattern;
/** Resolves Doris time-travel selectors to immutable Lance version IDs. */
public final class LanceSnapshotResolver {
@@ -36,8 +39,11 @@ public final class LanceSnapshotResolver {
try {
version = Long.parseLong(value);
} catch (NumberFormatException e) {
- throw new IllegalArgumentException(
- "Lance FOR VERSION AS OF requires a numeric version, but was '" + value + "'", e);
+ // Deliberately not chained: the catalog reports the root cause message, and the
+ // NumberFormatException text would replace this one.
+ throw new IllegalArgumentException(isVersionNumber(value)
+ ? "Lance FOR VERSION AS OF version " + value + " is out of range"
+ : "Lance FOR VERSION AS OF requires a numeric version, but was '" + value + "'");
}
if (version <= 0) {
throw new IllegalArgumentException(
@@ -46,26 +52,123 @@ public final class LanceSnapshotResolver {
return version;
}
+ private static final Pattern VERSION_NUMBER = Pattern.compile("[+-]?[0-9]+");
+
/**
- * Gets the latest Lance version whose commit time does not exceed the requested timestamp.
- *
- * <p>Called by
- * {@link LanceExternalCatalog#loadTableMetadata(String, String, java.util.Optional)} to resolve
- * a {@code FOR TIME AS OF} clause before loading the selected metadata snapshot.
+ * Whether a {@code FOR VERSION AS OF} value is a version number rather than a tag name. A
+ * signed number counts as one, so that {@code '-1'} is reported as an invalid version.
+ */
+ public static boolean isVersionNumber(String value) {
+ return VERSION_NUMBER.matcher(value).matches();
+ }
+
+ /** No version of a chain was committed at or before the requested {@code FOR TIME AS OF} time. */
+ public static final class NoVersionAtOrBeforeException extends IllegalArgumentException {
+ private NoVersionAtOrBeforeException(String requestedText) {
+ super("Lance dataset has no version at or before '" + requestedText + "'");
+ }
+ }
+
+ /**
+ * Cleanup removed a version newer than every version committed at or before the requested
+ * time. Its commit time went with it, so it may have been the answer.
*/
- public static long getVersionAtOrBefore(Dataset dataset, long timestampMillis) {
- return versionAtOrBefore(dataset.listVersions(), timestampMillis);
+ public static final class HistoryRemovedException extends IllegalArgumentException {
+ private final long version;
+
+ private HistoryRemovedException(long version) {
+ super("Lance version " + version + " no longer exists");
+ this.version = version;
+ }
+
+ public long getVersion() {
+ return version;
+ }
+ }
+
+ /** The commit time a manifest records, at the precision it records it in (below a millisecond). */
+ private static Instant commitTime(Version version) {
+ return version.getDataTime().toInstant();
}
- /** Selects a version from the version list fetched by {@link #getVersionAtOrBefore}. */
static long versionAtOrBefore(List<Version> versions, long timestampMillis) {
- Instant requestedTime = Instant.ofEpochMilli(timestampMillis);
+ return versionAtOrBefore(versions, timestampMillis, String.valueOf(timestampMillis));
+ }
+
+ /**
+ * Selects the version committed last at or before the requested timestamp, from the commit
+ * times the manifests record, compared in full: a commit later within the requested
+ * millisecond is after it. Of versions committed at the same instant, the newest (Iceberg
+ * keeps the first of equal times instead). As in Iceberg, commit times are compared as
+ * recorded, without assuming they grow with version numbers.
+ *
+ * @param requestedText the user's {@code FOR TIME AS OF} text, echoed in the error message
+ * @throws NoVersionAtOrBeforeException if every version was committed after the timestamp
+ */
+ public static long versionAtOrBefore(Collection<Version> versions, long timestampMillis, String requestedText) {
+ Instant requested = Instant.ofEpochMilli(timestampMillis);
return versions.stream()
- .filter(version -> !version.getDataTime().toInstant().isAfter(requestedTime))
- .max(Comparator.comparing((Version version) -> version.getDataTime().toInstant())
- .thenComparingLong(Version::getId))
- .orElseThrow(() -> new IllegalArgumentException(
- "Lance dataset has no version at or before timestamp " + timestampMillis))
+ .filter(version -> !commitTime(version).isAfter(requested))
+ .max(Comparator.comparing(LanceSnapshotResolver::commitTime).thenComparingLong(Version::getId))
+ .orElseThrow(() -> new NoVersionAtOrBeforeException(requestedText))
.getId();
}
+
+ /**
+ * Resolves {@code FOR TIME AS OF} on one manifest chain with {@link #versionAtOrBefore}.
+ *
+ * <p>Commit times need not grow with version numbers, and a version removed by cleanup takes
+ * its commit time with it, so it could have been the answer. As Iceberg drops its snapshot log
+ * before a removed snapshot, only the versions newer than the newest removed one are
+ * candidates; a time none of them covers fails rather than reading an older snapshot.
+ *
+ * @param listed the chain's versions a storage listing shows
+ * @param recorded the versions a namespace records for a managed chain, which are the chain;
+ * null when the chain is what storage holds. Lance numbers a chain's commits consecutively,
+ * under a namespace too (DirectoryNamespace accepts only the next version), so a number the
+ * listing skips is a removed version, and on a managed chain so is a number between its
+ * oldest and newest recorded version that the namespace lacks, and a recorded version the
+ * listing lacks
+ * @throws HistoryRemovedException if no candidate qualifies and a removed version cut the history
+ * @throws NoVersionAtOrBeforeException if no candidate qualifies otherwise
+ */
+ public static long versionAtOrBefore(Collection<Version> listed, NavigableSet<Long> recorded,
+ long timestampMillis, String requestedText) {
+ NavigableMap<Long, Version> byId = new TreeMap<>();
+ for (Version version : listed) {
+ if (recorded == null || recorded.contains(version.getId())) {
+ byId.put(version.getId(), version);
+ }
+ }
+ List<Version> history = new ArrayList<>();
+ Long removed = null;
+ if (recorded == null) {
+ Long expected = byId.isEmpty() ? null : byId.lastKey();
+ for (Version version : byId.descendingMap().values()) {
+ if (version.getId() != expected) {
+ removed = expected;
+ break;
+ }
+ history.add(version);
+ expected = version.getId() - 1;
+ }
+ } else if (!recorded.isEmpty()) {
+ for (long id = recorded.last(); id >= recorded.first(); id--) {
+ Version version = recorded.contains(id) ? byId.get(id) : null;
+ if (version == null) {
+ removed = id;
+ break;
+ }
+ history.add(version);
+ }
+ }
+ try {
+ return versionAtOrBefore(history, timestampMillis, requestedText);
+ } catch (NoVersionAtOrBeforeException e) {
+ if (removed != null) {
+ throw new HistoryRemovedException(removed);
+ }
+ throw e;
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTableAccess.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTableAccess.java
index 6d6ee6103a5..2249430c106 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTableAccess.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTableAccess.java
@@ -17,25 +17,81 @@
package org.apache.doris.datasource.lance.metadata;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
-/** Access parameters resolved for one read; credentials are not versioned dataset metadata. */
+/**
+ * Access parameters resolved for one read; credentials are not versioned dataset metadata.
+ *
+ * <p>Every dataset is read by its URI and version, by the FE and by the BE alike: Lance resolves
+ * a version from the {@code _versions/} directory of the dataset itself. For a namespace-managed
+ * dataset ({@code DescribeTableResponse.managed_versioning = true}) the namespace decides which
+ * versions exist and which is the newest, so the FE asks it before choosing the version to read.
+ */
public final class LanceTableAccess {
private final String datasetUri;
private final Map<String, String> storageOptions;
+ private final List<String> namespaceTableId;
+ private final String branch;
+ /** A dataset whose versions live in its own {@code _versions/} directory. */
public LanceTableAccess(String datasetUri, Map<String, String> storageOptions) {
- this.datasetUri = datasetUri;
+ this(datasetUri, storageOptions, null, null);
+ }
+
+ private LanceTableAccess(String datasetUri, Map<String, String> storageOptions,
+ List<String> namespaceTableId, String branch) {
+ this.datasetUri = Objects.requireNonNull(datasetUri, "datasetUri");
this.storageOptions = Collections.unmodifiableMap(new HashMap<>(storageOptions));
+ this.namespaceTableId = namespaceTableId == null
+ ? null : Collections.unmodifiableList(new ArrayList<>(namespaceTableId));
+ this.branch = branch;
+ }
+
+ /**
+ * The same table on one of its branches. A branch is a separate manifest chain with its own
+ * root directory, {@code <table>/tree/<branch>}; a reader that opens by URI, such as the BE,
+ * addresses the branch by that root. The storage options and namespace
+ * identity are unchanged.
+ */
+ public LanceTableAccess onBranch(String branchName, String branchUri) {
+ return new LanceTableAccess(Objects.requireNonNull(branchUri, "branchUri"), storageOptions,
+ namespaceTableId, Objects.requireNonNull(branchName, "branchName"));
+ }
+
+ /** The branch this access addresses, if not the main chain. */
+ public Optional<String> getBranch() {
+ return Optional.ofNullable(branch);
+ }
+
+ /** A dataset whose versions are recorded by the namespace that owns {@code namespaceTableId}. */
+ public static LanceTableAccess managedByNamespace(String datasetUri, Map<String, String> storageOptions,
+ List<String> namespaceTableId) {
+ return new LanceTableAccess(datasetUri, storageOptions,
+ Objects.requireNonNull(namespaceTableId, "namespaceTableId"), null);
}
public String getDatasetUri() {
return datasetUri;
}
+ /** The options both the FE and the BE open the dataset with. */
public Map<String, String> getStorageOptions() {
return storageOptions;
}
+
+ /** Whether the namespace, rather than the dataset directory, records which versions exist. */
+ public boolean isManagedVersioning() {
+ return namespaceTableId != null;
+ }
+
+ /** The namespace table identifier of a managed dataset; null otherwise. */
+ public List<String> getNamespaceTableId() {
+ return namespaceTableId;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTableMetadata.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTableMetadata.java
index 4dd892751cb..a53a57af0bc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTableMetadata.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTableMetadata.java
@@ -28,6 +28,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.OptionalInt;
/** Immutable metadata resolved from one already-fixed Lance dataset version. */
@@ -96,6 +97,19 @@ public class LanceTableMetadata {
return version;
}
+ /**
+ * Whether {@link #getVersion()} was resolved through the namespace that manages the table's
+ * versions rather than from the dataset's own {@code _versions/} directory.
+ */
+ public boolean isManagedVersioning() {
+ return access.isManagedVersioning();
+ }
+
+ /** The branch {@link #getVersion()} belongs to, if the query selected one. */
+ public Optional<String> getBranch() {
+ return access.getBranch();
+ }
+
public Schema getSchema() {
return schema;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java
index 9607596e59e..a483ef52aa2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java
@@ -395,6 +395,10 @@ public class LanceScanNode extends FileQueryScanNode {
result.append(prefix).append("lanceCatalogType=")
.append(((LanceExternalCatalog) lanceTable.getCatalog()).getLanceCatalogType()).append("\n");
result.append(prefix).append("lanceVersion=").append(scanPlan.version).append("\n");
+ result.append(prefix).append("lanceManagedVersioning=")
+ .append(plannedMetadata.isManagedVersioning()).append("\n");
+ plannedMetadata.getBranch().ifPresent(branch ->
+ result.append(prefix).append("lanceBranch=").append(branch).append("\n"));
result.append(prefix).append("lanceFragments=").append(scanPlan.fragmentCount).append("\n");
if (scanPlan.fragmentsPerSplit > 0) {
result.append(prefix).append("lanceFragmentGrouping=DEBUG\n");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LanceExternalSearchTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LanceExternalSearchTableValuedFunction.java
index be2589edfbd..12d078e1c9a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LanceExternalSearchTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LanceExternalSearchTableValuedFunction.java
@@ -58,6 +58,7 @@ import java.util.OptionalInt;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.regex.Pattern;
/** Common immutable planning state and validation for Lance external-search relation TVFs. */
abstract class LanceExternalSearchTableValuedFunction extends TableValuedFunctionIf {
@@ -70,6 +71,9 @@ abstract class LanceExternalSearchTableValuedFunction extends TableValuedFunctio
private static final String FULLY_QUALIFIED_TABLE_NAME_ERROR =
"'table' must be a fully qualified catalog.database.table name";
private static final long UINT32_MAX = 0xFFFF_FFFFL;
+ /** A trailing @tag/@branch/... or FOR VERSION/TIME AS OF on the 'table' argument. */
+ private static final Pattern SELECTOR_SUFFIX = Pattern.compile(
+ "(?i)(@\\s*(tag|branch|incr|options)\\b|\\sfor\\s+(version|time)\\b)");
private final String displayName;
private final TableName sourceTableName;
@@ -257,15 +261,27 @@ abstract class LanceExternalSearchTableValuedFunction extends TableValuedFunctio
return result;
}
+ /**
+ * The search TVFs always read the latest version of the main chain. A name that only fails to
+ * parse because it carries a selector gets a message saying so; a backquoted name containing
+ * '@' or ' for ' parses and is never affected.
+ */
+ private static String tableNameError(String value) {
+ return SELECTOR_SUFFIX.matcher(value).find()
+ ? "'table' of a Lance search function cannot select a version, tag or branch;"
+ + " it always searches the latest version"
+ : FULLY_QUALIFIED_TABLE_NAME_ERROR;
+ }
+
protected static TableName parseTableName(String value) throws AnalysisException {
Expression expression;
try {
expression = new NereidsParser().parseExpression(value);
} catch (ParseException e) {
- throw new AnalysisException(FULLY_QUALIFIED_TABLE_NAME_ERROR, e);
+ throw new AnalysisException(tableNameError(value), e);
}
if (!(expression instanceof UnboundSlot)) {
- throw new AnalysisException(FULLY_QUALIFIED_TABLE_NAME_ERROR);
+ throw new AnalysisException(tableNameError(value));
}
List<String> names = ((UnboundSlot) expression).getNameParts();
if (names.size() != 3) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManagedAccessTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManagedAccessTest.java
new file mode 100644
index 00000000000..94c23cc904c
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManagedAccessTest.java
@@ -0,0 +1,168 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance;
+
+import org.apache.doris.datasource.lance.metadata.LanceTableAccess;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.lance.namespace.LanceNamespace;
+import org.lance.namespace.model.DescribeTableResponse;
+import org.lance.namespace.model.ListTableVersionsRequest;
+import org.lance.namespace.model.ListTableVersionsResponse;
+import org.lance.namespace.model.TableVersion;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+/** How a managed table's access, newest version and branch directory are resolved, without JNI. */
+public class LanceManagedAccessTest {
+ private static final String LOCATION = "s3://example-bucket/items.lance";
+
+ /** A table_uri may add a query, such as presigned credentials, to the location; Doris opens it. */
+ @Test
+ public void testTableUriMayAddAQueryToTheLocation() {
+ LanceNamespace namespace = describing(LOCATION, LOCATION + "?sig=abc");
+ LanceTableAccess access = client(namespace).resolveTableAccess("default", "items");
+ Assertions.assertTrue(access.isManagedVersioning());
+ Assertions.assertEquals(LOCATION + "?sig=abc", access.getDatasetUri());
+
+ RuntimeException elsewhere = Assertions.assertThrows(RuntimeException.class,
+ () -> client(describing(LOCATION, "s3://example-bucket/other.lance?sig=abc"))
+ .resolveTableAccess("default", "items"));
+ Assertions.assertTrue(elsewhere.getMessage().contains("differs from location"), elsewhere.getMessage());
+
+ // A fragment is not a query: Lance would join a branch directory after it.
+ RuntimeException fragment = Assertions.assertThrows(RuntimeException.class,
+ () -> client(describing(LOCATION, LOCATION + "#f")).resolveTableAccess("default", "items"));
+ Assertions.assertTrue(fragment.getMessage().contains("differs from location"), fragment.getMessage());
+ }
+
+ /**
+ * Managed tables are described on every read, so their describes must not queue behind each
+ * other: each describe below waits until the other one has started.
+ */
+ @Test
+ public void testManagedDescribesRunConcurrently() throws Exception {
+ CountDownLatch bothStarted = new CountDownLatch(2);
+ LanceNamespace namespace = Mockito.mock(LanceNamespace.class);
+ Mockito.when(namespace.describeTable(Mockito.any())).thenAnswer(invocation -> {
+ bothStarted.countDown();
+ if (!bothStarted.await(10, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("the other describe never started");
+ }
+ return new DescribeTableResponse().location(LOCATION).tableUri(LOCATION).managedVersioning(true);
+ });
+ LanceNamespaceClient client = client(namespace);
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ Future<LanceTableAccess> first = executor.submit(() -> client.resolveTableAccess("default", "a"));
+ Future<LanceTableAccess> second = executor.submit(() -> client.resolveTableAccess("default", "b"));
+ Assertions.assertTrue(first.get(20, TimeUnit.SECONDS).isManagedVersioning());
+ Assertions.assertTrue(second.get(20, TimeUnit.SECONDS).isManagedVersioning());
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
+ /** An s3+ddb URI commits through DynamoDB, which would record and finalize versions on read. */
+ @Test
+ public void testManagedTableRejectsADynamoDbCommitUri() {
+ String ddb = "s3+ddb://example-bucket/items.lance";
+ RuntimeException rejected = Assertions.assertThrows(RuntimeException.class,
+ () -> client(describing(ddb, ddb + "?ddbTableName=commits")).resolveTableAccess("default", "items"));
+ Assertions.assertTrue(rejected.getMessage().contains("s3+ddb"), rejected.getMessage());
+ }
+
+ /** A limit is an upper bound: an empty page with a token does not mean an empty history. */
+ @Test
+ public void testNewestVersionFollowsPageTokens() {
+ LanceNamespace namespace = describing(LOCATION, LOCATION);
+ Mockito.when(namespace.listTableVersions(Mockito.any())).thenReturn(
+ new ListTableVersionsResponse().versions(Collections.emptyList()).pageToken("p2"),
+ new ListTableVersionsResponse().versions(Collections.singletonList(
+ new TableVersion().version(3L).manifestPath("items.lance/_versions/3.manifest"))));
+ LanceNamespaceClient client = client(namespace);
+ LanceTableAccess access = client.resolveTableAccess("default", "items");
+ Optional<TableVersion> newest = client.latestManagedVersion(access, Optional.of("dev"));
+ Assertions.assertEquals(Long.valueOf(3), newest.map(TableVersion::getVersion).orElse(null));
+ ArgumentCaptor<ListTableVersionsRequest> requests = ArgumentCaptor.forClass(ListTableVersionsRequest.class);
+ Mockito.verify(namespace, Mockito.times(2)).listTableVersions(requests.capture());
+ List<ListTableVersionsRequest> sent = requests.getAllValues();
+ Assertions.assertNull(sent.get(0).getPageToken());
+ Assertions.assertEquals("p2", sent.get(1).getPageToken());
+ Assertions.assertEquals("dev", sent.get(1).getBranch());
+ Assertions.assertEquals(Boolean.TRUE, sent.get(1).getDescending());
+
+ // Every page empty: the chain records no version.
+ LanceNamespace empty = describing(LOCATION, LOCATION);
+ Mockito.when(empty.listTableVersions(Mockito.any())).thenReturn(
+ new ListTableVersionsResponse().versions(Collections.emptyList()).pageToken("p2"),
+ new ListTableVersionsResponse().versions(Collections.emptyList()));
+ LanceNamespaceClient emptyClient = client(empty);
+ Assertions.assertEquals(Optional.empty(), emptyClient.latestManagedVersion(
+ emptyClient.resolveTableAccess("default", "items"), Optional.empty()));
+
+ // A token the namespace hands out twice would loop forever.
+ LanceNamespace looping = describing(LOCATION, LOCATION);
+ Mockito.when(looping.listTableVersions(Mockito.any())).thenReturn(
+ new ListTableVersionsResponse().versions(Collections.emptyList()).pageToken("p2"));
+ LanceNamespaceClient loopingClient = client(looping);
+ LanceTableAccess loopingAccess = loopingClient.resolveTableAccess("default", "items");
+ Assertions.assertThrows(IllegalStateException.class,
+ () -> loopingClient.latestManagedVersion(loopingAccess, Optional.empty()));
+ }
+
+ /** A branch's directory is joined as Lance joins it, and branchOf reads the branch back. */
+ @Test
+ public void testBranchUriFollowsLance() {
+ Assertions.assertEquals("s3://b/t.lance/tree/dev", LanceCatalogClient.branchUri("s3://b/t.lance", "dev"));
+ Assertions.assertEquals("s3://b/t.lance/tree/dev", LanceCatalogClient.branchUri("s3://b/t.lance/", "dev"));
+ Assertions.assertEquals("s3://b/t.lance/tree/feature/x",
+ LanceCatalogClient.branchUri("s3://b/t.lance", "feature/x"));
+ // Lance appends path segments before a query string.
+ Assertions.assertEquals("s3+ddb://b/t.lance/tree/dev?ddbTableName=t",
+ LanceCatalogClient.branchUri("s3+ddb://b/t.lance?ddbTableName=t", "dev"));
+ for (String root : new String[] {"s3://b/t.lance", "s3://b/t.lance?sig=abc", "file:///tmp/t.lance"}) {
+ for (String branch : new String[] {"dev", "feature/x"}) {
+ Assertions.assertEquals(Optional.of(branch),
+ LanceCatalogClient.branchOf(LanceCatalogClient.branchUri(root, branch), root), root);
+ }
+ }
+ }
+
+ private static LanceNamespace describing(String location, String tableUri) {
+ LanceNamespace namespace = Mockito.mock(LanceNamespace.class);
+ Mockito.when(namespace.describeTable(Mockito.any())).thenReturn(new DescribeTableResponse()
+ .location(location).tableUri(tableUri).managedVersioning(true));
+ return namespace;
+ }
+
+ private static LanceNamespaceClient client(LanceNamespace namespace) {
+ return new LanceNamespaceClient(namespace, "rest", "default", Collections.emptyList(),
+ Collections.emptyList(), 60, () -> TimeUnit.MILLISECONDS.toNanos(1_000_000));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManagedVersioningTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManagedVersioningTest.java
new file mode 100644
index 00000000000..6d5da9f669c
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManagedVersioningTest.java
@@ -0,0 +1,1460 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance;
+
+import org.apache.doris.analysis.TableSnapshot;
+import org.apache.doris.common.util.JsonUtil;
+import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.datasource.lance.metadata.LanceRefSelector;
+import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.io.ByteStreams;
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpServer;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.lance.Dataset;
+import org.lance.Fragment;
+import org.lance.FragmentMetadata;
+import org.lance.FragmentOperation;
+import org.lance.Ref;
+import org.lance.Version;
+import org.lance.WriteParams;
+import org.lance.cleanup.CleanupPolicy;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+
+/**
+ * Opens a namespace-managed Lance table through a REST namespace stub.
+ *
+ * <p>The stub owns the version list, the way a namespace with managed versioning does:
+ * DescribeTable reports {@code managed_versioning=true}, ListTableVersions and
+ * DescribeTableVersion answer from a static list, and the dataset itself is a real local Lance
+ * dataset with three versions written through the SDK. The test checks that the FE chooses
+ * versions by what the namespace records, that a version it does not record, or records
+ * somewhere Doris does not read, is unreachable, and that the FE reads the chosen version by URI
+ * with the storage options the BE is handed.
+ */
+@Disabled("Re-enable after fixing Arrow C Data JNI compatibility: CI libstdc++ lacks CXXABI_1.3.9")
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class LanceManagedVersioningTest {
+ private static final String BEARER_TOKEN = "managed-test-token";
+ private static final String FULL_TABLE = "managed_full";
+ private static final String PARTIAL_TABLE = "managed_partial";
+ private static final String PLAIN_TABLE = "plain";
+ private static final String DECLARED_TABLE = "declared_only";
+ private static final String EXPIRED_TABLE = "plain_expired";
+ /** Managed, records every version, but reports no commit times. */
+ private static final String UNTIMED_TABLE = "managed_untimed";
+ /** Managed, records only versions 1 and 2 while storage already holds version 3. */
+ private static final String LAGGING_TABLE = "managed_lagging";
+ /** Managed; its version 3 manifest is still at the staged path the namespace recorded. */
+ private static final String STAGED_TABLE = "managed_staged";
+ /** Managed; the namespace records more versions over time. */
+ private static final String GROWING_TABLE = "managed_growing";
+ /** Managed; DescribeTableVersion fails with a server error for version 2. */
+ private static final String FLAKY_TABLE = "managed_flaky";
+ /** Managed; DescribeTable returns a table_uri but no location. */
+ private static final String NO_LOCATION_TABLE = "managed_no_location";
+ /** Managed; DescribeTable returns a location that differs from table_uri. */
+ private static final String MISMATCH_TABLE = "managed_mismatch";
+ /** Managed; ListTableVersions without a page size returns one version per page with a token. */
+ private static final String PAGED_TABLE = "managed_paged";
+ /** Managed; the namespace lists no versions on main, and versions 2 and 3 on branch dev. */
+ private static final String EMPTY_TABLE = "managed_empty";
+ /** Managed; records every main version, but none on the branch storage holds. */
+ private static final String UNRECORDED_BRANCH_TABLE = "managed_branch_unrecorded";
+ /** Managed; reports a branch it lacks as a missing table (code 4), as Lance's Directory namespace does. */
+ private static final String DIRECTORY_ERRORS_TABLE = "managed_directory_errors";
+ /** Managed; DescribeTable moves to another dataset once {@link #relocated} is set. */
+ private static final String RELOCATED_TABLE = "managed_relocated";
+ /** Managed, untimed; its version 1 manifest is still at the staged path the namespace recorded. */
+ private static final String STAGED_UNTIMED_TABLE = "managed_staged_untimed";
+ /**
+ * Managed, untimed; records versions 1 to 3 of the expired dataset, where cleanup kept only 3.
+ * Version 1 is recorded at a staged path that no longer exists, version 2 at its canonical path.
+ */
+ private static final String UNTIMED_EXPIRED_TABLE = "managed_untimed_expired";
+ /** Storage-versioned; version 2 of its dataset was removed, versions 1 and 3 remain. */
+ private static final String PLAIN_GAP_TABLE = "plain_gap";
+ /** Managed; records versions 1 to 3 of the dataset whose version 2 was removed. */
+ private static final String MANAGED_GAP_TABLE = "managed_gap";
+ /** Managed; the namespace reports the first commit's time for every version, as after a copy. */
+ private static final String MISTIMED_TABLE = "managed_mistimed";
+ /** Managed; the namespace records version 9 as the head of branch dev, which storage lacks. */
+ private static final String MISSING_HEAD_TABLE = "managed_missing_head";
+ /** Managed; every DescribeTable vends a new value for {@link #GENERATION_OPTION}, as rotating credentials do. */
+ private static final String ROTATING_TABLE = "managed_rotating";
+ private static final String GENERATION_OPTION = "doris_test_describe_generation";
+ /** The suffix Lance's make_staging_manifest_path appends to a canonical manifest path. */
+ private static final String STAGED_SUFFIX = "-3c9d0e1f-2a4b-4c6d-8e0f-1a2b3c4d5e6f";
+ private static final String STAGED_ONLY = " cannot be read: the namespace records it at a staged manifest,"
+ + " and its canonical manifest, which Doris reads, does not exist (its commit was not finalized,"
+ + " or cleanup removed it)";
+ /** Managed; records version 3, on main and on dev, at a finalized manifest outside its canonical path. */
+ private static final String CUSTOM_MANIFEST_TABLE = "managed_custom_manifest";
+ /**
+ * Managed, untimed; right after answering a describe with {@link #moveOnDescribe} set, the
+ * namespace moves it to {@link #moveTo}.
+ */
+ private static final String MOVING_TABLE = "managed_moving";
+ /**
+ * Managed, untimed; time_travel.lance with a fourth version on main and on dev that the namespace
+ * records at a staged manifest that was never finalized. Version 3 of main is recorded at its
+ * staged manifest too, but was finalized.
+ */
+ private static final String PENDING_HEAD_TABLE = "managed_pending_head";
+ /**
+ * Managed; a dataset whose main manifests were all deleted, and whose branch dev (versions 3
+ * and 4) a namespace that keeps its own records still records. It records nothing on main.
+ */
+ private static final String BRANCH_ONLY_TABLE = "managed_branch_only";
+ /** A branch of time_travel.lance forked from version 2 with one extra append (row 100). */
+ private static final String BRANCH = "dev";
+ /** A tag pointing at version 3 of the branch. */
+ private static final String BRANCH_TAG = "rel";
+ private static final BigInteger U64_MAX = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE);
+
+ private final List<String> requestPaths = new CopyOnWriteArrayList<>();
+ private final List<String> requestQueries = new CopyOnWriteArrayList<>();
+ private HttpServer server;
+ private String restUri;
+ private String datasetUri;
+ private String datasetStorePath;
+ private List<Version> datasetVersions;
+ private long branchVersions;
+ private List<Version> branchDatasetVersions;
+ private String expiredDatasetUri;
+ private String stagedDatasetUri;
+ private Path stagedCanonicalManifest;
+ private String stagedManifestStorePath;
+ private String relocatedDatasetUri;
+ private String relocatedStorePath;
+ private volatile boolean relocated;
+ private String customManifestStorePath;
+ private String movingDatasetUri;
+ private String movingStorePath;
+ private volatile MoveTarget moveTo;
+ private volatile boolean moveOnDescribe;
+ private volatile boolean moved;
+ private final AtomicInteger rotatingDescribes = new AtomicInteger();
+ private String stagedUntimedDatasetUri;
+ private String stagedUntimedStorePath;
+ private String stagedUntimedV1StorePath;
+ private Path stagedUntimedV1Canonical;
+ private List<Version> stagedUntimedVersions;
+ private String gapDatasetUri;
+ private String gapStorePath;
+ private String branchOnlyDatasetUri;
+ private String branchOnlyStorePath;
+ private List<Version> gapVersions;
+ /** Versions the stub namespace records per managed table; null means storage-versioned. */
+ private final Map<String, List<Long>> namespaceVersions = new java.util.concurrent.ConcurrentHashMap<>();
+
+ private Path tempDir;
+
+ @BeforeAll
+ public void setUp() throws Exception {
+ LanceJniTestSupport.assumeJniBindingsLoadable();
+ tempDir = Files.createTempDirectory("lance_managed_versioning");
+ Path datasetDir = tempDir.resolve("time_travel.lance");
+ datasetUri = datasetDir.toUri().toString();
+ // Lance's object store addresses local files by their absolute path without the
+ // leading slash, which is the form a namespace records in TableVersion.manifest_path.
+ datasetStorePath = datasetDir.toAbsolutePath().toString().replaceFirst("^/", "");
+ datasetVersions = writeThreeVersions(datasetUri);
+ Assertions.assertEquals(3, datasetVersions.size());
+ try (BufferAllocator allocator = new RootAllocator();
+ Dataset dataset = Dataset.open(datasetUri, allocator)) {
+ dataset.tags().create("v2", 2);
+ try (Dataset branch = dataset.createBranch(BRANCH, Ref.ofMain(2))) {
+ Assertions.assertEquals(2, branch.version());
+ }
+ appendRows(datasetUri + "/tree/" + BRANCH, allocator, 2, 100, 100);
+ // A tag that points into the branch, not at main's version 3.
+ dataset.tags().create(BRANCH_TAG, 3, BRANCH);
+ }
+ branchVersions = 3;
+ try (BufferAllocator allocator = new RootAllocator();
+ Dataset branch = Dataset.open(datasetUri + "/tree/" + BRANCH, allocator)) {
+ branchDatasetVersions = branch.listVersions();
+ }
+ // A second dataset whose two oldest versions were removed by cleanup, so that an explicit
+ // selector can ask for a version that once existed but is no longer readable.
+ expiredDatasetUri = tempDir.resolve("expired.lance").toUri().toString();
+ writeThreeVersions(expiredDatasetUri);
+ try (BufferAllocator allocator = new RootAllocator();
+ Dataset dataset = Dataset.open(expiredDatasetUri, allocator)) {
+ dataset.cleanupWithPolicy(CleanupPolicy.builder().withBeforeVersion(3)
+ .withDeleteUnverified(true).build());
+ Assertions.assertEquals(1, dataset.listVersions().size(), "cleanup must leave only version 3");
+ }
+
+ // A third dataset whose version 3 manifest sits at the staged path a namespace records
+ // right after a commit, before anything finalizes it to the canonical path.
+ Path stagedDir = tempDir.resolve("staged.lance");
+ stagedDatasetUri = stagedDir.toUri().toString();
+ writeThreeVersions(stagedDatasetUri);
+ stagedCanonicalManifest = stagedDir.resolve("_versions")
+ .resolve(U64_MAX.subtract(BigInteger.valueOf(3)) + ".manifest");
+ Path stagedManifest = stagedDir.resolve("_versions")
+ .resolve(stagedCanonicalManifest.getFileName() + "-2f6a1c0e-5b7d-4e1a-9c3b-0d8e7f6a5b4c");
+ Files.move(stagedCanonicalManifest, stagedManifest);
+ Files.deleteIfExists(stagedDir.resolve("_versions").resolve("latest_version_hint.json"));
+ stagedManifestStorePath = stagedManifest.toAbsolutePath().toString().replaceFirst("^/", "");
+
+ // The dataset the relocated table moves to.
+ Path relocatedDir = tempDir.resolve("relocated.lance");
+ relocatedDatasetUri = relocatedDir.toUri().toString();
+ relocatedStorePath = relocatedDir.toAbsolutePath().toString().replaceFirst("^/", "");
+ writeThreeVersions(relocatedDatasetUri);
+
+ // A finalized copy of version 3's manifest outside its canonical path.
+ Path customManifest = Files.createDirectories(tempDir.resolve("manifests")).resolve("custom.manifest");
+ Files.copy(datasetDir.resolve("_versions").resolve(U64_MAX.subtract(BigInteger.valueOf(3)) + ".manifest"),
+ customManifest);
+ customManifestStorePath = customManifest.toAbsolutePath().toString().replaceFirst("^/", "");
+
+ // The dataset the moving table moves to: time_travel.lance's history plus a fourth version.
+ Path movingDir = tempDir.resolve("moving.lance");
+ movingDatasetUri = movingDir.toUri().toString();
+ movingStorePath = movingDir.toAbsolutePath().toString().replaceFirst("^/", "");
+ writeThreeVersions(movingDatasetUri);
+ try (BufferAllocator allocator = new RootAllocator()) {
+ Assertions.assertEquals(4, appendRows(movingDatasetUri, allocator, 3, 20, 22));
+ }
+
+ // A dataset whose oldest version is still staged: storage listings do not show it.
+ Path stagedUntimedDir = tempDir.resolve("staged_untimed.lance");
+ stagedUntimedDatasetUri = stagedUntimedDir.toUri().toString();
+ stagedUntimedStorePath = stagedUntimedDir.toAbsolutePath().toString().replaceFirst("^/", "");
+ stagedUntimedVersions = writeThreeVersions(stagedUntimedDatasetUri);
+ stagedUntimedV1Canonical = stagedUntimedDir.resolve("_versions")
+ .resolve(U64_MAX.subtract(BigInteger.ONE) + ".manifest");
+ Path stagedUntimedV1 = stagedUntimedDir.resolve("_versions")
+ .resolve(stagedUntimedV1Canonical.getFileName() + "-7c1d2e3f-4a5b-4c6d-8e9f-0a1b2c3d4e5f");
+ Files.move(stagedUntimedV1Canonical, stagedUntimedV1);
+ stagedUntimedV1StorePath = stagedUntimedV1.toAbsolutePath().toString().replaceFirst("^/", "");
+
+ // A dataset with a hole in its history: cleanup removed version 2 only.
+ Path gapDir = tempDir.resolve("gap.lance");
+ gapDatasetUri = gapDir.toUri().toString();
+ gapStorePath = gapDir.toAbsolutePath().toString().replaceFirst("^/", "");
+ gapVersions = writeThreeVersions(gapDatasetUri);
+ try (BufferAllocator allocator = new RootAllocator();
+ Dataset dataset = Dataset.open(gapDatasetUri, allocator)) {
+ dataset.cleanupWithPolicy(CleanupPolicy.builder().withVersions(Collections.singletonList(2L))
+ .withDeleteUnverified(true).build());
+ Assertions.assertEquals(Arrays.asList(1L, 3L), dataset.listVersions().stream().map(Version::getId)
+ .collect(java.util.stream.Collectors.toList()), "cleanup must leave versions 1 and 3");
+ }
+
+ // A dataset with a branch but no main manifests left.
+ Path branchOnlyDir = tempDir.resolve("branch_only.lance");
+ branchOnlyDatasetUri = branchOnlyDir.toUri().toString();
+ branchOnlyStorePath = branchOnlyDir.toAbsolutePath().toString().replaceFirst("^/", "");
+ writeThreeVersions(branchOnlyDatasetUri);
+ try (BufferAllocator allocator = new RootAllocator();
+ Dataset dataset = Dataset.open(branchOnlyDatasetUri, allocator)) {
+ try (Dataset branch = dataset.createBranch(BRANCH, Ref.ofMain(3))) {
+ Assertions.assertEquals(3, branch.version());
+ }
+ Assertions.assertEquals(4, appendRows(branchOnlyDatasetUri + "/tree/" + BRANCH, allocator, 3, 200, 201));
+ }
+ try (java.util.stream.Stream<Path> mainManifests = Files.list(branchOnlyDir.resolve("_versions"))) {
+ for (Path manifest : (Iterable<Path>) mainManifests::iterator) {
+ Files.delete(manifest);
+ }
+ }
+
+ namespaceVersions.put(FULL_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(PARTIAL_TABLE, Arrays.asList(1L, 3L));
+ namespaceVersions.put(UNTIMED_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(LAGGING_TABLE, Arrays.asList(1L, 2L));
+ namespaceVersions.put(STAGED_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(GROWING_TABLE, Arrays.asList(1L, 2L));
+ namespaceVersions.put(FLAKY_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(NO_LOCATION_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(MISMATCH_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(PAGED_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(EMPTY_TABLE, Collections.emptyList());
+ namespaceVersions.put(UNRECORDED_BRANCH_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(RELOCATED_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(STAGED_UNTIMED_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(UNTIMED_EXPIRED_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(MANAGED_GAP_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(MISTIMED_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(MISSING_HEAD_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(ROTATING_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(CUSTOM_MANIFEST_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(MOVING_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(PENDING_HEAD_TABLE, Arrays.asList(1L, 2L, 3L, 4L));
+ namespaceVersions.put(DIRECTORY_ERRORS_TABLE, Arrays.asList(1L, 2L, 3L));
+ namespaceVersions.put(BRANCH_ONLY_TABLE, Collections.emptyList());
+
+ server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+ server.createContext("/", this::handleRequest);
+ server.start();
+ restUri = "http://127.0.0.1:" + server.getAddress().getPort() + "/";
+ }
+
+ @AfterAll
+ public void tearDown() throws IOException {
+ if (server != null) {
+ server.stop(0);
+ }
+ if (tempDir != null) {
+ try (java.util.stream.Stream<Path> paths = Files.walk(tempDir)) {
+ paths.sorted(java.util.Comparator.reverseOrder()).forEach(path -> path.toFile().delete());
+ }
+ }
+ }
+
+ @Test
+ public void testManagedTableResolvesVersionsThroughNamespace() throws Exception {
+ LanceExternalCatalog catalog = newCatalog(300, "lance_managed_full");
+ try {
+ requestPaths.clear();
+ LanceTableMetadata latest = catalog.loadTableMetadata("default", FULL_TABLE);
+ Assertions.assertTrue(latest.isManagedVersioning());
+ Assertions.assertEquals(3, latest.getVersion());
+ Assertions.assertEquals(2, latest.getFragments().size());
+ Assertions.assertEquals(datasetUri, latest.getDatasetUri());
+ Assertions.assertTrue(requestPaths.stream().anyMatch(
+ path -> path.endsWith("/" + FULL_TABLE + "/version/list")),
+ "latest version must come from ListTableVersions: " + requestPaths);
+
+ requestPaths.clear();
+ LanceTableMetadata version2 = catalog.loadTableMetadata("default", FULL_TABLE,
+ Optional.of(new TableSnapshot("2", TableSnapshot.VersionType.VERSION)));
+ Assertions.assertTrue(version2.isManagedVersioning());
+ Assertions.assertEquals(2, version2.getVersion());
+ Assertions.assertEquals(1, version2.getFragments().size());
+ Assertions.assertTrue(requestPaths.stream().anyMatch(
+ path -> path.endsWith("/" + FULL_TABLE + "/version/describe")),
+ "explicit version must come from DescribeTableVersion: " + requestPaths);
+
+ // FOR TIME AS OF is parsed in the Doris session time zone; format the instant just
+ // after the second commit in that zone so the selector resolves to version 2.
+ String betweenSecondAndThird = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
+ .withZone(TimeUtils.getTimeZone().toZoneId())
+ .format(datasetVersions.get(1).getDataTime().toInstant().plusMillis(1));
+ LanceTableMetadata byTime = catalog.loadTableMetadata("default", FULL_TABLE,
+ Optional.of(new TableSnapshot(betweenSecondAndThird, TableSnapshot.VersionType.TIME)));
+ Assertions.assertEquals(2, byTime.getVersion());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testVersionMissingFromNamespaceIsReported() {
+ LanceExternalCatalog catalog = newCatalog(301, "lance_managed_partial");
+ try {
+ LanceTableMetadata latest = catalog.loadTableMetadata("default", PARTIAL_TABLE);
+ Assertions.assertEquals(3, latest.getVersion());
+ LanceTableMetadata version1 = catalog.loadTableMetadata("default", PARTIAL_TABLE,
+ Optional.of(new TableSnapshot("1", TableSnapshot.VersionType.VERSION)));
+ Assertions.assertEquals(1, version1.getVersion());
+ Assertions.assertTrue(version1.getFragments().isEmpty());
+
+ // Version 2 exists in storage, but the namespace does not record it.
+ RuntimeException exception = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PARTIAL_TABLE,
+ Optional.of(new TableSnapshot("2", TableSnapshot.VersionType.VERSION))));
+ Assertions.assertEquals(
+ "Lance version 2 of default." + PARTIAL_TABLE + " was not found in the namespace",
+ exception.getMessage());
+
+ // FOR TIME AS OF selects among the versions the namespace records, by their manifest
+ // commit times. The namespace no longer records version 2, which held the state after
+ // the second commit, so that instant fails instead of reading version 1, as it does
+ // after cleanup removed a version; an instant after the third commit selects version 3.
+ requestPaths.clear();
+ String betweenSecondAndThird = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
+ .withZone(TimeUtils.getTimeZone().toZoneId())
+ .format(datasetVersions.get(1).getDataTime().toInstant().plusMillis(1));
+ RuntimeException dropped = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PARTIAL_TABLE,
+ Optional.of(new TableSnapshot(betweenSecondAndThird, TableSnapshot.VersionType.TIME))));
+ Assertions.assertEquals("Lance cannot resolve FOR TIME AS OF '" + betweenSecondAndThird + "' on default."
+ + PARTIAL_TABLE + ": version 2, which may hold the state at that time, no longer exists",
+ dropped.getMessage());
+ Assertions.assertTrue(requestPaths.stream().anyMatch(
+ path -> path.endsWith("/" + PARTIAL_TABLE + "/version/list")),
+ "time travel must list the versions the namespace records: " + requestPaths);
+ Assertions.assertEquals(3, catalog.loadTableMetadata("default", PARTIAL_TABLE,
+ Optional.of(new TableSnapshot(timeAfter(datasetVersions.get(2)), TableSnapshot.VersionType.TIME)))
+ .getVersion());
+
+ // A namespace that lists versions without commit times resolves the same way: Doris
+ // only reads the commit times the manifests record.
+ LanceTableMetadata untimed = catalog.loadTableMetadata("default", UNTIMED_TABLE,
+ Optional.of(new TableSnapshot(betweenSecondAndThird, TableSnapshot.VersionType.TIME)));
+ Assertions.assertEquals(2, untimed.getVersion());
+ Assertions.assertTrue(untimed.isManagedVersioning());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testStorageVersionedTableStillOpensByUri() {
+ LanceExternalCatalog catalog = newCatalog(302, "lance_managed_plain");
+ try {
+ requestPaths.clear();
+ LanceTableMetadata latest = catalog.loadTableMetadata("default", PLAIN_TABLE);
+ Assertions.assertFalse(latest.isManagedVersioning());
+ Assertions.assertEquals(3, latest.getVersion());
+ Assertions.assertTrue(requestPaths.stream().noneMatch(path -> path.contains("/version/")),
+ "a storage-versioned table must not consult namespace versions: " + requestPaths);
+
+ // Version 2 is not recorded by the namespace either, but this table does not ask it.
+ LanceTableMetadata version2 = catalog.loadTableMetadata("default", PLAIN_TABLE,
+ Optional.of(new TableSnapshot("2", TableSnapshot.VersionType.VERSION)));
+ Assertions.assertEquals(2, version2.getVersion());
+
+ RuntimeException exception = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PLAIN_TABLE,
+ Optional.of(new TableSnapshot("99", TableSnapshot.VersionType.VERSION))));
+ Assertions.assertEquals("Lance version 99 of default." + PLAIN_TABLE + " was not found",
+ exception.getMessage());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testLatestVersionComesFromNamespaceNotStorage() {
+ LanceExternalCatalog catalog = newCatalog(305, "lance_managed_lagging");
+ try {
+ // Storage holds version 3, but the namespace has recorded only up to version 2, so
+ // the table's latest version, by number and by time, is 2.
+ LanceTableMetadata latest = catalog.loadTableMetadata("default", LAGGING_TABLE);
+ Assertions.assertEquals(2, latest.getVersion());
+ Assertions.assertEquals(1, latest.getFragments().size());
+ String afterThirdCommit = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
+ .withZone(TimeUtils.getTimeZone().toZoneId())
+ .format(datasetVersions.get(2).getDataTime().toInstant().plusMillis(1));
+ LanceTableMetadata byTime = catalog.loadTableMetadata("default", LAGGING_TABLE,
+ Optional.of(new TableSnapshot(afterThirdCommit, TableSnapshot.VersionType.TIME)));
+ Assertions.assertEquals(2, byTime.getVersion());
+ RuntimeException ahead = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", LAGGING_TABLE,
+ Optional.of(new TableSnapshot("3", TableSnapshot.VersionType.VERSION))));
+ Assertions.assertEquals("Lance version 3 of default." + LAGGING_TABLE + " was not found in the namespace",
+ ahead.getMessage());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ /**
+ * The namespace records version 3 at a staged manifest that was never copied to its canonical
+ * path: its commit reserved the version but did not finish. Doris reads a version at its
+ * canonical path, as the BE does, and does not finish another writer's commit, so the table
+ * and version 3 fail to read while version 2 still reads, and storage is left as it was.
+ */
+ @Test
+ public void testUnfinalizedHeadCannotBeRead() {
+ Assertions.assertFalse(Files.exists(stagedCanonicalManifest), "fixture must start with a staged manifest");
+ LanceExternalCatalog catalog = newCatalog(306, "lance_managed_staged");
+ try {
+ String pending = "Lance version 3 of default." + STAGED_TABLE + STAGED_ONLY;
+ RuntimeException latest = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", STAGED_TABLE));
+ Assertions.assertEquals(pending, latest.getMessage());
+ RuntimeException byVersion = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", STAGED_TABLE,
+ Optional.of(new TableSnapshot("3", TableSnapshot.VersionType.VERSION))));
+ Assertions.assertEquals(pending, byVersion.getMessage());
+ Assertions.assertEquals(2, catalog.loadTableMetadata("default", STAGED_TABLE,
+ Optional.of(new TableSnapshot("2", TableSnapshot.VersionType.VERSION))).getVersion());
+ } finally {
+ catalog.onClose();
+ }
+ Assertions.assertFalse(Files.exists(stagedCanonicalManifest), "a read must not finalize another commit");
+ }
+
+ /**
+ * The newest version of main and of dev was reserved but not finalized. Reads of that version
+ * fail, and so does FOR TIME AS OF, since its commit time is unknown and it may be the answer.
+ * Older versions still read, on main and inside dev, including main's version 3, which the
+ * namespace records at its staged manifest after it was finalized.
+ */
+ @Test
+ public void testUnfinalizedHeadsLeaveOlderVersionsReadable() {
+ LanceExternalCatalog catalog = newCatalog(326, "lance_managed_pending_head");
+ try {
+ RuntimeException latest = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PENDING_HEAD_TABLE));
+ Assertions.assertEquals("Lance version 4 of default." + PENDING_HEAD_TABLE + STAGED_ONLY,
+ latest.getMessage());
+ RuntimeException byTime = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PENDING_HEAD_TABLE,
+ Optional.of(new TableSnapshot("2100-01-01 00:00:00", TableSnapshot.VersionType.TIME))));
+ Assertions.assertEquals("Lance cannot resolve FOR TIME AS OF '2100-01-01 00:00:00' on default."
+ + PENDING_HEAD_TABLE + ": version 4, which may hold the state at that time," + STAGED_ONLY,
+ byTime.getMessage());
+ Assertions.assertEquals(3, catalog.loadTableMetadata("default", PENDING_HEAD_TABLE,
+ Optional.of(new TableSnapshot("3", TableSnapshot.VersionType.VERSION))).getVersion());
+
+ RuntimeException devLatest = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PENDING_HEAD_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.empty())));
+ Assertions.assertEquals("Lance version 4 of default." + PENDING_HEAD_TABLE + "@" + BRANCH + STAGED_ONLY,
+ devLatest.getMessage());
+ LanceTableMetadata dev3 = catalog.loadTableMetadata("default", PENDING_HEAD_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.of(new TableSnapshot("3",
+ TableSnapshot.VersionType.VERSION))));
+ Assertions.assertEquals(3, dev3.getVersion());
+ Assertions.assertEquals(datasetUri + "/tree/" + BRANCH, dev3.getDatasetUri());
+ RuntimeException dev9 = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PENDING_HEAD_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.of(new TableSnapshot("9",
+ TableSnapshot.VersionType.VERSION)))));
+ Assertions.assertEquals("Lance version 9 of default." + PENDING_HEAD_TABLE + "@" + BRANCH
+ + " was not found in the namespace", dev9.getMessage());
+ // FOR TIME AS OF on the branch fails the way it does on main.
+ RuntimeException devByTime = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PENDING_HEAD_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.of(new TableSnapshot("2100-01-01 00:00:00",
+ TableSnapshot.VersionType.TIME)))));
+ Assertions.assertEquals("Lance cannot resolve FOR TIME AS OF '2100-01-01 00:00:00' on default."
+ + PENDING_HEAD_TABLE + "@" + BRANCH + ": version 4, which may hold the state at that time,"
+ + STAGED_ONLY, devByTime.getMessage());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testNamespaceVersionsAreNotPinnedByTheAccessCache() {
+ LanceExternalCatalog catalog = newCatalog(307, "lance_managed_growing");
+ try {
+ // The version is resolved through the namespace on every read.
+ Assertions.assertEquals(2, catalog.loadTableMetadata("default", GROWING_TABLE).getVersion());
+ namespaceVersions.put(GROWING_TABLE, Arrays.asList(1L, 2L, 3L));
+ Assertions.assertEquals(3, catalog.loadTableMetadata("default", GROWING_TABLE).getVersion());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testNamespaceFailureIsNotReportedAsMissingVersion() {
+ LanceExternalCatalog catalog = newCatalog(308, "lance_managed_flaky");
+ try {
+ RuntimeException failure = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", FLAKY_TABLE,
+ Optional.of(new TableSnapshot("2", TableSnapshot.VersionType.VERSION))));
+ Assertions.assertTrue(failure.getMessage().startsWith(
+ "Failed to load Lance table metadata for default." + FLAKY_TABLE + ": "), failure.getMessage());
+ Assertions.assertFalse(failure.getMessage().contains("was not found"), failure.getMessage());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testManagedTableRequiresConsistentLocation() {
+ LanceExternalCatalog catalog = newCatalog(309, "lance_managed_location");
+ try {
+ RuntimeException noLocation = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", NO_LOCATION_TABLE));
+ Assertions.assertTrue(noLocation.getMessage().contains(
+ "returned no location for managed table"), noLocation.getMessage());
+ RuntimeException mismatch = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", MISMATCH_TABLE));
+ Assertions.assertTrue(mismatch.getMessage().contains(
+ "table_uri that differs from location"), mismatch.getMessage());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testTagSelectorsResolveToVersions() {
+ LanceExternalCatalog catalog = newCatalog(310, "lance_managed_tags");
+ try {
+ // Storage-versioned: the tag lives in the dataset's _refs/tags/.
+ Assertions.assertEquals(2, catalog.loadTableMetadata("default", PLAIN_TABLE,
+ LanceRefSelector.tag("v2")).getVersion());
+ RuntimeException plainMissing = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PLAIN_TABLE, LanceRefSelector.tag("nope")));
+ Assertions.assertEquals("Lance tag 'nope' of default." + PLAIN_TABLE + " was not found",
+ plainMissing.getMessage());
+
+ // Managed: tags live in the same _refs/tags/, and the namespace must record the
+ // version the tag points at, as for an explicit one.
+ requestPaths.clear();
+ LanceTableMetadata managed = catalog.loadTableMetadata("default", FULL_TABLE, LanceRefSelector.tag("v2"));
+ Assertions.assertEquals(2, managed.getVersion());
+ Assertions.assertTrue(managed.isManagedVersioning());
+ Assertions.assertTrue(requestPaths.stream().anyMatch(
+ path -> path.endsWith("/" + FULL_TABLE + "/version/describe")), requestPaths.toString());
+ RuntimeException managedMissing = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", FULL_TABLE, LanceRefSelector.tag("nope")));
+ Assertions.assertEquals("Lance tag 'nope' of default." + FULL_TABLE + " was not found",
+ managedMissing.getMessage());
+ // A tag that points at a version the namespace no longer records.
+ RuntimeException dropped = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PARTIAL_TABLE, LanceRefSelector.tag("v2")));
+ Assertions.assertEquals("Lance version 2 of default." + PARTIAL_TABLE
+ + " (tag 'v2') was not found in the namespace", dropped.getMessage());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testBranchSelectorsOpenTheBranchChain() {
+ LanceExternalCatalog catalog = newCatalog(311, "lance_managed_branches");
+ try {
+ // Storage-versioned: the branch is another manifest chain under <table>/tree/<branch>,
+ // which is what the BE receives as the dataset URI.
+ LanceTableMetadata dev = catalog.loadTableMetadata("default", PLAIN_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.empty()));
+ Assertions.assertEquals(3, dev.getVersion());
+ Assertions.assertEquals(2, dev.getFragments().size());
+ Assertions.assertEquals(Optional.of(BRANCH), dev.getBranch());
+ Assertions.assertTrue(dev.getDatasetUri().endsWith("/tree/" + BRANCH), dev.getDatasetUri());
+ LanceTableMetadata devV2 = catalog.loadTableMetadata("default", PLAIN_TABLE, LanceRefSelector.branch(
+ BRANCH, Optional.of(new TableSnapshot("2", TableSnapshot.VersionType.VERSION))));
+ Assertions.assertEquals(2, devV2.getVersion());
+ Assertions.assertEquals(1, devV2.getFragments().size());
+ RuntimeException missing = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PLAIN_TABLE,
+ LanceRefSelector.branch("nope", Optional.empty())));
+ Assertions.assertEquals("Lance branch 'nope' of default." + PLAIN_TABLE + " was not found",
+ missing.getMessage());
+ RuntimeException missingVersion = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PLAIN_TABLE, LanceRefSelector.branch(
+ BRANCH, Optional.of(new TableSnapshot("9", TableSnapshot.VersionType.VERSION)))));
+ Assertions.assertEquals("Lance version 9 of default." + PLAIN_TABLE + "@" + BRANCH + " was not found",
+ missingVersion.getMessage());
+
+ // Managed: the namespace records the branch's versions, and Doris asks it with the
+ // branch name.
+ requestQueries.clear();
+ LanceTableMetadata managedDev = catalog.loadTableMetadata("default", FULL_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.empty()));
+ Assertions.assertEquals(3, managedDev.getVersion());
+ Assertions.assertTrue(managedDev.isManagedVersioning());
+ Assertions.assertTrue(requestQueries.stream().anyMatch(q -> q.contains("branch=" + BRANCH)),
+ "the namespace must be asked for the branch: " + requestQueries);
+ RuntimeException managedMissing = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", FULL_TABLE,
+ LanceRefSelector.branch("nope", Optional.empty())));
+ // The namespace itself reports the branch missing, since the branch head comes from it.
+ Assertions.assertEquals("Lance branch 'nope' of default." + FULL_TABLE + " was not found in the namespace",
+ managedMissing.getMessage());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testTagOnBranchSelectsTheBranch() {
+ LanceExternalCatalog catalog = newCatalog(312, "lance_managed_branch_tags");
+ try {
+ // Storage-versioned: the tag's branch comes from _refs/tags/<tag>.json.
+ LanceTableMetadata plain = catalog.loadTableMetadata("default", PLAIN_TABLE,
+ LanceRefSelector.tag(BRANCH_TAG));
+ Assertions.assertEquals(3, plain.getVersion());
+ Assertions.assertEquals(Optional.of(BRANCH), plain.getBranch());
+ Assertions.assertTrue(plain.getDatasetUri().endsWith("/tree/" + BRANCH), plain.getDatasetUri());
+ Assertions.assertEquals(2, plain.getFragments().size(), "branch version 3, not main version 3");
+ // Managed: the same tag, on the branch the namespace records it for.
+ LanceTableMetadata managed = catalog.loadTableMetadata("default", FULL_TABLE,
+ LanceRefSelector.tag(BRANCH_TAG));
+ Assertions.assertEquals(3, managed.getVersion());
+ Assertions.assertEquals(Optional.of(BRANCH), managed.getBranch());
+ Assertions.assertEquals(2, managed.getFragments().size());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testTimeTravelInsideBranchUsesBranchCommitTimes() {
+ LanceExternalCatalog catalog = newCatalog(313, "lance_managed_branch_time");
+ try {
+ // Between the branch's version 2 (the fork point) and its own version 3.
+ String betweenBranchCommits = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
+ .withZone(TimeUtils.getTimeZone().toZoneId())
+ .format(branchDatasetVersions.stream().filter(v -> v.getId() == 3).findFirst()
+ .orElseThrow(() -> new AssertionError("branch must have version 3"))
+ .getDataTime().toInstant().minusMillis(1));
+ TableSnapshot time = new TableSnapshot(betweenBranchCommits, TableSnapshot.VersionType.TIME);
+ LanceTableMetadata plain = catalog.loadTableMetadata("default", PLAIN_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.of(time)));
+ Assertions.assertEquals(2, plain.getVersion());
+ Assertions.assertEquals(Optional.of(BRANCH), plain.getBranch());
+ // Managed: the stub lists the branch's versions without commit times, so the times
+ // come from the branch manifests, restricted to the versions the namespace lists.
+ requestQueries.clear();
+ LanceTableMetadata managed = catalog.loadTableMetadata("default", FULL_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.of(time)));
+ Assertions.assertEquals(2, managed.getVersion());
+ Assertions.assertTrue(requestQueries.stream().anyMatch(q -> q.contains("branch=" + BRANCH)),
+ requestQueries.toString());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testNamespaceVersionsArePagedToTheEnd() {
+ LanceExternalCatalog catalog = newCatalog(314, "lance_managed_paged");
+ try {
+ // Version 3 is only on the last page; any early stop or page size would miss it.
+ String afterThirdCommit = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
+ .withZone(TimeUtils.getTimeZone().toZoneId())
+ .format(datasetVersions.get(2).getDataTime().toInstant().plusMillis(1));
+ requestQueries.clear();
+ requestPaths.clear();
+ LanceTableMetadata byTime = catalog.loadTableMetadata("default", PAGED_TABLE,
+ Optional.of(new TableSnapshot(afterThirdCommit, TableSnapshot.VersionType.TIME)));
+ Assertions.assertEquals(3, byTime.getVersion());
+ long pages = requestQueries.stream().filter(q -> q.contains("page_token=")).count();
+ Assertions.assertEquals(2, pages, "every page after the first must be requested: " + requestQueries);
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testEmptyNamespaceVersionListIsReported() {
+ LanceExternalCatalog catalog = newCatalog(315, "lance_managed_empty");
+ try {
+ RuntimeException empty = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", EMPTY_TABLE,
+ Optional.of(new TableSnapshot("2030-01-01 00:00:00", TableSnapshot.VersionType.TIME))));
+ Assertions.assertEquals("Lance namespace lists no versions for default." + EMPTY_TABLE,
+ empty.getMessage());
+ // Storage still holds three versions, but the namespace published none of them: a
+ // plain query, a search and a metadata-only read must not fall back to storage.
+ List<Supplier<LanceTableMetadata>> latestReads = Arrays.asList(
+ () -> catalog.loadTableMetadata("default", EMPTY_TABLE),
+ () -> catalog.loadTableMetadataForSearch("default", EMPTY_TABLE),
+ () -> catalog.loadBasicTableMetadata("default", EMPTY_TABLE));
+ for (Supplier<LanceTableMetadata> latest : latestReads) {
+ RuntimeException unpublished = Assertions.assertThrows(RuntimeException.class, latest::get);
+ Assertions.assertEquals("Lance namespace lists no versions for default." + EMPTY_TABLE,
+ unpublished.getMessage());
+ }
+ // A branch the namespace records reads without any recorded main version: its head
+ // comes from the namespace, and the branch is read from its own directory. A tag that
+ // points into it is read through main, where the tag file lives.
+ LanceTableMetadata branchHead = catalog.loadTableMetadata("default", EMPTY_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.empty()));
+ Assertions.assertEquals(3, branchHead.getVersion());
+ Assertions.assertEquals(Optional.of(BRANCH), branchHead.getBranch());
+ LanceTableMetadata branchTag = catalog.loadTableMetadata("default", EMPTY_TABLE,
+ LanceRefSelector.tag(BRANCH_TAG));
+ Assertions.assertEquals(3, branchTag.getVersion());
+ Assertions.assertEquals(Optional.of(BRANCH), branchTag.getBranch());
+
+ // The same for a branch storage holds but the namespace records nothing on. A tag that
+ // points into it opens its version explicitly, which the namespace does not record.
+ RuntimeException unpublishedBranch = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", UNRECORDED_BRANCH_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.empty())));
+ Assertions.assertEquals("Lance namespace lists no versions for default." + UNRECORDED_BRANCH_TABLE
+ + "@" + BRANCH, unpublishedBranch.getMessage());
+ RuntimeException unpublishedTag = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", UNRECORDED_BRANCH_TABLE,
+ LanceRefSelector.tag(BRANCH_TAG)));
+ Assertions.assertEquals("Lance version 3 of default." + UNRECORDED_BRANCH_TABLE + "@" + BRANCH + " (tag '"
+ + BRANCH_TAG + "') was not found in the namespace", unpublishedTag.getMessage());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testRelocatedManagedTableIsReadAtItsNewLocation() {
+ LanceExternalCatalog catalog = newCatalog(318, "lance_managed_relocated");
+ try {
+ relocated = false;
+ Assertions.assertEquals(datasetUri, catalog.loadTableMetadata("default", RELOCATED_TABLE).getDatasetUri());
+ // The namespace moves the table. A managed table's access is not cached, so the next
+ // read describes the new location, and the metadata handed to the BE names it.
+ relocated = true;
+ LanceTableMetadata moved = catalog.loadTableMetadata("default", RELOCATED_TABLE);
+ Assertions.assertEquals(relocatedDatasetUri, moved.getDatasetUri());
+ Assertions.assertEquals(3, moved.getVersion());
+ Assertions.assertEquals(relocatedDatasetUri, catalog.loadTableMetadata("default", RELOCATED_TABLE,
+ Optional.of(new TableSnapshot("2", TableSnapshot.VersionType.VERSION))).getDatasetUri());
+ } finally {
+ relocated = false;
+ catalog.onClose();
+ }
+ }
+
+ /**
+ * The namespace moves the table to a dataset one version ahead right after the FE described
+ * it, so the versions it lists next are recorded at the new location. The read, which
+ * described the old one, would not read their manifests, so it fails and asks for a retry;
+ * the retry describes the new location and reads its newest version.
+ */
+ @Test
+ public void testTableMovedAfterItWasDescribedFailsTheRead() {
+ LanceExternalCatalog catalog = newCatalog(323, "lance_managed_moving");
+ try {
+ moveTo = new MoveTarget(movingDatasetUri, movingStorePath, Arrays.asList(1L, 2L, 3L, 4L));
+ moveOnDescribe = true;
+ RuntimeException moving = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", MOVING_TABLE));
+ Assertions.assertTrue(moved, "the namespace must have moved the table during the read");
+ Assertions.assertTrue(moving.getMessage().contains("if the table was moved during the query, retry it"),
+ moving.getMessage());
+ LanceTableMetadata retried = catalog.loadTableMetadata("default", MOVING_TABLE);
+ Assertions.assertEquals(movingDatasetUri, retried.getDatasetUri());
+ Assertions.assertEquals(4, retried.getVersion());
+ Assertions.assertEquals(9, retried.getRowCount());
+ } finally {
+ moveOnDescribe = false;
+ moved = false;
+ namespaceVersions.put(MOVING_TABLE, Arrays.asList(1L, 2L, 3L));
+ catalog.onClose();
+ }
+ }
+
+ /**
+ * Doris reads a version at its canonical path, as the BE does, so a version the namespace
+ * records at a finalized manifest elsewhere fails the read instead of reading another manifest.
+ */
+ @Test
+ public void testFinalizedManifestOutsideItsCanonicalPathFailsTheRead() {
+ LanceExternalCatalog catalog = newCatalog(324, "lance_managed_custom_manifest");
+ try {
+ RuntimeException error = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", CUSTOM_MANIFEST_TABLE));
+ Assertions.assertTrue(error.getMessage().contains("Lance namespace records version 3 of default."
+ + CUSTOM_MANIFEST_TABLE + " at manifest '" + customManifestStorePath + "'"), error.getMessage());
+ Assertions.assertEquals(2, catalog.loadTableMetadata("default", CUSTOM_MANIFEST_TABLE,
+ Optional.of(new TableSnapshot("2", TableSnapshot.VersionType.VERSION))).getVersion());
+
+ // FOR TIME AS OF compares the commit times of the manifests Doris reads, so every
+ // recorded version must be at one of them: even a time before version 3's commit,
+ // which would select version 2, fails, since the namespace's version 3 may have been
+ // committed at another time.
+ String beforeHead = timeAfter(datasetVersions.get(1));
+ RuntimeException byTime = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", CUSTOM_MANIFEST_TABLE,
+ Optional.of(new TableSnapshot(beforeHead, TableSnapshot.VersionType.TIME))));
+ Assertions.assertTrue(byTime.getMessage().contains("Lance namespace records version 3 of default."
+ + CUSTOM_MANIFEST_TABLE + " at manifest"), byTime.getMessage());
+
+ // The same on a branch.
+ RuntimeException devLatest = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", CUSTOM_MANIFEST_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.empty())));
+ Assertions.assertTrue(devLatest.getMessage().contains("Lance namespace records version 3 of default."
+ + CUSTOM_MANIFEST_TABLE + "@" + BRANCH + " at manifest"), devLatest.getMessage());
+ String beforeDevHead = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
+ .withZone(TimeUtils.getTimeZone().toZoneId())
+ .format(branchDatasetVersions.stream().filter(v -> v.getId() == 3).findFirst()
+ .orElseThrow(() -> new AssertionError("branch must have version 3"))
+ .getDataTime().toInstant().minusMillis(1));
+ RuntimeException devByTime = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", CUSTOM_MANIFEST_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.of(new TableSnapshot(beforeDevHead,
+ TableSnapshot.VersionType.TIME)))));
+ Assertions.assertTrue(devByTime.getMessage().contains("Lance namespace records version 3 of default."
+ + CUSTOM_MANIFEST_TABLE + "@" + BRANCH + " at manifest"), devByTime.getMessage());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testBeReadsWithTheOptionsOfTheReadsDescribe() {
+ LanceExternalCatalog catalog = newCatalog(320, "lance_managed_rotating");
+ try {
+ // Every read of a managed table describes it once (its access is not cached), and the
+ // FE and the BE both read with what that describe vends.
+ for (int read = 0; read < 2; read++) {
+ LanceTableMetadata metadata = catalog.loadTableMetadata("default", ROTATING_TABLE);
+ Assertions.assertEquals(3, metadata.getVersion());
+ Assertions.assertEquals(String.valueOf(rotatingDescribes.get()),
+ metadata.getLanceStorageOptions().get(GENERATION_OPTION));
+ }
+ LanceTableMetadata byVersion = catalog.loadTableMetadata("default", ROTATING_TABLE,
+ Optional.of(new TableSnapshot("2", TableSnapshot.VersionType.VERSION)));
+ Assertions.assertEquals(String.valueOf(rotatingDescribes.get()),
+ byVersion.getLanceStorageOptions().get(GENERATION_OPTION));
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ /**
+ * The namespace records version 1 at a staged manifest while its canonical one is missing: its
+ * commit was not finalized, or cleanup removed it, which the record does not tell apart. It
+ * cannot be read by version, a time in its range fails like removed history, and the read
+ * writes nothing.
+ */
+ @Test
+ public void testOlderVersionRecordedOnlyAsStagedCannotBeRead() {
+ Assertions.assertTrue(Files.notExists(stagedUntimedV1Canonical), "fixture must start with version 1 staged");
+ LanceExternalCatalog catalog = newCatalog(319, "lance_managed_staged_untimed");
+ try {
+ String betweenFirstAndSecond = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
+ .withZone(TimeUtils.getTimeZone().toZoneId())
+ .format(stagedUntimedVersions.get(0).getDataTime().toInstant().plusMillis(1));
+ RuntimeException removed = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", STAGED_UNTIMED_TABLE,
+ Optional.of(new TableSnapshot(betweenFirstAndSecond, TableSnapshot.VersionType.TIME))));
+ Assertions.assertEquals("Lance cannot resolve FOR TIME AS OF '" + betweenFirstAndSecond + "' on default."
+ + STAGED_UNTIMED_TABLE + ": version 1, which may hold the state at that time," + STAGED_ONLY,
+ removed.getMessage());
+ RuntimeException byVersion = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", STAGED_UNTIMED_TABLE,
+ Optional.of(new TableSnapshot("1", TableSnapshot.VersionType.VERSION))));
+ Assertions.assertEquals("Lance version 1 of default." + STAGED_UNTIMED_TABLE + STAGED_ONLY,
+ byVersion.getMessage());
+ Assertions.assertEquals(3, catalog.loadTableMetadata("default", STAGED_UNTIMED_TABLE).getVersion());
+ } finally {
+ catalog.onClose();
+ }
+ Assertions.assertTrue(Files.notExists(stagedUntimedV1Canonical), "a read must not finalize a manifest");
+
+ // Versions the namespace still records but cleanup removed are missing from storage as
+ // well; they stay out of the candidates instead of failing the query.
+ LanceExternalCatalog expired = newCatalog(320, "lance_managed_untimed_expired");
+ try {
+ Assertions.assertEquals(3, expired.loadTableMetadata("default", UNTIMED_EXPIRED_TABLE,
+ Optional.of(new TableSnapshot("2100-01-01 00:00:00", TableSnapshot.VersionType.TIME)))
+ .getVersion());
+ } finally {
+ expired.onClose();
+ }
+ }
+
+ @Test
+ public void testTimeTravelUsesManifestCommitTimes() {
+ LanceExternalCatalog catalog = newCatalog(321, "lance_managed_manifest_times");
+ try {
+ // The namespace reports the first commit's time for every version. Manifest commit
+ // times decide, as for Lance's own asof, so a time before the second commit is version 1.
+ Assertions.assertEquals(1, catalog.loadTableMetadata("default", MISTIMED_TABLE,
+ Optional.of(new TableSnapshot(timeAfter(datasetVersions.get(0)),
+ TableSnapshot.VersionType.TIME))).getVersion());
+
+ // Version 2 was removed, so a time between the first and the third commit has no known
+ // state; the query fails instead of reading version 1, for storage and managed tables.
+ for (String table : Arrays.asList(PLAIN_GAP_TABLE, MANAGED_GAP_TABLE)) {
+ String betweenFirstAndThird = timeAfter(gapVersions.get(1));
+ RuntimeException gap = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", table,
+ Optional.of(new TableSnapshot(betweenFirstAndThird, TableSnapshot.VersionType.TIME))));
+ Assertions.assertEquals("Lance cannot resolve FOR TIME AS OF '" + betweenFirstAndThird + "' on default."
+ + table + ": version 2, which may hold the state at that time, no longer exists",
+ gap.getMessage());
+ Assertions.assertEquals(3, catalog.loadTableMetadata("default", table,
+ Optional.of(new TableSnapshot(timeAfter(gapVersions.get(2)), TableSnapshot.VersionType.TIME)))
+ .getVersion(), table);
+ }
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ private static String timeAfter(Version version) {
+ return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS").withZone(TimeUtils.getTimeZone().toZoneId())
+ .format(version.getDataTime().toInstant().plusMillis(1));
+ }
+
+ @Test
+ public void testBranchOfACheckedOutUri() {
+ String root = "s3://bucket/t.lance";
+ Assertions.assertEquals(Optional.empty(), LanceCatalogClient.branchOf(root + "/", root));
+ Assertions.assertEquals(Optional.of("feature/x"), LanceCatalogClient.branchOf(root + "/tree/feature/x", root));
+ // Lance inserts the branch path before the query string.
+ Assertions.assertEquals(Optional.of(BRANCH),
+ LanceCatalogClient.branchOf(root + "/tree/" + BRANCH + "?sig=abc", root + "?sig=abc"));
+ Assertions.assertEquals(Optional.empty(), LanceCatalogClient.branchOf(root + "?sig=abc", root + "?sig=abc"));
+ IllegalStateException elsewhere = Assertions.assertThrows(IllegalStateException.class,
+ () -> LanceCatalogClient.branchOf("s3://bucket/other.lance?sig=abc", root + "?sig=abc"));
+ Assertions.assertFalse(elsewhere.getMessage().contains("sig="), elsewhere.getMessage());
+ }
+
+ @Test
+ public void testBranchTheNamespaceRecordsIsNotReportedMissing() {
+ LanceExternalCatalog catalog = newCatalog(322, "lance_managed_missing_head");
+ try {
+ // The namespace lists versions for dev, so dev exists; its head manifest is missing
+ // from storage, which is a missing version, not a missing branch.
+ RuntimeException broken = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", MISSING_HEAD_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.empty())));
+ Assertions.assertEquals("Lance version 9 of default." + MISSING_HEAD_TABLE + "@" + BRANCH + " was not found",
+ broken.getMessage());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testBranchMissingFromADirectoryNamespaceIsReported() {
+ LanceExternalCatalog catalog = newCatalog(327, "lance_managed_directory_errors");
+ try {
+ // The table was described in the same read, so a missing table from the branch's
+ // version request means the branch, whatever the selector.
+ String missing = "Lance branch 'nope' of default." + DIRECTORY_ERRORS_TABLE
+ + " was not found in the namespace";
+ List<Optional<TableSnapshot>> snapshots = Arrays.asList(Optional.empty(),
+ Optional.of(new TableSnapshot("2", TableSnapshot.VersionType.VERSION)),
+ Optional.of(new TableSnapshot("2100-01-01 00:00:00", TableSnapshot.VersionType.TIME)));
+ for (Optional<TableSnapshot> snapshot : snapshots) {
+ RuntimeException error = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", DIRECTORY_ERRORS_TABLE,
+ LanceRefSelector.branch("nope", snapshot)));
+ Assertions.assertEquals(missing, error.getMessage(), String.valueOf(snapshot));
+ }
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testBranchReadsWithoutAReadableMainChain() {
+ LanceExternalCatalog catalog = newCatalog(328, "lance_managed_branch_only");
+ try {
+ // A branch is read from its own directory, as the BE reads it, so neither a recorded
+ // main version nor a main manifest in storage is needed.
+ LanceTableMetadata head = catalog.loadTableMetadata("default", BRANCH_ONLY_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.empty()));
+ Assertions.assertEquals(4, head.getVersion());
+ Assertions.assertEquals(branchOnlyDatasetUri + "/tree/" + BRANCH, head.getDatasetUri());
+ Assertions.assertEquals(3, catalog.loadTableMetadata("default", BRANCH_ONLY_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.of(new TableSnapshot("3",
+ TableSnapshot.VersionType.VERSION)))).getVersion());
+ Assertions.assertEquals(4, catalog.loadTableMetadata("default", BRANCH_ONLY_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.of(new TableSnapshot("2100-01-01 00:00:00",
+ TableSnapshot.VersionType.TIME)))).getVersion());
+
+ RuntimeException main = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", BRANCH_ONLY_TABLE));
+ Assertions.assertEquals("Lance namespace lists no versions for default." + BRANCH_ONLY_TABLE,
+ main.getMessage());
+ RuntimeException missing = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", BRANCH_ONLY_TABLE,
+ LanceRefSelector.branch("nope", Optional.empty())));
+ Assertions.assertEquals("Lance branch 'nope' of default." + BRANCH_ONLY_TABLE
+ + " was not found in the namespace", missing.getMessage());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testCleanedUpVersionIsReportedAsNotFound() {
+ LanceExternalCatalog catalog = newCatalog(304, "lance_managed_expired");
+ try {
+ LanceTableMetadata latest = catalog.loadTableMetadata("default", EXPIRED_TABLE);
+ Assertions.assertEquals(3, latest.getVersion());
+ Assertions.assertEquals(3, catalog.loadTableMetadata("default", EXPIRED_TABLE,
+ Optional.of(new TableSnapshot("3", TableSnapshot.VersionType.VERSION))).getVersion());
+
+ // Version 1 was committed and later removed by cleanup: same error as a version that
+ // never existed, without the storage path of the missing manifest.
+ RuntimeException expired = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", EXPIRED_TABLE,
+ Optional.of(new TableSnapshot("1", TableSnapshot.VersionType.VERSION))));
+ Assertions.assertEquals("Lance version 1 of default." + EXPIRED_TABLE + " was not found",
+ expired.getMessage());
+ Assertions.assertFalse(String.valueOf(expired.getCause().getMessage()).contains(tempDir.toString()),
+ String.valueOf(expired.getCause().getMessage()));
+
+ // FOR TIME AS OF only sees the versions that still exist, so a time before the
+ // surviving version has nothing to select.
+ RuntimeException tooEarly = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", EXPIRED_TABLE,
+ Optional.of(new TableSnapshot("2000-01-01 00:00:00", TableSnapshot.VersionType.TIME))));
+ Assertions.assertEquals("Lance table default." + EXPIRED_TABLE
+ + " has no version at or before '2000-01-01 00:00:00'", tooEarly.getMessage());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ @Test
+ public void testSelectorErrorsKeepTheirMessages() {
+ LanceExternalCatalog catalog = newCatalog(303, "lance_managed_errors");
+ try {
+ RuntimeException nonNumeric = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PLAIN_TABLE,
+ Optional.of(new TableSnapshot("abc", TableSnapshot.VersionType.VERSION))));
+ Assertions.assertTrue(nonNumeric.getMessage().contains(
+ "Lance FOR VERSION AS OF requires a numeric version, but was 'abc'"), nonNumeric.getMessage());
+
+ RuntimeException tooEarly = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PLAIN_TABLE,
+ Optional.of(new TableSnapshot("2000-01-01 00:00:00", TableSnapshot.VersionType.TIME))));
+ Assertions.assertEquals("Lance table default." + PLAIN_TABLE
+ + " has no version at or before '2000-01-01 00:00:00'", tooEarly.getMessage());
+
+ RuntimeException unparsable = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", PLAIN_TABLE,
+ Optional.of(new TableSnapshot("not-a-time", TableSnapshot.VersionType.TIME))));
+ Assertions.assertTrue(unparsable.getMessage().contains(
+ "Cannot parse Lance FOR TIME AS OF value 'not-a-time'"), unparsable.getMessage());
+
+ RuntimeException declared = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadata("default", DECLARED_TABLE));
+ Assertions.assertTrue(declared.getMessage().startsWith(
+ "Failed to load Lance table metadata for default." + DECLARED_TABLE + ": "), declared.getMessage());
+ Assertions.assertTrue(declared.getMessage().endsWith(
+ "Lance table is declared in the namespace but has no data yet"), declared.getMessage());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
+ /** The dataset the moving table moves to, and the versions the namespace records for it there. */
+ private static final class MoveTarget {
+ private final String uri;
+ private final String storePath;
+ private final List<Long> versions;
+
+ private MoveTarget(String uri, String storePath, List<Long> versions) {
+ this.uri = uri;
+ this.storePath = storePath;
+ this.versions = versions;
+ }
+ }
+
+ private LanceExternalCatalog newCatalog(long id, String name) {
+ Map<String, String> properties = new HashMap<>();
+ properties.put("type", "lance");
+ properties.put(LanceExternalCatalog.LANCE_CATALOG_TYPE, LanceExternalCatalog.LANCE_REST);
+ properties.put(LanceExternalCatalog.REST_URI, restUri);
+ properties.put(LanceExternalCatalog.REST_SECURITY_TYPE, "bearer");
+ properties.put(LanceExternalCatalog.REST_BEARER_TOKEN, BEARER_TOKEN);
+ return new LanceExternalCatalog(id, name, null, properties, "");
+ }
+
+ /** Writes a dataset with three versions: an empty create, then two one-fragment appends. */
+ private static List<Version> writeThreeVersions(String uri) throws Exception {
+ Schema schema = new Schema(Collections.singletonList(
+ new Field("row_id", FieldType.notNullable(new ArrowType.Int(32, true)), null)));
+ try (BufferAllocator allocator = new RootAllocator()) {
+ WriteParams params = new WriteParams.Builder().withDataStorageVersion("2.0").build();
+ try (Dataset created = Dataset.create(allocator, uri, schema, params)) {
+ Assertions.assertEquals(1, created.version());
+ }
+ long readVersion = 1;
+ for (int append = 0; append < 2; append++) {
+ readVersion = appendRows(uri, allocator, readVersion, append * 3 + 1, append * 3 + 3);
+ Thread.sleep(20);
+ }
+ try (Dataset dataset = Dataset.open(uri, allocator)) {
+ return dataset.listVersions();
+ }
+ }
+ }
+
+ /** Appends one fragment holding row_id {@code first..last} and returns the committed version. */
+ private static long appendRows(String uri, BufferAllocator allocator, long readVersion, int first, int last)
+ throws Exception {
+ Schema schema = new Schema(Collections.singletonList(
+ new Field("row_id", FieldType.notNullable(new ArrowType.Int(32, true)), null)));
+ WriteParams params = new WriteParams.Builder().withDataStorageVersion("2.0").build();
+ int count = last - first + 1;
+ try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
+ IntVector rowId = (IntVector) root.getVector("row_id");
+ rowId.allocateNew(count);
+ for (int i = 0; i < count; i++) {
+ rowId.set(i, first + i);
+ }
+ rowId.setValueCount(count);
+ root.setRowCount(count);
+ List<FragmentMetadata> fragments = Fragment.create(uri, allocator, root, params);
+ try (Dataset committed = new FragmentOperation.Append(fragments)
+ .commit(allocator, uri, Optional.of(readVersion), Collections.emptyMap())) {
+ return committed.version();
+ }
+ }
+ }
+
+ // Three versions are written with V2 manifest naming, so version v is u64::MAX - v.
+ private String manifestPath(String table, long version, String branch) {
+ if (STAGED_TABLE.equals(table) && version == 3) {
+ return stagedManifestStorePath;
+ }
+ if (CUSTOM_MANIFEST_TABLE.equals(table) && version == 3) {
+ return customManifestStorePath;
+ }
+ if (PENDING_HEAD_TABLE.equals(table) && (version == 4 || (version == 3 && branch == null))) {
+ return datasetStorePath + (branch == null ? "" : "/tree/" + branch) + "/_versions/"
+ + U64_MAX.subtract(BigInteger.valueOf(version)) + ".manifest" + STAGED_SUFFIX;
+ }
+ if (STAGED_UNTIMED_TABLE.equals(table) && version == 1 && Files.notExists(stagedUntimedV1Canonical)) {
+ return stagedUntimedV1StorePath;
+ }
+ if (UNTIMED_EXPIRED_TABLE.equals(table) && version == 1) {
+ return Paths.get(java.net.URI.create(expiredDatasetUri)).toAbsolutePath().toString()
+ .replaceFirst("^/", "") + "/_versions/" + U64_MAX.subtract(BigInteger.ONE)
+ + ".manifest-00000000-0000-4000-8000-000000000000";
+ }
+ String root = STAGED_TABLE.equals(table)
+ ? Paths.get(java.net.URI.create(stagedDatasetUri)).toAbsolutePath().toString().replaceFirst("^/", "")
+ : datasetStorePath;
+ if (STAGED_UNTIMED_TABLE.equals(table)) {
+ root = stagedUntimedStorePath;
+ } else if (MANAGED_GAP_TABLE.equals(table)) {
+ root = gapStorePath;
+ } else if (BRANCH_ONLY_TABLE.equals(table)) {
+ root = branchOnlyStorePath;
+ } else if (UNTIMED_EXPIRED_TABLE.equals(table)) {
+ root = Paths.get(java.net.URI.create(expiredDatasetUri)).toAbsolutePath().toString()
+ .replaceFirst("^/", "");
+ } else if (RELOCATED_TABLE.equals(table) && relocated) {
+ root = relocatedStorePath;
+ } else if (MOVING_TABLE.equals(table) && moved) {
+ root = moveTo.storePath;
+ }
+ if (branch != null) {
+ root = root + "/tree/" + branch;
+ }
+ return root + "/_versions/" + U64_MAX.subtract(BigInteger.valueOf(version)) + ".manifest";
+ }
+
+ /** The error for a branch the namespace lacks; Lance's Directory namespace reports a missing table. */
+ private String missingBranchJson(String table, String branch) {
+ return DIRECTORY_ERRORS_TABLE.equals(table)
+ ? "{\"error\":\"branch '" + branch + "' not found for table at '" + locationOf(table)
+ + "'\",\"code\":4}"
+ : "{\"error\":\"table branch " + branch + " not found\",\"code\":22}";
+ }
+
+ /** The versions the stub namespace records on a branch. */
+ private List<Long> namespaceBranchVersions(String table, String branch) {
+ if ((EMPTY_TABLE.equals(table) || CUSTOM_MANIFEST_TABLE.equals(table)) && BRANCH.equals(branch)) {
+ return Arrays.asList(2L, 3L);
+ }
+ if (BRANCH_ONLY_TABLE.equals(table) && BRANCH.equals(branch)) {
+ return Arrays.asList(3L, 4L);
+ }
+ if (UNRECORDED_BRANCH_TABLE.equals(table) && BRANCH.equals(branch)) {
+ return Collections.emptyList();
+ }
+ if (MISSING_HEAD_TABLE.equals(table) && BRANCH.equals(branch)) {
+ return Arrays.asList(2L, 3L, 9L);
+ }
+ if (PENDING_HEAD_TABLE.equals(table) && BRANCH.equals(branch)) {
+ return Arrays.asList(2L, 3L, 4L);
+ }
+ if (FULL_TABLE.equals(table) && BRANCH.equals(branch)) {
+ List<Long> versions = new java.util.ArrayList<>();
+ for (long v = 2; v <= branchVersions; v++) {
+ versions.add(v);
+ }
+ return versions;
+ }
+ return null;
+ }
+
+ /** The location DescribeTable reports; the expired table has its own dataset. */
+ private String locationOf(String table) {
+ if (PLAIN_GAP_TABLE.equals(table) || MANAGED_GAP_TABLE.equals(table)) {
+ return gapDatasetUri;
+ }
+ if (BRANCH_ONLY_TABLE.equals(table)) {
+ return branchOnlyDatasetUri;
+ }
+ if (EXPIRED_TABLE.equals(table) || UNTIMED_EXPIRED_TABLE.equals(table)) {
+ return expiredDatasetUri;
+ }
+ if (STAGED_TABLE.equals(table)) {
+ return stagedDatasetUri;
+ }
+ if (STAGED_UNTIMED_TABLE.equals(table)) {
+ return stagedUntimedDatasetUri;
+ }
+ if (RELOCATED_TABLE.equals(table) && relocated) {
+ return relocatedDatasetUri;
+ }
+ if (MOVING_TABLE.equals(table) && moved) {
+ return moveTo.uri;
+ }
+ if (NO_LOCATION_TABLE.equals(table)) {
+ return "";
+ }
+ return datasetUri;
+ }
+
+ private void handleRequest(HttpExchange exchange) throws IOException {
+ byte[] body = ByteStreams.toByteArray(exchange.getRequestBody());
+ String path = exchange.getRequestURI().getPath();
+ String query = String.valueOf(exchange.getRequestURI().getQuery());
+ requestPaths.add(path);
+ requestQueries.add(query);
+
+ int status = 200;
+ String response;
+ if (!("Bearer " + BEARER_TOKEN).equals(exchange.getRequestHeaders().getFirst("Authorization"))) {
+ status = 401;
+ response = "{\"error\":\"unauthorized\",\"code\":16}";
+ } else if (path.matches("/v1/table/[^/]+/version/list")) {
+ String table = path.split("/")[3];
+ java.util.regex.Matcher branchMatcher = java.util.regex.Pattern.compile("(?:^|&)branch=([^&]+)")
+ .matcher(query);
+ String branch = branchMatcher.find()
+ ? java.net.URLDecoder.decode(branchMatcher.group(1), "UTF-8") : null;
+ List<Long> versions = branch == null ? namespaceVersions.get(table)
+ : namespaceBranchVersions(table, branch);
+ if (branch != null && versions == null) {
+ status = 404;
+ response = missingBranchJson(table, branch);
+ } else if (versions == null) {
+ status = 400;
+ response = "{\"error\":\"versions are not managed\",\"code\":13}";
+ } else if (PAGED_TABLE.equals(table) && !query.contains("limit=")) {
+ // One version per page, oldest first, with the index of the next one as the token.
+ java.util.regex.Matcher tokenMatcher = java.util.regex.Pattern.compile("(?:^|&)page_token=(\\d+)")
+ .matcher(query);
+ int start = tokenMatcher.find() ? Integer.parseInt(tokenMatcher.group(1)) : 0;
+ response = "{\"versions\":[" + tableVersionJson(table, versions.get(start), branch) + "]"
+ + (start + 1 < versions.size() ? ",\"page_token\":\"" + (start + 1) + "\"" : "") + "}";
+ } else {
+ boolean descending = query.contains("descending=true");
+ java.util.regex.Matcher limitMatcher = java.util.regex.Pattern.compile("(?:^|&)limit=(\\d+)")
+ .matcher(query);
+ int limit = limitMatcher.find() ? Integer.parseInt(limitMatcher.group(1)) : versions.size();
+ StringBuilder entries = new StringBuilder();
+ for (int i = 0; i < Math.min(limit, versions.size()); i++) {
+ long version = versions.get(descending ? versions.size() - 1 - i : i);
+ if (entries.length() > 0) {
+ entries.append(',');
+ }
+ entries.append(tableVersionJson(table, version, branch));
+ }
+ response = "{\"versions\":[" + entries + "]}";
+ }
+ } else if (path.matches("/v1/table/[^/]+/version/describe")) {
+ String table = path.split("/")[3];
+ JsonNode request = JsonUtil.readTree(new String(body, StandardCharsets.UTF_8));
+ JsonNode requested = request.get("version");
+ String branch = request.hasNonNull("branch") ? request.get("branch").asText() : null;
+ List<Long> versions = branch == null ? namespaceVersions.get(table)
+ : namespaceBranchVersions(table, branch);
+ long version = requested == null ? -1 : requested.asLong();
+ if (branch != null && versions == null) {
+ status = 404;
+ response = missingBranchJson(table, branch);
+ } else if (FLAKY_TABLE.equals(table) && version == 2) {
+ status = 500;
+ response = "{\"error\":\"version store unavailable\",\"code\":18}";
+ } else if (versions == null || !versions.contains(version)) {
+ status = 404;
+ response = "{\"error\":\"table version " + version + " not found\",\"code\":11}";
+ } else {
+ response = "{\"version\":" + tableVersionJson(table, version, branch) + "}";
+ }
+ } else if (path.matches("/v1/table/[^/]+/describe")) {
+ String table = path.split("/")[3];
+ boolean managed = namespaceVersions.containsKey(table);
+ boolean declared = DECLARED_TABLE.equals(table);
+ boolean plain = PLAIN_TABLE.equals(table) || EXPIRED_TABLE.equals(table) || PLAIN_GAP_TABLE.equals(table);
+ if (!managed && !plain && !declared) {
+ status = 404;
+ response = "{\"error\":\"table not found\",\"code\":4}";
+ } else {
+ String location = locationOf(table);
+ String tableUri = MISMATCH_TABLE.equals(table) || NO_LOCATION_TABLE.equals(table)
+ ? expiredDatasetUri : location;
+ if (MOVING_TABLE.equals(table) && moveOnDescribe && !moved) {
+ namespaceVersions.put(MOVING_TABLE, moveTo.versions);
+ moved = true;
+ }
+ response = "{\"table\":\"" + table + "\",\"namespace\":[],"
+ + "\"location\":\"" + location + "\",\"table_uri\":\"" + tableUri + "\","
+ + "\"storage_options\":{" + (ROTATING_TABLE.equals(table)
+ ? "\"" + GENERATION_OPTION + "\":\"" + rotatingDescribes.incrementAndGet() + "\"" : "")
+ + "},"
+ + "\"managed_versioning\":" + managed + ","
+ + "\"is_only_declared\":" + declared + "}";
+ }
+ } else if (path.endsWith("/table/list")) {
+ response = "{\"tables\":[]}";
+ } else if (path.endsWith("/list")) {
+ response = "{\"namespaces\":[]}";
+ } else {
+ status = 404;
+ response = "{\"error\":\"not found\",\"code\":4}";
+ }
+ byte[] responseBytes = response.getBytes(StandardCharsets.UTF_8);
+ exchange.getResponseHeaders().set("Content-Type", "application/json");
+ exchange.sendResponseHeaders(status, responseBytes.length);
+ exchange.getResponseBody().write(responseBytes);
+ exchange.close();
+ }
+
+ private String tableVersionJson(String table, long version, String branch) {
+ String manifestPath = manifestPath(table, version, branch);
+ long size;
+ try {
+ size = Files.size(Paths.get("/" + manifestPath));
+ } catch (IOException e) {
+ size = 0;
+ }
+ // Branch versions are reported without commit times; the manifests in storage give them.
+ String commitTime = UNTIMED_TABLE.equals(table) || STAGED_UNTIMED_TABLE.equals(table)
+ || UNTIMED_EXPIRED_TABLE.equals(table) || MOVING_TABLE.equals(table)
+ || PENDING_HEAD_TABLE.equals(table) || branch != null
+ ? "" : ",\"timestamp_millis\":"
+ + datasetVersions.get(MISTIMED_TABLE.equals(table) ? 0 : (int) version - 1).getDataTime().toInstant()
+ .toEpochMilli();
+ return "{\"version\":" + version + ",\"manifest_path\":\"" + manifestPath
+ + "\",\"manifest_size\":" + size + commitTime + "}";
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManifestPathsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManifestPathsTest.java
new file mode 100644
index 00000000000..6f08d5ae464
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManifestPathsTest.java
@@ -0,0 +1,120 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance;
+
+import org.apache.doris.datasource.lance.LanceManifestPaths.Recorded;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.Locale;
+import java.util.Optional;
+
+public class LanceManifestPathsTest {
+ private static final String S3_URI = "s3://bucket/table.lance";
+ /** Version 3 in the V2 naming scheme is u64::MAX - 3. */
+ private static final String CANONICAL = "table.lance/_versions/18446744073709551612.manifest";
+
+ private static Recorded check(String location, String branch, String recorded) {
+ return LanceManifestPaths.check(location, Optional.ofNullable(branch), 3, recorded, "db.t");
+ }
+
+ /** The canonical path in either naming scheme, however many surrounding slashes. */
+ @Test
+ public void testCanonicalPaths() {
+ Assertions.assertEquals(Recorded.CANONICAL, check(S3_URI, null, CANONICAL));
+ Assertions.assertEquals(Recorded.CANONICAL, check(S3_URI, null, "/" + CANONICAL));
+ Assertions.assertEquals(Recorded.CANONICAL, check(S3_URI, null, "table.lance/_versions/3.manifest"));
+ Assertions.assertEquals(Recorded.CANONICAL,
+ check(S3_URI, "dev", "table.lance/tree/dev/_versions/18446744073709551612.manifest"));
+ Assertions.assertEquals(Recorded.CANONICAL,
+ check(S3_URI, "feature/x", "table.lance/tree/feature/x/_versions/18446744073709551612.manifest"));
+ // A table at the root of its bucket, and one whose location Java's URI cannot parse.
+ Assertions.assertEquals(Recorded.CANONICAL, check("s3://bucket", null, "_versions/18446744073709551612.manifest"));
+ Assertions.assertEquals(Recorded.CANONICAL,
+ check("s3://bucket/sales data/t.lance", null, "sales data/t.lance/_versions/3.manifest"));
+ }
+
+ /** A staged manifest beside the canonical path, named as Lance names one. */
+ @Test
+ public void testStagedPaths() {
+ Assertions.assertEquals(Recorded.STAGED, check(S3_URI, null, CANONICAL + "-2f6a1c0e"));
+ Assertions.assertEquals(Recorded.STAGED, check(S3_URI, null, "table.lance/_versions/3.manifest-2f6a1c0e"));
+ }
+
+ /**
+ * Any other path names a manifest Doris would not read: another name, another version,
+ * another directory, a branch version at main's path and the other way round, a staged
+ * manifest in a subdirectory, or a path whose trailing slash Lance drops before reading it.
+ */
+ @Test
+ public void testOtherPathsAreRejected() {
+ for (String elsewhere : Arrays.asList("table.lance/_versions/custom.manifest",
+ "table.lance/_versions/custom.manifest/",
+ "other.lance/_versions/18446744073709551612.manifest",
+ "other.lance/_versions/18446744073709551612.manifest-2f6a1c0e",
+ "table.lance/_versions/18446744073709551613.manifest",
+ "table.lance/_versions/18446744073709551612.manifest-x/y",
+ "table.lance/_versions/foo.json",
+ "table.lance/tree/dev/_versions/18446744073709551612.manifest",
+ "")) {
+ IllegalStateException rejected = Assertions.assertThrows(IllegalStateException.class,
+ () -> check(S3_URI, null, elsewhere), elsewhere);
+ Assertions.assertTrue(rejected.getMessage().contains("'" + CANONICAL + "'"), rejected.getMessage());
+ Assertions.assertTrue(rejected.getMessage().contains("retry"), rejected.getMessage());
+ }
+ Assertions.assertThrows(IllegalStateException.class, () -> check(S3_URI, "dev", CANONICAL));
+ Assertions.assertThrows(IllegalStateException.class, () -> check(S3_URI, null, null));
+ }
+
+ /** Lance writes ASCII digits; the FE's locale must not change the name it expects. */
+ @Test
+ public void testCanonicalNameDoesNotDependOnTheLocale() {
+ Locale previous = Locale.getDefault(Locale.Category.FORMAT);
+ Locale.setDefault(Locale.Category.FORMAT, Locale.forLanguageTag("ar-SA-u-nu-arab"));
+ try {
+ Assertions.assertEquals(Recorded.CANONICAL, check(S3_URI, null, CANONICAL));
+ } finally {
+ Locale.setDefault(Locale.Category.FORMAT, previous);
+ }
+ }
+
+ /**
+ * The path lance-io addresses a location by: after the bucket, percent-decoded for a URL, as
+ * is without a scheme.
+ */
+ @Test
+ public void testObjectStorePathFollowsLance() {
+ Assertions.assertEquals("sales data/t.lance",
+ LanceManifestPaths.objectStorePath("s3://bucket/sales data/t.lance"));
+ Assertions.assertEquals("sales data/t.lance",
+ LanceManifestPaths.objectStorePath("s3://bucket/sales%20data/t.lance"));
+ Assertions.assertEquals("a|b/t.lance", LanceManifestPaths.objectStorePath("s3://bucket/a|b/t.lance/"));
+ Assertions.assertEquals("", LanceManifestPaths.objectStorePath("s3://bucket"));
+ Assertions.assertEquals("", LanceManifestPaths.objectStorePath("s3://bucket/"));
+ Assertions.assertEquals("t.lance",
+ LanceManifestPaths.objectStorePath("s3+ddb://bucket/t.lance?ddbTableName=x"));
+ Assertions.assertEquals("p/t.lance",
+ LanceManifestPaths.objectStorePath("abfss://container@account.dfs.core.windows.net/p/t.lance"));
+ Assertions.assertEquals("tmp/a b/t.lance", LanceManifestPaths.objectStorePath("file:///tmp/a%20b/t.lance"));
+ Assertions.assertEquals("tmp/t.lance", LanceManifestPaths.objectStorePath("file:/tmp/t.lance"));
+ Assertions.assertEquals("tmp/a%20b/t.lance", LanceManifestPaths.objectStorePath("/tmp/a%20b/t.lance"));
+ Assertions.assertEquals("t%zz.lance", LanceManifestPaths.objectStorePath("s3://bucket/t%zz.lance"));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSharedSessionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSharedSessionTest.java
index e17b6f3b5cb..721f793186a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSharedSessionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSharedSessionTest.java
@@ -89,12 +89,14 @@ public class LanceSharedSessionTest {
Assertions.assertEquals(6, group.getCounterMap().get("MetadataReadCalls").getValue());
Assertions.assertEquals(1, group.getCounterMap().get("MetadataReadFailures").getValue());
Assertions.assertEquals(6, group.getCounterMap().get("TableAccessResolveCalls").getValue());
- Assertions.assertEquals(6, group.getCounterMap().get("DatasetOpenCalls").getValue());
+ // FOR TIME AS OF checks the selected version out of the open latest dataset, which
+ // counts as version resolution rather than a second open.
+ Assertions.assertEquals(5, group.getCounterMap().get("DatasetOpenCalls").getValue());
Assertions.assertEquals(5, group.getCounterMap().get("SchemaReadCalls").getValue());
Assertions.assertEquals(4, group.getCounterMap().get("FragmentMetadataReadCalls").getValue());
Assertions.assertEquals(3, group.getCounterMap().get("FieldIdsReadCalls").getValue());
Assertions.assertEquals(3, group.getCounterMap().get("IndexMetadataReadCalls").getValue());
- Assertions.assertEquals(1, group.getCounterMap().get("VersionResolveCalls").getValue());
+ Assertions.assertEquals(2, group.getCounterMap().get("VersionResolveCalls").getValue());
Assertions.assertFalse(group.getCounterMap().containsKey("SplitPlanningTime"));
} finally {
catalog.onClose();
@@ -171,8 +173,9 @@ public class LanceSharedSessionTest {
freshUuid = fresh.getIndexes().get(0).getSegments().get(0).getUuid();
}
Assertions.assertNotEquals(oldUuid, freshUuid);
- // The padded manifest prevents open() from opportunistically replacing the cached index list.
- Assertions.assertEquals(oldUuid,
+ // Lance keys its index metadata cache on the manifest's etag (lance#8904, in lance-core
+ // 12), so the recreated dataset's indexes are read even before the catalog is refreshed.
+ Assertions.assertEquals(freshUuid,
catalog.loadTableMetadata("default", "table").getIndexes().get(0).getSegments().get(0).getUuid());
catalog.onRefreshCache(true);
Assertions.assertEquals(freshUuid,
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceTableAccessCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceTableAccessCacheTest.java
index b4e3603f927..447e57f97af 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceTableAccessCacheTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceTableAccessCacheTest.java
@@ -86,6 +86,23 @@ public class LanceTableAccessCacheTest {
Mockito.verify(namespace, Mockito.times(2)).describeTable(Mockito.any());
}
+ /**
+ * A managed table is described on every read even without vended options: the version list
+ * a read asks for next is the namespace's current one, and must be checked against the
+ * location the namespace reports now.
+ */
+ @Test
+ public void testManagedTableIsDescribedForEveryRead() {
+ LanceNamespace namespace = namespace();
+ Mockito.when(namespace.describeTable(Mockito.any())).thenReturn(new DescribeTableResponse()
+ .location("s3://example-bucket/items.lance").tableUri("s3://example-bucket/items.lance")
+ .managedVersioning(true));
+ LanceNamespaceClient client = client(namespace, "rest", 60, new AtomicLong(1_000_000));
+ Assertions.assertTrue(client.resolveTableAccess("default", "items").isManagedVersioning());
+ client.resolveTableAccess("default", "items");
+ Mockito.verify(namespace, Mockito.times(2)).describeTable(Mockito.any());
+ }
+
@Test
public void testVendedCredentialsAreResolvedForEveryRead() {
LanceNamespace namespace = namespace();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/metadata/LanceSnapshotTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/metadata/LanceSnapshotTest.java
index 7653abdcfd9..b4dbaac4e2c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/metadata/LanceSnapshotTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/metadata/LanceSnapshotTest.java
@@ -29,11 +29,17 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.lance.Version;
+import java.time.Instant;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.Random;
import java.util.TreeMap;
+import java.util.TreeSet;
public class LanceSnapshotTest {
@@ -62,6 +68,211 @@ public class LanceSnapshotTest {
Arrays.asList(version1, version2, version3), first.minusNanos(1).toInstant().toEpochMilli()));
}
+ /**
+ * Lance records commit times below a millisecond. A commit later within the requested
+ * millisecond is after it, and two commits within one millisecond keep their order.
+ */
+ @Test
+ public void testTimeSelectorComparesCommitTimesInFull() {
+ Instant commit2 = Instant.parse("2026-09-19T13:06:09.113997Z");
+ Version version1 = versionAt(1, Instant.parse("2026-09-19T13:06:07.597965Z"));
+ Version version2 = versionAt(2, commit2);
+ long requested = Instant.parse("2026-09-19T13:06:09.113Z").toEpochMilli();
+ Assertions.assertEquals(1, LanceSnapshotResolver.versionAtOrBefore(Arrays.asList(version1, version2),
+ requested));
+ Assertions.assertEquals(2, LanceSnapshotResolver.versionAtOrBefore(Arrays.asList(version1, version2),
+ requested + 1));
+ // Version 3 was committed half a millisecond before version 2, in the same millisecond.
+ Version version3 = versionAt(3, commit2.minusNanos(500_000));
+ Assertions.assertEquals(2, LanceSnapshotResolver.versionAtOrBefore(
+ Arrays.asList(version1, version2, version3), requested + 1));
+ }
+
+ @Test
+ public void testTimeSelectorComparesCommitTimesAcrossTheHistory() {
+ // Commit times that go back: version 2 reports an earlier time than version 1.
+ Assertions.assertEquals(1, LanceSnapshotResolver.versionAtOrBefore(
+ Arrays.asList(version(1, 100), version(2, 90), version(3, 200)), null, 150, "150"));
+ // A managed chain selects among the versions the namespace records, not every one storage lists:
+ // storage's version 3 is not published yet.
+ Assertions.assertEquals(2, LanceSnapshotResolver.versionAtOrBefore(
+ Arrays.asList(version(1, 10), version(2, 20), version(3, 30)), recorded(1, 2), 100, "100"));
+ // A version the namespace no longer records was removed, whether or not storage still holds
+ // its manifest: nothing older is a candidate.
+ LanceSnapshotResolver.HistoryRemovedException dropped = Assertions.assertThrows(
+ LanceSnapshotResolver.HistoryRemovedException.class, () -> LanceSnapshotResolver.versionAtOrBefore(
+ Arrays.asList(version(1, 10), version(2, 20), version(3, 30)), recorded(1, 3), 25, "25"));
+ Assertions.assertEquals(2, dropped.getVersion());
+ LanceSnapshotResolver.HistoryRemovedException gap = Assertions.assertThrows(
+ LanceSnapshotResolver.HistoryRemovedException.class, () -> LanceSnapshotResolver.versionAtOrBefore(
+ Arrays.asList(version(1, 10), version(2, 20), version(8, 80), version(9, 90)),
+ recorded(1, 2, 8, 9), 50, "50"));
+ Assertions.assertEquals(7, gap.getVersion());
+ Assertions.assertEquals(9, LanceSnapshotResolver.versionAtOrBefore(
+ Arrays.asList(version(1, 10), version(2, 20), version(8, 80), version(9, 90)),
+ recorded(1, 2, 8, 9), 95, "95"));
+ // A branch's chain starts where it was forked; the versions before are not a gap.
+ Assertions.assertEquals(3, LanceSnapshotResolver.versionAtOrBefore(
+ Arrays.asList(version(2, 20), version(3, 30)), recorded(2, 3), 35, "35"));
+ // The newest recorded version has no manifest yet: its commit time is unknown.
+ LanceSnapshotResolver.HistoryRemovedException pending = Assertions.assertThrows(
+ LanceSnapshotResolver.HistoryRemovedException.class, () -> LanceSnapshotResolver.versionAtOrBefore(
+ Arrays.asList(version(1, 10), version(2, 20), version(3, 30)), recorded(1, 2, 3, 4), 100,
+ "100"));
+ Assertions.assertEquals(4, pending.getVersion());
+ // A recorded version storage lacks was removed: nothing older is a candidate.
+ LanceSnapshotResolver.HistoryRemovedException recordedRemoved = Assertions.assertThrows(
+ LanceSnapshotResolver.HistoryRemovedException.class, () -> LanceSnapshotResolver.versionAtOrBefore(
+ Arrays.asList(version(1, 10), version(3, 30)), recorded(1, 2, 3), 20, "20"));
+ Assertions.assertEquals(2, recordedRemoved.getVersion());
+ // Cleanup removed version 2: nothing from before it is a candidate, even version 1.
+ LanceSnapshotResolver.HistoryRemovedException removed = Assertions.assertThrows(
+ LanceSnapshotResolver.HistoryRemovedException.class, () -> LanceSnapshotResolver.versionAtOrBefore(
+ Arrays.asList(version(1, 10), version(3, 30)), null, 20, "20"));
+ Assertions.assertEquals(2, removed.getVersion());
+ Assertions.assertEquals(3, LanceSnapshotResolver.versionAtOrBefore(
+ Arrays.asList(version(1, 10), version(3, 30)), null, 30, "30"));
+ }
+
+ /**
+ * How a version appears: listed in storage and recorded (for a managed chain), removed from
+ * storage but still recorded, listed but not recorded, or neither.
+ */
+ private enum State { LISTED, REMOVED, UNRECORDED, DROPPED }
+
+ /**
+ * Checks the selection against a direct reading of its rule on random histories: times that
+ * repeat, go back, and fall inside a requested millisecond, removed, unrecorded and dropped
+ * versions, on storage and managed chains, including a managed chain whose newest recorded
+ * version storage lacks or whose newest listed version the namespace does not record. Commit
+ * times are in microseconds, requested times in milliseconds.
+ */
+ @Test
+ public void testTimeSelectorMatchesItsRuleOnRandomHistories() {
+ Random random = new Random(20260927L);
+ for (int round = 0; round < 20000; round++) {
+ boolean managed = random.nextBoolean();
+ int count = 1 + random.nextInt(7);
+ long[] times = new long[count + 1];
+ State[] states = new State[count + 1];
+ for (int id = 1; id <= count; id++) {
+ times[id] = random.nextInt(10) * 1000L + random.nextInt(3) * 500L;
+ State[] choices = managed ? State.values() : new State[] {State.LISTED, State.REMOVED};
+ // The newest storage version is the open dataset, so it is always there; a managed
+ // chain's records can end before or after it.
+ states[id] = id == count && !managed ? State.LISTED : choices[random.nextInt(choices.length)];
+ }
+ long timestamp = random.nextInt(11) - 1;
+
+ List<Version> listed = new ArrayList<>();
+ NavigableSet<Long> recorded = managed ? new TreeSet<>() : null;
+ for (int id = 1; id <= count; id++) {
+ if (states[id] == State.LISTED || states[id] == State.UNRECORDED) {
+ listed.add(versionAtMicros(id, times[id]));
+ }
+ if (managed && (states[id] == State.LISTED || states[id] == State.REMOVED)) {
+ recorded.add((long) id);
+ }
+ }
+ Collections.shuffle(listed, random);
+
+ // The rule: the newest removed version cuts the history, and the latest commit at or
+ // before the time wins. For storage, a removed version is one the listing shows a gap
+ // for; for a managed chain, any number between its oldest and newest recorded version
+ // that is not both recorded and listed.
+ Long removed = null;
+ if (managed) {
+ long newest = recorded.isEmpty() ? 0 : recorded.last();
+ long oldest = recorded.isEmpty() ? 1 : recorded.first();
+ for (long id = newest; id >= oldest && removed == null; id--) {
+ if (states[(int) id] != State.LISTED) {
+ removed = id;
+ }
+ }
+ } else {
+ int oldestListed = count;
+ for (int id = count; id >= 1; id--) {
+ if (states[id] == State.LISTED) {
+ oldestListed = id;
+ }
+ }
+ for (int id = count; id >= 1 && removed == null; id--) {
+ if (states[id] == State.REMOVED && id > oldestListed) {
+ removed = (long) id;
+ }
+ }
+ }
+ Long expected = null;
+ for (int id = count; removed == null || id > removed; id--) {
+ if (id < 1) {
+ break;
+ }
+ boolean candidate = states[id] == State.LISTED;
+ if (candidate && times[id] <= timestamp * 1000
+ && (expected == null || times[id] > times[expected.intValue()])) {
+ expected = (long) id;
+ }
+ }
+
+ String context = "round " + round + (managed ? " managed" : " storage") + " times "
+ + Arrays.toString(times) + " states " + Arrays.toString(states) + " at " + timestamp;
+ try {
+ long actual = LanceSnapshotResolver.versionAtOrBefore(listed, recorded, timestamp,
+ String.valueOf(timestamp));
+ Assertions.assertEquals(expected, Long.valueOf(actual), context);
+ } catch (LanceSnapshotResolver.HistoryRemovedException e) {
+ Assertions.assertNull(expected, context);
+ Assertions.assertEquals(removed, Long.valueOf(e.getVersion()), context);
+ } catch (LanceSnapshotResolver.NoVersionAtOrBeforeException e) {
+ Assertions.assertNull(expected, context);
+ Assertions.assertNull(removed, context);
+ }
+ }
+ }
+
+ private static Version version(long id, long commitMillis) {
+ return versionAt(id, Instant.ofEpochMilli(commitMillis));
+ }
+
+ private static Version versionAtMicros(long id, long commitMicros) {
+ return versionAt(id, Instant.EPOCH.plusNanos(commitMicros * 1000));
+ }
+
+ private static Version versionAt(long id, Instant commitTime) {
+ return new Version(id, ZonedDateTime.ofInstant(commitTime, ZoneOffset.UTC), new TreeMap<>());
+ }
+
+ private static NavigableSet<Long> recorded(long... ids) {
+ NavigableSet<Long> result = new TreeSet<>();
+ for (long id : ids) {
+ result.add(id);
+ }
+ return result;
+ }
+
+ @Test
+ public void testVersionNumbersAndTagNames() {
+ Assertions.assertTrue(LanceSnapshotResolver.isVersionNumber("2"));
+ Assertions.assertTrue(LanceSnapshotResolver.isVersionNumber("007"));
+ Assertions.assertEquals(7, LanceSnapshotResolver.parseVersion("007"));
+ // A signed number is a version, so '-1' is reported as invalid rather than as a missing tag.
+ Assertions.assertTrue(LanceSnapshotResolver.isVersionNumber("-1"));
+ Assertions.assertTrue(LanceSnapshotResolver.isVersionNumber("+3"));
+ Assertions.assertEquals(3, LanceSnapshotResolver.parseVersion("+3"));
+ IllegalArgumentException negative = Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceSnapshotResolver.parseVersion("-1"));
+ Assertions.assertEquals("Lance FOR VERSION AS OF requires a positive version, but was -1",
+ negative.getMessage());
+ // Anything else names a tag, as for Iceberg and Paimon tables.
+ for (String tag : new String[] {"v2", "-", "+", " 2", "2 ", "", "1.5", "1e3"}) {
+ Assertions.assertFalse(LanceSnapshotResolver.isVersionNumber(tag), tag);
+ }
+ IllegalArgumentException outOfRange = Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceSnapshotResolver.parseVersion("99999999999999999999"));
+ Assertions.assertEquals("Lance FOR VERSION AS OF version 99999999999999999999 is out of range",
+ outOfRange.getMessage());
+ }
+
@Test
public void testBoundSnapshotCarriesItsOwnSchema() {
LanceTableMetadata intMetadata = metadata(10,
diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java
index 25c63f52126..1fe7caefa33 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java
@@ -44,6 +44,24 @@ public class VectorSearchTableValuedFunctionTest {
Assert.assertEquals("items", tableName.getTbl());
}
+ @Test
+ public void testBackquotedTableNameMayContainSelectorCharacters() throws Exception {
+ TableName at = VectorSearchTableValuedFunction.parseTableName("c.d.`user@corp`");
+ Assert.assertEquals("user@corp", at.getTbl());
+ TableName forName = VectorSearchTableValuedFunction.parseTableName("c.d.`sales for version 2024`");
+ Assert.assertEquals("sales for version 2024", forName.getTbl());
+ }
+
+ @Test
+ public void testTableNameCannotSelectVersionTagOrBranch() {
+ for (String table : new String[] {"c.d.t@tag(v1)", "c.d.t@branch(dev)", "c.d.t FOR VERSION AS OF 2"}) {
+ AnalysisException exception = Assert.assertThrows(AnalysisException.class,
+ () -> VectorSearchTableValuedFunction.parseTableName(table));
+ Assert.assertTrue(exception.getMessage(),
+ exception.getMessage().contains("cannot select a version, tag or branch"));
+ }
+ }
+
@Test
public void testRejectAmbiguousUnquotedMultiLevelNamespace() {
AnalysisException exception = Assert.assertThrows(AnalysisException.class,
diff --git a/fe/pom.xml b/fe/pom.xml
index 3322689eaae..002e3e60552 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -349,7 +349,7 @@ under the License.
<!-- Please modify iceberg.version and avro.version together,
you can find avro version info in iceberg mvn repository -->
<iceberg.version>1.11.0</iceberg.version>
- <lance.version>11.0.0</lance.version>
+ <lance.version>12.0.0</lance.version>
<substrait.version>0.40.0</substrait.version>
<!-- 0.56.1 has bug that "SplitMode" in query response may not be set-->
<maxcompute.version>0.53.2-public</maxcompute.version>
diff --git a/regression-test/data/external_table_p0/lance/test_lance_rest_catalog.out b/regression-test/data/external_table_p0/lance/test_lance_rest_catalog.out
index 5dc72192239..17e65ed4f29 100644
--- a/regression-test/data/external_table_p0/lance/test_lance_rest_catalog.out
+++ b/regression-test/data/external_table_p0/lance/test_lance_rest_catalog.out
@@ -7,6 +7,15 @@ mysql
-- !rest_tables --
all_types
all_types_unprefixed
+time_travel
+time_travel_managed
+time_travel_managed_lagging
+time_travel_managed_misrecorded
+time_travel_managed_partial
+time_travel_managed_pending
+time_travel_managed_staged
+time_travel_managed_unprefixed
+time_travel_managed_untimed
-- !rest_scan --
12 12 1 12 78
@@ -17,3 +26,4 @@ all_types_unprefixed
-- !rest_predicate_pushdown --
8
9
+
diff --git a/regression-test/data/external_table_p0/lance/test_lance_rest_time_travel.out b/regression-test/data/external_table_p0/lance/test_lance_rest_time_travel.out
new file mode 100644
index 00000000000..ea9662a455a
--- /dev/null
+++ b/regression-test/data/external_table_p0/lance/test_lance_rest_time_travel.out
@@ -0,0 +1,134 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !tables --
+all_types
+all_types_unprefixed
+time_travel
+time_travel_managed
+time_travel_managed_lagging
+time_travel_managed_misrecorded
+time_travel_managed_partial
+time_travel_managed_pending
+time_travel_managed_staged
+time_travel_managed_unprefixed
+time_travel_managed_untimed
+
+-- !plain_latest --
+9 9
+
+-- !plain_version_2 --
+1 v1
+2 v1
+3 v1
+4 v2
+5 v2
+6 v2
+
+-- !plain_time --
+6 6
+
+-- !managed_latest --
+9 9
+
+-- !managed_version_1 --
+1 v1
+2 v1
+3 v1
+
+-- !managed_version_2 --
+1 v1
+2 v1
+3 v1
+4 v2
+5 v2
+6 v2
+
+-- !managed_version_with_predicate --
+3
+
+-- !managed_time --
+6 6
+
+-- !managed_self_join --
+1 v1 v1
+2 v1 v1
+3 v1 v1
+
+-- !partial_latest --
+9 9
+
+-- !partial_version_3 --
+1 v1
+2 v1
+3 v1
+4 v2
+5 v2
+6 v2
+7 v3
+8 v3
+9 v3
+
+-- !partial_version_1 --
+1 v1
+2 v1
+3 v1
+
+-- !partial_time_version_3 --
+9 9
+
+-- !lagging_latest --
+6 6
+
+-- !lagging_time_after_storage_version_3 --
+6 6
+
+-- !managed_tag_v1 --
+1 v1
+2 v1
+3 v1
+
+-- !plain_tag_v3 --
+7
+8
+9
+
+-- !managed_branch_dev --
+7 100
+
+-- !plain_branch_dev_version_2 --
+6 6
+
+-- !managed_branch_dev_time --
+7 100
+
+-- !managed_tag_on_branch --
+7 100
+
+-- !untimed_latest --
+9 9
+
+-- !untimed_time_between_2_and_3 --
+6 6
+
+-- !unprefixed_latest --
+9 9
+
+-- !unprefixed_version_2 --
+1 v1
+2 v1
+3 v1
+4 v2
+5 v2
+6 v2
+
+-- !pending_version_3 --
+9 9
+
+-- !pending_branch_dev_version_3 --
+7 100
+
+-- !staged_latest --
+9 9
+
+-- !misrecorded_version_2 --
+6 6
+
diff --git a/regression-test/data/external_table_p0/lance/test_lance_time_travel.out b/regression-test/data/external_table_p0/lance/test_lance_time_travel.out
new file mode 100644
index 00000000000..b2429e42b09
--- /dev/null
+++ b/regression-test/data/external_table_p0/lance/test_lance_time_travel.out
@@ -0,0 +1,141 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !latest --
+9 1 9
+
+-- !version_1 --
+1 v1
+2 v1
+3 v1
+
+-- !version_2 --
+1 v1
+2 v1
+3 v1
+4 v2
+5 v2
+6 v2
+
+-- !version_3 --
+1 v1
+2 v1
+3 v1
+4 v2
+5 v2
+6 v2
+7 v3
+8 v3
+9 v3
+
+-- !version_with_predicate --
+3
+
+-- !version_with_predicate_empty --
+0
+
+-- !self_join --
+1 v1 v1
+2 v1 v1
+3 v1 v1
+
+-- !union_versions --
+1 3
+2 6
+3 9
+
+-- !time_before_version_2 --
+3 3
+
+-- !time_before_version_3 --
+6 6
+
+-- !time_after_version_3 --
+9 9
+
+-- !time_millisecond_precision --
+6 6
+
+-- !time_just_after_commit_2 --
+6 6
+
+-- !time_within_commit_2_millisecond --
+3 3
+
+-- !time_session_time_zone --
+6 6
+
+-- !multi_frag_latest --
+27
+
+-- !multi_frag_before_delete --
+30
+
+-- !multi_frag_deleted_rows_before_delete --
+15
+25
+5
+
+-- !multi_frag_deleted_rows_after_delete --
+
+-- !multi_frag_first_fragment_only --
+10 1 10
+
+-- !tag_v2 --
+1 v1
+2 v1
+3 v1
+4 v2
+5 v2
+6 v2
+
+-- !version_as_of_tag_name --
+1 v1
+2 v1
+3 v1
+4 v2
+5 v2
+6 v2
+
+-- !version_as_of_tag_on_branch --
+7 100
+
+-- !branch_dev --
+1 v1
+100 dev
+2 v1
+3 v1
+4 v2
+5 v2
+6 v2
+
+-- !branch_dev_version_2 --
+1 v1
+2 v1
+3 v1
+4 v2
+5 v2
+6 v2
+
+-- !branch_vs_main --
+\N 100
+7 \N
+8 \N
+9 \N
+
+-- !branch_main --
+9 9
+
+-- !branch_dev_map_form --
+7 100
+
+-- !branch_dev_time --
+7 100
+
+-- !tag_on_branch --
+1 v1
+100 dev
+2 v1
+3 v1
+4 v2
+5 v2
+6 v2
+
diff --git a/regression-test/suites/external_table_p0/lance/test_lance_rest_catalog.groovy b/regression-test/suites/external_table_p0/lance/test_lance_rest_catalog.groovy
index 75f2c100793..64681600433 100644
--- a/regression-test/suites/external_table_p0/lance/test_lance_rest_catalog.groovy
+++ b/regression-test/suites/external_table_p0/lance/test_lance_rest_catalog.groovy
@@ -26,8 +26,8 @@ suite("test_lance_rest_catalog", "p0,external") {
* -> FE pins the Lance dataset version and fragments
* -> BE lance-c reads the real dataset from MinIO
*
- * managed_versioning is false because the current BE opens storage-native
- * Lance versions by URI and does not resolve REST-managed external manifests.
+ * managed_versioning is false here: the table's versions live in its own
+ * _versions/ directory. test_lance_rest_time_travel covers namespace-managed versions.
*/
String enabled = context.config.otherConfigs.get("enableIcebergTest")
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
diff --git a/regression-test/suites/external_table_p0/lance/test_lance_rest_time_travel.groovy b/regression-test/suites/external_table_p0/lance/test_lance_rest_time_travel.groovy
new file mode 100644
index 00000000000..c92a03f7b9e
--- /dev/null
+++ b/regression-test/suites/external_table_p0/lance/test_lance_rest_time_travel.groovy
@@ -0,0 +1,294 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_lance_rest_time_travel", "p0,external") {
+ /*
+ * Time travel through a Lance REST Namespace, with and without managed versioning.
+ *
+ * All these tables are the same time_travel.lance dataset in MinIO (three versions, see
+ * test_lance_time_travel). They differ in what the REST fixture says about it:
+ *
+ * time_travel managed_versioning=false: versions come from _versions/
+ * time_travel_managed managed_versioning=true, namespace records versions 1, 2, 3
+ * time_travel_managed_partial managed_versioning=true, namespace records versions 1, 3
+ * time_travel_managed_lagging managed_versioning=true, namespace records versions 1, 2
+ * time_travel_managed_untimed managed_versioning=true, records 1, 2, 3 without commit times
+ * time_travel_managed_unprefixed as time_travel_managed, credentials vended unprefixed
+ * time_travel_managed_pending records 1, 2, 3, and 4 (main and dev) at a staged manifest
+ * whose commit never finalized
+ * time_travel_managed_staged records 1, 2, 3, with 3 at its staged manifest
+ * time_travel_managed_misrecorded records 1, 2, 3, with 3 at a manifest Doris does not read
+ *
+ * For a managed table the namespace decides which versions exist: the latest version is what
+ * ListTableVersions returns, FOR VERSION AS OF is checked with DescribeTableVersion, and
+ * FOR TIME AS OF picks, by the commit times the manifests record, among the versions
+ * ListTableVersions lists; a version it no longer records between recorded ones counts as
+ * removed history. A version the namespace does not record is unreachable even though its
+ * manifest is still in storage. The FE and the BE then both open the chosen version by URI
+ * with the vended credentials, exactly as for a storage-versioned table, so the namespace has
+ * to record the version at its canonical manifest, or at a staged manifest beside it once the
+ * canonical one exists.
+ */
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable Lance REST test because the Iceberg MinIO environment is disabled.")
+ return
+ }
+
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ String lanceRestPort = context.config.otherConfigs.get("lance_rest_port")
+ String catalogName = "test_lance_rest_time_travel"
+ String plain = "`${catalogName}`.`default`.`time_travel`"
+ String managed = "`${catalogName}`.`default`.`time_travel_managed`"
+ String partial = "`${catalogName}`.`default`.`time_travel_managed_partial`"
+ String lagging = "`${catalogName}`.`default`.`time_travel_managed_lagging`"
+ String untimed = "`${catalogName}`.`default`.`time_travel_managed_untimed`"
+ String unprefixed = "`${catalogName}`.`default`.`time_travel_managed_unprefixed`"
+ String pending = "`${catalogName}`.`default`.`time_travel_managed_pending`"
+ String staged = "`${catalogName}`.`default`.`time_travel_managed_staged`"
+ String misrecorded = "`${catalogName}`.`default`.`time_travel_managed_misrecorded`"
+ def scannerV2Rows = sql """SHOW VARIABLES LIKE 'enable_file_scanner_v2'"""
+ String originalScannerV2 = scannerV2Rows[0][1].toString()
+ String originalTimeZone = (sql """SHOW VARIABLES LIKE 'time_zone'""")[0][1].toString()
+
+ sql """DROP CATALOG IF EXISTS `${catalogName}`"""
+ try {
+ sql """SET enable_file_scanner_v2 = true"""
+ sql """SET time_zone = 'UTC'"""
+ // No static access key or secret key: every read below uses vended credentials.
+ sql """
+ CREATE CATALOG `${catalogName}` PROPERTIES (
+ "type" = "lance",
+ "lance.catalog.type" = "rest",
+ "lance.rest.uri" = "http://${externalEnvIp}:${lanceRestPort}",
+ "lance.rest.security.type" = "bearer",
+ "lance.rest.bearer-token" = "doris-lance-rest-test-token",
+ "lance.namespace.root_database" = "default",
+ "s3.endpoint" = "http://${externalEnvIp}:${minioPort}",
+ "s3.region" = "us-east-1",
+ "use_path_style" = "true",
+ "test_connection" = "true"
+ )
+ """
+ order_qt_tables """SHOW TABLES FROM `${catalogName}`.`default`"""
+
+ // Storage-native versions through REST.
+ qt_plain_latest """SELECT count(*), max(row_id) FROM ${plain}"""
+ order_qt_plain_version_2 """SELECT * FROM ${plain} FOR VERSION AS OF 2"""
+ qt_plain_time """
+ SELECT count(*), max(row_id) FROM ${plain} FOR TIME AS OF '2026-09-19 13:06:10'
+ """
+ explain {
+ sql("SELECT row_id FROM ${plain} FOR VERSION AS OF 2")
+ contains "lanceCatalogType=rest"
+ contains "lanceVersion=2"
+ contains "lanceManagedVersioning=false"
+ }
+
+ // Namespace-managed versions: the namespace records all three.
+ qt_managed_latest """SELECT count(*), max(row_id) FROM ${managed}"""
+ explain {
+ sql("SELECT row_id FROM ${managed}")
+ contains "lanceVersion=3"
+ contains "lanceManagedVersioning=true"
+ }
+ order_qt_managed_version_1 """SELECT * FROM ${managed} FOR VERSION AS OF 1"""
+ order_qt_managed_version_2 """SELECT * FROM ${managed} FOR VERSION AS OF 2"""
+ explain {
+ sql("SELECT row_id FROM ${managed} FOR VERSION AS OF 2 WHERE row_id > 3")
+ contains "lanceVersion=2"
+ contains "lanceManagedVersioning=true"
+ contains "lancePushdownPredicate="
+ }
+ qt_managed_version_with_predicate """
+ SELECT count(*) FROM ${managed} FOR VERSION AS OF 2 WHERE row_id > 3
+ """
+ qt_managed_time """
+ SELECT count(*), max(row_id) FROM ${managed} FOR TIME AS OF '2026-09-19 13:06:10'
+ """
+ order_qt_managed_self_join """
+ SELECT a.row_id, a.tag, b.tag
+ FROM ${managed} FOR VERSION AS OF 1 a
+ JOIN ${managed} FOR VERSION AS OF 3 b ON a.row_id = b.row_id
+ """
+ test {
+ sql """SELECT count(*) FROM ${managed} FOR VERSION AS OF 99"""
+ exception "Lance version 99 of default.time_travel_managed was not found in the namespace"
+ }
+
+ // Namespace-managed versions where the namespace no longer records version 2.
+ qt_partial_latest """SELECT count(*), max(row_id) FROM ${partial}"""
+ order_qt_partial_version_3 """SELECT * FROM ${partial} FOR VERSION AS OF 3"""
+ order_qt_partial_version_1 """SELECT * FROM ${partial} FOR VERSION AS OF 1"""
+ test {
+ sql """SELECT count(*) FROM ${partial} FOR VERSION AS OF 2"""
+ exception "Lance version 2 of default.time_travel_managed_partial was not found in the namespace"
+ }
+ // FOR TIME AS OF only selects among the versions the namespace records. Version 2 held
+ // the state between the second and third commit, and the namespace no longer records it,
+ // so that time fails as it would after cleanup removed version 2, instead of reading
+ // version 1.
+ test {
+ sql """SELECT count(*) FROM ${partial} FOR TIME AS OF '2026-09-19 13:06:10'"""
+ exception "version 2, which may hold the state at that time, no longer exists"
+ }
+ qt_partial_time_version_3 """
+ SELECT count(*), max(row_id) FROM ${partial} FOR TIME AS OF '2026-09-19 13:06:11'
+ """
+ // History before the dropped version is not used, as after cleanup, so a time before the
+ // first commit fails the same way; with every version recorded, it has no version.
+ test {
+ sql """SELECT count(*) FROM ${partial} FOR TIME AS OF '2026-09-19 13:06:07'"""
+ exception "version 2, which may hold the state at that time, no longer exists"
+ }
+ test {
+ sql """SELECT count(*) FROM ${managed} FOR TIME AS OF '2026-09-19 13:06:07'"""
+ exception "Lance table default.time_travel_managed has no version at or before '2026-09-19 13:06:07'"
+ }
+
+ // Namespace-managed versions where the namespace has not recorded version 3 yet even
+ // though its manifest is in storage: the table's latest version is the namespace's
+ // latest, by number and by time, and version 3 is not reachable.
+ qt_lagging_latest """SELECT count(*), max(row_id) FROM ${lagging}"""
+ explain {
+ sql("SELECT row_id FROM ${lagging}")
+ contains "lanceVersion=2"
+ contains "lanceManagedVersioning=true"
+ }
+ qt_lagging_time_after_storage_version_3 """
+ SELECT count(*), max(row_id) FROM ${lagging} FOR TIME AS OF '2026-09-19 13:06:11'
+ """
+ test {
+ sql """SELECT count(*) FROM ${lagging} FOR VERSION AS OF 3"""
+ exception "Lance version 3 of default.time_travel_managed_lagging was not found in the namespace"
+ }
+
+ // Tags live in the dataset's _refs/tags/ for managed tables too; the version a tag points
+ // at is then resolved through the namespace, so the partial table's tag v2 points at a
+ // version the namespace no longer records.
+ order_qt_managed_tag_v1 """SELECT * FROM ${managed}@tag(v1)"""
+ order_qt_plain_tag_v3 """SELECT row_id FROM ${plain}@tag(v3) WHERE row_id > 6"""
+ test {
+ sql """SELECT count(*) FROM ${managed}@tag(nope)"""
+ exception "Lance tag 'nope' of default.time_travel_managed was not found"
+ }
+ test {
+ sql """SELECT count(*) FROM ${partial}@tag(v2)"""
+ exception "Lance version 2 of default.time_travel_managed_partial (tag 'v2') was not found in the namespace"
+ }
+
+ // Branches: the managed table's branch "dev" is recorded by the namespace (versions 2
+ // and 3 under tree/dev/), the storage-versioned table's by its own tree/dev/ directory.
+ qt_managed_branch_dev """SELECT count(*), max(row_id) FROM ${managed}@branch(dev)"""
+ explain {
+ sql("SELECT row_id FROM ${managed}@branch(dev)")
+ contains "lanceBranch=dev"
+ contains "lanceManagedVersioning=true"
+ contains "lanceVersion=3"
+ }
+ qt_plain_branch_dev_version_2 """
+ SELECT count(*), max(row_id) FROM ${plain}@branch(dev) FOR VERSION AS OF 2
+ """
+ qt_managed_branch_dev_time """
+ SELECT count(*), max(row_id) FROM ${managed}@branch(dev) FOR TIME AS OF '2030-01-01 00:00:00'
+ """
+ // Tag "rel" points at version 3 of branch dev, which the namespace records for the branch.
+ qt_managed_tag_on_branch """SELECT count(*), max(row_id) FROM ${managed}@tag(rel)"""
+ test {
+ sql """SELECT count(*) FROM ${managed}@branch(nope)"""
+ exception "Lance branch 'nope' of default.time_travel_managed was not found"
+ }
+
+ // A namespace that lists versions without commit times resolves the same way: Doris only
+ // reads the commit times the manifests record, so the instant between commits 2 and 3
+ // selects version 2.
+ qt_untimed_latest """SELECT count(*), max(row_id) FROM ${untimed}"""
+ qt_untimed_time_between_2_and_3 """
+ SELECT count(*), max(row_id) FROM ${untimed} FOR TIME AS OF '2026-09-19 13:06:10'
+ """
+ explain {
+ sql("SELECT row_id FROM ${untimed} FOR TIME AS OF '2026-09-19 13:06:10'")
+ contains "lanceVersion=2"
+ contains "lanceManagedVersioning=true"
+ }
+
+ // Managed versioning with credentials vended under the unprefixed spelling: Doris
+ // normalizes them, and both readers must still be able to open the dataset.
+ qt_unprefixed_latest """SELECT count(*), max(row_id) FROM ${unprefixed}"""
+ order_qt_unprefixed_version_2 """SELECT * FROM ${unprefixed} FOR VERSION AS OF 2"""
+
+ // A commit the namespace reserved but never finalized: version 4 is recorded at its staged
+ // manifest, and its canonical manifest, which both readers open, does not exist. Reading it
+ // fails instead of falling back to version 3, and nothing is written; the versions before
+ // it stay readable.
+ test {
+ sql """SELECT count(*) FROM ${pending}"""
+ exception "Lance version 4 of default.time_travel_managed_pending cannot be read"
+ }
+ qt_pending_version_3 """SELECT count(*), max(row_id) FROM ${pending} FOR VERSION AS OF 3"""
+ // Its commit time is unknown until it is finalized, so FOR TIME AS OF cannot tell whether
+ // version 4 or an older one held the state at any time.
+ test {
+ sql """SELECT count(*) FROM ${pending} FOR TIME AS OF '2030-01-01 00:00:00'"""
+ exception "version 4, which may hold the state at that time, cannot be read"
+ }
+ test {
+ sql """SELECT count(*) FROM ${pending} FOR TIME AS OF '2026-09-19 13:06:08'"""
+ exception "version 4, which may hold the state at that time, cannot be read"
+ }
+ test {
+ sql """SELECT count(*) FROM ${pending}@branch(dev)"""
+ exception "Lance version 4 of default.time_travel_managed_pending@dev cannot be read"
+ }
+ test {
+ sql """SELECT count(*) FROM ${pending}@branch(dev) FOR TIME AS OF '2030-01-01 00:00:00'"""
+ exception "on default.time_travel_managed_pending@dev: version 4, which may hold the state at that time, cannot be read"
+ }
+ qt_pending_branch_dev_version_3 """
+ SELECT count(*), max(row_id) FROM ${pending}@branch(dev) FOR VERSION AS OF 3
+ """
+ test {
+ sql """SELECT count(*) FROM ${pending}@branch(dev) FOR VERSION AS OF 9"""
+ exception "Lance version 9 of default.time_travel_managed_pending@dev was not found in the namespace"
+ }
+
+ // Version 3 recorded at its staged manifest after its commit was finalized: the canonical
+ // manifest exists, so the version reads as usual.
+ qt_staged_latest """SELECT count(*), max(row_id) FROM ${staged}"""
+ explain {
+ sql("SELECT row_id FROM ${staged}")
+ contains "lanceVersion=3"
+ contains "lanceManagedVersioning=true"
+ }
+
+ // Version 3 recorded at a manifest Doris would not read: the read fails rather than read
+ // a different manifest than the namespace names.
+ test {
+ sql """SELECT count(*) FROM ${misrecorded}"""
+ exception "Lance namespace records version 3 of default.time_travel_managed_misrecorded at manifest 'lance/time_travel.lance/_versions/custom.manifest'"
+ }
+ qt_misrecorded_version_2 """SELECT count(*), max(row_id) FROM ${misrecorded} FOR VERSION AS OF 2"""
+
+ String showCreate = sql("""SHOW CREATE CATALOG `${catalogName}`""")[0][1].toString()
+ assertFalse(showCreate.contains("doris-lance-rest-test-token"))
+ } finally {
+ sql """SET enable_file_scanner_v2 = ${originalScannerV2}"""
+ sql """SET time_zone = '${originalTimeZone}'"""
+ sql """DROP CATALOG IF EXISTS `${catalogName}`"""
+ }
+}
diff --git a/regression-test/suites/external_table_p0/lance/test_lance_time_travel.groovy b/regression-test/suites/external_table_p0/lance/test_lance_time_travel.groovy
new file mode 100644
index 00000000000..29cda7d1ee4
--- /dev/null
+++ b/regression-test/suites/external_table_p0/lance/test_lance_time_travel.groovy
@@ -0,0 +1,267 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_lance_time_travel", "p0,external") {
+ /*
+ * FOR VERSION AS OF / FOR TIME AS OF on a Lance filesystem catalog.
+ *
+ * time_travel.lance keeps three versions (see lance_build_time_travel.py):
+ * version 1 row_id 1..3 committed 2026-09-19 13:06:07.597965 UTC
+ * version 2 row_id 4..6 committed 2026-09-19 13:06:09.113997 UTC
+ * version 3 row_id 7..9 committed 2026-09-19 13:06:10.621196 UTC
+ * The timestamps below are those commit times; regenerating the fixture changes them.
+ * Every version carries a Lance tag of the same name, and a branch "dev" forks from
+ * version 2 with one extra append (row_id 100).
+ */
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable Lance test because the Iceberg MinIO environment is disabled.")
+ return
+ }
+
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ String catalogName = "test_lance_time_travel"
+ String table = "`${catalogName}`.`default`.`time_travel`"
+ def scannerV2Rows = sql """SHOW VARIABLES LIKE 'enable_file_scanner_v2'"""
+ String originalScannerV2 = scannerV2Rows[0][1].toString()
+ String originalTimeZone = (sql """SHOW VARIABLES LIKE 'time_zone'""")[0][1].toString()
+
+ sql """DROP CATALOG IF EXISTS `${catalogName}`"""
+ try {
+ sql """SET enable_file_scanner_v2 = true"""
+ sql """SET time_zone = 'UTC'"""
+ sql """
+ CREATE CATALOG `${catalogName}` PROPERTIES (
+ "type" = "lance",
+ "lance.catalog.type" = "filesystem",
+ "warehouse" = "s3://warehouse/lance",
+ "s3.endpoint" = "http://${externalEnvIp}:${minioPort}",
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.region" = "us-east-1",
+ "use_path_style" = "true"
+ )
+ """
+
+ // Latest version without a selector.
+ qt_latest """SELECT count(*), min(row_id), max(row_id) FROM ${table}"""
+ explain {
+ sql("SELECT row_id FROM ${table}")
+ contains "lanceVersion=3"
+ contains "lanceManagedVersioning=false"
+ contains "lanceFragments=3"
+ }
+
+ // Explicit versions: each one is a fixed snapshot that cannot see later appends.
+ order_qt_version_1 """SELECT * FROM ${table} FOR VERSION AS OF 1"""
+ order_qt_version_2 """SELECT * FROM ${table} FOR VERSION AS OF 2"""
+ order_qt_version_3 """SELECT * FROM ${table} FOR VERSION AS OF 3"""
+ explain {
+ sql("SELECT row_id FROM ${table} FOR VERSION AS OF 2")
+ contains "lanceVersion=2"
+ contains "lanceFragments=2"
+ }
+
+ // Pushed-down predicates and LIMIT are evaluated on the selected version.
+ qt_version_with_predicate """
+ SELECT count(*) FROM ${table} FOR VERSION AS OF 3 WHERE row_id > 6
+ """
+ explain {
+ sql("SELECT row_id FROM ${table} FOR VERSION AS OF 2 WHERE row_id > 6")
+ contains "lanceVersion=2"
+ contains "lancePushdownPredicate="
+ }
+ qt_version_with_predicate_empty """
+ SELECT count(*) FROM ${table} FOR VERSION AS OF 2 WHERE row_id > 6
+ """
+
+ // Two references to the same table pin two different versions in one statement.
+ order_qt_self_join """
+ SELECT a.row_id, a.tag, b.tag
+ FROM ${table} FOR VERSION AS OF 1 a
+ JOIN ${table} FOR VERSION AS OF 3 b ON a.row_id = b.row_id
+ """
+ order_qt_union_versions """
+ SELECT 1 AS v, count(*) FROM ${table} FOR VERSION AS OF 1
+ UNION ALL SELECT 2, count(*) FROM ${table} FOR VERSION AS OF 2
+ UNION ALL SELECT 3, count(*) FROM ${table} FOR VERSION AS OF 3
+ """
+
+ // FOR TIME AS OF picks the latest version committed at or before the timestamp,
+ // parsed in the session time zone.
+ qt_time_before_version_2 """
+ SELECT count(*), max(row_id) FROM ${table} FOR TIME AS OF '2026-09-19 13:06:08'
+ """
+ qt_time_before_version_3 """
+ SELECT count(*), max(row_id) FROM ${table} FOR TIME AS OF '2026-09-19 13:06:10'
+ """
+ qt_time_after_version_3 """
+ SELECT count(*), max(row_id) FROM ${table} FOR TIME AS OF '2030-01-01 00:00:00'
+ """
+ qt_time_millisecond_precision """
+ SELECT count(*), max(row_id) FROM ${table} FOR TIME AS OF '2026-09-19 13:06:09.500'
+ """
+ // Commit times are compared in full (version 2 is 09.113997): the millisecond version 2
+ // lands in is still before it, the next one is not.
+ qt_time_just_after_commit_2 """
+ SELECT count(*), max(row_id) FROM ${table} FOR TIME AS OF '2026-09-19 13:06:09.114'
+ """
+ qt_time_within_commit_2_millisecond """
+ SELECT count(*), max(row_id) FROM ${table} FOR TIME AS OF '2026-09-19 13:06:09.113'
+ """
+ sql """SET time_zone = '+08:00'"""
+ qt_time_session_time_zone """
+ SELECT count(*), max(row_id) FROM ${table} FOR TIME AS OF '2026-09-19 21:06:10'
+ """
+ sql """SET time_zone = 'UTC'"""
+
+ // Time travel across a delete: multi_frag.lance appends three fragments (versions 1-3)
+ // and then deletes one row per fragment (version 4). An older version must show the
+ // rows that a later deletion file removes, and COUNT(*) must follow the same version.
+ String multiFrag = "`${catalogName}`.`default`.`multi_frag`"
+ qt_multi_frag_latest """SELECT count(*) FROM ${multiFrag}"""
+ qt_multi_frag_before_delete """SELECT count(*) FROM ${multiFrag} FOR VERSION AS OF 3"""
+ order_qt_multi_frag_deleted_rows_before_delete """
+ SELECT row_id FROM ${multiFrag} FOR VERSION AS OF 3 WHERE row_id IN (5, 15, 25)
+ """
+ order_qt_multi_frag_deleted_rows_after_delete """
+ SELECT row_id FROM ${multiFrag} FOR VERSION AS OF 4 WHERE row_id IN (5, 15, 25)
+ """
+ order_qt_multi_frag_first_fragment_only """
+ SELECT count(*), min(row_id), max(row_id) FROM ${multiFrag} FOR VERSION AS OF 1
+ """
+ explain {
+ sql("SELECT row_id FROM ${multiFrag} FOR VERSION AS OF 2")
+ contains "lanceVersion=2"
+ contains "lanceFragments=2"
+ }
+
+ // Error cases.
+ test {
+ sql """SELECT count(*) FROM ${table} FOR VERSION AS OF 0"""
+ exception "Lance FOR VERSION AS OF requires a positive version, but was 0"
+ }
+ // A signed number is still a version, not a tag name.
+ test {
+ sql """SELECT count(*) FROM ${table} FOR VERSION AS OF '-1'"""
+ exception "Lance FOR VERSION AS OF requires a positive version, but was -1"
+ }
+ // As for Paimon, and for tag names in Iceberg, a non-numeric FOR VERSION AS OF names a tag.
+ test {
+ sql """SELECT count(*) FROM ${table} FOR VERSION AS OF 'abc'"""
+ exception "Lance tag 'abc' of default.time_travel was not found"
+ }
+ test {
+ sql """SELECT count(*) FROM ${table}@branch(dev) FOR VERSION AS OF 'abc'"""
+ exception "FOR VERSION AS OF 'abc' names a tag, which cannot be combined with @branch"
+ }
+ test {
+ sql """SELECT count(*) FROM ${table} FOR VERSION AS OF '99999999999999999999'"""
+ exception "Lance FOR VERSION AS OF version 99999999999999999999 is out of range"
+ }
+ // The branch's chain starts where it was forked; earlier timestamps select nothing there.
+ test {
+ sql """SELECT count(*) FROM ${table}@branch(dev) FOR TIME AS OF '2026-09-19 13:06:10'"""
+ exception "Lance branch 'dev' of default.time_travel has no version at or before '2026-09-19 13:06:10'"
+ }
+ test {
+ sql """SELECT count(*) FROM ${table} FOR VERSION AS OF 99"""
+ exception "Lance version 99 of default.time_travel was not found"
+ }
+ test {
+ sql """SELECT count(*) FROM ${table} FOR TIME AS OF '2026-09-19 13:06:07'"""
+ exception "Lance table default.time_travel has no version at or before '2026-09-19 13:06:07'"
+ }
+ test {
+ sql """SELECT count(*) FROM ${table} FOR TIME AS OF 'not-a-time'"""
+ exception "Cannot parse Lance FOR TIME AS OF value 'not-a-time'"
+ }
+ // A Lance tag (stored under _refs/tags/) resolves to the version it points at.
+ order_qt_tag_v2 """SELECT * FROM ${table}@tag(v2)"""
+ order_qt_version_as_of_tag_name """SELECT * FROM ${table} FOR VERSION AS OF 'v2'"""
+ // A tag determines its own branch, so it is never combined with @branch, @branch(main)
+ // included: "rel" points into dev and must not be read under an explicit main.
+ test {
+ sql """SELECT count(*) FROM ${table}@branch(main) FOR VERSION AS OF 'v2'"""
+ exception "FOR VERSION AS OF 'v2' names a tag, which cannot be combined with @branch"
+ }
+ test {
+ sql """SELECT count(*) FROM ${table}@branch(main) FOR VERSION AS OF 'rel'"""
+ exception "FOR VERSION AS OF 'rel' names a tag, which cannot be combined with @branch"
+ }
+ qt_version_as_of_tag_on_branch """SELECT count(*), max(row_id) FROM ${table} FOR VERSION AS OF 'rel'"""
+ test {
+ sql """SELECT count(*) FROM ${table}@tag('name'='v1', 'x'='y')"""
+ exception "Lance @tag takes exactly one name"
+ }
+ explain {
+ sql("SELECT row_id FROM ${table}@tag(v2)")
+ contains "lanceVersion=2"
+ }
+ test {
+ sql """SELECT count(*) FROM ${table}@tag(no_such_tag)"""
+ exception "Lance tag 'no_such_tag' of default.time_travel was not found"
+ }
+ // A Lance branch (a separate manifest chain under tree/<branch>/): "dev" forks from
+ // version 2 and appends row 100, so its version 3 differs from main's version 3.
+ order_qt_branch_dev """SELECT * FROM ${table}@branch(dev)"""
+ order_qt_branch_dev_version_2 """SELECT * FROM ${table}@branch(dev) FOR VERSION AS OF 2"""
+ explain {
+ sql("SELECT row_id FROM ${table}@branch(dev)")
+ contains "lanceBranch=dev"
+ contains "lanceVersion=3"
+ contains "lanceFragments=3"
+ }
+ order_qt_branch_vs_main """
+ SELECT m.row_id AS main_row, d.row_id AS dev_row
+ FROM ${table} FOR VERSION AS OF 3 m
+ FULL OUTER JOIN ${table}@branch(dev) d ON m.row_id = d.row_id
+ WHERE m.row_id IS NULL OR d.row_id IS NULL
+ """
+ // "main" is Lance's name for the main chain, and a branch can also be given in map form.
+ qt_branch_main """SELECT count(*), max(row_id) FROM ${table}@branch(main)"""
+ qt_branch_dev_map_form """SELECT count(*), max(row_id) FROM ${table}@branch('name'='dev')"""
+ // FOR TIME AS OF inside a branch resolves against the branch's own commit times.
+ qt_branch_dev_time """
+ SELECT count(*), max(row_id) FROM ${table}@branch(dev) FOR TIME AS OF '2030-01-01 00:00:00'
+ """
+ // A tag that points into a branch selects that branch, not the same version number on main.
+ order_qt_tag_on_branch """SELECT * FROM ${table}@tag(rel)"""
+ explain {
+ sql("SELECT row_id FROM ${table}@tag(rel)")
+ contains "lanceBranch=dev"
+ contains "lanceVersion=3"
+ }
+ test {
+ sql """SELECT count(*) FROM ${table}@tag(v2) FOR VERSION AS OF 1"""
+ exception "@tag cannot be combined with FOR VERSION AS OF or FOR TIME AS OF"
+ }
+ test {
+ sql """SELECT count(*) FROM ${table}@branch(nope)"""
+ exception "Lance branch 'nope' of default.time_travel was not found"
+ }
+ test {
+ sql """SELECT count(*) FROM ${table}@branch(dev) FOR VERSION AS OF 9"""
+ exception "Lance version 9 of default.time_travel@dev was not found"
+ }
+ } finally {
+ sql """SET enable_file_scanner_v2 = ${originalScannerV2}"""
+ sql """SET time_zone = '${originalTimeZone}'"""
+ sql """DROP CATALOG IF EXISTS `${catalogName}`"""
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 32/35: [improvement](lance) Push down array membership and boolean scalar index predicates (#68687)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 238cf2c2538fd27d53c65705718a46e72700df39
Author: Gabriel <li...@selectdb.com>
AuthorDate: Thu Oct 8 09:57:40 2026 +0800
[improvement](lance) Push down array membership and boolean scalar index predicates (#68687)
### What problem does this PR solve?
Ordinary Lance scans currently retain array label membership predicates
in Doris, and scalar index planning misses usable boolean drivers. This
adds Substrait pushdown for built-in `array_contains` on a direct
`List<Utf8>` column with a non-NULL constant string, and for nonempty
constant-string `arrays_overlap` (including the normal Nereids rewrite
of three or more membership disjuncts). Positive indexable boolean
conditions can select a LabelList/BTree/Bitmap segment. OR requires
usable drivers on both branches of the same selected index. Literal
starts_with prefixes and LIKE filters with a usable leading prefix
remain eligible for BTree planning. OR branches requiring a refine
filter retain fragment scans. Missing FE field/index metadata and
competing logical indexes on one column preserve native automatic index
selection and fragment ownership. For example, two label conditions
joined by AND now reach Lance together and can reduce the index
candidate set before row materialization.
NULL needles, other array types, dynamic operands, UDFs, and
`array_contains_all` remain residual. Doris `array_contains_all` tests a
contiguous ordered subsequence, so translating it to Lance set
containment would change results. Each task still selects one logical
index; this change does not implement multi-column index intersection.
Fragment coverage, residual evaluation, and limit handling are
preserved. Complement-only predicates and expressions exceeding the
pinned native node/depth budget use parallel fragment scans with
scalar-index use disabled. An independent positive condition retains one
search per index segment even when another conjunct is a complement.
This avoids repeated segment-wide searches and prevents large overlap or
unindexable OR filters from serializing fallback scans.
Dependency:
[lance-format/lance-c#93](https://github.com/lance-format/lance-c/pull/93),
merged in upstream main at `cd63420bfbe27f6f0a1edcc873b9191af7d52852`,
which is now the pinned dependency. The existing Foyer payload is
reapplied without implementation changes because
[lance-c#73](https://github.com/lance-format/lance-c/pull/73) is still
open and supplies cache APIs used by Doris. Remove that remaining patch
after #73 merges. The build now records and validates the installed
Lance source/archive/patch fingerprint, detects incomplete publication,
and refuses stale external build output. Before removing an existing
installation, it verifies required rebuild inputs and rejects external
definitions that differ from the checkout. The third-party CI script job
runs the installation harness under the existing workflow path filters.
These third-party/build changes are synchronized to #68689 for master.
This PR targets `branch-4.1`.
Documentation: https://github.com/apache/doris-website/pull/4184
(English and Chinese; draft pending implementation).
### Validation
- 70 focused FE JUnit tests passed after compiling the changed
converter/planner/scan sources and tests against an existing FE
dependency build. Coverage includes metadata fallbacks, competing
same-column indexes in both metadata orders, balanced wide conjunctions,
short IN expansion, literal/refined prefixes, nested OR guards, and
existing complement/overlap limits. The new regression assertions failed
before the fixes.
- FE Checkstyle passed with zero violations; the updated Groovy
regression suite passed syntax validation.
- Executed 12 plans produced by the actual FE converter and scan planner
through 30 pinned lance-c scans. Verified result rows and index counters
for unavailable field IDs, omitted index metadata, unequal same-column
index coverage, a 34-predicate conjunction, 62/63-label overlap combined
with two-value IN, literal prefixes, refined LIKE, and direct/nested OR
fallback. No segment fallback occurred; metadata/ambiguity cases
retained native index loads. Previous integration also covered fully
indexed, partially indexed, and unindexed datasets.
- The installation harness passed on master and branch-4.1 using
independent external source definitions: legacy/matching installs,
missing helper/vars/patch/downloader/builder, mismatched and
synchronized pin/patch updates, incomplete artifacts, stale builder
output, interrupted publication, and retry. Rejected preflight cases
verify that the entire installation and builder invocation count remain
unchanged. The missing-helper regression failed before the fix. Shell
syntax, workflow YAML, and CI path-filter checks passed.
- The unchanged pinned upstream main plus Foyer dependency previously
passed 75 Rust unit tests, 376 C API tests, and all 3 native
C/C++/static OSS consumer tests. Third-party
extraction/cache-refresh/patch-failure checks also passed on both
dependency definitions.
- SQL regressions assert pushed/residual EXPLAIN output, fragment versus
segment grouping, actual index searches/candidates/fallbacks, partial
coverage, 64/65-label boundaries, mixed complements, and selective
LIMIT. Full Doris compilation and distributed SQL execution on this
revision are pending CI.
### Release note
Support ordinary Lance array-label membership pushdown and safe boolean
scalar index planning.
### Check List (For Author)
- Test
- [x] Regression test added (SQL execution pending CI)
- [x] Unit Test
- [x] Manual integration test (FE-produced Substrait through lance-c,
described above)
- Behavior changed:
- [x] Yes. Compatible label predicates are evaluated in Lance; eligible
positive boolean drivers use scalar segments, while broad complements
and unsupported or oversized expressions retain fragment parallelism.
- Does this need documentation?
- [x] Yes: https://github.com/apache/doris-website/pull/4184
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Confirm the upstream dependency is ready
---
.github/workflows/build-thirdparty.yml | 3 +
build.sh | 25 +-
.../scripts/lance_build_array_predicates.py | 106 ++++++++
.../scripts/lance_build_preinstalled_catalog.py | 4 +
.../bitmap_page_lookup.lance | Bin 0 -> 993 bytes
.../page_data.lance | Bin 0 -> 769 bytes
.../page_lookup.lance | Bin 0 -> 1259 bytes
.../0-a4640b8d-55a7-47f8-9392-5ec3d0ed19c9.txn | Bin 0 -> 325 bytes
.../1-d21db0fc-7e46-4699-9948-ab6f8d66098c.txn | Bin 0 -> 184 bytes
.../2-8498550d-87e2-42c8-931c-1ed0d8bc7371.txn | Bin 0 -> 197 bytes
.../_versions/18446744073709551612.manifest | Bin 0 -> 901 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 735 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 729 bytes
.../_versions/latest_version_hint.json | 1 +
...0010111110100106a0464445da0b5a4b7c3465eb5.lance | Bin 0 -> 1223 bytes
...101111100111010027ce24037932857b1f2b85e24.lance | Bin 0 -> 1223 bytes
.../bitmap_page_lookup.lance | Bin 0 -> 865 bytes
.../page_data.lance | Bin 0 -> 704 bytes
.../page_lookup.lance | Bin 0 -> 1259 bytes
.../0-58691d36-bff2-4a01-ad25-ae3642019067.txn | Bin 0 -> 244 bytes
.../1-c343d785-33b7-4448-b0da-160c970f78a1.txn | Bin 0 -> 182 bytes
.../2-bcdae8c1-73a4-4b25-ad8b-887bf393af0f.txn | Bin 0 -> 195 bytes
.../3-8a4b0277-3a05-4eca-8787-2f5ad24763fb.txn | Bin 0 -> 124 bytes
.../_versions/18446744073709551611.manifest | Bin 0 -> 824 bytes
.../_versions/18446744073709551612.manifest | Bin 0 -> 812 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 648 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 565 bytes
.../_versions/latest_version_hint.json | 1 +
...011101101000101d214d7401ba969a49532b0ae8b.lance | Bin 0 -> 1223 bytes
...00100001001011139a9b14af698a84d7cb598d6a7.lance | Bin 0 -> 1223 bytes
.../0-7c18f76d-3524-4079-835f-7404de102fa9.txn | Bin 0 -> 325 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 729 bytes
.../_versions/latest_version_hint.json | 1 +
...0010010101001015f6f6345d896c3324132a6d562.lance | Bin 0 -> 1223 bytes
...1100100010111008b02e34ab199273cee3ff3688e.lance | Bin 0 -> 1223 bytes
.../lance/source/LancePredicateConverter.java | 75 +++++-
.../lance/source/LanceScalarIndexPlanner.java | 233 ++++++++++++++--
.../datasource/lance/source/LanceScanPlanner.java | 6 +-
.../lance/source/LanceSubstraitSerializer.java | 18 +-
.../main/resources/substrait/lance_functions.yaml | 27 ++
.../lance/LancePredicateConverterTest.java | 115 ++++++++
.../lance/source/LanceScalarIndexPlannerTest.java | 292 +++++++++++++++++++++
.../datasource/lance/source/LanceScanNodeTest.java | 69 ++++-
.../test_lance_array_predicate_pushdown.groovy | 154 +++++++++++
thirdparty/build-thirdparty.sh | 7 +
thirdparty/lance-install.sh | 38 +++
thirdparty/patches/lance-c-foyer.patch | 31 ++-
thirdparty/test/lance-install-test.sh | 176 +++++++++++++
thirdparty/vars.sh | 10 +-
49 files changed, 1343 insertions(+), 49 deletions(-)
diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml
index 3d3d11ff27d..df6339c8ab8 100644
--- a/.github/workflows/build-thirdparty.yml
+++ b/.github/workflows/build-thirdparty.yml
@@ -80,6 +80,9 @@ jobs:
run: |
thirdparty/test/adbc-jni-config-test.sh
+ - name: Test Lance installation
+ run: bash thirdparty/test/lance-install-test.sh
+
build_linux:
name: Build Third Party Libraries (Linux)
needs: changes
diff --git a/build.sh b/build.sh
index 8449eab271a..5eac08e4078 100755
--- a/build.sh
+++ b/build.sh
@@ -441,6 +441,7 @@ if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && ${BUILD_
fi
# build thirdparty libraries if necessary. check last thirdparty lib installation
+source "${DORIS_HOME}/thirdparty/lance-install.sh"
if [[ "${TARGET_SYSTEM}" == 'Darwin' ]]; then
LAST_THIRDPARTY_LIB='libbrotlienc.a'
else
@@ -456,10 +457,21 @@ if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ||
! -f "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ||
! -s "${DORIS_THIRDPARTY}/installed/lib64/libpaimon_c.a" ||
! -s "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ||
- -e "${DORIS_THIRDPARTY}/installed/lib64/.paimon-installing" ]]; then
- # Compilation images may contain only installed artifacts; never erase them without a rebuild source.
- if [[ ! -f "${DORIS_THIRDPARTY}/build-thirdparty.sh" ]]; then
- echo "Third-party dependencies require a rebuild, but build-thirdparty.sh is missing." >&2
+ -e "${DORIS_THIRDPARTY}/installed/lib64/.paimon-installing" ]] ||
+ ! lance_c_install_is_current "${DORIS_HOME}/thirdparty" "${DORIS_THIRDPARTY}/installed"; then
+ # External trees can be partially updated or pinned to another revision. Preserve
+ # the existing prefix unless their build inputs can produce the requested Lance version.
+ for input in build-thirdparty.sh download-thirdparty.sh vars.sh lance-install.sh patches/lance-c-foyer.patch; do
+ if [[ ! -f "${DORIS_THIRDPARTY}/${input}" || ! -r "${DORIS_THIRDPARTY}/${input}" ]]; then
+ echo "Third-party dependencies require a rebuild, but ${input} is missing or unreadable." >&2
+ echo "Refresh the compilation image or set DORIS_THIRDPARTY to a complete third-party source tree." >&2
+ exit 1
+ fi
+ done
+ if ! expected_lance_fingerprint="$(lance_c_install_fingerprint "${DORIS_HOME}/thirdparty")" ||
+ ! rebuild_lance_fingerprint="$(lance_c_install_fingerprint "${DORIS_THIRDPARTY}")" ||
+ [[ "${rebuild_lance_fingerprint}" != "${expected_lance_fingerprint}" ]]; then
+ echo "Lance rebuild sources do not match this checkout; installed dependencies have been preserved." >&2
echo "Refresh the compilation image or set DORIS_THIRDPARTY to a complete third-party source tree." >&2
exit 1
fi
@@ -472,6 +484,11 @@ if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ||
else
bash "${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}" --clean
fi
+ # An external build script can itself be stale. Never link its old output silently.
+ if ! lance_c_install_is_current "${DORIS_HOME}/thirdparty" "${DORIS_THIRDPARTY}/installed"; then
+ echo "Lance dependency revision does not match this checkout. Refresh the third-party build tree." >&2
+ exit 1
+ fi
fi
update_submodule() {
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_array_predicates.py b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_array_predicates.py
new file mode 100644
index 00000000000..114e5a8a128
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_array_predicates.py
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Build isolated array pushdown fixtures with lance_fixture_requirements.txt.
+
+The standard MinIO mirror publishes these below warehouse/lance/predicate_arrays.
+Each dataset uses relative data/index paths and two eight-row fragments. The
+partial dataset appends one unindexed fragment after creating its LabelList index.
+"""
+
+import argparse
+from pathlib import Path
+
+import lance
+import pyarrow as pa
+
+
+LABELS = [["red"], ["blue"], ["red", "blue"], [], None,
+ ["red", "red"], [None, "blue"], ["blue", "red"]] * 2
+
+
+def build(output):
+ if not __debug__:
+ raise RuntimeError("Fixture verification requires assertions; do not use python -O")
+ if lance.__version__ != "7.0.0":
+ raise RuntimeError("Use the pinned pylance 7.0.0 fixture writer")
+ output.mkdir(parents=True, exist_ok=True)
+ table = pa.table({"id": pa.array(range(16), type=pa.int64()),
+ "labels": pa.array(LABELS, type=pa.list_(pa.string())),
+ "category": pa.array([i % 3 for i in range(16)], type=pa.int32())})
+ for name in ["indexed", "partial", "unindexed"]:
+ path = output / (name + ".lance")
+ if path.exists():
+ raise FileExistsError(path)
+ first = table.slice(0, 8) if name == "partial" else table
+ dataset = lance.write_dataset(first, str(path), max_rows_per_file=8, max_rows_per_group=8)
+ if name != "unindexed":
+ dataset.create_scalar_index("labels", "LABEL_LIST", name="labels_idx")
+ dataset.create_scalar_index("category", "BTREE", name="category_idx")
+ if name == "partial":
+ lance.write_dataset(table.slice(8), str(path), mode="append", max_rows_per_file=8)
+ check(output)
+
+
+def check(output):
+ if not __debug__:
+ raise RuntimeError("Fixture verification requires assertions; do not use python -O")
+ table = pa.table({"id": pa.array(range(16), type=pa.int64()),
+ "labels": pa.array(LABELS, type=pa.list_(pa.string())),
+ "category": pa.array([i % 3 for i in range(16)], type=pa.int32())})
+ for name in ["indexed", "partial", "unindexed"]:
+ path = output / (name + ".lance")
+ dataset = lance.dataset(str(path))
+ assert dataset.to_table().to_pydict() == table.to_pydict()
+ fragments = dataset.get_fragments()
+ assert len(fragments) == 2
+ assert [f.to_table(columns=["id"])["id"].to_pylist() for f in fragments] == [
+ list(range(8)), list(range(8, 16))]
+ indexes = {index["name"]: index for index in dataset.list_indices()}
+ assert set(indexes) == (set() if name == "unindexed" else {"labels_idx", "category_idx"})
+ expected_coverage = {f.fragment_id for f in (fragments[:1] if name == "partial" else fragments)}
+ # Names alone do not prove the mixed indexed/unindexed fixture: verify each
+ # physical domain so rebuilding both fragments into an index cannot pass silently.
+ for index_name, field, index_type in [("labels_idx", "labels", "LabelList"),
+ ("category_idx", "category", "BTree")]:
+ if name == "unindexed":
+ continue
+ index = indexes[index_name]
+ assert index["fields"] == [field] and index["type"] == index_type
+ assert index["fragment_ids"] == expected_coverage, (name, index_name, index)
+ if name == "partial":
+ assert len({f.fragment_id for f in fragments} - expected_coverage) == 1
+ for predicate, expected in [
+ ("array_contains(labels, 'red')", [0, 2, 5, 7, 8, 10, 13, 15]),
+ ("array_contains(labels, 'red') AND array_contains(labels, 'blue')", [2, 7, 10, 15]),
+ ("array_contains(labels, 'red') OR array_contains(labels, 'blue')",
+ [0, 1, 2, 5, 6, 7, 8, 9, 10, 13, 14, 15])]:
+ for use_index in [False, True]:
+ ids = dataset.to_table(columns=["id"], filter=predicate,
+ use_scalar_index=use_index)["id"].to_pylist()
+ assert sorted(ids) == expected, (name, predicate, ids)
+ assert len(dataset.list_indices()) == (0 if name == "unindexed" else 2)
+ print("Verified array predicate fixtures (indexed, partial, unindexed)")
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--output", type=Path, default=Path(__file__).parent /
+ "preinstalled_data/lance/predicate_arrays")
+ parser.add_argument("--check", action="store_true", help="Verify committed fixtures without rebuilding")
+ args = parser.parse_args()
+ (check if args.check else build)(args.output)
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py
index 54560857370..2645e8748d8 100644
--- a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py
@@ -89,6 +89,7 @@ import lance
import lance_namespace
import pyarrow as pa
import pyarrow.ipc as ipc
+from lance_build_array_predicates import build as build_array_predicates, check as check_array_predicates
from lance_build_multivector import build as build_multivector, check as check_multivector
from lance_build_nested_null import build as build_nested_null, check as check_nested_null
from lance_build_time_travel import check as check_time_travel
@@ -1671,6 +1672,8 @@ def check_fts_dataset(location: str, *, table_name: str, index_name: str,
def check_catalog(root: Path) -> None:
+ # The SQL pushdown suite also relies on these independently published datasets.
+ check_array_predicates(root / "predicate_arrays")
check_data_shapes()
namespace = lance_namespace.connect("dir", {"root": str(root)})
tables = namespace.list_tables(ListTablesRequest(id=[NAMESPACE]))
@@ -1810,6 +1813,7 @@ def main() -> int:
staging = Path(staging_name) / "lance"
staging.mkdir()
build(staging, all_types_source, time_travel_source)
+ build_array_predicates(staging / "predicate_arrays")
check_catalog(staging)
backup = output.with_name(output.name + ".old")
if backup.exists():
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_indices/026150a9-7d9e-4859-a412-fa0582e40071/bitmap_page_lookup.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_indices/026150a9-7d9e-4859-a412-fa0582e40071/bitmap_page_lookup.lance
new file mode 100644
index 00000000000..8b817459d44
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_indices/026150a9-7d9e-4859-a412-fa0582e40071/bitmap_page_lookup.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_indices/79b28d78-5490-40ba-a969-399822c12126/page_data.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_indices/79b28d78-5490-40ba-a969-399822c12126/page_data.lance
new file mode 100644
index 00000000000..af115f4fb6e
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_indices/79b28d78-5490-40ba-a969-399822c12126/page_data.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_indices/79b28d78-5490-40ba-a969-399822c12126/page_lookup.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_indices/79b28d78-5490-40ba-a969-399822c12126/page_lookup.lance
new file mode 100644
index 00000000000..bb72eed316c
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_indices/79b28d78-5490-40ba-a969-399822c12126/page_lookup.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_transactions/0-a4640b8d-55a7-47f8-9392-5ec3d0ed19c9.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_transactions/0-a4640b8d-55a7-47f8-9392-5ec3d0ed19c9.txn
new file mode 100644
index 00000000000..2db5ba7fb9b
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_transactions/0-a4640b8d-55a7-47f8-9392-5ec3d0ed19c9.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_transactions/1-d21db0fc-7e46-4699-9948-ab6f8d66098c.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_transactions/1-d21db0fc-7e46-4699-9948-ab6f8d66098c.txn
new file mode 100644
index 00000000000..38fdb4d0534
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_transactions/1-d21db0fc-7e46-4699-9948-ab6f8d66098c.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_transactions/2-8498550d-87e2-42c8-931c-1ed0d8bc7371.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_transactions/2-8498550d-87e2-42c8-931c-1ed0d8bc7371.txn
new file mode 100644
index 00000000000..75a68117ca4
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_transactions/2-8498550d-87e2-42c8-931c-1ed0d8bc7371.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/18446744073709551612.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/18446744073709551612.manifest
new file mode 100644
index 00000000000..b96a888dd78
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/18446744073709551612.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/18446744073709551613.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/18446744073709551613.manifest
new file mode 100644
index 00000000000..4e0e7ac5ed3
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/18446744073709551613.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/18446744073709551614.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/18446744073709551614.manifest
new file mode 100644
index 00000000000..74c137306c7
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/18446744073709551614.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/latest_version_hint.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/latest_version_hint.json
new file mode 100644
index 00000000000..f93d3984472
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/_versions/latest_version_hint.json
@@ -0,0 +1 @@
+{"version":3}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/data/1001010000010111110100106a0464445da0b5a4b7c3465eb5.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/data/1001010000010111110100106a0464445da0b5a4b7c3465eb5.lance
new file mode 100644
index 00000000000..82104af9df6
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/data/1001010000010111110100106a0464445da0b5a4b7c3465eb5.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/data/111000011101111100111010027ce24037932857b1f2b85e24.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/data/111000011101111100111010027ce24037932857b1f2b85e24.lance
new file mode 100644
index 00000000000..dfe133e40e7
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/indexed.lance/data/111000011101111100111010027ce24037932857b1f2b85e24.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_indices/9b6b2f0a-bd44-450f-9a3e-ab3f4ace7990/bitmap_page_lookup.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_indices/9b6b2f0a-bd44-450f-9a3e-ab3f4ace7990/bitmap_page_lookup.lance
new file mode 100644
index 00000000000..dfd2305285a
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_indices/9b6b2f0a-bd44-450f-9a3e-ab3f4ace7990/bitmap_page_lookup.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_indices/db755621-13aa-4ca9-a495-c0745e34f7c4/page_data.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_indices/db755621-13aa-4ca9-a495-c0745e34f7c4/page_data.lance
new file mode 100644
index 00000000000..7e9051906cd
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_indices/db755621-13aa-4ca9-a495-c0745e34f7c4/page_data.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_indices/db755621-13aa-4ca9-a495-c0745e34f7c4/page_lookup.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_indices/db755621-13aa-4ca9-a495-c0745e34f7c4/page_lookup.lance
new file mode 100644
index 00000000000..bb72eed316c
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_indices/db755621-13aa-4ca9-a495-c0745e34f7c4/page_lookup.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/0-58691d36-bff2-4a01-ad25-ae3642019067.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/0-58691d36-bff2-4a01-ad25-ae3642019067.txn
new file mode 100644
index 00000000000..b8e46a2b19d
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/0-58691d36-bff2-4a01-ad25-ae3642019067.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/1-c343d785-33b7-4448-b0da-160c970f78a1.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/1-c343d785-33b7-4448-b0da-160c970f78a1.txn
new file mode 100644
index 00000000000..3eb9059d63e
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/1-c343d785-33b7-4448-b0da-160c970f78a1.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/2-bcdae8c1-73a4-4b25-ad8b-887bf393af0f.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/2-bcdae8c1-73a4-4b25-ad8b-887bf393af0f.txn
new file mode 100644
index 00000000000..cbf508f445e
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/2-bcdae8c1-73a4-4b25-ad8b-887bf393af0f.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/3-8a4b0277-3a05-4eca-8787-2f5ad24763fb.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/3-8a4b0277-3a05-4eca-8787-2f5ad24763fb.txn
new file mode 100644
index 00000000000..ace146f66c0
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_transactions/3-8a4b0277-3a05-4eca-8787-2f5ad24763fb.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551611.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551611.manifest
new file mode 100644
index 00000000000..9e186aac8a1
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551611.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551612.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551612.manifest
new file mode 100644
index 00000000000..ca0a927adc0
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551612.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551613.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551613.manifest
new file mode 100644
index 00000000000..b5cd6bb01d7
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551613.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551614.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551614.manifest
new file mode 100644
index 00000000000..971179fb5c1
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/18446744073709551614.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/latest_version_hint.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/latest_version_hint.json
new file mode 100644
index 00000000000..205c7a40a84
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/_versions/latest_version_hint.json
@@ -0,0 +1 @@
+{"version":4}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/data/011010010011101101000101d214d7401ba969a49532b0ae8b.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/data/011010010011101101000101d214d7401ba969a49532b0ae8b.lance
new file mode 100644
index 00000000000..82104af9df6
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/data/011010010011101101000101d214d7401ba969a49532b0ae8b.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/data/11001100100100001001011139a9b14af698a84d7cb598d6a7.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/data/11001100100100001001011139a9b14af698a84d7cb598d6a7.lance
new file mode 100644
index 00000000000..dfe133e40e7
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/partial.lance/data/11001100100100001001011139a9b14af698a84d7cb598d6a7.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/_transactions/0-7c18f76d-3524-4079-835f-7404de102fa9.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/_transactions/0-7c18f76d-3524-4079-835f-7404de102fa9.txn
new file mode 100644
index 00000000000..7bb4aa01bae
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/_transactions/0-7c18f76d-3524-4079-835f-7404de102fa9.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/_versions/18446744073709551614.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/_versions/18446744073709551614.manifest
new file mode 100644
index 00000000000..b0e2e53d706
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/_versions/18446744073709551614.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/_versions/latest_version_hint.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/_versions/latest_version_hint.json
new file mode 100644
index 00000000000..491d734467a
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/_versions/latest_version_hint.json
@@ -0,0 +1 @@
+{"version":1}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/data/0011111110010010101001015f6f6345d896c3324132a6d562.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/data/0011111110010010101001015f6f6345d896c3324132a6d562.lance
new file mode 100644
index 00000000000..82104af9df6
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/data/0011111110010010101001015f6f6345d896c3324132a6d562.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/data/1001011111100100010111008b02e34ab199273cee3ff3688e.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/data/1001011111100100010111008b02e34ab199273cee3ff3688e.lance
new file mode 100644
index 00000000000..dfe133e40e7
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/predicate_arrays/unindexed.lance/data/1001011111100100010111008b02e34ab199273cee3ff3688e.lance differ
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LancePredicateConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LancePredicateConverter.java
index 2e1cd8b890c..2b23992c9dd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LancePredicateConverter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LancePredicateConverter.java
@@ -17,6 +17,7 @@
package org.apache.doris.datasource.lance.source;
+import org.apache.doris.analysis.ArrayLiteral;
import org.apache.doris.analysis.BinaryPredicate;
import org.apache.doris.analysis.BoolLiteral;
import org.apache.doris.analysis.CompoundPredicate;
@@ -48,6 +49,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
+import java.io.InputStream;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.time.DateTimeException;
@@ -67,6 +69,8 @@ import java.util.stream.Collectors;
/** Converts predicates with identical Doris and Lance semantics to a Substrait ExtendedExpression. */
public class LancePredicateConverter {
private static final TypeCreator REQUIRED = TypeCreator.of(false);
+ private static final String LANCE_FUNCTIONS = "https://github.com/apache/doris/blob/master/"
+ + "fe/fe-core/src/main/resources/substrait/lance_functions.yaml";
private static final SimpleExtension.ExtensionCollection EXTENSIONS = loadExtensions();
private final Schema schema;
@@ -125,7 +129,7 @@ public class LancePredicateConverter {
return convertLike((LikePredicate) expr);
}
if (expr instanceof FunctionCallExpr) {
- return convertStringFunction((FunctionCallExpr) expr);
+ return convertFunction((FunctionCallExpr) expr);
}
if (expr instanceof SlotRef) {
return convertBooleanSlot((SlotRef) expr);
@@ -264,7 +268,7 @@ public class LancePredicateConverter {
return convertStringPredicate("like:str_str", predicate.getChild(0), predicate.getChild(1), true);
}
- private Optional<Expression> convertStringFunction(FunctionCallExpr function) {
+ private Optional<Expression> convertFunction(FunctionCallExpr function) {
if (function.getFnName() == null || function.getFn() == null
|| function.getFn().getBinaryType() != TFunctionBinaryType.BUILTIN
|| function.getChildren().size() != 2) {
@@ -272,6 +276,10 @@ public class LancePredicateConverter {
}
String functionName = function.getFnName().getFunction().toLowerCase(Locale.ROOT);
switch (functionName) {
+ case "arrays_overlap":
+ return convertArraysOverlap(function);
+ case "array_contains":
+ return convertArrayContains(function);
case "like":
return convertStringPredicate(
"like:str_str", function.getChild(0), function.getChild(1), true);
@@ -286,6 +294,64 @@ public class LancePredicateConverter {
}
}
+ private Optional<Expression> convertArraysOverlap(FunctionCallExpr function) {
+ Expr input = function.getChild(0);
+ Expr values = function.getChild(1);
+ if (input instanceof ArrayLiteral) {
+ input = function.getChild(1);
+ values = function.getChild(0);
+ }
+ SlotRef slot = directSlot(input);
+ ResolvedField field = slot == null ? null : findField(slot);
+ if (field == null || !LanceSubstraitSerializer.supportsStringList(field.field)
+ || !(values instanceof ArrayLiteral) || values.getChildren().isEmpty()) {
+ return Optional.empty();
+ }
+ // Nereids folds three or more membership disjuncts into arrays_overlap. Expand
+ // only non-NULL string needles: Doris also matches NULL elements to NULL needles.
+ List<Expression> level = new ArrayList<>();
+ for (Expr needle : values.getChildren()) {
+ if (!(needle instanceof StringLiteral)) {
+ return Optional.empty();
+ }
+ level.add(arrayMembership(field, needle.getStringValue()));
+ }
+ // Keep expression depth logarithmic for large constant label lists.
+ while (level.size() > 1) {
+ List<Expression> next = new ArrayList<>();
+ for (int i = 0; i < level.size(); i += 2) {
+ next.add(i + 1 == level.size() ? level.get(i)
+ : booleanFunction("or:bool", Arrays.asList(level.get(i), level.get(i + 1))));
+ }
+ level = next;
+ }
+ return Optional.of(level.get(0));
+ }
+
+ private Optional<Expression> convertArrayContains(FunctionCallExpr function) {
+ SlotRef slot = directSlot(function.getChild(0));
+ ResolvedField field = slot == null ? null : findField(slot);
+ Expr needle = function.getChild(1);
+ if (field == null || !LanceSubstraitSerializer.supportsStringList(field.field)
+ || !(needle instanceof StringLiteral)) {
+ return Optional.empty();
+ }
+ // Doris matches a NULL needle against NULL elements; DataFusion does not. Also,
+ // array_contains_all is a contiguous subsequence test, not DataFusion's set containment.
+ // Only non-NULL scalar membership is interchangeable across the two engines.
+ return Optional.of(arrayMembership(field, needle.getStringValue()));
+ }
+
+ private Expression arrayMembership(ResolvedField field, String needle) {
+ return Expression.ScalarFunctionInvocation.builder()
+ .declaration(EXTENSIONS.getScalarFunction(
+ SimpleExtension.FunctionAnchor.of(LANCE_FUNCTIONS, "array_has:list_str")))
+ .outputType(TypeCreator.of(field.field.isNullable()).BOOLEAN)
+ .arguments(Arrays.asList(fieldReference(field),
+ ExpressionCreator.string(false, needle)))
+ .build();
+ }
+
private Optional<Expression> convertStringPredicate(
String function, Expr input, Expr pattern, boolean rejectEscapedPattern) {
SlotRef slot = directSlot(input);
@@ -497,8 +563,9 @@ public class LancePredicateConverter {
}
private static SimpleExtension.ExtensionCollection loadExtensions() {
- try {
- return SimpleExtension.loadDefaults();
+ try (InputStream definitions =
+ LancePredicateConverter.class.getResourceAsStream("/substrait/lance_functions.yaml")) {
+ return SimpleExtension.loadDefaults().merge(SimpleExtension.load(LANCE_FUNCTIONS, definitions));
} catch (Exception e) {
throw new IllegalStateException("Failed to load Substrait extension definitions", e);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java
index ab259a05fe2..4d6319242f6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java
@@ -17,9 +17,17 @@
package org.apache.doris.datasource.lance.source;
+import org.apache.doris.analysis.ArrayLiteral;
+import org.apache.doris.analysis.BinaryPredicate;
import org.apache.doris.analysis.CompoundPredicate;
import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.FunctionCallExpr;
+import org.apache.doris.analysis.InPredicate;
+import org.apache.doris.analysis.IsNullPredicate;
+import org.apache.doris.analysis.LikePredicate;
+import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.SlotRef;
+import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.datasource.lance.index.LanceIndexSegmentGroup;
import org.apache.doris.datasource.lance.index.LanceIndexSegmentInfo;
import org.apache.doris.datasource.lance.metadata.LanceFragmentInfo;
@@ -28,6 +36,7 @@ import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
import org.lance.index.IndexType;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -35,6 +44,11 @@ import java.util.Set;
/** Assigns one BTree/Bitmap/LabelList segment and a disjoint fragment domain to each ordinary scan task. */
final class LanceScalarIndexPlanner {
+ // Match the pinned lance-c scoped-expression bounds. Exceeding either bound makes
+ // native planning scan the whole domain, so do not coalesce fragments in that case.
+ private static final int MAX_EXPRESSION_NODES = 128;
+ private static final int MAX_EXPRESSION_DEPTH = 32;
+
static final class Plan {
final String indexName;
final LanceSplitBuilder splits;
@@ -53,23 +67,39 @@ final class LanceScalarIndexPlanner {
|| !metadata.getIndexMetadataState().canPlanIndexSegments()) {
return null;
}
- Set<Integer> filterFields = collectFilterFields(metadata, pushedConjuncts);
- if (filterFields.isEmpty()) {
+ if (exceedsExpressionBudget(pushedConjuncts)) {
return null;
}
// Metadata already groups physical segments by logical index. Name order
// provides a stable winner when multiple indices cover the same number of rows.
List<LanceIndexSegmentGroup> indices = new ArrayList<>(metadata.getIndexes());
indices.sort(java.util.Comparator.comparing(LanceIndexSegmentGroup::getName));
+ // Native chooses the first matching parser, not the index with the widest
+ // coverage. Without its dispatch order, leave competing logical indexes to native.
+ Set<Integer> indexedFields = new HashSet<>();
+ Set<Integer> ambiguousFields = new HashSet<>();
+ for (LanceIndexSegmentGroup index : indices) {
+ for (Integer field : index.getSegments().get(0).getFieldIds()) {
+ if (!indexedFields.add(field)) {
+ ambiguousFields.add(field);
+ }
+ }
+ }
Plan selected = null;
+ Map<IndexType, Set<Integer>> fieldsByIndexType = new HashMap<>();
for (LanceIndexSegmentGroup logicalIndex : indices) {
List<LanceIndexSegmentInfo> segments = logicalIndex.getSegments();
- // PR #79 supports one top-level key in BTree/Bitmap/LabelList indices. Lance
- // performs the final typed driver selection and falls back within the same domain.
+ // Only coalesce a segment when a positive, indexable necessary condition
+ // exists. Broad complements keep fragment parallelism without repeating a
+ // segment-wide index search once per fragment.
LanceIndexSegmentInfo index = segments.get(0);
if ((index.getIndexType() != IndexType.BTREE && index.getIndexType() != IndexType.BITMAP
&& index.getIndexType() != IndexType.LABEL_LIST)
- || index.getFieldIds().size() != 1 || !filterFields.contains(index.getFieldIds().get(0))) {
+ || index.getFieldIds().size() != 1
+ || ambiguousFields.contains(index.getFieldIds().get(0))
+ || !fieldsByIndexType.computeIfAbsent(index.getIndexType(),
+ type -> collectFilterFields(metadata, pushedConjuncts, type))
+ .contains(index.getFieldIds().get(0))) {
continue;
}
Plan candidate = groupFragments(metadata, segments, visibleFragments);
@@ -80,26 +110,199 @@ final class LanceScalarIndexPlanner {
return selected;
}
- private static Set<Integer> collectFilterFields(LanceTableMetadata metadata, List<Expr> pushedConjuncts) {
- Set<SlotRef> slots = new HashSet<>();
- pushedConjuncts.forEach(expr -> collectDriverSlots(expr, slots));
+ private static Set<Integer> collectFilterFields(LanceTableMetadata metadata,
+ List<Expr> pushedConjuncts, IndexType indexType) {
Set<Integer> fields = new HashSet<>();
+ for (Expr expr : pushedConjuncts) {
+ fields.addAll(collectDriverFields(metadata, expr, indexType, false));
+ }
+ return fields;
+ }
+
+ private static Set<Integer> collectDriverFields(LanceTableMetadata metadata, Expr expr,
+ IndexType indexType, boolean requireExact) {
+ Set<Integer> fields = new HashSet<>();
+ if (expr instanceof CompoundPredicate) {
+ CompoundPredicate.Operator op = ((CompoundPredicate) expr).getOp();
+ if (op == CompoundPredicate.Operator.NOT) {
+ return fields;
+ }
+ boolean exactBranches = requireExact || op == CompoundPredicate.Operator.OR;
+ Set<Integer> left = collectDriverFields(metadata, expr.getChild(0), indexType, exactBranches);
+ Set<Integer> right = collectDriverFields(metadata, expr.getChild(1), indexType, exactBranches);
+ if (op == CompoundPredicate.Operator.AND) {
+ // A refine-only conjunct anywhere below OR makes native reject the
+ // union, even if its sibling could independently drive this index.
+ if (requireExact && (left.isEmpty() || right.isEmpty())) {
+ return fields;
+ }
+ left.addAll(right);
+ } else {
+ // Sharing a slot is insufficient: both OR branches must actually be
+ // indexable. A suffix LIKE, for example, cannot supply BTree candidates.
+ left.retainAll(right);
+ }
+ return left;
+ }
+ if (!isPositiveIndexLeaf(expr, indexType, requireExact)) {
+ return fields;
+ }
+ Set<SlotRef> slots = new HashSet<>();
+ expr.collect(SlotRef.class, slots);
for (SlotRef slot : slots) {
metadata.getLanceFieldId(slot.getColumnName()).ifPresent(fields::add);
}
return fields;
}
- private static void collectDriverSlots(Expr expr, Set<SlotRef> slots) {
- // A predicate below OR or NOT is not a necessary condition of the whole filter.
- // Do not select its index and then force every task into a non-indexed fallback.
+ private static boolean isPositiveIndexLeaf(Expr expr, IndexType indexType, boolean requireExact) {
+ if (indexType == IndexType.LABEL_LIST) {
+ if (!(expr instanceof FunctionCallExpr) || expr.getChildren().size() != 2) {
+ return false;
+ }
+ String name = ((FunctionCallExpr) expr).getFnName().getFunction();
+ if ("array_contains".equalsIgnoreCase(name)) {
+ return expr.getChild(0) instanceof SlotRef && expr.getChild(1) instanceof LiteralExpr;
+ }
+ return "arrays_overlap".equalsIgnoreCase(name) && overlapSize(expr) > 0;
+ }
+ if (expr instanceof BinaryPredicate) {
+ BinaryPredicate.Operator op = ((BinaryPredicate) expr).getOp();
+ return (op == BinaryPredicate.Operator.EQ || op == BinaryPredicate.Operator.GT
+ || op == BinaryPredicate.Operator.GE || op == BinaryPredicate.Operator.LT
+ || op == BinaryPredicate.Operator.LE)
+ && ((expr.getChild(0) instanceof SlotRef && expr.getChild(1) instanceof LiteralExpr)
+ || (expr.getChild(1) instanceof SlotRef && expr.getChild(0) instanceof LiteralExpr));
+ }
+ if (expr instanceof InPredicate) {
+ return !((InPredicate) expr).isNotIn() && expr.getChild(0) instanceof SlotRef;
+ }
+ if (expr instanceof IsNullPredicate) {
+ return !((IsNullPredicate) expr).isNotNull() && expr.getChild(0) instanceof SlotRef;
+ }
+ // Bitmap has no prefix-query support. A refined LIKE can drive an AND,
+ // but native's OR planner rejects branches that need a residual recheck.
+ if (indexType != IndexType.BTREE || expr.getChildren().size() != 2
+ || !(expr.getChild(0) instanceof SlotRef) || !(expr.getChild(1) instanceof StringLiteral)) {
+ return false;
+ }
+ String name = expr instanceof FunctionCallExpr
+ ? ((FunctionCallExpr) expr).getFnName().getFunction() : "";
+ String prefix = ((StringLiteral) expr.getChild(1)).getStringValue();
+ if ("starts_with".equalsIgnoreCase(name)) {
+ // Unlike LIKE, every character in starts_with is literal, including % and _.
+ return !prefix.isEmpty();
+ }
+ boolean like = (expr instanceof LikePredicate && ((LikePredicate) expr).getOp() == LikePredicate.Operator.LIKE)
+ || "like".equalsIgnoreCase(name);
+ if (!like || prefix.isEmpty() || prefix.indexOf('\\') >= 0) {
+ return false;
+ }
+ for (int i = 0; i < prefix.length(); i++) {
+ char character = prefix.charAt(i);
+ if (character == '%' || character == '_') {
+ return i > 0 && (!requireExact || (character == '%' && i == prefix.length() - 1));
+ }
+ }
+ return true;
+ }
+
+ private static int overlapSize(Expr expr) {
+ if (!(expr instanceof FunctionCallExpr) || expr.getChildren().size() != 2
+ || !"arrays_overlap".equalsIgnoreCase(((FunctionCallExpr) expr).getFnName().getFunction())) {
+ return 0;
+ }
+ for (int i = 0; i < 2; i++) {
+ if (expr.getChild(i) instanceof ArrayLiteral && expr.getChild(1 - i) instanceof SlotRef) {
+ return expr.getChild(i).getChildren().size();
+ }
+ }
+ return 0;
+ }
+
+ static boolean shouldDisableFragmentIndex(List<Expr> pushedConjuncts) {
+ if (pushedConjuncts.isEmpty()) {
+ return false;
+ }
+ // Missing metadata or an ambiguous index name is not evidence against native
+ // index use. Disable only known expensive shapes, independent of FE discovery.
+ return exceedsExpressionBudget(pushedConjuncts)
+ || pushedConjuncts.stream().noneMatch(LanceScalarIndexPlanner::hasPositivePredicate);
+ }
+
+ private static boolean hasPositivePredicate(Expr expr) {
if (expr instanceof CompoundPredicate) {
- if (((CompoundPredicate) expr).getOp() == CompoundPredicate.Operator.AND) {
- expr.getChildren().forEach(child -> collectDriverSlots(child, slots));
+ return ((CompoundPredicate) expr).getOp() != CompoundPredicate.Operator.NOT
+ && expr.getChildren().stream().anyMatch(LanceScalarIndexPlanner::hasPositivePredicate);
+ }
+ if (expr instanceof BinaryPredicate) {
+ return ((BinaryPredicate) expr).getOp() != BinaryPredicate.Operator.NE;
+ }
+ if (expr instanceof InPredicate) {
+ return !((InPredicate) expr).isNotIn();
+ }
+ if (expr instanceof IsNullPredicate) {
+ return !((IsNullPredicate) expr).isNotNull();
+ }
+ return true;
+ }
+
+ private static boolean exceedsExpressionBudget(List<Expr> conjuncts) {
+ return conjunctionNodes(conjuncts, 0, conjuncts.size(), 0) > MAX_EXPRESSION_NODES;
+ }
+
+ private static int conjunctionNodes(List<Expr> conjuncts, int begin, int end, int depth) {
+ if (begin == end) {
+ return 0;
+ }
+ if (end - begin == 1) {
+ return expressionNodes(conjuncts.get(begin), depth);
+ }
+ // The converter emits n-ary and:bool; DataFusion splits its arguments in
+ // halves. Preserve each leaf's actual depth instead of assuming a left-deep AND.
+ int middle = begin + (end - begin) / 2;
+ return 1 + conjunctionNodes(conjuncts, begin, middle, depth + 1)
+ + conjunctionNodes(conjuncts, middle, end, depth + 1);
+ }
+
+ private static int expressionNodes(Expr expr, int depth) {
+ if (depth > MAX_EXPRESSION_DEPTH) {
+ return MAX_EXPRESSION_NODES + 1;
+ }
+ if (expr instanceof CompoundPredicate) {
+ int nodes = 1;
+ for (Expr child : expr.getChildren()) {
+ nodes += expressionNodes(child, depth + 1);
+ if (nodes > MAX_EXPRESSION_NODES) {
+ break;
+ }
+ }
+ return nodes;
+ }
+ int labels = overlapSize(expr);
+ // The converter emits a balanced OR tree: N memberships require 2*N-1 nodes.
+ if (labels > 0) {
+ int levels = 32 - Integer.numberOfLeadingZeros(labels - 1);
+ if (labels > MAX_EXPRESSION_NODES / 2 || depth + levels > MAX_EXPRESSION_DEPTH) {
+ return MAX_EXPRESSION_NODES + 1;
+ }
+ return 2 * labels - 1;
+ }
+ if (expr instanceof InPredicate) {
+ InPredicate in = (InPredicate) expr;
+ int values = in.getInElementNum();
+ // DataFusion 54 expands up to three values into a left-deep OR (or
+ // AND of negated equalities for NOT IN) before the native budget check.
+ if (values > 0 && values <= 3) {
+ int negation = in.isNotIn() ? 1 : 0;
+ if (depth + values - 1 + negation > MAX_EXPRESSION_DEPTH) {
+ return MAX_EXPRESSION_NODES + 1;
+ }
+ return (2 + negation) * values - 1;
}
- } else {
- expr.collect(SlotRef.class, slots);
+ return in.isNotIn() ? 2 : 1;
}
+ return 1;
}
private static Plan groupFragments(LanceTableMetadata metadata, List<LanceIndexSegmentInfo> segments,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java
index e95420582b6..23fc3eeb5dd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java
@@ -195,7 +195,11 @@ final class LanceScanPlanner {
} else {
plan = new LanceSplitBuilder(metadata.getDatasetUri(), metadata.getVersion(), 0);
}
- plan.addUncoveredFragments(visibleFragments.values(), 1, scalarIndexPlan != null);
+ // Uncovered fragments must not repeat the selected segment's search. When FE
+ // could not select a segment, retain native selection unless the filter itself
+ // is known to cause expensive repeated searches (a complement or oversized tree).
+ plan.addUncoveredFragments(visibleFragments.values(), 1, scalarIndexPlan != null
+ || LanceScalarIndexPlanner.shouldDisableFragmentIndex(lancePushedConjuncts));
return plan.buildSplits();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceSubstraitSerializer.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceSubstraitSerializer.java
index e5323bde815..79d3f479261 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceSubstraitSerializer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceSubstraitSerializer.java
@@ -108,10 +108,19 @@ final class LanceSubstraitSerializer {
|| type instanceof ArrowType.LargeUtf8;
}
+ static boolean supportsStringList(Field field) {
+ // Keep this separate from scalar comparability: list serialization must not enable
+ // array comparisons, IN, or null checks without their own semantic coverage.
+ return field.getType() instanceof ArrowType.List && field.getChildren().size() == 1
+ && field.getChildren().get(0).getType() instanceof ArrowType.Utf8;
+ }
+
static Type fieldType(Field field) {
TypeCreator creator = TypeCreator.of(field.isNullable());
ArrowType type = field.getType();
- if (type instanceof ArrowType.Bool) {
+ if (supportsStringList(field)) {
+ return creator.list(fieldType(field.getChildren().get(0)));
+ } else if (type instanceof ArrowType.Bool) {
return creator.BOOLEAN;
} else if (type instanceof ArrowType.Int) {
switch (((ArrowType.Int) type).getBitWidth()) {
@@ -198,6 +207,13 @@ final class LanceSubstraitSerializer {
}
private static Optional<io.substrait.proto.Type> toSubstraitProtoType(Field field) {
+ if (supportsStringList(field)) {
+ return Optional.of(io.substrait.proto.Type.newBuilder()
+ .setList(io.substrait.proto.Type.List.newBuilder()
+ .setNullability(nullability(field))
+ .setType(toSubstraitProtoType(field.getChildren().get(0)).get()))
+ .build());
+ }
if (!supportsType(field.getType())) {
return Optional.empty();
}
diff --git a/fe/fe-core/src/main/resources/substrait/lance_functions.yaml b/fe/fe-core/src/main/resources/substrait/lance_functions.yaml
new file mode 100644
index 00000000000..3e6f64ecbb9
--- /dev/null
+++ b/fe/fe-core/src/main/resources/substrait/lance_functions.yaml
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+%YAML 1.2
+---
+scalar_functions:
+ - name: array_has
+ description: Non-null UTF-8 label membership in a List<Utf8> column.
+ impls:
+ - args:
+ - value: list<string>
+ - value: string
+ return: boolean
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LancePredicateConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LancePredicateConverterTest.java
index 64e6fd70724..148fbf1e53d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LancePredicateConverterTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LancePredicateConverterTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.datasource.lance;
+import org.apache.doris.analysis.ArrayLiteral;
import org.apache.doris.analysis.BinaryPredicate;
import org.apache.doris.analysis.CompoundPredicate;
import org.apache.doris.analysis.DateLiteral;
@@ -29,8 +30,10 @@ import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.LargeIntLiteral;
import org.apache.doris.analysis.LikePredicate;
+import org.apache.doris.analysis.NullLiteral;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StringLiteral;
+import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.ScalarFunction;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
@@ -73,6 +76,118 @@ public class LancePredicateConverterTest {
Field.nullable("timestamp_ns", new ArrowType.Timestamp(TimeUnit.NANOSECOND, null)),
Field.nullable("timestamp_us_utc", new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC")))));
+ @Test
+ public void testStringArrayMembershipSchemaAndFunction() throws Exception {
+ // An unsupported preceding field must not shift the array's field reference.
+ Schema schema = new Schema(Arrays.asList(
+ Field.nullable("nested", ArrowType.Struct.INSTANCE),
+ new Field("labels", FieldType.nullable(ArrowType.List.INSTANCE),
+ Collections.singletonList(Field.nullable("item", ArrowType.Utf8.INSTANCE)))));
+ Expr predicate = arrayFunction("array_contains", new SlotRef(null, "labels"), new StringLiteral("red"));
+ LancePredicateConverter.ConversionResult result =
+ new LancePredicateConverter(schema).convert(Collections.singletonList(predicate));
+ Assertions.assertEquals(1, result.getPushedConjuncts().size());
+ Assertions.assertTrue(result.getResidualConjuncts().isEmpty());
+ ExtendedExpression envelope = ExtendedExpression.parseFrom(result.getSubstraitFilter());
+ Assertions.assertEquals(Arrays.asList("__unlikely_name_placeholder_doris_0", "labels"),
+ envelope.getBaseSchema().getNamesList());
+ io.substrait.proto.Type.List list = envelope.getBaseSchema().getStruct().getTypes(1).getList();
+ Assertions.assertEquals(io.substrait.proto.Type.Nullability.NULLABILITY_NULLABLE, list.getNullability());
+ Assertions.assertTrue(list.getType().hasString());
+ Assertions.assertEquals(io.substrait.proto.Type.Nullability.NULLABILITY_NULLABLE,
+ list.getType().getString().getNullability());
+ io.substrait.proto.Expression.ScalarFunction function =
+ envelope.getReferredExpr(0).getExpression().getScalarFunction();
+ Assertions.assertEquals(1, function.getArguments(0).getValue().getSelection()
+ .getDirectReference().getStructField().getField());
+ Assertions.assertEquals("red", function.getArguments(1).getValue().getLiteral().getString());
+ Assertions.assertEquals(io.substrait.proto.Type.Nullability.NULLABILITY_NULLABLE,
+ function.getOutputType().getBool().getNullability());
+ Assertions.assertTrue(envelope.getExtensionsList().stream()
+ .anyMatch(extension -> extension.hasExtensionFunction()
+ && extension.getExtensionFunction().getName().startsWith("array_has:")));
+ }
+
+ @Test
+ public void testArrayMembershipBooleanCombinations() {
+ LancePredicateConverter arrays = arrayConverter(ArrowType.List.INSTANCE, ArrowType.Utf8.INSTANCE);
+ Expr red = arrayFunction("array_contains", new SlotRef(null, "labels"), new StringLiteral("red"));
+ Expr blue = arrayFunction("array_contains", new SlotRef(null, "labels"), new StringLiteral("blue"));
+ for (Expr predicate : Arrays.asList(red,
+ new CompoundPredicate(CompoundPredicate.Operator.AND, red, blue),
+ new CompoundPredicate(CompoundPredicate.Operator.OR, red, blue),
+ new CompoundPredicate(CompoundPredicate.Operator.NOT, red, null))) {
+ Assertions.assertEquals(1, arrays.convert(Collections.singletonList(predicate))
+ .getPushedConjuncts().size());
+ }
+ }
+
+ @Test
+ public void testRewrittenArrayOverlapPushdown() throws Exception {
+ LancePredicateConverter arrays = arrayConverter(ArrowType.List.INSTANCE, ArrowType.Utf8.INSTANCE);
+ Expr labels = new SlotRef(null, "labels");
+ Expr values = new ArrayLiteral(ArrayType.create(Type.STRING, true),
+ new StringLiteral("red"), new StringLiteral("blue"), new StringLiteral("green"));
+ for (Expr predicate : Arrays.asList(arrayFunction("arrays_overlap", labels, values),
+ arrayFunction("arrays_overlap", values, labels),
+ new CompoundPredicate(CompoundPredicate.Operator.NOT,
+ arrayFunction("arrays_overlap", labels, values), null))) {
+ LancePredicateConverter.ConversionResult result = arrays.convert(Collections.singletonList(predicate));
+ Assertions.assertTrue(result.getResidualConjuncts().isEmpty());
+ String encoded = ExtendedExpression.parseFrom(result.getSubstraitFilter()).toString();
+ Assertions.assertTrue(encoded.contains("array_has:list_str"));
+ Assertions.assertTrue(encoded.contains("or:bool"));
+ Assertions.assertTrue(encoded.contains("green"));
+ }
+ // A NULL needle can match a NULL element in Doris and must not become array_has.
+ for (Expr valuesWithDifferentSemantics : Arrays.asList(
+ new ArrayLiteral(ArrayType.create(Type.STRING, true), new StringLiteral("red"), new NullLiteral()),
+ new ArrayLiteral(), new SlotRef(null, "labels"))) {
+ Expr predicate = arrayFunction("arrays_overlap", labels, valuesWithDifferentSemantics);
+ Assertions.assertEquals(Collections.singletonList(predicate),
+ arrays.convert(Collections.singletonList(predicate)).getResidualConjuncts());
+ }
+ }
+
+ @Test
+ public void testUnsafeArrayMembershipRemainsResidual() {
+ LancePredicateConverter arrays = arrayConverter(ArrowType.List.INSTANCE, ArrowType.Utf8.INSTANCE);
+ Expr red = arrayFunction("array_contains", new SlotRef(null, "labels"), new StringLiteral("red"));
+ Expr nullNeedle = arrayFunction("array_contains", new SlotRef(null, "labels"), new NullLiteral());
+ FunctionCallExpr udf = arrayFunction("array_contains", new SlotRef(null, "labels"), new StringLiteral("red"));
+ udf.getFn().setBinaryType(TFunctionBinaryType.JAVA_UDF);
+ for (Expr predicate : Arrays.asList(nullNeedle, udf,
+ arrayFunction("array_contains_all", new SlotRef(null, "labels"), new StringLiteral("red")),
+ arrayFunction("array_contains", new SlotRef(null, "labels"), new SlotRef(null, "needle")),
+ new CompoundPredicate(CompoundPredicate.Operator.OR, red, nullNeedle),
+ new CompoundPredicate(CompoundPredicate.Operator.NOT, nullNeedle, null),
+ new IsNullPredicate(new SlotRef(null, "labels"), false))) {
+ Assertions.assertEquals(Collections.singletonList(predicate),
+ arrays.convert(Collections.singletonList(predicate)).getResidualConjuncts());
+ }
+ Assertions.assertEquals(1, arrays.convert(Arrays.asList(red, nullNeedle)).getPushedConjuncts().size());
+ for (LancePredicateConverter unsupported : Arrays.asList(
+ arrayConverter(ArrowType.List.INSTANCE, new ArrowType.Int(64, true)),
+ arrayConverter(ArrowType.List.INSTANCE, ArrowType.LargeUtf8.INSTANCE),
+ arrayConverter(ArrowType.LargeList.INSTANCE, ArrowType.Utf8.INSTANCE),
+ arrayConverter(new ArrowType.FixedSizeList(2), ArrowType.Utf8.INSTANCE))) {
+ Assertions.assertEquals(1, unsupported.convert(Collections.singletonList(red)).getResidualConjuncts().size());
+ }
+ }
+
+ private static LancePredicateConverter arrayConverter(ArrowType container, ArrowType element) {
+ return new LancePredicateConverter(new Schema(Collections.singletonList(
+ new Field("labels", FieldType.nullable(container),
+ Collections.singletonList(Field.nullable("item", element))))));
+ }
+
+ private static FunctionCallExpr arrayFunction(String name, Expr... arguments) {
+ FunctionCallExpr function = new FunctionCallExpr(name, Arrays.asList(arguments));
+ function.setFn(new ScalarFunction(new FunctionName(name),
+ Arrays.asList(ArrayType.create(Type.STRING, true), Type.STRING), Type.BOOLEAN, false, true));
+ return function;
+ }
+
@Test
public void testComparisonAndStringLiteralEncoding() throws Exception {
Expr rowId = new SlotRef(null, "row_id");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlannerTest.java
new file mode 100644
index 00000000000..d94e2f55474
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlannerTest.java
@@ -0,0 +1,292 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.source;
+
+import org.apache.doris.analysis.ArrayLiteral;
+import org.apache.doris.analysis.BinaryPredicate;
+import org.apache.doris.analysis.CompoundPredicate;
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.FunctionCallExpr;
+import org.apache.doris.analysis.FunctionName;
+import org.apache.doris.analysis.InPredicate;
+import org.apache.doris.analysis.IntLiteral;
+import org.apache.doris.analysis.LikePredicate;
+import org.apache.doris.analysis.SlotRef;
+import org.apache.doris.analysis.StringLiteral;
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.ScalarFunction;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.datasource.lance.index.LanceIndexSegmentInfo;
+import org.apache.doris.datasource.lance.metadata.LanceFragmentInfo;
+import org.apache.doris.datasource.lance.metadata.LanceTableAccess;
+import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
+
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.lance.index.IndexType;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+public class LanceScalarIndexPlannerTest {
+ @Test
+ public void testBooleanDriverSelectionPreservesAllBranches() {
+ Expr key = equal("key", 1);
+ Expr otherKey = equal("key", 2);
+ Expr residual = equal("other", 3);
+ for (Expr filter : Arrays.asList(or(key, otherKey),
+ or(and(key, residual), and(otherKey, residual)), and(not(and(key, residual)), key))) {
+ LanceScalarIndexPlanner.Plan plan = plan(filter);
+ Assertions.assertNotNull(plan, filter.toSql());
+ Assertions.assertEquals("key_idx", plan.indexName);
+ Assertions.assertEquals(1, plan.splits.splitCount());
+ Assertions.assertTrue(plan.splits.isCoveredByIndexSegment(1));
+ Assertions.assertTrue(plan.splits.isCoveredByIndexSegment(2));
+ }
+ // Pruning either an OR branch or an AND below NOT could drop matching rows.
+ for (Expr filter : Arrays.asList(not(key), not(or(key, otherKey)), or(key, residual), not(and(key, residual)),
+ not(or(key, residual)), or(and(key, residual), residual))) {
+ Assertions.assertNull(plan(filter), filter.toSql());
+ }
+ }
+
+ @Test
+ public void testConvertedArrayPredicatesSelectLabelList() {
+ Schema schema = new Schema(Collections.singletonList(new Field("labels",
+ FieldType.nullable(ArrowType.List.INSTANCE),
+ Collections.singletonList(Field.nullable("item", ArrowType.Utf8.INSTANCE)))));
+ LanceFragmentInfo fragment = new LanceFragmentInfo(1, 10, 10);
+ LanceTableMetadata metadata = LanceTableMetadata.createSnapshotWithIndexes(
+ new LanceTableAccess("s3://bucket/labels.lance", Collections.emptyMap()), 42, schema,
+ Collections.singletonList(fragment), Collections.singletonMap("labels", 9),
+ Collections.singletonList(new LanceIndexSegmentInfo(UUID.randomUUID(), "labels_idx",
+ Collections.singletonList(9), Collections.singletonList(1L), IndexType.LABEL_LIST, null)));
+ FunctionCallExpr red = contains("red");
+ FunctionCallExpr blue = contains("blue");
+ for (Expr filter : Arrays.asList(red, and(red, blue), or(red, blue))) {
+ LancePredicateConverter.ConversionResult converted =
+ new LancePredicateConverter(schema).convert(Collections.singletonList(filter));
+ Assertions.assertTrue(converted.getResidualConjuncts().isEmpty());
+ LanceScalarIndexPlanner.Plan plan = LanceScalarIndexPlanner.plan(metadata,
+ converted.getPushedConjuncts(), Collections.singletonMap(1L, fragment));
+ Assertions.assertNotNull(plan);
+ Assertions.assertEquals("labels_idx", plan.indexName);
+ Assertions.assertEquals(1, plan.splits.splitCount());
+ }
+ }
+
+ @Test
+ public void testPositiveDriverWithComplementSearchesSegmentOnce() {
+ Expr key = equal("key", 1);
+ Expr otherNotEqual = new BinaryPredicate(BinaryPredicate.Operator.NE,
+ new SlotRef(null, "other"), new IntLiteral(0));
+ Assertions.assertEquals(1, plan(Arrays.asList(key, otherNotEqual), false).splits.splitCount());
+ Assertions.assertEquals(1, plan(and(key, otherNotEqual)).splits.splitCount());
+ Assertions.assertEquals(1, plan(and(key, not(equal("key", 2)))).splits.splitCount());
+ }
+
+ @Test
+ public void testUnindexableOrBranchDoesNotGroupFragments() {
+ Expr suffix = new LikePredicate(LikePredicate.Operator.LIKE,
+ new SlotRef(null, "key"), new StringLiteral("%y%"));
+ Expr key = new BinaryPredicate(BinaryPredicate.Operator.EQ,
+ new SlotRef(null, "key"), new StringLiteral("x"));
+ Assertions.assertNull(plan(Collections.singletonList(or(key, suffix)), true));
+ Assertions.assertNotNull(plan(Collections.singletonList(and(key, suffix)), true));
+ Expr prefix = new LikePredicate(LikePredicate.Operator.LIKE,
+ new SlotRef(null, "key"), new StringLiteral("y%"));
+ Assertions.assertEquals(1, plan(Collections.singletonList(or(key, prefix)), true).splits.splitCount());
+ }
+
+ @Test
+ public void testExpressionDepthBudget() {
+ Expr filter = equal("key", 1);
+ for (int depth = 1; depth <= 32; depth++) {
+ filter = and(filter, equal("key", depth));
+ }
+ Assertions.assertNotNull(plan(filter));
+ Assertions.assertNull(plan(and(filter, equal("key", 33))));
+ }
+
+ @Test
+ public void testOverlapExpressionBudget() throws Exception {
+ Schema schema = new Schema(Collections.singletonList(new Field("labels",
+ FieldType.nullable(ArrowType.List.INSTANCE),
+ Collections.singletonList(Field.nullable("item", ArrowType.Utf8.INSTANCE)))));
+ LanceFragmentInfo first = new LanceFragmentInfo(1, 10, 10);
+ LanceFragmentInfo second = new LanceFragmentInfo(2, 10, 10);
+ Map<Long, LanceFragmentInfo> fragments = new HashMap<>();
+ fragments.put(1L, first);
+ fragments.put(2L, second);
+ LanceTableMetadata metadata = LanceTableMetadata.createSnapshotWithIndexes(
+ new LanceTableAccess("s3://bucket/labels.lance", Collections.emptyMap()), 42, schema,
+ Arrays.asList(first, second), Collections.singletonMap("labels", 9),
+ Collections.singletonList(new LanceIndexSegmentInfo(UUID.randomUUID(), "labels_idx",
+ Collections.singletonList(9), Arrays.asList(1L, 2L), IndexType.LABEL_LIST, null)));
+ for (int count : Arrays.asList(64, 65)) {
+ StringLiteral[] labels = new StringLiteral[count];
+ for (int i = 0; i < count; i++) {
+ labels[i] = new StringLiteral("label_" + i);
+ }
+ FunctionCallExpr overlap = new FunctionCallExpr("arrays_overlap", Arrays.asList(
+ new SlotRef(null, "labels"), new ArrayLiteral(ArrayType.create(Type.STRING, true), labels)));
+ overlap.setFn(new ScalarFunction(new FunctionName("arrays_overlap"),
+ Arrays.asList(ArrayType.create(Type.STRING, true), ArrayType.create(Type.STRING, true)),
+ Type.BOOLEAN, false, true));
+ LancePredicateConverter.ConversionResult converted = new LancePredicateConverter(schema)
+ .convert(Collections.singletonList(overlap));
+ Assertions.assertTrue(converted.getResidualConjuncts().isEmpty());
+ LanceScalarIndexPlanner.Plan selected = LanceScalarIndexPlanner.plan(metadata,
+ converted.getPushedConjuncts(), fragments);
+ if (count == 64) {
+ Assertions.assertNotNull(selected);
+ Assertions.assertEquals(1, selected.splits.splitCount());
+ // Count the enclosing AND and its other leaf, not just overlap's 127 nodes.
+ Assertions.assertNull(LanceScalarIndexPlanner.plan(metadata,
+ Arrays.asList(overlap, contains("extra")), fragments));
+ } else {
+ Assertions.assertNull(selected);
+ }
+ }
+ }
+
+ @Test
+ public void testWideConjunctionUsesBalancedDepth() {
+ List<Expr> filters = new ArrayList<>();
+ filters.add(equal("key", 1));
+ for (int i = 0; i < 33; i++) {
+ filters.add(equal("filter_" + i, i));
+ }
+ Assertions.assertNotNull(plan(filters, false));
+ }
+
+ @Test
+ public void testShortInExpansionAtOverlapBudgetBoundary() throws Exception {
+ Schema schema = new Schema(Arrays.asList(new Field("labels",
+ FieldType.nullable(ArrowType.List.INSTANCE),
+ Collections.singletonList(Field.nullable("item", ArrowType.Utf8.INSTANCE))),
+ Field.nullable("category", new ArrowType.Int(64, true))));
+ List<LanceFragmentInfo> fragments = Arrays.asList(
+ new LanceFragmentInfo(1, 10, 10), new LanceFragmentInfo(2, 10, 10));
+ Map<String, Integer> fields = new HashMap<>();
+ fields.put("labels", 9);
+ fields.put("category", 10);
+ LanceTableMetadata metadata = LanceTableMetadata.createSnapshotWithIndexes(
+ new LanceTableAccess("s3://bucket/labels.lance", Collections.emptyMap()), 42, schema,
+ fragments, fields, Arrays.asList(
+ new LanceIndexSegmentInfo(UUID.randomUUID(), "labels_idx", Collections.singletonList(9),
+ Arrays.asList(1L, 2L), IndexType.LABEL_LIST, null),
+ new LanceIndexSegmentInfo(UUID.randomUUID(), "category_idx", Collections.singletonList(10),
+ Arrays.asList(1L, 2L), IndexType.BTREE, null)));
+ Map<Long, LanceFragmentInfo> visible = new HashMap<>();
+ fragments.forEach(fragment -> visible.put(fragment.getId(), fragment));
+ for (int count : Arrays.asList(62, 63)) {
+ StringLiteral[] labels = new StringLiteral[count];
+ for (int i = 0; i < count; i++) {
+ labels[i] = new StringLiteral("label_" + i);
+ }
+ Expr overlap = new FunctionCallExpr("arrays_overlap", Arrays.asList(
+ new SlotRef(null, "labels"), new ArrayLiteral(ArrayType.create(Type.STRING, true), labels)));
+ Expr in = new InPredicate(new SlotRef(null, "category"),
+ Arrays.asList(new IntLiteral(0), new IntLiteral(1)), false);
+ LanceScalarIndexPlanner.Plan selected = LanceScalarIndexPlanner.plan(metadata,
+ Arrays.asList(overlap, in), visible);
+ if (count == 62) {
+ Assertions.assertNotNull(selected);
+ } else {
+ Assertions.assertNull(selected);
+ }
+ }
+ }
+
+ @Test
+ public void testLiteralAndRefinedPrefixes() {
+ Expr key = new BinaryPredicate(BinaryPredicate.Operator.EQ,
+ new SlotRef(null, "key"), new StringLiteral("x"));
+ for (String prefix : Arrays.asList("test_ns$", "literal%value", "literal\\value")) {
+ Expr startsWith = new FunctionCallExpr("starts_with",
+ Arrays.asList(new SlotRef(null, "key"), new StringLiteral(prefix)));
+ Assertions.assertNotNull(plan(Collections.singletonList(startsWith), true));
+ Assertions.assertNotNull(plan(Collections.singletonList(or(key, startsWith)), true));
+ }
+ for (String pattern : Arrays.asList("foo%bar%", "foo_bar%")) {
+ Expr like = new LikePredicate(LikePredicate.Operator.LIKE,
+ new SlotRef(null, "key"), new StringLiteral(pattern));
+ Assertions.assertNotNull(plan(Collections.singletonList(like), true));
+ // Native cannot union an index branch that still carries a refine predicate.
+ Assertions.assertNull(plan(Collections.singletonList(or(key, like)), true));
+ Assertions.assertNull(plan(Collections.singletonList(or(and(key, like), key)), true));
+ }
+ }
+
+ private static FunctionCallExpr contains(String label) {
+ FunctionCallExpr function = new FunctionCallExpr("array_contains",
+ Arrays.asList(new SlotRef(null, "labels"), new StringLiteral(label)));
+ function.setFn(new ScalarFunction(new FunctionName("array_contains"),
+ Arrays.asList(ArrayType.create(Type.STRING, true), Type.STRING), Type.BOOLEAN, false, true));
+ return function;
+ }
+
+ private static LanceScalarIndexPlanner.Plan plan(Expr filter) {
+ return plan(Collections.singletonList(filter), false);
+ }
+
+ private static LanceScalarIndexPlanner.Plan plan(List<Expr> filters, boolean stringKey) {
+ LanceFragmentInfo first = new LanceFragmentInfo(1, 10, 10);
+ LanceFragmentInfo second = new LanceFragmentInfo(2, 10, 10);
+ Map<Long, LanceFragmentInfo> fragments = new HashMap<>();
+ fragments.put(1L, first);
+ fragments.put(2L, second);
+ Map<String, Integer> fields = new HashMap<>();
+ fields.put("key", 9);
+ fields.put("other", 10);
+ LanceTableMetadata metadata = LanceTableMetadata.createSnapshotWithIndexes(
+ new LanceTableAccess("s3://bucket/labels.lance", Collections.emptyMap()), 42,
+ new Schema(Arrays.asList(Field.nullable("key", stringKey ? ArrowType.Utf8.INSTANCE : new ArrowType.Int(64, true)),
+ Field.nullable("other", new ArrowType.Int(64, true)))),
+ Arrays.asList(first, second), fields, Collections.singletonList(new LanceIndexSegmentInfo(
+ UUID.randomUUID(), "key_idx", Collections.singletonList(9),
+ Arrays.asList(1L, 2L), IndexType.BTREE, null)));
+ return LanceScalarIndexPlanner.plan(metadata, filters, fragments);
+ }
+
+ private static Expr equal(String column, int value) {
+ return new BinaryPredicate(BinaryPredicate.Operator.EQ, new SlotRef(null, column), new IntLiteral(value));
+ }
+
+ private static Expr and(Expr left, Expr right) {
+ return new CompoundPredicate(CompoundPredicate.Operator.AND, left, right);
+ }
+
+ private static Expr or(Expr left, Expr right) {
+ return new CompoundPredicate(CompoundPredicate.Operator.OR, left, right);
+ }
+
+ private static Expr not(Expr child) {
+ return new CompoundPredicate(CompoundPredicate.Operator.NOT, child, null);
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java
index b68ca1f437c..ac1f0f7218f 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java
@@ -234,7 +234,7 @@ public class LanceScanNodeTest {
}
@Test
- public void testScalarSegmentSelectionDoesNotDescendThroughOrOrNot() throws Exception {
+ public void testScalarSegmentSelectionSupportsSameColumnOrAndNot() throws Exception {
Expr predicate = scalarPredicate();
for (Expr filter : Arrays.asList(
new CompoundPredicate(CompoundPredicate.Operator.OR, predicate, predicate),
@@ -243,11 +243,29 @@ public class LanceScanNodeTest {
setMetadata(node, scalarMetadata(Collections.singletonList(
scalarSegment(UUID.randomUUID(), IndexType.BTREE, Arrays.asList(1L, 2L, 3L, 4L)))));
setPushedConjuncts(node, filter);
- Assert.assertEquals(4, node.getSplits(20).size());
+ int expectedSplits = ((CompoundPredicate) filter).getOp() == CompoundPredicate.Operator.NOT ? 4 : 1;
+ List<Split> initialSplits = node.getSplits(20);
+ Assert.assertEquals(expectedSplits, initialSplits.size());
+ if (expectedSplits == 4) {
+ for (Split split : initialSplits) {
+ TFileRangeDesc range = new TFileRangeDesc();
+ node.setScanParams(range, split);
+ TLanceFileDesc params = range.getTableFormatParams().getLanceParams();
+ Assert.assertFalse(params.isSetIndexSegmentUuids());
+ Assert.assertTrue(params.isSetUseScalarIndex());
+ Assert.assertFalse(params.isUseScalarIndex());
+ }
+ }
setPushedConjuncts(node, new CompoundPredicate(CompoundPredicate.Operator.AND, filter, predicate));
List<Split> splits = node.getSplits(20);
Assert.assertEquals(1, splits.size());
- Assert.assertTrue(((LanceSplit) splits.get(0)).getIndexSegmentUuid().isPresent());
+ List<Long> assignedFragments = new ArrayList<>();
+ for (Split split : splits) {
+ assignedFragments.addAll(((LanceSplit) split).getFragmentIds());
+ Assert.assertTrue(((LanceSplit) split).getIndexSegmentUuid().isPresent());
+ }
+ Collections.sort(assignedFragments);
+ Assert.assertEquals(Arrays.asList(1L, 2L, 3L, 4L), assignedFragments);
}
}
@@ -286,6 +304,51 @@ public class LanceScanNodeTest {
Assert.assertFalse(range.getTableFormatParams().getLanceParams().isSetLimit());
}
+ @Test
+ public void testMissingSegmentMetadataPreservesNativeIndexSelection() throws Exception {
+ List<LanceIndexSegmentInfo> segments = Collections.singletonList(
+ scalarSegment(UUID.randomUUID(), IndexType.BTREE, Arrays.asList(1L, 2L, 3L, 4L)));
+ LanceTableMetadata complete = scalarMetadata(segments);
+ LanceTableAccess access = new LanceTableAccess("s3://bucket/scalar.lance", Collections.emptyMap());
+ for (LanceTableMetadata metadata : Arrays.asList(
+ LanceTableMetadata.createSnapshotWithUnavailableFieldIds(access, 42,
+ complete.getSchema(), complete.getFragments(), segments),
+ LanceTableMetadata.createSnapshotWithIndexes(access, 42,
+ complete.getSchema(), complete.getFragments(), Collections.singletonMap("key", 9),
+ Collections.emptyList()))) {
+ assertNativeFragmentSelection(metadata);
+ }
+ }
+
+ @Test
+ public void testCompetingLogicalIndexesPreserveNativeSelection() throws Exception {
+ List<LanceIndexSegmentInfo> segments = Arrays.asList(
+ new LanceIndexSegmentInfo(UUID.randomUUID(), "a_key_idx", Collections.singletonList(9),
+ Collections.singletonList(1L), IndexType.BTREE, null),
+ new LanceIndexSegmentInfo(UUID.randomUUID(), "z_key_idx", Collections.singletonList(9),
+ Arrays.asList(1L, 2L, 3L, 4L), IndexType.BTREE, null));
+ assertNativeFragmentSelection(scalarMetadata(segments));
+ Collections.reverse(segments);
+ assertNativeFragmentSelection(scalarMetadata(segments));
+ }
+
+ private static void assertNativeFragmentSelection(LanceTableMetadata metadata) throws Exception {
+ LanceScanNode node = newNode();
+ setMetadata(node, metadata);
+ setPushedConjuncts(node, new CompoundPredicate(CompoundPredicate.Operator.OR,
+ new BinaryPredicate(BinaryPredicate.Operator.EQ, new SlotRef(null, "key"), new IntLiteral(1)),
+ new BinaryPredicate(BinaryPredicate.Operator.EQ, new SlotRef(null, "key"), new IntLiteral(2))));
+ List<Split> splits = node.getSplits(20);
+ Assert.assertEquals(4, splits.size());
+ for (Split split : splits) {
+ TFileRangeDesc range = new TFileRangeDesc();
+ node.setScanParams(range, split);
+ TLanceFileDesc params = range.getTableFormatParams().getLanceParams();
+ Assert.assertFalse(params.isSetIndexSegmentUuids());
+ Assert.assertFalse(params.isSetUseScalarIndex());
+ }
+ }
+
private static Expr scalarPredicate() {
return new BinaryPredicate(BinaryPredicate.Operator.GE, new SlotRef(null, "key"), new IntLiteral(2));
}
diff --git a/regression-test/suites/external_table_p0/lance/test_lance_array_predicate_pushdown.groovy b/regression-test/suites/external_table_p0/lance/test_lance_array_predicate_pushdown.groovy
new file mode 100644
index 00000000000..7c2d21a8e02
--- /dev/null
+++ b/regression-test/suites/external_table_p0/lance/test_lance_array_predicate_pushdown.groovy
@@ -0,0 +1,154 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.action.ProfileAction
+
+suite("test_lance_array_predicate_pushdown", "p0,external") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("Lance array pushdown requires the Iceberg MinIO environment")
+ return
+ }
+ String catalog = "test_lance_array_predicate_pushdown"
+ String endpoint = "http://${context.config.otherConfigs.get('externalEnvIp')}:" +
+ context.config.otherConfigs.get('iceberg_minio_port')
+ sql "DROP CATALOG IF EXISTS ${catalog}"
+ try {
+ sql """CREATE CATALOG ${catalog} PROPERTIES (
+ "type" = "lance", "lance.catalog.type" = "filesystem",
+ "warehouse" = "s3://warehouse/lance/predicate_arrays",
+ "s3.endpoint" = "${endpoint}", "s3.access_key" = "admin",
+ "s3.secret_key" = "password", "s3.region" = "us-east-1",
+ "use_path_style" = "true")"""
+ sql "SET enable_profile = true"
+ sql "SET profile_level = 2"
+ def profiles = new ProfileAction(context)
+ String red = "array_contains(labels, 'red')"
+ String blue = "array_contains(labels, 'blue')"
+ String green = "array_contains(labels, 'green')"
+ def cases = [
+ [red, [0, 2, 5, 7, 8, 10, 13, 15], 8, 4],
+ ["${red} AND ${blue}", [2, 7, 10, 15], 4, 2],
+ ["${red} OR ${blue}", [0, 1, 2, 5, 6, 7, 8, 9, 10, 13, 14, 15], 12, 6],
+ // Nereids rewrites this shape to arrays_overlap before Lance conversion.
+ ["${red} OR ${blue} OR ${green}", [0, 1, 2, 5, 6, 7, 8, 9, 10, 13, 14, 15], 12, 6],
+ ["arrays_overlap(labels, ['red', 'blue', 'green'])",
+ [0, 1, 2, 5, 6, 7, 8, 9, 10, 13, 14, 15], 12, 6],
+ ["NOT arrays_overlap(labels, ['red', 'blue', 'green'])", [3, 11], -1, -1],
+ ["NOT (${red})", [1, 3, 6, 9, 11, 14], -1, -1],
+ ["NOT (${green})", [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15], -1, -1],
+ ["category IN (0, 1)", [0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15], 11, 6],
+ ["category = 0 OR category = 1", [0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15], 11, 6],
+ ["${red} AND category = 1", [7, 10, 13], null],
+ ["${red} AND category <> 1", [0, 2, 5, 8, 15], null]
+ ]
+ for (int size : [64, 65]) {
+ String needles = (["'red'"] + (1..<size).collect { "'absent_${it}'" }).join(", ")
+ cases.add(["arrays_overlap(labels, [${needles}])", [0, 2, 5, 7, 8, 10, 13, 15],
+ size == 64 ? 8 : -1, size == 64 ? 4 : -1])
+ }
+ // Native expands a two-value IN into three index-expression nodes. With
+ // 63 memberships the combined tree exceeds 128 nodes and must stay parallel.
+ for (int size : [62, 63]) {
+ String needles = (["'red'"] + (1..<size).collect { "'absent_${it}'" }).join(", ")
+ cases.add(["arrays_overlap(labels, [${needles}]) AND category IN (0, 1)",
+ [0, 7, 10, 13, 15], size == 62 ? null : -1, -1])
+ }
+ for (String table : ["indexed", "partial", "unindexed"]) {
+ String relation = "${catalog}.`default`.`${table}`"
+ cases.eachWithIndex { c, caseId ->
+ String token = "lance_array_${table}_${caseId}_" + UUID.randomUUID().toString()
+ String query = "SELECT /* ${token} */ id FROM ${relation} WHERE ${c[0]} ORDER BY id"
+ explain {
+ sql(query)
+ contains "lancePushdownPredicate="
+ notContains "predicates:"
+ if (table != "unindexed" && c[2] != -1) {
+ contains "lanceScalarIndexScan=SEGMENT"
+ } else {
+ notContains "lanceScalarIndexScan=SEGMENT"
+ contains "lanceFragmentGrouping=FRAGMENT"
+ contains "lanceFragments=2"
+ }
+ }
+ assertEquals(c[1], sql(query).collect { (it[0] as Number).intValue() })
+ // A pushed predicate is not necessarily indexed: also verify runtime searches
+ // and candidate counts, so a non-indexed fallback cannot satisfy this test.
+ if (table != "unindexed") {
+ String profile = profiles.getProfileBySql(token,
+ ["LanceScalarIndexSegmentsSearched", "LanceScalarIndexCandidateRows",
+ "LanceScalarIndexSegmentFallbacks"])
+ def counter = { String name ->
+ def matches = profile =~ /${name}: (?:sum )?(\d+)\b/
+ assertTrue(matches.find(), "Missing counter ${name}: ${profile}")
+ return matches.group(1).toLong()
+ }
+ // Broad complements stay as parallel scans without repeated segment searches.
+ long expectedSearches = c[2] == -1 ? 0L : 1L
+ assertEquals(expectedSearches, counter("LanceScalarIndexSegmentsSearched"))
+ long expectedCandidates = c[2] == null ? -1L :
+ (c[table == "partial" ? 3 : 2] as Number).longValue()
+ if (expectedCandidates >= 0) {
+ assertEquals(expectedCandidates, counter("LanceScalarIndexCandidateRows"))
+ }
+ assertEquals(0L, counter("LanceScalarIndexSegmentFallbacks"))
+ }
+ }
+ // NULL needle matching and ordered subsequence matching have Doris-specific semantics.
+ def residualCases = [
+ ["array_contains(labels, NULL)", [6, 14]],
+ ["arrays_overlap(labels, ['green', NULL])", [6, 14]],
+ ["arrays_overlap(labels, [])", []],
+ ["array_contains_all(labels, ['red', 'blue'])", [2, 10]],
+ ["array_contains_all(labels, ['red', 'red'])", [5, 13]],
+ ["${red} OR array_contains(labels, NULL)", [0, 2, 5, 6, 7, 8, 10, 13, 14, 15]]
+ ]
+ for (def c : residualCases) {
+ String query = "SELECT id FROM ${relation} WHERE ${c[0]} ORDER BY id"
+ explain {
+ sql(query)
+ contains "predicates:"
+ notContains "lancePushdownPredicate="
+ }
+ assertEquals(c[1], sql(query).collect { (it[0] as Number).intValue() })
+ }
+ String mixed = "SELECT id FROM ${relation} WHERE ${blue} AND array_contains(labels, NULL) ORDER BY id"
+ explain {
+ sql(mixed)
+ contains "lancePushdownPredicate="
+ contains "predicates:"
+ }
+ assertEquals([6, 14], sql(mixed).collect { (it[0] as Number).intValue() })
+ // An unordered LIMIT reaches the scanner; TopN's bound does not. The two
+ // selective predicates force results from the indexed and uncovered domains.
+ for (int expectedId : [5, 13]) {
+ String limited = "SELECT id FROM ${relation} WHERE ${red} AND id = ${expectedId} LIMIT 1"
+ explain {
+ sql(limited)
+ contains "lanceLimit=1"
+ contains "lancePushdownPredicate="
+ notContains "predicates:"
+ }
+ assertEquals([[(long) expectedId]], sql(limited))
+ }
+ // The appended fragment must still contribute matching rows, including with LIMIT.
+ assertEquals([[15L]], sql("SELECT id FROM ${relation} WHERE ${red} ORDER BY id DESC LIMIT 1"))
+ }
+ } finally {
+ sql "DROP CATALOG IF EXISTS ${catalog}"
+ }
+}
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index a5e209a0aed..89c69090e04 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -156,6 +156,8 @@ if [[ ! -f "${TP_DIR}/vars.sh" ]]; then
fi
. "${TP_DIR}/vars.sh"
+. "${TP_DIR}/lance-install.sh"
+LANCE_C_INSTALL_FINGERPRINT="$(lance_c_install_fingerprint "${TP_DIR}")"
cd "${TP_DIR}"
@@ -2228,9 +2230,14 @@ build_lance_c() {
env "${cargo_env[@]}" "${cargo_bin}" "${cargo_args[@]}"
mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64"
+ # Invalidate before publishing either file so interrupted installs cannot reuse
+ # a matching marker with a partial header/archive pair.
+ rm -f "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint"
rm -rf "${TP_INSTALL_DIR}/include/lance"
cp -av include/lance "${TP_INSTALL_DIR}/include/"
install_rust_archive "${BUILD_DIR}/release/liblance_c.a"
+ printf '%s\n' "${LANCE_C_INSTALL_FINGERPRINT}" > "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint.tmp"
+ mv "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint.tmp" "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint"
}
# paimon-rust
diff --git a/thirdparty/lance-install.sh b/thirdparty/lance-install.sh
new file mode 100644
index 00000000000..28594d61c79
--- /dev/null
+++ b/thirdparty/lance-install.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Keep the check independent of the installed prefix: an external compilation image
+# can carry old vars.sh alongside an ABI-compatible but behaviorally stale archive.
+lance_c_install_fingerprint() (
+ local definitions="$1"
+ local TP_DIR="${definitions}"
+ source "${definitions}/vars.sh" || return 1
+ local patch_checksum
+ patch_checksum="$(cksum < "${definitions}/patches/lance-c-foyer.patch")" || return 1
+ printf '%s\n' "${LANCE_C_SOURCE}" "${LANCE_C_MD5SUM}" "${patch_checksum}"
+)
+
+lance_c_install_is_current() {
+ local definitions="$1" installed="$2" expected
+ [[ -s "${installed}/lib64/liblance_c.a" &&
+ -s "${installed}/include/lance/lance.h" &&
+ -s "${installed}/include/lance/lance.hpp" &&
+ -s "${installed}/lib64/.lance-c-fingerprint" ]] || return 1
+ expected="$(lance_c_install_fingerprint "${definitions}")" || return 1
+ [[ "$(cat "${installed}/lib64/.lance-c-fingerprint")" == "${expected}" ]]
+}
diff --git a/thirdparty/patches/lance-c-foyer.patch b/thirdparty/patches/lance-c-foyer.patch
index 73f244c49fd..bb2104ab32d 100644
--- a/thirdparty/patches/lance-c-foyer.patch
+++ b/thirdparty/patches/lance-c-foyer.patch
@@ -1,8 +1,7 @@
# Foyer data-cache integration for lance-format/lance-c#73.
-# Base: 9bd730add2ac70316c1d642b8459011e2dd92022
+# Base: cd63420bfbe27f6f0a1edcc873b9191af7d52852 (lance-format/lance-c main, including #93)
# Source: https://github.com/Gabriel39/lance-c/commit/24c7ca4bcb9422c113b0d3e07e4efe1173b0bc9f
-# Regenerate the payload in lance-c; do not maintain separate downstream edits:
-# git diff --full-index --binary 9bd730add2ac70316c1d642b8459011e2dd92022 24c7ca4bcb9422c113b0d3e07e4efe1173b0bc9f -- . | sed 's/^ $//'
+# The unchanged Foyer payload is reapplied to the newer upstream base with git apply --3way.
diff --git a/Cargo.lock b/Cargo.lock
index 78812f1330e146db295a14276f90f9e28654f399..daf9f85daf3e0ea59bb906e8e3c32470519b084b 100644
--- a/Cargo.lock
@@ -199,7 +198,7 @@ index 18654b862440a90aa6bf0dd846e8f1ad6036fd7c..fd134a7151e1df61da201d867a642e9b
prost = "0.14"
snafu = "0.9"
diff --git a/README.md b/README.md
-index 9ceccdc4f3f6b5d13dc271be3d2659c7be400306..94de4d1c26d8ab869d1726e2b416d21f21661d6d 100644
+index 9f0871b17b13ebfd96916947b4bee0dd06f6a468..441719c9f3904058127e55d1cd29ea1afc878935 100644
--- a/README.md
+++ b/README.md
@@ -68,6 +68,7 @@ Based on the [liblance RFC](https://github.com/lance-format/lance/discussions/60
@@ -208,9 +207,9 @@ index 9ceccdc4f3f6b5d13dc271be3d2659c7be400306..94de4d1c26d8ab869d1726e2b416d21f
| [x] | Filter pushdown | `lance_scanner_set_substrait_filter()` accepts a serialized Substrait `ExtendedExpression`; `lance_scanner_additional_sql_filter()` adds SQL predicates with AND before scanning starts |
+| [x] | Data-file cache | Optional Foyer memory/disk cache for immutable `data/*.lance` reads |
- ## Multi-vector search
+ ## Segment-scoped array label filters
-@@ -231,6 +232,35 @@ auto ds = lance::Dataset::open_with_session(session, "data.lance");
+@@ -362,6 +363,35 @@ auto ds = lance::Dataset::open_with_session(session, "data.lance");
auto stats = session.cache_stats();
```
@@ -247,7 +246,7 @@ index 9ceccdc4f3f6b5d13dc271be3d2659c7be400306..94de4d1c26d8ab869d1726e2b416d21f
`lance_dataset_open` takes a `version` argument — `0` means the latest, any
diff --git a/include/lance/lance.h b/include/lance/lance.h
-index 7630913fd7849d0bff2cb0f31e1ab1199cb9f0fb..1c1d0752efdda5023b29eafe5aa9a87e03dc5303 100644
+index 94772e2a14bf9a8a950dfb2d39112136e27197c0..fe48eb0d9afd1c7157a3bd2230cce7e769120bac 100644
--- a/include/lance/lance.h
+++ b/include/lance/lance.h
@@ -214,6 +214,36 @@ typedef struct LanceSessionCacheStats {
@@ -335,7 +334,7 @@ index 7630913fd7849d0bff2cb0f31e1ab1199cb9f0fb..1c1d0752efdda5023b29eafe5aa9a87e
void lance_dataset_close(LanceDataset* dataset);
diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp
-index 286724e96736e354785e046f2fcfe5ae7c65147b..5070d423453a5d60d31d03ce758ad7dbe83a6ea2 100644
+index edcd39fc3caba387ab490fa84139f24e9376130a..17b71957d0f5c419fe0b3392ce11faa757ddbfe1 100644
--- a/include/lance/lance.hpp
+++ b/include/lance/lance.hpp
@@ -176,6 +176,13 @@ struct SqlColumn {
@@ -2244,7 +2243,7 @@ index 1971510de4a13ee6f04fc39c159b088e78e21d55..ba51c87a8cebb6897c48fb55509f71da
// SAFETY: `out_dataset` is non-NULL (checked above) and the caller
// guarantees it points to caller-owned, writable storage of size
diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index 1a25e34a8ca50623812ffedaa7bb5a8c54f7850e..c341859e9434d4578b2833eca6ec313a28f31f32 100644
+index d3bea44529a842fe3ae28ee3f90cd8d45fe0b394..98e6901a2cfeb2cc98df6606e26b04788d4218b2 100644
--- a/tests/c_api_test.rs
+++ b/tests/c_api_test.rs
@@ -100,10 +100,83 @@ fn create_large_dataset(num_rows: i32) -> (tempfile::TempDir, String) {
@@ -2503,7 +2502,7 @@ index 1a25e34a8ca50623812ffedaa7bb5a8c54f7850e..c341859e9434d4578b2833eca6ec313a
fn test_dataset_restore_to_current_latest_writes_new_manifest() {
// Restoring to the current latest still writes a new manifest. The
diff --git a/tests/cpp/test_c_api.c b/tests/cpp/test_c_api.c
-index 5df9cde4d8fa971823b9e6c974a8a0db6c9ec9ca..1444e55161631ac8ad976f427d867ad91d358195 100644
+index efad5ed55987976a3f1820220e42fc58bef42638..050543aebc02454cab09f6e0bd38ec10b1d21720 100644
--- a/tests/cpp/test_c_api.c
+++ b/tests/cpp/test_c_api.c
@@ -171,6 +171,37 @@ static void test_shared_session(const char *uri) {
@@ -2544,16 +2543,16 @@ index 5df9cde4d8fa971823b9e6c974a8a0db6c9ec9ca..1444e55161631ac8ad976f427d867ad9
static void test_scan(const char *uri) {
printf(" test_scan... ");
-@@ -1323,6 +1354,7 @@ int main(int argc, char **argv) {
-
+@@ -1471,6 +1502,7 @@ int main(int argc, char **argv) {
+ test_batch_nearest(uri);
test_open_and_metadata(uri);
test_shared_session(uri);
+ test_data_cache_session(uri, write_uri);
test_scan(uri);
+ test_distance_range(uri);
test_scan_with_limit(uri);
- test_scanner_blob_handling(blob_uri);
diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp
-index 0332d1a2687354984d0551e10716df08344a4013..ef64ab0d02bed2f1750c3d388792e4aa44265e24 100644
+index e2d541e4430889aba89ab85c01cc7e3eacf97baa..ac1c10bccfcf399bbbfcfe908d3188484aff0add 100644
--- a/tests/cpp/test_cpp_api.cpp
+++ b/tests/cpp/test_cpp_api.cpp
@@ -131,6 +131,28 @@ static void test_shared_session(const std::string& uri) {
@@ -2585,11 +2584,11 @@ index 0332d1a2687354984d0551e10716df08344a4013..ef64ab0d02bed2f1750c3d388792e4aa
static void test_dataset_schema(const std::string& uri) {
TEST(test_dataset_schema);
-@@ -1211,6 +1233,7 @@ int main(int argc, char** argv) {
+@@ -1353,6 +1375,7 @@ int main(int argc, char** argv) {
test_dataset_open(uri);
test_shared_session(uri);
+ test_data_cache_session(uri, write_uri);
test_dataset_schema(uri);
test_scanner_fluent(uri);
- test_scanner_async_stream_ownership(uri);
+ test_distance_range(uri);
diff --git a/thirdparty/test/lance-install-test.sh b/thirdparty/test/lance-install-test.sh
new file mode 100644
index 00000000000..db93c355c0f
--- /dev/null
+++ b/thirdparty/test/lance-install-test.sh
@@ -0,0 +1,176 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+work="$(mktemp -d)"
+trap 'rm -rf "${work}"' EXIT
+mkdir -p "${work}/repo/thirdparty/patches" "${work}/external"
+cp "${ROOT}/thirdparty/vars.sh" "${work}/repo/thirdparty/"
+cp "${ROOT}/thirdparty/patches/lance-c-foyer.patch" "${work}/repo/thirdparty/patches/"
+# Extract the real build gate; all destructive operations stay inside this temporary install.
+echo 'set -eo pipefail' > "${work}/gate.sh"
+sed -n '/^# build thirdparty libraries if necessary/,/^update_submodule()/p' "${ROOT}/build.sh" \
+ | sed '$d' >> "${work}/gate.sh"
+export DORIS_HOME="${work}/repo" DORIS_THIRDPARTY="${work}/external"
+export TARGET_SYSTEM=Linux CLEAN=0 PARALLEL=1
+export TEST_HELPER="${ROOT}/thirdparty/lance-install.sh"
+cp "${TEST_HELPER}" "${work}/repo/thirdparty/"
+cp -r "${DORIS_HOME}/thirdparty/." "${DORIS_THIRDPARTY}/"
+printf '#!/usr/bin/env bash\nexit 0\n' > "${DORIS_THIRDPARTY}/download-thirdparty.sh"
+cat > "${DORIS_THIRDPARTY}/build-thirdparty.sh" <<'BUILDER'
+set -euo pipefail
+TP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${TP_DIR}/lance-install.sh"
+fingerprint="$(lance_c_install_fingerprint "${TP_DIR}")"
+echo rebuilt >> "${DORIS_THIRDPARTY}/builds"
+installed="${DORIS_THIRDPARTY}/installed"
+mkdir -p "${installed}/lib/hadoop_hdfs/native" "${installed}/lib/hadoop_hdfs_3_4/native" "${installed}/lib64" \
+ "${installed}/include/lance" "${installed}/include/paimon_rust"
+for file in lib/hadoop_hdfs/native/libhdfs.a lib/hadoop_hdfs_3_4/native/libhdfs.a \
+ lib64/liblance_c.a lib64/libpaimon_c.a \
+ include/lance/lance.h include/lance/lance.hpp include/paimon_rust/paimon.h; do
+ echo artifact > "${installed}/${file}"
+done
+printf '%s\n' "${fingerprint}" > "${installed}/lib64/.lance-c-fingerprint"
+BUILDER
+# A complete legacy image has all old sentinels but no Lance revision marker.
+installed="${DORIS_THIRDPARTY}/installed"
+mkdir -p "${installed}/lib/hadoop_hdfs/native" "${installed}/lib/hadoop_hdfs_3_4/native" "${installed}/lib64" \
+ "${installed}/include/lance" "${installed}/include/paimon_rust"
+for file in lib/hadoop_hdfs/native/libhdfs.a lib/hadoop_hdfs_3_4/native/libhdfs.a \
+ lib64/liblance_c.a lib64/libpaimon_c.a \
+ include/lance/lance.h include/lance/lance.hpp include/paimon_rust/paimon.h; do
+ echo legacy > "${installed}/${file}"
+done
+bash "${work}/gate.sh"
+[[ -f "${DORIS_THIRDPARTY}/builds" ]] || { echo 'FAIL: reused unversioned Lance archive'; exit 1; }
+source "${TEST_HELPER}"
+lance_c_install_is_current "${DORIS_HOME}/thirdparty" "${installed}"
+check_builds() {
+ [[ "$(wc -l < "${DORIS_THIRDPARTY}/builds")" -eq "$1" ]]
+}
+check_builds 1
+bash "${work}/gate.sh"
+check_builds 1
+echo 'PASS: legacy image rebuilt; matching install reused'
+# Failed preflight must preserve every dependency, not just the Lance archive.
+expect_preserved_install() {
+ cp -a "${installed}" "${work}/saved-install"
+ cp "${DORIS_THIRDPARTY}/builds" "${work}/saved-builds"
+ if bash "${work}/gate.sh" > "${work}/preflight.log" 2>&1; then
+ echo 'FAIL: accepted incomplete or mismatched rebuild sources'; exit 1
+ fi
+ if ! diff -r "${work}/saved-install" "${installed}"; then
+ echo 'FAIL: rejected rebuild sources after changing installed dependencies'; exit 1
+ fi
+ cmp "${work}/saved-builds" "${DORIS_THIRDPARTY}/builds"
+ rm -rf "${work}/saved-install"
+}
+echo stale > "${installed}/lib64/.lance-c-fingerprint"
+for file in lance-install.sh vars.sh patches/lance-c-foyer.patch download-thirdparty.sh build-thirdparty.sh; do
+ mv "${DORIS_THIRDPARTY}/${file}" "${work}/missing-input"
+ expect_preserved_install
+ mv "${work}/missing-input" "${DORIS_THIRDPARTY}/${file}"
+done
+echo 'PASS: incomplete external sources rejected before changing installed dependencies'
+echo stale > "${installed}/lib64/.lance-c-fingerprint"
+bash "${work}/gate.sh"
+check_builds 2
+# Pin and patch updates both invalidate installed artifacts, even with identical ABI.
+sed 's/^LANCE_C_SOURCE=.*/LANCE_C_SOURCE="lance-c-test-revision"/' "${DORIS_HOME}/thirdparty/vars.sh" \
+ > "${work}/new-vars.sh"
+mv "${work}/new-vars.sh" "${DORIS_HOME}/thirdparty/vars.sh"
+expect_preserved_install
+cp "${DORIS_HOME}/thirdparty/vars.sh" "${DORIS_THIRDPARTY}/vars.sh"
+bash "${work}/gate.sh"
+check_builds 3
+echo '# test patch update' >> "${DORIS_HOME}/thirdparty/patches/lance-c-foyer.patch"
+expect_preserved_install
+cp "${DORIS_HOME}/thirdparty/patches/lance-c-foyer.patch" "${DORIS_THIRDPARTY}/patches/"
+bash "${work}/gate.sh"
+check_builds 4
+echo 'PASS: mismatched sources rejected; synchronized pin and patch updates rebuild'
+rm "${installed}/include/lance/lance.h"
+bash "${work}/gate.sh"
+check_builds 5
+: > "${installed}/lib64/liblance_c.a"
+bash "${work}/gate.sh"
+check_builds 6
+echo 'PASS: incomplete header/archive install rebuilt'
+# A legacy external builder may exit successfully without installing the new revision.
+cp "${DORIS_THIRDPARTY}/build-thirdparty.sh" "${work}/good-builder.sh"
+printf '#!/usr/bin/env bash\nexit 0\n' > "${DORIS_THIRDPARTY}/build-thirdparty.sh"
+rm "${installed}/lib64/.lance-c-fingerprint"
+if bash "${work}/gate.sh" > "${work}/old-builder.log" 2>&1; then
+ echo 'FAIL: accepted output from a stale external builder'; exit 1
+fi
+grep -q 'Lance dependency revision does not match' "${work}/old-builder.log"
+cp "${work}/good-builder.sh" "${DORIS_THIRDPARTY}/build-thirdparty.sh"
+bash "${DORIS_THIRDPARTY}/build-thirdparty.sh"
+echo 'PASS: stale external builder cannot silently satisfy the revision gate'
+# Images without rebuild sources must fail before deleting installed dependencies.
+rm "${DORIS_THIRDPARTY}/build-thirdparty.sh" "${installed}/lib64/.lance-c-fingerprint"
+if bash "${work}/gate.sh" > "${work}/missing-source.log" 2>&1; then
+ echo 'FAIL: accepted stale compilation image without build sources'; exit 1
+fi
+[[ -s "${installed}/lib64/liblance_c.a" ]]
+echo 'PASS: missing rebuild source fails without deleting installed artifacts'
+# Execute the actual publication function with a tiny Cargo stand-in. A failed
+# archive copy must invalidate the old marker, and only a complete retry may stamp it.
+sed -n '/^build_lance_c()/,/^}/p' "${ROOT}/thirdparty/build-thirdparty.sh" > "${work}/publish-function.sh"
+export TP_DIR="${DORIS_HOME}/thirdparty"
+source "${TP_DIR}/vars.sh"
+export LANCE_C_SOURCE TP_SOURCE_DIR TP_INSTALL_DIR
+export LANCE_C_INSTALL_FINGERPRINT="$(lance_c_install_fingerprint "${TP_DIR}")"
+mkdir -p "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}/include/lance" "${TP_INSTALL_DIR}/bin"
+echo header > "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}/include/lance/lance.h"
+echo header > "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}/include/lance/lance.hpp"
+printf '#!/bin/sh\nexit 0\n' > "${TP_INSTALL_DIR}/bin/protoc"
+chmod +x "${TP_INSTALL_DIR}/bin/protoc"
+cat > "${work}/cargo" <<'CARGO'
+#!/usr/bin/env bash
+set -eu
+if [[ "$1" == --version ]]; then
+ echo 'cargo 1.94.0'; exit 0
+fi
+mkdir -p "${CARGO_TARGET_DIR}/release"
+echo archive > "${CARGO_TARGET_DIR}/release/liblance_c.a"
+CARGO
+chmod +x "${work}/cargo"
+export LANCE_C_CARGO="${work}/cargo" RUSTUP_TOOLCHAIN=1.94.0
+export BUILD_DIR=build KERNEL=Linux LANCE_C_CARGO_OFFLINE=OFF
+cat > "${work}/publish.sh" <<'PUBLISH'
+set -eo pipefail
+check_if_source_exist() { :; }
+install_rust_archive() {
+ if [[ "${FAIL_INSTALL:-0}" == 1 ]]; then return 1; fi
+ cp "$1" "${TP_INSTALL_DIR}/lib64/liblance_c.a"
+}
+source "$1"
+build_lance_c
+PUBLISH
+bash "${work}/publish.sh" "${work}/publish-function.sh" > "${work}/publish.log" 2>&1
+lance_c_install_is_current "${TP_DIR}" "${TP_INSTALL_DIR}"
+if FAIL_INSTALL=1 bash "${work}/publish.sh" "${work}/publish-function.sh" >> "${work}/publish.log" 2>&1; then
+ echo 'FAIL: expected archive publication failure'; exit 1
+fi
+[[ ! -e "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint" ]]
+bash "${work}/publish.sh" "${work}/publish-function.sh" >> "${work}/publish.log" 2>&1
+lance_c_install_is_current "${TP_DIR}" "${TP_INSTALL_DIR}"
+echo 'PASS: successful install stamped; failed publication invalidated; retry repaired'
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index fc5e9818c32..147e4e4b2c4 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -580,11 +580,11 @@ PUGIXML_SOURCE=pugixml-1.15
PUGIXML_MD5SUM="3b894c29455eb33a40b165c6e2de5895"
# lance-c
-# Complete-segment prefilter fixes are supplied by upstream lance-c, not local patches.
-LANCE_C_DOWNLOAD="https://codeload.github.com/lance-format/lance-c/tar.gz/9bd730add2ac70316c1d642b8459011e2dd92022"
-LANCE_C_NAME="lance-c-9bd730add2ac70316c1d642b8459011e2dd92022.tar.gz"
-LANCE_C_SOURCE="lance-c-9bd730add2ac70316c1d642b8459011e2dd92022"
-LANCE_C_MD5SUM="63851b09bf1689032579f1a094ff2f37"
+# Includes lance-c #93: scoped boolean scalar-index expressions and Substrait label filters.
+LANCE_C_DOWNLOAD="https://codeload.github.com/lance-format/lance-c/tar.gz/cd63420bfbe27f6f0a1edcc873b9191af7d52852"
+LANCE_C_NAME="lance-c-cd63420bfbe27f6f0a1edcc873b9191af7d52852.tar.gz"
+LANCE_C_SOURCE="lance-c-cd63420bfbe27f6f0a1edcc873b9191af7d52852"
+LANCE_C_MD5SUM="37d82907559c4fdb8ed6e5b767b0d309"
# paimon-rust
PAIMON_RUST_DOWNLOAD="https://github.com/apache/paimon-rust/archive/refs/tags/v0.4.0-rc1.tar.gz"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/35: branch-4.1: [improve](profile) Skip _init_profile in load_v1 when enable_profile is false #61131 (#62801)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 48f05f67dd153e4effb9b931e18fd727d46c44b6
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Sep 29 10:00:41 2026 +0800
branch-4.1: [improve](profile) Skip _init_profile in load_v1 when enable_profile is false #61131 (#62801)
Cherry-picked from #61131
Co-authored-by: 黄瑞鑫 <11...@users.noreply.github.com>
---
be/src/cloud/cloud_tablets_channel.cpp | 4 +++-
be/src/load/channel/load_channel.cpp | 19 ++++++++++++++-----
be/src/load/channel/tablets_channel.cpp | 10 ++++++++--
be/src/load/delta_writer/delta_writer.cpp | 6 +++++-
be/src/load/memtable/memtable_writer.cpp | 3 +++
be/src/storage/rowset_builder.cpp | 6 +++++-
6 files changed, 38 insertions(+), 10 deletions(-)
diff --git a/be/src/cloud/cloud_tablets_channel.cpp b/be/src/cloud/cloud_tablets_channel.cpp
index 7b74e9b1e07..3efedcc6b16 100644
--- a/be/src/cloud/cloud_tablets_channel.cpp
+++ b/be/src/cloud/cloud_tablets_channel.cpp
@@ -46,7 +46,9 @@ Status CloudTabletsChannel::add_batch(const PTabletWriterAddBlockRequest& reques
// FIXME(plat1ko): Too many duplicate code with `TabletsChannel`
SCOPED_TIMER(_add_batch_timer);
int64_t cur_seq = 0;
- _add_batch_number_counter->update(1);
+ if (_add_batch_number_counter != nullptr) {
+ _add_batch_number_counter->update(1);
+ }
auto status = _get_current_seq(cur_seq, request);
if (UNLIKELY(!status.ok())) {
diff --git a/be/src/load/channel/load_channel.cpp b/be/src/load/channel/load_channel.cpp
index 4a76d1a5697..1b6af15240c 100644
--- a/be/src/load/channel/load_channel.cpp
+++ b/be/src/load/channel/load_channel.cpp
@@ -72,7 +72,9 @@ LoadChannel::LoadChannel(const UniqueId& load_id, int64_t timeout_s, bool is_hig
// _load_channels in load_channel_mgr, or it may be erased
// immediately by gc thread.
_last_updated_time.store(time(nullptr));
- _init_profile();
+ if (enable_profile) {
+ _init_profile();
+ }
}
LoadChannel::~LoadChannel() {
@@ -88,6 +90,7 @@ LoadChannel::~LoadChannel() {
}
void LoadChannel::_init_profile() {
+ DCHECK(_enable_profile);
_profile = std::make_unique<RuntimeProfile>("LoadChannels");
_mgr_add_batch_timer = ADD_TIMER(_profile, "LoadChannelMgrAddBatchTime");
_handle_mem_limit_timer = ADD_TIMER(_profile, "HandleMemLimitTime");
@@ -179,7 +182,9 @@ Status LoadChannel::add_batch(const PTabletWriterAddBlockRequest& request,
DBUG_EXECUTE_IF("LoadChannel.add_batch.failed",
{ return Status::InternalError("fault injection"); });
SCOPED_TIMER(_add_batch_timer);
- COUNTER_UPDATE(_add_batch_times, 1);
+ if (_enable_profile) {
+ COUNTER_UPDATE(_add_batch_times, 1);
+ }
SCOPED_ATTACH_TASK(_resource_ctx);
int64_t index_id = request.index_id();
// 1. get tablets channel
@@ -193,7 +198,9 @@ Status LoadChannel::add_batch(const PTabletWriterAddBlockRequest& request,
// 2. add block to tablets channel
if (request.has_block()) {
RETURN_IF_ERROR(channel->add_batch(request, response));
- _add_batch_number_counter->update(1);
+ if (_enable_profile) {
+ _add_batch_number_counter->update(1);
+ }
}
// 3. handle eos
@@ -204,7 +211,7 @@ Status LoadChannel::add_batch(const PTabletWriterAddBlockRequest& request,
if (!st.ok()) {
return st;
}
- } else if (_add_batch_number_counter->value() % 100 == 1) {
+ } else if (_enable_profile && _add_batch_number_counter->value() % 100 == 1) {
_report_profile(response);
}
_last_updated_time.store(time(nullptr));
@@ -214,7 +221,9 @@ Status LoadChannel::add_batch(const PTabletWriterAddBlockRequest& request,
Status LoadChannel::_handle_eos(BaseTabletsChannel* channel,
const PTabletWriterAddBlockRequest& request,
PTabletWriterAddBlockResult* response) {
- _self_profile->add_info_string("EosHost", fmt::format("{}", request.backend_id()));
+ if (_enable_profile) {
+ _self_profile->add_info_string("EosHost", fmt::format("{}", request.backend_id()));
+ }
bool finished = false;
auto index_id = request.index_id();
diff --git a/be/src/load/channel/tablets_channel.cpp b/be/src/load/channel/tablets_channel.cpp
index 10ffb91ed1e..708ca52bff0 100644
--- a/be/src/load/channel/tablets_channel.cpp
+++ b/be/src/load/channel/tablets_channel.cpp
@@ -66,7 +66,9 @@ BaseTabletsChannel::BaseTabletsChannel(const TabletsChannelKey& key, const Uniqu
_closed_senders(64),
_is_high_priority(is_high_priority) {
static std::once_flag once_flag;
- _init_profile(profile);
+ if (profile != nullptr) {
+ _init_profile(profile);
+ }
std::call_once(once_flag, [] {
REGISTER_HOOK_METRIC(tablet_writer_count, [&]() { return _s_tablet_writer_count.load(); });
});
@@ -102,6 +104,7 @@ Status BaseTabletsChannel::_get_current_seq(int64_t& cur_seq,
}
void BaseTabletsChannel::_init_profile(RuntimeProfile* profile) {
+ DCHECK(profile != nullptr);
_profile =
profile->create_child(fmt::format("TabletsChannel {}", _key.to_string()), true, true);
_add_batch_number_counter = ADD_COUNTER(_profile, "NumberBatchAdded", TUnit::UNIT);
@@ -122,6 +125,7 @@ void BaseTabletsChannel::_init_profile(RuntimeProfile* profile) {
}
void TabletsChannel::_init_profile(RuntimeProfile* profile) {
+ DCHECK(profile != nullptr);
BaseTabletsChannel::_init_profile(profile);
_slave_replica_timer = ADD_TIMER(_profile, "SlaveReplicaTime");
}
@@ -847,7 +851,9 @@ Status TabletsChannel::add_batch(const PTabletWriterAddBlockRequest& request,
PTabletWriterAddBlockResult* response) {
SCOPED_TIMER(_add_batch_timer);
int64_t cur_seq = 0;
- _add_batch_number_counter->update(1);
+ if (_add_batch_number_counter) {
+ _add_batch_number_counter->update(1);
+ }
auto status = _get_current_seq(cur_seq, request);
if (UNLIKELY(!status.ok())) {
diff --git a/be/src/load/delta_writer/delta_writer.cpp b/be/src/load/delta_writer/delta_writer.cpp
index 16aae6bb0ca..184f22f9624 100644
--- a/be/src/load/delta_writer/delta_writer.cpp
+++ b/be/src/load/delta_writer/delta_writer.cpp
@@ -64,7 +64,9 @@ using namespace ErrorCode;
BaseDeltaWriter::BaseDeltaWriter(const WriteRequest& req, RuntimeProfile* profile,
const UniqueId& load_id)
: _req(req), _memtable_writer(new MemTableWriter(req)) {
- _init_profile(profile);
+ if (profile != nullptr) {
+ _init_profile(profile);
+ }
}
DeltaWriter::DeltaWriter(StorageEngine& engine, const WriteRequest& req, RuntimeProfile* profile,
@@ -74,12 +76,14 @@ DeltaWriter::DeltaWriter(StorageEngine& engine, const WriteRequest& req, Runtime
}
void BaseDeltaWriter::_init_profile(RuntimeProfile* profile) {
+ DCHECK(profile != nullptr);
_profile = profile->create_child(fmt::format("DeltaWriter {}", _req.tablet_id), true, true);
_close_wait_timer = ADD_TIMER(_profile, "CloseWaitTime");
_wait_flush_limit_timer = ADD_TIMER(_profile, "WaitFlushLimitTime");
}
void DeltaWriter::_init_profile(RuntimeProfile* profile) {
+ DCHECK(profile != nullptr);
BaseDeltaWriter::_init_profile(profile);
_commit_txn_timer = ADD_TIMER(_profile, "CommitTxnTime");
}
diff --git a/be/src/load/memtable/memtable_writer.cpp b/be/src/load/memtable/memtable_writer.cpp
index d0edf0bf6dc..a000206990b 100644
--- a/be/src/load/memtable/memtable_writer.cpp
+++ b/be/src/load/memtable/memtable_writer.cpp
@@ -303,6 +303,9 @@ Status MemTableWriter::_do_close_wait() {
}
void MemTableWriter::_update_profile(RuntimeProfile* profile) {
+ if (!profile) {
+ return;
+ }
// NOTE: MemTableWriter may be accessed when profile is out of scope, in MemTableMemoryLimiter.
// To avoid accessing dangling pointers, we cannot make profile as a member of MemTableWriter.
auto child =
diff --git a/be/src/storage/rowset_builder.cpp b/be/src/storage/rowset_builder.cpp
index c62ccfd6f6d..3db62d12dd9 100644
--- a/be/src/storage/rowset_builder.cpp
+++ b/be/src/storage/rowset_builder.cpp
@@ -66,7 +66,9 @@ using namespace ErrorCode;
BaseRowsetBuilder::BaseRowsetBuilder(const WriteRequest& req, RuntimeProfile* profile)
: _req(req), _tablet_schema(std::make_shared<TabletSchema>()) {
- _init_profile(profile);
+ if (profile != nullptr) {
+ _init_profile(profile);
+ }
}
RowsetBuilder::RowsetBuilder(StorageEngine& engine, const WriteRequest& req,
@@ -74,6 +76,7 @@ RowsetBuilder::RowsetBuilder(StorageEngine& engine, const WriteRequest& req,
: BaseRowsetBuilder(req, profile), _engine(engine) {}
void BaseRowsetBuilder::_init_profile(RuntimeProfile* profile) {
+ DCHECK(profile != nullptr);
_profile = profile->create_child(fmt::format("RowsetBuilder {}", _req.tablet_id), true, true);
_build_rowset_timer = ADD_TIMER(_profile, "BuildRowsetTime");
_submit_delete_bitmap_timer = ADD_TIMER(_profile, "DeleteBitmapSubmitTime");
@@ -81,6 +84,7 @@ void BaseRowsetBuilder::_init_profile(RuntimeProfile* profile) {
}
void RowsetBuilder::_init_profile(RuntimeProfile* profile) {
+ DCHECK(profile != nullptr);
BaseRowsetBuilder::_init_profile(profile);
_commit_txn_timer = ADD_TIMER(_profile, "CommitTxnTime");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 15/35: branch-4.1: [fix](function) Skip the rows with a NULL argument in array_range #68620 (#68644)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 761c243c71ad3051a86ebbaa70a12bf84efb2a79
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Sep 29 23:34:36 2026 +0800
branch-4.1: [fix](function) Skip the rows with a NULL argument in array_range #68620 (#68644)
Cherry-picked from #68620
Co-authored-by: Chenyang Sun <su...@selectdb.com>
Co-authored-by: Claude Opus 5.5 <no...@anthropic.com>
---
.../exprs/function/array/function_array_range.cpp | 12 +++++
.../exprs/function/function_array_range_test.cpp | 12 +++++
.../test_array_range_null_argument.out | 15 +++++++
.../test_array_range_null_argument.groovy | 51 ++++++++++++++++++++++
4 files changed, 90 insertions(+)
diff --git a/be/src/exprs/function/array/function_array_range.cpp b/be/src/exprs/function/array/function_array_range.cpp
index 24d4de938bd..c24e6afe527 100644
--- a/be/src/exprs/function/array/function_array_range.cpp
+++ b/be/src/exprs/function/array/function_array_range.cpp
@@ -41,6 +41,7 @@
#include "core/pod_array_fwd.h"
#include "core/types.h"
#include "core/value/vdatetime_value.h"
+#include "exec/common/util.hpp"
#include "exprs/aggregate/aggregate_function.h"
#include "exprs/function/function.h"
#include "exprs/function/function_date_or_datetime_computation.h"
@@ -79,6 +80,10 @@ public:
return make_nullable(res);
}
+ // range_execute skips the rows with a NULL argument. The default NULL handling would build a
+ // range from the value under the NULL, which can be larger than the array size limit.
+ bool use_default_implementation_for_nulls() const override { return false; }
+
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const override {
return Impl::execute_impl(context, block, arguments, result, input_rows_count);
@@ -147,6 +152,13 @@ struct RangeImplUtil {
for (int i = 0; i < 3; ++i) {
argument_columns[i] =
block.get_by_position(arguments[i]).column->convert_to_full_column_if_const();
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(*argument_columns[i])) {
+ // Read the null map before replacing the column, because the replacement can free
+ // the nullable column.
+ VectorizedUtils::update_null_map(args_null_map->get_data(),
+ nullable->get_null_map_data());
+ argument_columns[i] = nullable->get_nested_column_ptr();
+ }
}
auto start_column =
assert_cast<const ColumnVector<SourceDataPType>*>(argument_columns[0].get());
diff --git a/be/test/exprs/function/function_array_range_test.cpp b/be/test/exprs/function/function_array_range_test.cpp
index d6026f84005..c91d17918ca 100644
--- a/be/test/exprs/function/function_array_range_test.cpp
+++ b/be/test/exprs/function/function_array_range_test.cpp
@@ -193,5 +193,17 @@ TEST(FunctionArrayRangeTest, ArraySizeLimit) {
}
}
+TEST(FunctionArrayRangeTest, NullStartSkipsTheRange) {
+ // The value under a NULL start is 0. A range from 0 to this end is larger than the limit, so
+ // the row must give NULL without building the range, even next to rows that are not NULL.
+ const auto large_end = static_cast<Int32>(max_array_size_as_field) * 4;
+ for (const auto* name : {"array_range", "sequence"}) {
+ check_range(name, {{1, 3, 1}, {std::nullopt, large_end, 1}, {2, 6, 2}},
+ {std::vector<Int32> {1, 2}, std::nullopt, std::vector<Int32> {2, 4}});
+ check_range(name, {{1, 3, 0}, {std::nullopt, large_end, 0}},
+ {std::vector<Int32> {1, 2}, std::nullopt}, 0, 2);
+ }
+}
+
} // namespace
} // namespace doris
diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_range_null_argument.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_range_null_argument.out
new file mode 100644
index 00000000000..89f0569b5f3
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_range_null_argument.out
@@ -0,0 +1,15 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !int_range --
+1 [1, 2, 3] [1, 2, 3] [1, 2, 3]
+2 \N \N \N
+3 \N \N \N
+4 [2, 3, 4, 5, 6, 7] \N \N
+5 [0, 1, 2] [0, 2] [0, 2]
+
+-- !datetime_range --
+1 ["2024-01-01 00:00:00.123456", "2024-01-01 00:00:01.123456", "2024-01-01 00:00:02.123456"]
+2 \N
+3 \N
+4 ["2024-01-01 00:00:00.000000", "2024-01-01 00:00:01.000000"]
+5 \N
+
diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_range_null_argument.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_range_null_argument.groovy
new file mode 100644
index 00000000000..6336e214313
--- /dev/null
+++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_range_null_argument.groovy
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_array_range_null_argument") {
+ sql "DROP TABLE IF EXISTS test_array_range_null_argument"
+ sql """
+ CREATE TABLE test_array_range_null_argument (
+ id INT,
+ s INT NULL,
+ e INT NULL,
+ st INT NULL,
+ ds DATETIME(6) NULL,
+ de DATETIME(6) NULL
+ ) DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ // The rows are read in one block. A row with a NULL argument gives NULL, and must not build a
+ // range from the value under the NULL, which can be larger than the array size limit.
+ sql """
+ INSERT INTO test_array_range_null_argument VALUES
+ (1, 1, 4, 1, '2024-01-01 00:00:00.123456', '2024-01-01 00:00:03.123456'),
+ (2, NULL, 5000000, 1, NULL, '2024-01-02 00:00:00'),
+ (3, 2, NULL, 1, '2024-01-01 00:00:00', NULL),
+ (4, 2, 8, NULL, '2024-01-01 00:00:00', '2024-01-01 00:00:02'),
+ (5, 0, 3, 2, NULL, NULL)
+ """
+
+ order_qt_int_range """
+ SELECT id, array_range(s, e), array_range(s, e, st), sequence(s, e, st)
+ FROM test_array_range_null_argument
+ """
+ order_qt_datetime_range """
+ SELECT id, sequence(ds, de, INTERVAL 1 SECOND)
+ FROM test_array_range_null_argument
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 10/35: [fix](regression) Backport regression stability fixes to branch-4.1 (#68449) (#68618)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a8f3b9dd7dc6dae943247fda7aea588929545e84
Author: chunping <zh...@selectdb.com>
AuthorDate: Tue Sep 29 21:36:07 2026 +0800
[fix](regression) Backport regression stability fixes to branch-4.1 (#68449) (#68618)
### What problem does this PR solve?
Issue Number: None
Related PR: #68449
Problem Summary:
Backport #68449 (`50a39819e185f54fbb3a82e67182b1ec8f78433b`) to
`branch-4.1`. The changes stabilize regression clients and orchestration
for TLS/mTLS, Docker isolation, multi-cluster state cleanup, cloud
cache/compaction, asynchronous metadata visibility, and delete-bitmap
restart/concurrency tests. Doris Compose uses exact container-name
matching and retries concurrent removal races.
Resolved 27 conflict paths with these branch adaptations:
- Keep the `nereids_p0` suite paths, generated database names, planner
settings, existing SHOW columns and 4.1-only test logic.
- Preserve configured-master JDBC/HTTP endpoints in the
insert-visible-timeout suite, including mapped ports from #65340.
- Keep the routine-load WHERE expectation `(k00 = 8)`: 4.1's
`PlanUtils.ExpressionToExpr` stores unquoted slot labels, unlike master.
Retain the source topic fix, diagnostic fields and independent polling
counters.
- Keep `nereids_p0/hbo` in the non-concurrent exclusions while excluding
all `cloud_p0` suites as intended by the source fix.
- Combine the shared TLS/compaction helpers with 4.1's existing
framework behavior; do not import unrelated scenarios present only in
the source commit's parent.
- Omit `test_partition_cloud_add_build_index_validation.groovy`: the
partition index-format rollout feature from #66475 and its required
properties/configuration are absent from 4.1.
- Reuse the original PR's result files without manually creating
expected output; retain the deterministic variant ordering fix.
The resulting backport changes 350 files; no FE implementation,
protocol, or storage-format change is added.
### Release note
None
### Check List (For Author)
- Test:
- [x] Regression test on an existing 4.1.4 cloud cluster (1 FE / 3 BE):
127 passed, 3 failed, 186 skipped across all 316 changed suites.
- Retained failures: `test_create_view_nereids` (SHOW VIEW column-name
quoting mismatch), `test_streaming_insert_job_priv` (one successful task
and one pending task after the 300-second wait), and
`test_load_channel_profile` (S3 TVF query reports unknown column `c1`).
Their failed assertions/queries are unchanged by this backport. Root
causes are not established by this run.
- Python support is disabled: 19 attempts hit that prerequisite and 23
more suites were skipped. One JDBC fixture was unavailable, and the
single-BE Java static-load case returned early on this three-BE cluster;
these are counted as skipped, not passes.
- Other skipped suites require dedicated Docker/multi-compute-group
fixtures, restarts, TLS/external services, global configuration/fault
injection, or stress capacity. Four packed-file injection suites were
excluded before launch.
- Arrow Flight authentication passed after correcting only the
task-local endpoint configuration. Three small-table TTL/row-store
compaction follow-ups also passed.
- The failed streaming job and its test user were cleaned up. All FE/BE
nodes remained alive and HTTP health returned 200 after execution.
- [x] Unit Test: `run-regression-test.sh --compile` completed on the
remote validation host; all 35 regression-framework tests passed (0
failures, 0 errors, 0 skips).
- [x] Manual test:
- Diff whitespace and conflict-marker checks passed.
- All 340 changed existing Groovy files passed conversion-phase parsing
with Groovy 4.0.19 / JDK 17; both changed Python files passed
`py_compile`.
- `build-support/check-format.sh` passed with clang-format 16.0.6 (4,400
C/C++ files).
- Compared conflict resolutions with the source patch and branch-4.1
implementations, including RPC/debug-point dependencies and routine-load
expression serialization.
- BE/FE module builds, deployment and clang-tidy were not run, per the
requested existing-cluster validation scope. The live enterprise cluster
is not a build of this backport, so BE debug-point additions and
excluded scenarios still require CI.
- Behavior changed: Yes. Regression framework/test behavior and Doris
Compose discovery match the applicable source fixes; BE additions are
gated debug points.
- Does this need documentation: No.
<details>
<summary>Backport self-review checkpoints</summary>
| Checkpoint | Conclusion |
| --- | --- |
| Goal and tests | Applicable source fixes preserved; syntax/static
checks and 35 framework unit tests pass. Existing-cluster suite outcomes
and remaining CI scope are listed above. |
| Scope | Source patch plus documented branch adaptations; unsupported
feature suite omitted. |
| Concurrency and locking | Existing production lock order retained;
debug-only post-calculation barrier releases the rowset lock before
waiting. Suite worker errors propagate to their owners. |
| Lifecycle | Source connection/process cleanup and shared-state
restoration retained. |
| Configuration | Existing TLS and debug configuration used; referenced
cloud sync settings exist in 4.1. |
| Compatibility | Target RPC/HTTP/debug APIs verified; branch endpoint
mappings and result schemas retained. No wire/storage format changes. |
| Parallel paths | Relevant plaintext/TLS and local/cloud helper paths
retained. |
| Conditional checks | Source bounded polling and negative assertions
retained; branch adaptations checked against implementation. |
| Test coverage | Source coverage retained; 35 framework tests and 127
existing-cluster suites passed, with 3 retained failures and 186
documented skips. |
| Expected results | Source result files reused; no hand-written golden
output. |
| Observability | Source timeout/last-state diagnostics and
token/tablet/transaction debug logs retained. |
| Persistence and failover | No production persistence or replay change.
|
| Writes and atomicity | Production write behavior unchanged outside
enabled test debug points. |
| FE/BE propagation | Existing target sync-load RPC and metadata
endpoints support new helpers. |
| Performance | Production additions are gated debug checks; regression
polling is bounded. |
| Other findings | Focused framework/BE, TLS-suite and cloud-suite
reviews found no unresolved backport-specific blocker. |
</details>
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases and CI results
- [ ] Confirm document
- [ ] Add branch pick label
---
.../cloud/cloud_engine_calc_delete_bitmap_task.cpp | 26 +
be/src/cloud/cloud_meta_mgr.cpp | 10 +
docker/runtime/doris-compose/utils.py | 43 +-
.../cache/ttl/st04_alter_ttl_n_to_0_runtime.out | 7 +-
.../cache/ttl/st06_warmup_ttl_type_assert.out | 4 +-
.../data/cloud_p0/cache/ttl/test_show_ttl.out | 16 -
.../test_double_write_when_schema_change.out | 4 +-
.../org/apache/doris/regression/Config.groovy | 139 ++--
.../doris/regression/action/ProfileAction.groovy | 38 +-
.../doris/regression/suite/ScriptContext.groovy | 11 +
.../org/apache/doris/regression/suite/Suite.groovy | 231 ++++++-
.../doris/regression/suite/SuiteContext.groovy | 55 +-
.../apache/doris/regression/util/DebugPoint.groovy | 12 +-
.../org/apache/doris/regression/util/Http.groovy | 32 +-
.../regression/util/MultiClusterStateGuard.groovy | 195 ++++++
.../doris/regression/util/MySqlClient.groovy | 70 ++
.../doris/regression/util/RowsetMetaUtils.groovy | 79 +++
.../regression/util/WarmupMetricsUtils.groovy | 96 ++-
.../regression/ConfigArrowFlightSqlUrlTest.groovy | 61 ++
.../doris/regression/ConfigRootPasswordTest.groovy | 61 ++
.../suite/CompactionPluginCompatibilityTest.groovy | 154 +++++
.../regression/suite/SuiteSshExecTest.groovy} | 24 +-
.../regression/util/HttpAuthenticationTest.groovy | 72 ++
.../util/MultiClusterStateGuardTest.groovy | 256 ++++++++
.../doris/regression/util/MySqlClientTest.groovy | 96 +++
.../regression/util/RowsetMetaUtilsTest.groovy | 81 +++
.../nonConcurrent/conf/regression-conf.groovy | 2 +-
.../plugins/cloud_multi_cluster_state.groovy | 136 ++++
regression-test/plugins/plugin_compaction.groovy | 134 ++--
.../plugins/plugin_curl_requester.groovy | 3 +-
regression-test/plugins/plugin_index_change.groovy | 5 +-
.../suites/account_p0/test_auth_show.groovy | 4 +-
.../suites/account_p0/test_grant_priv.groovy | 10 +-
.../account_p0/test_grant_priv_resource.groovy | 4 +-
.../account_p0/test_grant_priv_workload.groovy | 4 +-
.../account_p0/test_nereids_authentication.groovy | 14 +-
.../account_p0/test_nereids_row_policy.groovy | 4 +-
.../suites/account_p0/test_set_password.groovy | 6 +-
.../suites/account_p0/test_view_row_policy.groovy | 6 +-
.../ann_topn_small_candidate_fallback.groovy | 21 +-
.../arrow_flight_sql_p0/test_auth_remote_ip.groovy | 3 +-
.../suites/audit/test_audit_log_queue_time.groovy | 212 +++---
.../auth_call/test_dml_select_udf_auth.groovy | 1 +
.../test_dml_stream_load_be_auth_docker.groovy | 49 +-
.../auth_p0/test_role_mapping_system_table.groovy | 2 +-
.../cloud_p0/auth/test_set_default_cluster.groovy | 6 +-
.../balance/test_cross_cg_bvar_metrics.groovy | 27 +-
.../multi_cluster/read_write/sync_insert.groovy | 330 ++++++----
.../read_write/test_multi_stale_rowset.groovy | 70 +-
.../cluster/test_warm_up_cluster_batch.groovy | 409 +++++++-----
...est_warm_up_cluster_event_cancel_expired.groovy | 92 ++-
.../cluster/test_warm_up_compute_group.groovy | 209 +++---
.../warm_up/table/test_warm_up_partition.groovy | 399 +++++++-----
.../test_warm_up_same_table_multi_times.groovy | 43 +-
.../warm_up/table/test_warm_up_table.groovy | 200 ++++--
.../warm_up/table/test_warm_up_tables.groovy | 486 ++++++++------
.../test_clear_file_cache_on_load_failure.groovy | 121 ++--
.../cloud_p0/cache/test_file_cache_info.groovy | 318 +++++----
.../suites/cloud_p0/cache/ttl/alter_ttl_2.groovy | 231 +++----
.../suites/cloud_p0/cache/ttl/alter_ttl_4.groovy | 489 ++++++++------
.../cloud_p0/cache/ttl/alter_ttl_random.groovy | 21 +-
.../cloud_p0/cache/ttl/alter_ttl_seconds.groovy | 170 -----
.../cache/ttl/st04_alter_ttl_n_to_0_runtime.groovy | 233 ++++---
.../cache/ttl/st06_warmup_ttl_type_assert.groovy | 231 ++++---
.../suites/cloud_p0/cache/ttl/test_show_ttl.groovy | 57 ++
.../suites/cloud_p0/cache/ttl/test_show_ttl.sql | 40 --
.../suites/cloud_p0/cache/ttl/test_ttl.groovy | 433 +++++++-----
...st_file_cache_warmup_read_metrics_docker.groovy | 4 +-
.../cache/warm_up/test_warmup_table_docker.groovy | 6 +-
...test_cloud_alter_disable_auto_compaction.groovy | 8 +-
.../async_copy_into/async_load.groovy | 10 +-
.../multi_cluster/copy_into/sync_laod.groovy | 124 ++--
.../cloud_p0/multi_cluster/default_cluster.groovy | 38 +-
.../multi_cluster_s3_load/load.groovy | 66 +-
.../cloud_p0/multi_cluster/rename_cluster.groovy | 10 +-
.../routine_load/test_routine_load.groovy | 6 +-
...test_materialized_view_with_drop_cluster.groovy | 8 +-
...est_materialized_view_with_readd_cluster.groovy | 8 +-
.../test_rollup_with_drop_cluster.groovy | 8 +-
.../test_rollup_with_readd_cluster.groovy | 8 +-
.../test_schema_change_with_drop_cluster.groovy | 8 +-
.../test_schema_change_with_readd_cluster.groovy | 8 +-
.../smooth_upgrade/smooth_upgrade.groovy | 10 +
.../multi_cluster/stream_load/stream_load.groovy | 67 +-
.../stream_load/stream_load_2pc.groovy | 3 +
.../stream_load/stream_load_lb.groovy | 12 +-
.../cloud_p0/multi_cluster/table_rebalance.groovy | 30 +-
.../test_apsaradb_internal_stage.groovy | 31 +-
.../multi_cluster/test_cloud_cluster.groovy | 19 +-
.../multi_cluster/test_drop_cluster.groovy | 98 +--
.../cloud_p0/multi_cluster/test_drop_userid.groovy | 2 +-
.../test_group_commit_multi_cluster.groovy | 12 +-
.../multi_cluster/test_overdue_instance.groovy | 54 +-
.../suites/cloud_p0/multi_cluster/test_tvf.groovy | 11 +-
.../default_vcg_auto_failover.groovy | 19 +-
.../use_vcg_read_write_routine_load.groovy | 722 +++++++++++----------
.../cloud_p0/snapshot/test_snapshot_command.groovy | 12 +-
.../test_clean_tablet_when_drop_force_table.groovy | 144 +++-
.../test_clean_tablet_when_rebalance.groovy | 109 ++--
.../test_create_partition_idempotence.groovy | 3 +-
.../test_insert_overwrite_idempotence.groovy | 3 +-
.../test_schema_change_with_compaction10.groovy | 66 +-
.../test_schema_change_with_compaction11.groovy | 91 +--
.../test_schema_change_with_compaction5.groovy | 25 +-
.../test_schema_change_with_compaction6.groovy | 67 +-
.../test_schema_change_with_compaction9.groovy | 24 +-
.../test_inverted_index_sc_version1_race.groovy | 2 +-
.../test_sc_compaction_cross_v1_retry.groovy | 73 ++-
...c_compaction_cross_v1_stale_base_refresh.groovy | 30 +-
.../test_sc_compaction_optimization.groovy | 50 +-
...est_sc_compaction_optimization_with_load.groovy | 52 +-
.../compaction_width_array_column.groovy | 2 +-
.../compaction/test_admin_compact_tablet.groovy | 88 ++-
...paction_with_dup_key_max_file_size_limit.groovy | 148 ++---
.../test_compaction_uniq_keys_row_store.groovy | 4 +-
.../test_compaction_uniq_keys_row_store_ck.groovy | 4 +-
...test_compaction_update_big_delete_bitmap.groovy | 17 +-
.../test_compaction_with_visible_version.groovy | 3 +
.../test_config_prune_delete_sign.groovy | 129 ++--
...ilecache_with_base_compaction_thresthold.groovy | 150 +++--
.../test_mow_compact_multi_segments.groovy | 48 +-
...paction_agg_and_remove_pre_delete_bitmap.groovy | 17 +-
...test_mow_compaction_and_rowset_not_exist.groovy | 17 +-
.../test_mow_compaction_and_schema_change.groovy | 17 +-
.../test_mow_stale_rowset_delete_bitmap.groovy | 17 +-
.../test_non_mow_key_bounds_aggregation.groovy | 25 +-
.../test_create_view_nereids.groovy | 1 +
.../suites/ddl_p0/test_drop_view_nereids.groovy | 1 +
.../delete/batch-delete-manual.md.groovy | 12 +-
.../import-way/group-commit-manual.md.groovy | 11 +-
.../import/import-way/mysql-load-manual.md.groovy | 20 +-
.../update/unique-update-transaction.md.groovy | 6 +-
.../update/update-of-aggregate-model.md.groovy | 2 +-
.../update/update-of-unique-model.md.groovy | 2 +-
.../test_iceberg_arrow_flight_split_source.groovy | 3 +-
.../select/test_mysql_varbinary_with_udf.groovy | 1 +
.../test_cloud_full_compaction_do_lease.groovy | 142 ++--
.../test_cloud_mow_delete_bitmap_lock_case.groovy | 412 +++++++++---
.../test_cloud_mow_partial_update_retry.groovy | 99 ++-
..._cloud_mow_stale_resp_load_load_conflict.groovy | 105 ++-
.../test_delete_bitmap_lock_with_restart.groovy | 545 ++++++++--------
...est_cold_data_compaction_fault_injection.groovy | 146 +++--
...ion_clone_missing_rowset_fault_injection.groovy | 29 +-
.../test_incomplete_commit_info.groovy | 117 ++--
...st_skip_index_compaction_fault_injection.groovy | 90 +--
.../http_rest_api/get/test_schema_api.groovy | 4 +-
.../http_rest_api/post/test_query_stmt.groovy | 3 +-
.../insert_p0/insert_group_commit_into.groovy | 10 +-
.../mow_insert_with_partition_drop.groovy | 24 +-
.../test_insert_visible_timeout_return_mode.groovy | 16 +-
regression-test/suites/insert_p0/test_jdbc.groovy | 4 +-
.../transaction/test_publish_exception.groovy | 5 +
.../suites/insert_p0/transaction/txn_insert.groovy | 5 +-
.../operate_internal_schema.groovy | 2 +-
.../show_data/test_show_index_data_p2.groovy | 104 +--
.../collect/test_javaudf_groupcount.groovy | 1 +
.../collect/test_javaudf_murmurhash3.groovy | 1 +
.../collect/test_javaudf_sessionize.groovy | 2 +-
.../javaudf_p0/date/test_javaudf_adddays.groovy | 1 +
.../date/test_javaudf_addisoperioud.groovy | 1 +
.../javaudf_p0/date/test_javaudf_daydiff.groovy | 1 +
.../function_meta/function_query_test.groovy | 10 +-
.../suites/javaudf_p0/function_meta/load.groovy | 19 +-
.../json/test_javaudf_convertfromcamelcase.groovy | 1 +
.../json/test_javaudf_converttocamelcase.groovy | 1 +
regression-test/suites/javaudf_p0/null/load.groovy | 1 +
.../sanity/test_javaudf_assertequal.groovy | 1 +
.../sanity/test_javaudf_assertlessthan.groovy | 1 +
.../sanity/test_javaudf_assertudf.groovy | 1 +
.../javaudf_p0/sketch/test_javaudf_md5.groovy | 1 +
.../test_javaudaf_my_date_datetime.groovy | 1 +
.../test_javaudaf_mygroupconcat_string.groovy | 1 +
.../javaudf_p0/test_javaudaf_mysum_array.groovy | 1 +
.../javaudf_p0/test_javaudaf_mysum_decimal.groovy | 1 +
.../test_javaudaf_mysum_float_double.groovy | 1 +
.../javaudf_p0/test_javaudaf_mysum_int.groovy | 1 +
.../javaudf_p0/test_javaudaf_null_test.groovy | 1 +
.../javaudf_p0/test_javaudaf_return_map.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_agg_map.groovy | 1 +
.../javaudf_p0/test_javaudf_all_types.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_array.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_auth.groovy | 5 +-
.../suites/javaudf_p0/test_javaudf_boolean.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_case.groovy | 1 +
.../javaudf_p0/test_javaudf_const_test.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_date.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_decimal.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_float.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_int.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_map.groovy | 1 +
.../javaudf_p0/test_javaudf_multi_evaluate.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_no_input.groovy | 2 +-
.../suites/javaudf_p0/test_javaudf_override.groovy | 1 +
.../suites/javaudf_p0/test_javaudf_ret_map.groovy | 1 +
.../test_javaudf_static_load_test.groovy | 2 +-
.../suites/javaudf_p0/test_javaudf_string.groovy | 1 +
.../javaudf_p0/test_javaudf_struct_type.groovy | 1 +
.../javaudf_p0/test_javaudf_with_decimal.groovy | 1 +
.../javaudf_p0/test_javaudtf_all_types.groovy | 1 +
.../javaudf_p0/test_javaudtf_arrayint.groovy | 1 +
.../suites/javaudf_p0/test_javaudtf_decimal.groovy | 1 +
.../suites/javaudf_p0/test_javaudtf_double.groovy | 1 +
.../suites/javaudf_p0/test_javaudtf_int.groovy | 1 +
.../suites/javaudf_p0/test_javaudtf_string.groovy | 1 +
.../test_streaming_insert_job_priv.groovy | 8 +-
.../test_insert_statistic_after_fe_restart.groovy | 6 +-
.../test_routine_load_advanced_mapping.groovy | 56 +-
.../routine_load/test_routine_load_with_udf.groovy | 1 +
.../scripts/stream_load_redirect_chunked_e2e.py | 16 +-
.../test_packed_file_stream_load_case1.groovy | 2 +-
.../test_packed_file_stream_load_case2.groovy | 2 +-
.../test_packed_file_stream_load_case3.groovy | 2 +-
.../test_packed_file_stream_load_case4.groovy | 2 +-
.../test_packed_file_stream_load_case5.groovy | 11 +-
.../test_packed_file_stream_load_case6.groovy | 11 +-
.../test_packed_file_stream_load_case7.groovy | 11 +-
.../test_packed_file_stream_load_case8.groovy | 11 +-
.../load_p0/stream_load/test_sink_tolerate.groovy | 3 +-
.../load_p0/stream_load/test_stream_load.groovy | 38 +-
.../stream_load/test_stream_load_endpoint.groovy | 141 ++--
...test_stream_load_fe_redirect_chunked_e2e.groovy | 10 +-
.../stream_load/test_stream_load_with_udf.groovy | 1 +
.../suites/manager/test_manager_interface_3.groovy | 317 +++------
.../suites/mtmv_p0/test_expand_star_mtmv.groovy | 1 +
.../mysql_ssl_p0/test_mysql_connection.groovy | 66 +-
.../cache/parse_sql_from_sql_cache.groovy | 21 +-
.../insert_into_table/insert_auth.groovy | 6 +-
.../javaudf/collect/test_javaudf_groupcount.groovy | 1 +
.../collect/test_javaudf_murmurhash3.groovy | 1 +
.../javaudf/collect/test_javaudf_sessionize.groovy | 2 +-
.../javaudf/date/test_javaudf_adddays.groovy | 1 +
.../javaudf/date/test_javaudf_addisoperioud.groovy | 1 +
.../javaudf/date/test_javaudf_daydiff.groovy | 1 +
.../json/test_javaudf_convertfromcamelcase.groovy | 1 +
.../json/test_javaudf_converttocamelcase.groovy | 1 +
.../javaudf/sanity/test_javaudf_assertequal.groovy | 1 +
.../sanity/test_javaudf_assertlessthan.groovy | 1 +
.../javaudf/sanity/test_javaudf_assertudf.groovy | 1 +
.../javaudf/sketch/test_javaudf_md5.groovy | 1 +
.../javaudf/test_javaudaf_my_date_datetime.groovy | 1 +
.../test_javaudaf_mygroupconcat_string.groovy | 1 +
.../javaudf/test_javaudaf_mysum_decimal.groovy | 1 +
.../test_javaudaf_mysum_float_double.groovy | 1 +
.../javaudf/test_javaudaf_mysum_int.groovy | 1 +
.../javaudf/test_javaudaf_null_test.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_boolean.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_case.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_date.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_decimal.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_float.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_int.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_ip.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_null.groovy | 1 +
.../nereids_p0/javaudf/test_javaudf_string.groovy | 1 +
.../agg_state/test_outfile_agg_state.groovy | 2 +-
.../agg_state_array/test_outfile_agg_array.groovy | 2 +-
.../test_outfile_agg_state_bitmap.groovy | 2 +-
.../nereids_p0/outfile/hll/test_outfile_hll.groovy | 2 +-
.../test_outfile_quantile_state.groovy | 2 +-
.../show/test_nereids_show_functions.groovy | 13 +-
.../stats/col_stats/hot_value_analyze_sync.groovy | 2 +-
.../test_partition_create_tablet_rr.groovy | 1 +
.../auto_partition/sql/multi_thread_load.groovy | 30 +-
.../sql/stress_test_insert_into.groovy | 22 +-
.../diff_data/stress_test_diff_date_list.groovy | 16 +-
.../stress_test_high_concurrency_load.groovy | 19 +-
.../same_data/stress_test_same_date_range.groovy | 19 +-
.../stress_test_two_stream_load.groovy | 19 +-
.../suites/point_query_p0/test_point_query.groovy | 10 +-
.../point_query_p0/test_point_query_ck.groovy | 17 +-
.../test_point_query_partition.groovy | 6 +-
.../suites/point_query_p0/test_rowstore.groovy | 14 +-
.../suites/point_query_p0/test_rowstore_ck.groovy | 14 +-
.../test_python_raise_error_propagation.groovy | 3 +
.../pythonudaf_p0/test_pythonudaf_basic.groovy | 1 +
...st_pythonudaf_complex_aggregation_module.groovy | 1 +
..._pythonudaf_complex_state_objects_module.groovy | 1 +
.../pythonudaf_p0/test_pythonudaf_drop.groovy | 2 +
.../test_pythonudaf_forbidden_module.groovy | 1 +
.../test_pythonudaf_pkg_isolation.groovy | 2 +
.../pythonudaf_p0/test_pythonudaf_simple.groovy | 1 +
.../test_pythonudaf_window_advanced_module.groovy | 1 +
.../test_python_udaf_complex.groovy | 1 +
.../test_python_udf_business_logic.groovy | 1 +
.../test_python_udf_external_api.groovy | 3 +-
.../test_python_udf_nlp_chinese.groovy | 1 +
.../test_python_udtf_complex.groovy | 1 +
.../sanity/test_pythonudf_assertequal.groovy | 1 +
.../sanity/test_pythonudf_assertlessthan.groovy | 1 +
.../pythonudf_p0/test_pythonudf_array.groovy | 1 +
.../test_pythonudf_base_data_type.groovy | 1 +
.../pythonudf_p0/test_pythonudf_boolean.groovy | 1 +
.../test_pythonudf_complex_data_type.groovy | 1 +
.../suites/pythonudf_p0/test_pythonudf_drop.groovy | 2 +
.../test_pythonudf_file_protocol.groovy | 1 +
.../pythonudf_p0/test_pythonudf_float.groovy | 1 +
.../test_pythonudf_forbidden_module.groovy | 1 +
.../suites/pythonudf_p0/test_pythonudf_int.groovy | 1 +
.../suites/pythonudf_p0/test_pythonudf_map.groovy | 1 +
.../pythonudf_p0/test_pythonudf_module.groovy | 1 +
.../test_pythonudf_module_advanced.groovy | 1 +
.../test_pythonudf_module_scalar.groovy | 1 +
.../test_pythonudf_module_vector.groovy | 1 +
.../test_pythonudf_pkg_isolation.groovy | 2 +
.../pythonudf_p0/test_pythonudf_ret_map.groovy | 1 +
.../pythonudf_p0/test_pythonudf_string.groovy | 1 +
.../test_pythonudtf_basic_module.groovy | 1 +
.../test_pythonudtf_data_types_module.groovy | 1 +
.../pythonudtf_p0/test_pythonudtf_drop.groovy | 2 +
.../test_pythonudtf_edge_cases_module.groovy | 1 +
.../test_pythonudtf_exceptions_module.groovy | 1 +
.../test_pythonudtf_forbidden_module.groovy | 1 +
.../test_pythonudtf_io_patterns_module.groovy | 1 +
.../test_pythonudtf_pkg_isolation.groovy | 2 +
.../test_pythonudtf_sql_integration_module.groovy | 1 +
.../query_p0/cache/query_cache_with_context.groovy | 3 +-
.../query_p0/system/test_partitions_schema.groovy | 6 +-
.../query_p0/system/test_table_options.groovy | 6 +-
.../query_p0/system/test_table_properties.groovy | 6 +-
.../suites/query_p0/test_row_policy.groovy | 4 +-
.../query_p2/big_join_build/big_join_build.groovy | 2 +-
.../test_set_operation_large_string.groovy | 2 +-
.../suites/query_profile/profile_safe.groovy | 5 +-
.../query_profile/s3_load_profile_test.groovy | 55 +-
.../query_profile/test_load_channel_profile.groovy | 6 +-
...e_column_with_delete_drop_column_dup_key.groovy | 82 +--
...olumn_with_delete_drop_column_unique_key.groovy | 63 +-
.../test_filecache_with_alter_table.groovy | 36 +-
.../test_alter_table_column_rename.groovy | 2 +-
.../test_schema_change_fail.groovy | 11 +-
...test_show_create_table_and_views_nereids.groovy | 89 ++-
.../suites/show_p0/test_show_data.groovy | 2 +-
.../suites/show_p0/test_show_processlist.groovy | 6 +-
.../suites/statistics/test_analyze_all_null.groovy | 2 +-
.../suites/statistics/test_hot_value.groovy | 2 +-
.../test_skip_agg_table_value_column.groovy | 2 +-
.../test_partial_update_2pc_schema_change.groovy | 38 +-
.../legacy/test_f_2pc_schema_change.groovy | 38 +-
.../publish/test_f_auto_inc_compaction.groovy | 35 +-
.../test_f_seq_publish_read_from_old.groovy | 33 +-
...xble_partial_update_publish_conflict_seq.groovy | 35 +-
...flexible_partial_update_publish_conflict.groovy | 35 +-
.../test_key_bounds_truncation_basic.groovy | 27 +-
...est_key_bounds_truncation_read_scenarios.groovy | 39 +-
.../test_partial_update_publish_all_del.groovy | 35 +-
.../publish/test_partial_update_publish_seq.groovy | 35 +-
...partial_update_delete_sign_with_conflict.groovy | 33 +-
.../test_partial_update_multi_stmt.groovy | 13 +-
.../test_double_write_when_schema_change.groovy | 4 +-
.../suites/variant_p0/variant_with_rowstore.groovy | 4 +-
350 files changed, 8633 insertions(+), 5157 deletions(-)
diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
index ffa7529d172..32c96474be7 100644
--- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
+++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
@@ -213,6 +213,15 @@ Status CloudTabletCalcDeleteBitmapTask::handle(int64_t queue_time_us) const {
}
int64_t t3 = MonotonicMicros();
+ DBUG_EXECUTE_IF("CloudEngineCalcDeleteBitmapTask.handle.block_for_restart", {
+ if (dp->param<int64_t>("tablet_id", -1) == _tablet_id) {
+ // The restart regression pairs this marker with the persisted MS lock owner.
+ LOG(INFO) << "delete bitmap restart barrier token="
+ << dp->param<std::string>("token", "") << " lock_id=" << _transaction_id
+ << " tablet_id=" << _tablet_id;
+ DBUG_BLOCK;
+ }
+ });
DBUG_EXECUTE_IF("CloudEngineCalcDeleteBitmapTask.handle.inject_sleep", {
auto p = dp->param("percent", 0.01);
// 100s > Config.calculate_delete_bitmap_task_timeout_seconds = 60s
@@ -275,6 +284,23 @@ Status CloudTabletCalcDeleteBitmapTask::handle(int64_t queue_time_us) const {
DCHECK(invisible_rowsets.size() == i + 1 - empty_rowset_count);
}
}
+ DBUG_EXECUTE_IF("CloudTabletCalcDeleteBitmapTask.handle.block_after_calc", {
+ auto target_tablet_id = dp->param<int64_t>("tablet_id", 0);
+ auto target_transaction_id = dp->param<int64_t>("transaction_id", 0);
+ if (status.ok() && target_tablet_id == _tablet_id &&
+ (target_transaction_id == 0 || target_transaction_id == _transaction_id)) {
+ // Delay the response after calculation without preventing another transaction from
+ // calculating the delete bitmap on the same tablet.
+ wrlock.unlock();
+ LOG(INFO) << "start debug block after calculating delete bitmap"
+ << ", tablet_id=" << _tablet_id << ", txn_id=" << _transaction_id;
+ while (DebugPoints::instance()->is_enable(DP_NAME)) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(50));
+ }
+ LOG(INFO) << "finish debug block after calculating delete bitmap"
+ << ", tablet_id=" << _tablet_id << ", txn_id=" << _transaction_id;
+ }
+ });
DBUG_EXECUTE_IF("CloudCalcDbmTask.handle.return.block",
auto target_tablet_id = dp->param<int64_t>("tablet_id", 0);
if (target_tablet_id == tablet->tablet_id()) {DBUG_BLOCK});
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index d6293997e77..f9cf4b31609 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -2338,6 +2338,16 @@ Status CloudMetaMgr::get_delete_bitmap_update_lock(const CloudTablet& tablet, in
"initiator {}",
tablet.table_id(), lock_id, initiator);
}
+ DBUG_EXECUTE_IF("CloudMetaMgr.get_delete_bitmap_update_lock.block_for_restart", {
+ if (st.ok() && res.status().code() == MetaServiceCode::OK &&
+ dp->param<int64_t>("table_id", -1) == tablet.table_id()) {
+ // Pause only a successful acquisition, never a conflict or failed RPC.
+ LOG(INFO) << "delete bitmap restart barrier token="
+ << dp->param<std::string>("token", "") << " lock_id=" << lock_id
+ << " tablet_id=" << tablet.tablet_id() << " initiator=" << initiator;
+ DBUG_BLOCK;
+ }
+ });
return st;
}
diff --git a/docker/runtime/doris-compose/utils.py b/docker/runtime/doris-compose/utils.py
index 0f41728d3f2..ff7f5e4b033 100644
--- a/docker/runtime/doris-compose/utils.py
+++ b/docker/runtime/doris-compose/utils.py
@@ -21,6 +21,7 @@ import jsonpickle
import logging
import os
import pwd
+import re
import socket
import subprocess
import sys
@@ -145,24 +146,46 @@ def is_container_running(container):
# return all doris containers when cluster_names is empty
def get_doris_containers(cluster_names):
- if cluster_names:
- if type(cluster_names) == type(""):
- filter_names = "{}{}-*".format(DORIS_PREFIX, cluster_names)
- else:
- filter_names = "|".join([
- "{}{}-*".format(DORIS_PREFIX, name) for name in cluster_names
- ])
+ import cluster
+ if isinstance(cluster_names, str):
+ requested_names = {cluster_names} if cluster_names else set()
else:
- filter_names = "{}*".format(DORIS_PREFIX)
+ requested_names = set(cluster_names or [])
+
+ # Docker's name filter matches substrings, so anchor the complete service
+ # name before the SDK inspects each matching container. Otherwise a lookup
+ # for "test_vcg" also inspects "test_vcg_warmup..." containers.
+ name_pattern = ("|".join(re.escape(name) for name in sorted(requested_names))
+ if requested_names else ".+")
+ node_pattern = "|".join(re.escape(node_type)
+ for node_type in cluster.Node.TYPE_ALL)
+ filter_names = r"^/{}({})-({})-[0-9]+$".format(
+ re.escape(DORIS_PREFIX), name_pattern, node_pattern)
clusters = {}
client = docker.client.from_env()
- containers = client.containers.list(filters={"name": filter_names})
+ max_list_attempts = 3
+ for attempt in range(1, max_list_attempts + 1):
+ try:
+ # A container may be removed between Docker's list and inspect.
+ containers = client.containers.list(filters={"name": filter_names},
+ ignore_removed=True)
+ break
+ except docker.errors.NotFound as err:
+ if attempt == max_list_attempts:
+ raise
+ LOG.warning(
+ "Docker returned 404 while listing Doris containers; "
+ "retrying enumeration (%d/%d): %s",
+ attempt,
+ max_list_attempts,
+ err,
+ )
for container in containers:
cluster_name, _, _ = parse_service_name(container.name)
if not cluster_name:
continue
- if cluster_names and cluster_name not in cluster_names:
+ if requested_names and cluster_name not in requested_names:
continue
if cluster_name not in clusters:
clusters[cluster_name] = []
diff --git a/regression-test/data/cloud_p0/cache/ttl/st04_alter_ttl_n_to_0_runtime.out b/regression-test/data/cloud_p0/cache/ttl/st04_alter_ttl_n_to_0_runtime.out
index 475b9ff01e9..eaa6918709a 100644
--- a/regression-test/data/cloud_p0/cache/ttl/st04_alter_ttl_n_to_0_runtime.out
+++ b/regression-test/data/cloud_p0/cache/ttl/st04_alter_ttl_n_to_0_runtime.out
@@ -1,4 +1,7 @@
-- This file is automatically generated. You should know what you did if you want to edit this
--- !sql --
-400
+-- !before_alter --
+400 79800 3490
+
+-- !after_alter --
+400 79800 3490
diff --git a/regression-test/data/cloud_p0/cache/ttl/st06_warmup_ttl_type_assert.out b/regression-test/data/cloud_p0/cache/ttl/st06_warmup_ttl_type_assert.out
index a8419e02ed9..0db2a386610 100644
--- a/regression-test/data/cloud_p0/cache/ttl/st06_warmup_ttl_type_assert.out
+++ b/regression-test/data/cloud_p0/cache/ttl/st06_warmup_ttl_type_assert.out
@@ -1,7 +1,7 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !source_preheat --
-200
+19900 2690
-- !target_query --
-200
+19900 2690
diff --git a/regression-test/data/cloud_p0/cache/ttl/test_show_ttl.out b/regression-test/data/cloud_p0/cache/ttl/test_show_ttl.out
deleted file mode 100644
index 1d2ad674a77..00000000000
--- a/regression-test/data/cloud_p0/cache/ttl/test_show_ttl.out
+++ /dev/null
@@ -1,16 +0,0 @@
--- This file is automatically generated. You should know what you did if you want to edit this
--- !test_show_ttl --
-0
-
--- !test_show_ttl_2 --
-0
-
--- !test_show_ttl_3 --
-customer_ttl CREATE TABLE `customer_ttl` (\n `C_CUSTKEY` int NOT NULL,\n `C_NAME` varchar(25) NOT NULL,\n `C_ADDRESS` varchar(40) NOT NULL,\n `C_NATIONKEY` int NOT NULL,\n `C_PHONE` char(15) NOT NULL,\n `C_ACCTBAL` decimal(15,2) NOT NULL,\n `C_MKTSEGMENT` char(10) NOT NULL,\n `C_COMMENT` varchar(117) NOT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`C_CUSTKEY`, `C_NAME`)\nDISTRIBUTED BY HASH(`C_CUSTKEY`) BUCKETS 32\nPROPERTIES (\n"file_cache_ttl_seconds" = "300",\n"is_being_synced" = "fals [...]
-
--- !test_show_ttl_4 --
-0
-
--- !test_show_ttl_5 --
-customer_ttl CREATE TABLE `customer_ttl` (\n `C_CUSTKEY` int NOT NULL,\n `C_NAME` varchar(25) NOT NULL,\n `C_ADDRESS` varchar(40) NOT NULL,\n `C_NATIONKEY` int NOT NULL,\n `C_PHONE` char(15) NOT NULL,\n `C_ACCTBAL` decimal(15,2) NOT NULL,\n `C_MKTSEGMENT` char(10) NOT NULL,\n `C_COMMENT` varchar(117) NOT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`C_CUSTKEY`, `C_NAME`)\nDISTRIBUTED BY HASH(`C_CUSTKEY`) BUCKETS 32\nPROPERTIES (\n"file_cache_ttl_seconds" = "0",\n"is_being_synced" = "false" [...]
-
diff --git a/regression-test/data/variant_p0/schema_change/test_double_write_when_schema_change.out b/regression-test/data/variant_p0/schema_change/test_double_write_when_schema_change.out
index 1190a5f30bd..37aba8eb64c 100644
--- a/regression-test/data/variant_p0/schema_change/test_double_write_when_schema_change.out
+++ b/regression-test/data/variant_p0/schema_change/test_double_write_when_schema_change.out
@@ -28,7 +28,7 @@ PullRequestEvent 2489395770 2015-01-01T01:00:02Z
1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/1310570?","gravatar_id":"","id":1310570,"login":"soumith","url":"https://api.github.com/users/soumith"},"created_at":"2015-01-01T01:00:00Z","id":"2489395767","payload":{"before":"2d06657267b32e0c8e193c617039da200f710195","commits":[{"sha":"dbd68d30ee1f7b60d404553fc1c6226ebb374c8e","author":{"email":"88de463b5797707cf3425f85a415c3d869db732b@gmail.com","name":"Soumith Chintala"},"message":"back to old structure, except lua f [...]
1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/6799218?","gravatar_id":"","id":6799218,"login":"gyuho","url":"https://api.github.com/users/gyuho"},"created_at":"2015-01-01T02:00:00Z","id":"2489418154","payload":{"before":"79d9868b8a1afbc5c6c42d33fcc7b97524f888ff","commits":[{"sha":"cf79996e894954b28db73a6bed54ba490c63f174","author":{"email":"0dfbe0912134938594f432abfeb9cb2f2e9092c9@gmail.com","name":"Gyu-Ho Lee"},"message":"Commit with markdownx at 2014-12-31 17:59:58" [...]
1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/4188511?","gravatar_id":"","id":4188511,"login":"iganari","url":"https://api.github.com/users/iganari"},"created_at":"2015-01-01T03:00:01Z","id":"2489436569","payload":{"action":"opened","number":44,"pull_request":{"_links":{"comments":{"href":"https://api.github.com/repos/iganari/mrtg/issues/44/comments"},"commits":{"href":"https://api.github.com/repos/iganari/mrtg/pulls/44/commits"},"html":{"href":"https://github.com/iga [...]
-1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/100067519?","display_login":"savorfamily","gravatar_id":"123","id":100067519,"login":"savorfamily","url":"https://api.github.com/users/savorfamily"},"created_at":"2022-11-07T02:00:00Z","id":"25061216922","payload":{"before":"2bf823a4febcf809da126828ecef7617c8cc48ea","commits":[{"sha":"81106d369f763cb729d9d77610ace252c9db53f0","author":{"email":"73812536+upptime-bot@users.noreply.github.com","name":"Upptime Bot"},"message": [...]
+1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/100067519?","display_login":"savorfamily","gravatar_id":"123","id":100067519,"login":"savorfamily","url":"https://api.github.com/users/savorfamily"},"created_at":"2022-11-07T02:00:00Z","id":"25061216922","payload":{"before":"2bf823a4febcf809da126828ecef7617c8cc48ea","commits":[{"sha":"81106d369f763cb729d9d77610ace252c9db53f0","author":{"email":"73812536+upptime-bot@users.noreply.github.com","name":"Upptime Bot"},"message": [...]
1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/100067519?","display_login":"savorfamily","gravatar_id":"123","id":100067519,"login":"savorfamily","url":"https://api.github.com/users/savorfamily"},"created_at":"2022-11-07T02:00:00Z","id":"25061216922","payload":{"before":"2bf823a4febcf809da126828ecef7617c8cc48ea","commits":[{"sha":"81106d369f763cb729d9d77610ace252c9db53f0","author":{"email":"73812536+upptime-bot@users.noreply.github.com","name":"Upptime Bot"},"message": [...]
1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/100067519?","display_login":"savorfamily","gravatar_id":"123","id":100067519,"login":"savorfamily","url":"https://api.github.com/users/savorfamily"},"created_at":"2022-11-07T02:00:00Z","id":"25061216922","payload":{"before":"2bf823a4febcf809da126828ecef7617c8cc48ea","commits":[{"sha":"81106d369f763cb729d9d77610ace252c9db53f0","author":{"email":"73812536+upptime-bot@users.noreply.github.com","name":"Upptime Bot"},"message": [...]
1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/100067519?","display_login":"savorfamily","gravatar_id":"123","id":100067519,"login":"savorfamily","url":"https://api.github.com/users/savorfamily"},"created_at":"2022-11-07T02:00:00Z","id":"25061216922","payload":{"before":"2bf823a4febcf809da126828ecef7617c8cc48ea","commits":[{"sha":"81106d369f763cb729d9d77610ace252c9db53f0","author":{"email":"73812536+upptime-bot@users.noreply.github.com","name":"Upptime Bot"},"message": [...]
@@ -40,7 +40,7 @@ PullRequestEvent 2489395770 2015-01-01T01:00:02Z
1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/1310570?","gravatar_id":"","id":1310570,"login":"soumith","url":"https://api.github.com/users/soumith"},"created_at":"2015-01-01T01:00:00Z","id":"2489395767","payload":{"before":"2d06657267b32e0c8e193c617039da200f710195","commits":[{"sha":"dbd68d30ee1f7b60d404553fc1c6226ebb374c8e","author":{"email":"88de463b5797707cf3425f85a415c3d869db732b@gmail.com","name":"Soumith Chintala"},"message":"back to old structure, except lua f [...]
1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/6799218?","gravatar_id":"","id":6799218,"login":"gyuho","url":"https://api.github.com/users/gyuho"},"created_at":"2015-01-01T02:00:00Z","id":"2489418154","payload":{"before":"79d9868b8a1afbc5c6c42d33fcc7b97524f888ff","commits":[{"sha":"cf79996e894954b28db73a6bed54ba490c63f174","author":{"email":"0dfbe0912134938594f432abfeb9cb2f2e9092c9@gmail.com","name":"Gyu-Ho Lee"},"message":"Commit with markdownx at 2014-12-31 17:59:58" [...]
1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/4188511?","gravatar_id":"","id":4188511,"login":"iganari","url":"https://api.github.com/users/iganari"},"created_at":"2015-01-01T03:00:01Z","id":"2489436569","payload":{"action":"opened","number":44,"pull_request":{"_links":{"comments":{"href":"https://api.github.com/repos/iganari/mrtg/issues/44/comments"},"commits":{"href":"https://api.github.com/repos/iganari/mrtg/pulls/44/commits"},"html":{"href":"https://github.com/iga [...]
-1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/100067519?","display_login":"savorfamily","gravatar_id":"123","id":100067519,"login":"savorfamily","url":"https://api.github.com/users/savorfamily"},"created_at":"2022-11-07T02:00:00Z","id":"25061216922","payload":{"before":"2bf823a4febcf809da126828ecef7617c8cc48ea","commits":[{"sha":"81106d369f763cb729d9d77610ace252c9db53f0","author":{"email":"73812536+upptime-bot@users.noreply.github.com","name":"Upptime Bot"},"message": [...]
+1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/100067519?","display_login":"savorfamily","gravatar_id":"123","id":100067519,"login":"savorfamily","url":"https://api.github.com/users/savorfamily"},"created_at":"2022-11-07T02:00:00Z","id":"25061216922","payload":{"before":"2bf823a4febcf809da126828ecef7617c8cc48ea","commits":[{"sha":"81106d369f763cb729d9d77610ace252c9db53f0","author":{"email":"73812536+upptime-bot@users.noreply.github.com","name":"Upptime Bot"},"message": [...]
1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/100067519?","display_login":"savorfamily","gravatar_id":"123","id":100067519,"login":"savorfamily","url":"https://api.github.com/users/savorfamily"},"created_at":"2022-11-07T02:00:00Z","id":"25061216922","payload":{"before":"2bf823a4febcf809da126828ecef7617c8cc48ea","commits":[{"sha":"81106d369f763cb729d9d77610ace252c9db53f0","author":{"email":"73812536+upptime-bot@users.noreply.github.com","name":"Upptime Bot"},"message": [...]
1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/100067519?","display_login":"savorfamily","gravatar_id":"123","id":100067519,"login":"savorfamily","url":"https://api.github.com/users/savorfamily"},"created_at":"2022-11-07T02:00:00Z","id":"25061216922","payload":{"before":"2bf823a4febcf809da126828ecef7617c8cc48ea","commits":[{"sha":"81106d369f763cb729d9d77610ace252c9db53f0","author":{"email":"73812536+upptime-bot@users.noreply.github.com","name":"Upptime Bot"},"message": [...]
1 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/100067519?","display_login":"savorfamily","gravatar_id":"123","id":100067519,"login":"savorfamily","url":"https://api.github.com/users/savorfamily"},"created_at":"2022-11-07T02:00:00Z","id":"25061216922","payload":{"before":"2bf823a4febcf809da126828ecef7617c8cc48ea","commits":[{"sha":"81106d369f763cb729d9d77610ace252c9db53f0","author":{"email":"73812536+upptime-bot@users.noreply.github.com","name":"Upptime Bot"},"message": [...]
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index b4be1a88c21..85fa5a3ef4e 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -1145,7 +1145,9 @@ class Config {
void fetchCloudMode() {
if (runMode == RunMode.UNKNOWN) {
try {
- def result = JdbcUtils.executeToMapArray(getRootConnection(), "SHOW FRONTEND CONFIG LIKE 'cloud_unique_id'")
+ def result = getRootConnection().withCloseable { conn ->
+ JdbcUtils.executeToMapArray(conn, "SHOW FRONTEND CONFIG LIKE 'cloud_unique_id'")
+ }
runMode = result[0].Value.toString().isEmpty() ? RunMode.NOT_CLOUD : RunMode.CLOUD
} catch (Throwable t) {
throw new IllegalStateException("Fetch server config 'cloud_unique_id' failed, jdbcUrl: ${jdbcUrl}", t)
@@ -1155,7 +1157,9 @@ class Config {
boolean isClusterKeyEnabled() {
try {
- def result = JdbcUtils.executeToMapArray(getRootConnection(), "SHOW FRONTEND CONFIG LIKE 'random_add_order_by_keys_for_mow'")
+ def result = getRootConnection().withCloseable { conn ->
+ JdbcUtils.executeToMapArray(conn, "SHOW FRONTEND CONFIG LIKE 'random_add_order_by_keys_for_mow'")
+ }
log.info("show random_add_order_by_keys_for_mow config: ${result}".toString())
return result[0].Value.toString().equalsIgnoreCase("true")
} catch (Throwable t) {
@@ -1199,8 +1203,12 @@ class Config {
return DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword)
}
+ String getRootPassword() {
+ return (jdbcUser == 'root' && jdbcPassword) ? jdbcPassword : ''
+ }
+
Connection getRootConnection() {
- return DriverManager.getConnection(jdbcUrl, 'root', '')
+ return DriverManager.getConnection(jdbcUrl, 'root', getRootPassword())
}
Connection getConnectionByDbName(String dbName) {
@@ -1219,14 +1227,7 @@ class Config {
Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver")
String arrowFlightSqlHost = otherConfigs.get("extArrowFlightSqlHost")
String arrowFlightSqlPort = otherConfigs.get("extArrowFlightSqlPort")
- String arrowFlightSqlUrl = "jdbc:arrow-flight-sql://${arrowFlightSqlHost}:${arrowFlightSqlPort}" +
- "/?useServerPrepStmts=false&useSSL=false&useEncryption=false"
- // Arrow 17.0.0-rc03 support jdbc:arrow-flight-sql connect db
- // https://github.com/apache/arrow/issues/41947
- if (dbName?.trim()) {
- arrowFlightSqlUrl = "jdbc:arrow-flight-sql://${arrowFlightSqlHost}:${arrowFlightSqlPort}" +
- "/catalog=" + dbName + "?useServerPrepStmts=false&useSSL=false&useEncryption=false"
- }
+ String arrowFlightSqlUrl = getArrowFlightSqlJdbcUrl(arrowFlightSqlHost, arrowFlightSqlPort, dbName)
tryCreateDbIfNotExist(dbName)
log.info("connect to ${arrowFlightSqlUrl}".toString())
String arrowFlightSqlJdbcUser = otherConfigs.get("extArrowFlightSqlUser")
@@ -1234,6 +1235,27 @@ class Config {
return DriverManager.getConnection(arrowFlightSqlUrl, arrowFlightSqlJdbcUser, arrowFlightSqlJdbcPassword)
}
+ String getArrowFlightSqlJdbcUrl(String host, String port, String dbName = null) {
+ // Arrow 17.0.0-rc03 supports selecting the database with /catalog=<dbName>.
+ // Flight JDBC uses useEncryption and PEM paths rather than MySQL's useSSL/key stores.
+ String catalog = dbName?.trim() ? "catalog=${dbName}" : ""
+ boolean enableTls = otherConfigs.get("enableTLS")?.toString()?.equalsIgnoreCase("true") ?: false
+ String url = "jdbc:arrow-flight-sql://${host}:${port}/${catalog}?useServerPrepStmts=false&useEncryption=${enableTls}"
+ if (enableTls) {
+ String verifyMode = otherConfigs.get("tlsVerifyMode")?.toString()?.toLowerCase() ?: "strict"
+ if (verifyMode == "none") {
+ url += "&disableCertificateVerification=true"
+ } else if (otherConfigs.get("trustCACert")) {
+ url += "&tlsRootCerts=${otherConfigs.get('trustCACert')}"
+ }
+ if (otherConfigs.get("trustCert") && otherConfigs.get("trustCAKey")) {
+ url += "&clientCertificate=${otherConfigs.get('trustCert')}" +
+ "&clientKey=${otherConfigs.get('trustCAKey')}"
+ }
+ }
+ return url
+ }
+
Connection getDownstreamConnection() {
return DriverManager.getConnection(ccrDownstreamUrl, ccrDownstreamUser, ccrDownstreamPassword)
}
@@ -1300,24 +1322,17 @@ class Config {
return jdbcUrl
}
- String urlWithDb = jdbcUrl
- String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3)
- if (urlWithoutSchema.indexOf("/") >= 0) {
- if (jdbcUrl.contains("?")) {
- // e.g: jdbc:mysql://locahost:8080/?a=b
- urlWithDb = jdbcUrl.substring(0, jdbcUrl.lastIndexOf("?"))
- urlWithDb = urlWithDb.substring(0, urlWithDb.lastIndexOf("/"))
- urlWithDb += ("/" + dbName) + jdbcUrl.substring(jdbcUrl.lastIndexOf("?"))
- } else {
- // e.g: jdbc:mysql://locahost:8080/
- urlWithDb += dbName
- }
- } else {
- // e.g: jdbc:mysql://locahost:8080
- urlWithDb += ("/" + dbName)
+ int authorityStart = jdbcUrl.indexOf("://") + 3
+ int queryStart = jdbcUrl.indexOf("?", authorityStart)
+ int pathStart = jdbcUrl.indexOf("/", authorityStart)
+ if (pathStart >= 0 && (queryStart < 0 || pathStart < queryStart)) {
+ return jdbcUrl.substring(0, pathStart + 1) + dbName +
+ (queryStart >= 0 ? jdbcUrl.substring(queryStart) : "")
}
-
- return urlWithDb
+ if (queryStart >= 0) {
+ return jdbcUrl.substring(0, queryStart) + "/" + dbName + jdbcUrl.substring(queryStart)
+ }
+ return jdbcUrl + "/" + dbName
}
public static String buildUrlWithDb(String jdbcUrl, String dbName) {
@@ -1330,8 +1345,7 @@ class Config {
public static String buildUrlWithDb (String jdbcUrl, String dbName, String keyStorePath, String keyStorePassword, String trustStorePath, String trustStorePassword) {
String urlWithDb = buildUrlWithDbImpl(jdbcUrl, dbName);
- urlWithDb = addTlsUrl(urlWithDb, keyStorePath, keyStorePassword, trustStorePath, trustStorePassword);
- urlWithDb = addTimeoutUrl(urlWithDb);
+ urlWithDb = buildTlsJdbcUrl(urlWithDb, keyStorePath, keyStorePassword, trustStorePath, trustStorePassword);
return urlWithDb
}
@@ -1343,8 +1357,27 @@ class Config {
return url
}
+ public static String buildUrlWithDb(String host, int queryPort, String dbName,
+ String keyStorePath, String keyStorePassword,
+ String trustStorePath, String trustStorePassword) {
+ def url = String.format(
+ "jdbc:mysql://%s:%s/?useLocalSessionState=true&allowLoadLocalInfile=false",
+ host, queryPort)
+ url = buildUrlWithDb(url, dbName, keyStorePath, keyStorePassword, trustStorePath, trustStorePassword)
+ return url
+ }
+
+ public static String buildTlsJdbcUrl(String jdbcUrl, String keyStorePath, String keyStorePassword,
+ String trustStorePath, String trustStorePassword) {
+ String tlsUrl = addTlsUrl(jdbcUrl, keyStorePath, keyStorePassword, trustStorePath, trustStorePassword)
+ return addTimeoutUrl(tlsUrl)
+ }
+
private static String addSslUrl(String url) {
- if (url.contains("TLS")) {
+ // A configured TLS URL already has its own verification policy and key stores.
+ // Appending the optional-SSL defaults would override requireSSL and certificate verification.
+ if (url.contains("useSSL=") || url.contains("sslMode=") ||
+ url.contains("clientCertificateKeyStoreUrl=") || url.contains("trustCertificateKeyStoreUrl=")) {
return url
}
// ssl-mode = PREFERRED
@@ -1366,25 +1399,33 @@ class Config {
}
private static String addTlsUrl(String url, String keyStorePath, String keyStorePassword, String trustStorePath, String trustStorePassword) {
- // ssl-mode = PREFERRED
- String useSsl = "true"
- String requireSsl = "true"
- String useSslconfig = "useSSL=" + useSsl + "&requireSSL=" + requireSsl + "&verifyServerCertificate=true"
- String clientCAKey = "clientCertificateKeyStoreUrl=file:" + keyStorePath
- String clientCAPwd = "clientCertificateKeyStorePassword=" + keyStorePassword
- String trustCAKey = "trustCertificateKeyStoreUrl=file:" + trustStorePath
- String trustCAPwd = "trustCertificateKeyStorePassword=" + trustStorePassword
- String tlsUrl = useSslconfig + "&" + clientCAKey + "&" + clientCAPwd + "&" + trustCAKey + "&" + trustCAPwd
- // e.g: jdbc:mysql://locahost:8080/dbname?
- if (url.charAt(url.length() - 1) == '?') {
- return url + tlsUrl
- // e.g: jdbc:mysql://locahost:8080/dbname?a=b
- } else if (url.contains('?')) {
- return url + '&' + tlsUrl
- // e.g: jdbc:mysql://locahost:8080/dbname
- } else {
- return url + '?' + tlsUrl
+ int queryStart = url.indexOf('?')
+ String baseUrl = queryStart >= 0 ? url.substring(0, queryStart) : url
+ List<String> options = new ArrayList<>()
+ Set<String> tlsOptions = new HashSet<>(Arrays.asList(
+ "sslMode", "useSSL", "requireSSL", "verifyServerCertificate",
+ "clientCertificateKeyStoreUrl", "clientCertificateKeyStorePassword",
+ "trustCertificateKeyStoreUrl", "trustCertificateKeyStorePassword"))
+ if (queryStart >= 0) {
+ for (String option : url.substring(queryStart + 1).split('&')) {
+ if (option.isEmpty()) {
+ continue
+ }
+ int equals = option.indexOf('=')
+ String name = equals >= 0 ? option.substring(0, equals) : option
+ if (!tlsOptions.contains(name)) {
+ options.add(option)
+ }
+ }
}
+ options.add("useSSL=true")
+ options.add("requireSSL=true")
+ options.add("verifyServerCertificate=true")
+ options.add("clientCertificateKeyStoreUrl=file:" + keyStorePath)
+ options.add("clientCertificateKeyStorePassword=" + keyStorePassword)
+ options.add("trustCertificateKeyStoreUrl=file:" + trustStorePath)
+ options.add("trustCertificateKeyStorePassword=" + trustStorePassword)
+ return baseUrl + "?" + String.join("&", options)
}
private static String addTimeoutUrl(String url) {
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy
index b73ef8e196c..40deeec8930 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ProfileAction.groovy
@@ -36,6 +36,7 @@ class ProfileAction implements SuiteAction {
private Runnable runCallback
private Closure<String> check
private SuiteContext context
+ private InetSocketAddress profileAddress
ProfileAction(SuiteContext context, String tag) {
this.context = context
@@ -47,6 +48,13 @@ class ProfileAction implements SuiteAction {
this.tag = null
}
+ /** Query profiles are local to the FE that executed the SQL. */
+ ProfileAction(SuiteContext context, InetSocketAddress profileAddress) {
+ this.context = context
+ this.tag = null
+ this.profileAddress = Objects.requireNonNull(profileAddress, "profileAddress can not be null")
+ }
+
void run(@ClosureParams(value = FromString, options = []) Runnable run) {
runCallback = run
}
@@ -58,17 +66,13 @@ class ProfileAction implements SuiteAction {
List getProfileList() {
def httpCli = new HttpCliAction(context)
- def addr = context.getFeHttpAddress()
+ def addr = profileAddress ?: context.getFeHttpAddress()
httpCli.endpoint("${addr.hostString}:${addr.port}")
httpCli.uri("/rest/v1/query_profile")
httpCli.op("get")
httpCli.printResponse(false)
- if (context.config.isCloudMode()) {
- httpCli.basicAuthorization(context.config.feCloudHttpUser, context.config.feCloudHttpPassword)
- } else {
- httpCli.basicAuthorization(context.config.feHttpUser, context.config.feHttpPassword)
- }
+ httpCli.basicAuthorization(context.config.feHttpUser, context.config.feHttpPassword)
List profileData = []
httpCli.check { code, body ->
if (code != 200) {
@@ -104,17 +108,13 @@ class ProfileAction implements SuiteAction {
String getProfile(String profileId) {
def profileCli = new HttpCliAction(context)
- def addr = context.getFeHttpAddress()
+ def addr = profileAddress ?: context.getFeHttpAddress()
profileCli.endpoint("${addr.hostString}:${addr.port}")
profileCli.uri("/rest/v1/query_profile/${profileId}")
profileCli.op("get")
profileCli.printResponse(false)
- if (context.config.isCloudMode()) {
- profileCli.basicAuthorization(context.config.feCloudHttpUser, context.config.feCloudHttpPassword)
- } else {
- profileCli.basicAuthorization(context.config.feHttpUser, context.config.feHttpPassword)
- }
+ profileCli.basicAuthorization(context.config.feHttpUser, context.config.feHttpPassword)
def result = [text: ""]
profileCli.check { profileCode, profileResp ->
if (profileCode != 200) {
@@ -222,17 +222,13 @@ class ProfileAction implements SuiteAction {
}
def httpCli = new HttpCliAction(context)
- def addr = context.getFeHttpAddress()
+ def addr = profileAddress ?: context.getFeHttpAddress()
httpCli.endpoint("${addr.hostString}:${addr.port}")
httpCli.uri("/rest/v1/query_profile")
httpCli.op("get")
httpCli.printResponse(false)
- if (context.config.isCloudMode()) {
- httpCli.basicAuthorization(context.config.feCloudHttpUser, context.config.feCloudHttpPassword)
- } else {
- httpCli.basicAuthorization(context.config.feHttpUser, context.config.feHttpPassword)
- }
+ httpCli.basicAuthorization(context.config.feHttpUser, context.config.feHttpPassword)
httpCli.check { code, body ->
if (code != 200) {
throw new IllegalStateException("Get profile list failed, code: ${code}, body:\n${body}")
@@ -252,11 +248,7 @@ class ProfileAction implements SuiteAction {
profileCli.op("get")
profileCli.printResponse(false)
- if (context.config.isCloudMode()) {
- profileCli.basicAuthorization(context.config.feCloudHttpUser, context.config.feCloudHttpPassword)
- } else {
- profileCli.basicAuthorization(context.config.feHttpUser, context.config.feHttpPassword)
- }
+ profileCli.basicAuthorization(context.config.feHttpUser, context.config.feHttpPassword)
profileCli.check { profileCode, profileResp ->
if (profileCode != 200) {
throw new IllegalStateException("Get profile failed, url: ${"/rest/v1/query_profile/${profileId}"}, code: ${profileCode}, body:\n${profileResp}")
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptContext.groovy
index c8b64fb9807..16790e43309 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptContext.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptContext.groovy
@@ -21,6 +21,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.doris.regression.Config
import org.apache.doris.regression.suite.event.EventListener
+import org.apache.doris.regression.util.Http
import java.lang.reflect.UndeclaredThrowableException
import java.util.concurrent.ExecutorService
@@ -62,6 +63,16 @@ class ScriptContext implements Closeable {
def outputRelativePath = path.substring(0, path.lastIndexOf(".")) + ".out"
this.outputFile = new File(new File(config.dataPath), outputRelativePath)
this.dataPath = this.outputFile.getParentFile().getCanonicalFile()
+ Http.configure(
+ (config.otherConfigs.get("enableTLS")?.toString()?.equalsIgnoreCase("true")) ?: false,
+ config.otherConfigs.get("tlsVerifyMode")?.toString()?.toLowerCase() ?: "strict",
+ config.otherConfigs.get("trustStorePath")?.toString(),
+ config.otherConfigs.get("trustStorePassword")?.toString(),
+ config.otherConfigs.get("trustStoreType")?.toString() ?: "PKCS12",
+ config.otherConfigs.get("keyStorePath")?.toString(),
+ config.otherConfigs.get("keyStorePassword")?.toString(),
+ config.otherConfigs.get("keyStoreType")?.toString() ?: "PKCS12"
+ )
}
private final synchronized Suite newSuite(String suiteName, String group) {
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index d280c880874..9899da208c1 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -35,6 +35,10 @@ import groovy.json.JsonSlurper
import groovy.util.logging.Slf4j
import org.apache.commons.lang3.ObjectUtils
import org.apache.doris.regression.Config
+import org.apache.doris.regression.suite.client.BackendClientImpl
+import org.apache.doris.regression.util.RowsetMetaUtils
+import org.apache.doris.thrift.TNetworkAddress
+import org.apache.doris.thrift.TSyncLoadForTabletsRequest
import org.apache.doris.regression.RegressionTest
import org.apache.doris.regression.action.FlightRecordAction
import org.apache.doris.regression.action.BenchmarkAction
@@ -134,6 +138,30 @@ class Suite implements GroovyInterceptable {
return getConf("suites." + name + "." + key, defaultValue)
}
+ boolean isDorisTlsEnabled() {
+ return Boolean.parseBoolean(getConf("enableTLS", "false"))
+ }
+
+ String getDorisHttpScheme() {
+ return isDorisTlsEnabled() ? "https" : "http"
+ }
+
+ String getDorisCurlTlsOptions() {
+ if (!isDorisTlsEnabled()) {
+ return ""
+ }
+ return " --cert ${quoteShellArgument(getConf('trustCert'))}" +
+ " --key ${quoteShellArgument(getConf('trustCAKey'))}" +
+ " --cacert ${quoteShellArgument(getConf('trustCACert'))}"
+ }
+
+ private static String quoteShellArgument(String value) {
+ if (!value) {
+ throw new IllegalArgumentException("Missing TLS certificate path for curl")
+ }
+ return "'" + value.replace("'", "'\"'\"'") + "'"
+ }
+
List<String> getDorisConnectorTlsArgs() {
if (!Boolean.parseBoolean(getConf("enableTLS", "false"))) {
return Collections.emptyList()
@@ -299,6 +327,21 @@ class Suite implements GroovyInterceptable {
return context.connect(user, password, url, actionSupplier)
}
+ public <T> T connectToDoris(String user = context.config.jdbcUser, String password = context.config.jdbcPassword,
+ String url = context.getJdbcUrl(), Closure<T> actionSupplier) {
+ return context.connectToDoris(user, password, url, actionSupplier)
+ }
+
+ public <T> T connectWithDockerCluster(
+ SuiteCluster cluster,
+ Boolean connectToFollower = false,
+ String user = context.config.jdbcUser,
+ String password = context.config.jdbcPassword,
+ Closure<T> actionSupplier) {
+ def jdbcUrl = cluster.getJdbcUrl(connectToFollower)
+ return context.connect(user, password, jdbcUrl, actionSupplier)
+ }
+
public <T> T connectInDocker(String user = context.config.jdbcUser, String password = context.config.jdbcPassword,
Closure<T> actionSupplier) {
def connInfo = context.threadLocalConn.get()
@@ -314,6 +357,119 @@ class Suite implements GroovyInterceptable {
// }
// )
// }
+ /** Wait for continuous version coverage on each tablet's serving BE, with lazy commit enabled. */
+ void syncAndWaitTabletVersion(Collection<Map> tablets, long version, int timeoutSeconds = 60) {
+ Assertions.assertFalse(tablets.isEmpty(), "no tablets to synchronize")
+ List<Map> tabletList = tablets.toList()
+ Map<String, List<Map>> tabletGroups = tabletList.groupBy { it.BackendId.toString() }
+ Map<String, BackendClientImpl> backendClients = [:]
+ Set<Integer> ready = [] as Set
+ Map<String, Object> lastStates = [:]
+ int pollCount = 0
+ try {
+ if (isCloudMode()) {
+ def backendById = sql_return_maparray("SHOW BACKENDS").collectEntries {
+ [(it.BackendId.toString()): it]
+ }
+ tabletGroups.keySet().each { backendId ->
+ def backend = backendById[backendId]
+ Assertions.assertNotNull(backend,
+ "backend ${backendId} for tablets ${tabletGroups[backendId]*.TabletId} was not found")
+ backendClients[backendId] = new BackendClientImpl(
+ new TNetworkAddress(backend.Host.toString(), backend.BePort as int),
+ backend.HttpPort as int)
+ }
+ }
+
+ awaitUntil(timeoutSeconds, 0.5) {
+ // The BE RPC is asynchronous. Retry it periodically in case an earlier queued
+ // task observed no advancement while Meta Service was finalizing lazy commit.
+ if (!backendClients.isEmpty() && pollCount++ % 10 == 0) {
+ tabletGroups.each { backendId, backendTablets ->
+ backendClients[backendId].client.syncLoadForTablets(
+ new TSyncLoadForTabletsRequest(
+ backendTablets.collect { it.TabletId as long }))
+ }
+ }
+ tabletList.eachWithIndex { tablet, index ->
+ if (!ready.contains(index)) {
+ def status = Http.GET(tablet.CompactionStatus.toString(), true, false,
+ context.config.feHttpUser, context.config.feHttpPassword)
+ lastStates["${tablet.TabletId}@${tablet.BackendId}"] = status
+ if (RowsetMetaUtils.coversVersion(status, version)) {
+ ready.add(index)
+ }
+ }
+ }
+ return ready.size() == tabletList.size()
+ }
+ } catch (org.awaitility.core.ConditionTimeoutException e) {
+ throw new IllegalStateException(
+ "Waiting for tablet version ${version} timed out; last BE states: ${lastStates}", e)
+ } finally {
+ backendClients.values().each { it.close() }
+ }
+ }
+
+ /** Fetch an exact physical rowset after the serving BE has caught up. */
+ Map syncAndWaitCloudRowsetMeta(Map tablet, long version, int timeoutSeconds = 60) {
+ syncAndWaitTabletVersion([tablet], version, timeoutSeconds)
+ return getRowsetMetaAtVersion(tablet, version)
+ }
+
+ /**
+ * Call after syncAndWaitTabletVersion. Cloud headers omit rs_metas, so read the committed
+ * MS rowset key instead. The caller must prevent compaction from removing the exact version.
+ */
+ Map getRowsetMetaAtVersion(Map tablet, long version) {
+ if (!isCloudMode()) {
+ String metaUrl = tablet.MetaUrl.toString()
+ metaUrl += (metaUrl.contains('?') ? '&' : '?') + 'byte_to_base64=true'
+ def header = Http.GET(metaUrl, true, false,
+ context.config.feHttpUser, context.config.feHttpPassword)
+ Assertions.assertTrue(header.rs_metas instanceof List, "tablet header is missing rs_metas")
+ def meta = header.rs_metas.find { (it.end_version as long) == version }
+ Assertions.assertNotNull(meta, "rowset not found: tablet=${tablet.TabletId}, version=${version}")
+ return RowsetMetaUtils.decodeKeyBounds(meta)
+ }
+ def endpoint = context.config.metaServiceHttpAddress
+ def token = context.config.metaServiceToken
+ // CI deployment writes multiClusterInstanceId as a custom property; the legacy
+ // multiClusterInstance can still contain the template's default_instance_id.
+ def instanceId = context.config.otherConfigs.get("multiClusterInstanceId")?.toString()?.trim() ?:
+ context.config.multiClusterInstance?.trim()
+ Assertions.assertTrue(endpoint?.trim() && token?.trim() && instanceId?.trim(),
+ "metaServiceHttpAddress, metaServiceToken and multiClusterInstanceId (or multiClusterInstance) must be configured")
+ def params = [token: token, key_type: "MetaRowsetKey", instance_id: instanceId,
+ tablet_id: tablet.TabletId, version: version]
+ def query = params.collect { key, value ->
+ "${key}=${java.net.URLEncoder.encode(value.toString(), 'UTF-8')}"
+ }.join('&')
+ // Do not use Http.GET here: it logs the URL, including the MS token.
+ def baseUrl = endpoint.contains('://') ? endpoint : "http://${endpoint}"
+ HttpURLConnection conn = new URL("${baseUrl}/MetaService/http/get_value?${query}").openConnection()
+ conn.connectTimeout = 5000
+ conn.readTimeout = 10000
+ conn.instanceFollowRedirects = false
+ try {
+ int code = conn.responseCode
+ Assertions.assertEquals(200, code,
+ "MS rowset read failed: instance=${instanceId}, tablet=${tablet.TabletId}, version=${version}")
+ def meta = new JsonSlurper().parseText(conn.inputStream.getText('UTF-8'))
+ Assertions.assertNotNull(meta.end_version,
+ "MS response is not rowset metadata: tablet=${tablet.TabletId}, version=${version}, code=${meta.code}")
+ Assertions.assertEquals(version, meta.end_version as long)
+ Assertions.assertEquals(tablet.TabletId as long, meta.tablet_id as long)
+ return RowsetMetaUtils.decodeKeyBounds(meta)
+ } catch (IOException e) {
+ // Network exception messages may contain the credential-bearing URL.
+ throw new IOException("MS rowset read failed: tablet=${tablet.TabletId}, version=${version}, " +
+ "error=${e.class.simpleName}")
+ } finally {
+ conn.disconnect()
+ }
+ }
+
public void awaitUntil(int atMostSeconds, double intervalSecond = 1, Closure actionSupplier) {
Awaitility
.with().pollInSameThread()
@@ -1453,8 +1609,12 @@ class Suite implements GroovyInterceptable {
return outBuf.toString()
}
+ static String buildSshCommand(String username, String host, String cmd) {
+ return "ssh -o StrictHostKeyChecking=no ${username}@${host} '${cmd}'"
+ }
+
void sshExec(String username, String host, String cmd, boolean alert=true) {
- String command = "ssh ${username}@${host} '${cmd}'"
+ String command = buildSshCommand(username, host, cmd)
def cmds = ["/bin/bash", "-c", command]
logger.info("Execute: ${cmds}".toString())
Process p = cmds.execute()
@@ -1980,27 +2140,34 @@ class Suite implements GroovyInterceptable {
}
String getServerPrepareJdbcUrl(String jdbcUrl, String database, boolean useMasterIp) {
- String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3)
- def sql_ip = useMasterIp ? getMasterIp() : urlWithoutSchema.substring(0, urlWithoutSchema.indexOf(":"))
- def sql_port
- if (urlWithoutSchema.indexOf("/") >= 0) {
- // e.g: jdbc:mysql://locahost:8080/?a=b
- sql_port = urlWithoutSchema.substring(urlWithoutSchema.indexOf(":") + 1, urlWithoutSchema.indexOf("/"))
- } else {
- // e.g: jdbc:mysql://locahost:8080
- sql_port = urlWithoutSchema.substring(urlWithoutSchema.indexOf(":") + 1)
- }
- String tlsUrl = ""
- // set server side prepared statement url
+ String scheme = "jdbc:mysql://"
+ if (!jdbcUrl.startsWith(scheme)) {
+ throw new IllegalArgumentException("Expected a MySQL JDBC URL")
+ }
+ String endpointAndPath = jdbcUrl.substring(scheme.length())
+ int pathStart = endpointAndPath.indexOf("/")
+ int queryStart = endpointAndPath.indexOf("?")
+ int endpointEnd = pathStart >= 0 && (queryStart < 0 || pathStart < queryStart)
+ ? pathStart : (queryStart >= 0 ? queryStart : endpointAndPath.length())
+ String endpoint = endpointAndPath.substring(0, endpointEnd)
+ int portStart = endpoint.lastIndexOf(":")
+ if (portStart < 0) {
+ throw new IllegalArgumentException("MySQL JDBC URL has no port")
+ }
+ String host = useMasterIp ? getMasterIp() : endpoint.substring(0, portStart)
+ String suffix = endpointAndPath.substring(endpointEnd)
+ if (!suffix.startsWith("/")) {
+ suffix = "/" + suffix
+ }
+ String url = Config.buildUrlWithDbImpl(scheme + host + endpoint.substring(portStart) + suffix, database)
if ((context.config.otherConfigs.get("enableTLS")?.toString()?.equalsIgnoreCase("true")) ?: false) {
- String useSslconfig = "useSSL=true&requireSSL=true&verifyServerCertificate=true"
- String clientCAKey = "clientCertificateKeyStoreUrl=file:" + context.config.otherConfigs.get("keyStorePath")
- String clientCAPwd = "clientCertificateKeyStorePassword=" + context.config.otherConfigs.get("keyStorePassword")
- String trustCAKey = "trustCertificateKeyStoreUrl=file:" + context.config.otherConfigs.get("trustStorePath")
- String trustCAPwd = "trustCertificateKeyStorePassword=" + context.config.otherConfigs.get("trustStorePassword")
- tlsUrl = "&" + useSslconfig + "&" + clientCAKey + "&" + clientCAPwd + "&" + trustCAKey + "&" + trustCAPwd
+ url = Config.buildTlsJdbcUrl(url,
+ context.config.otherConfigs.get("keyStorePath")?.toString(),
+ context.config.otherConfigs.get("keyStorePassword")?.toString(),
+ context.config.otherConfigs.get("trustStorePath")?.toString(),
+ context.config.otherConfigs.get("trustStorePassword")?.toString())
}
- return "jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + database + "?&useServerPrepStmts=true" + tlsUrl
+ return url + (url.contains("?") ? "&" : "?") + "useServerPrepStmts=true"
}
DebugPoint GetDebugPoint() {
@@ -3394,7 +3561,7 @@ class Suite implements GroovyInterceptable {
endpoint feEndPoint
uri "/rest/v1/query_profile"
check check_func
- basicAuthorization "${context.config.feCloudHttpUser}","${context.config.feCloudHttpPassword}"
+ basicAuthorization "${context.config.feHttpUser}","${context.config.feHttpPassword}"
}
}
@@ -3417,7 +3584,7 @@ class Suite implements GroovyInterceptable {
endpoint feEndPoint
uri "/api/profile?query_id=${query_id}"
check check_func
- basicAuthorization "${context.config.feCloudHttpUser}","${context.config.feCloudHttpPassword}"
+ basicAuthorization "${context.config.feHttpUser}","${context.config.feHttpPassword}"
}
}
@@ -3626,6 +3793,26 @@ class Suite implements GroovyInterceptable {
}
}
+ def scp_udf_file_to_all_fe = { udf_file_path ->
+ def udf_file = new File(udf_file_path).absoluteFile
+ assertTrue(udf_file.isFile(), "UDF file does not exist: ${udf_file}")
+ def fe_hosts = sql_return_maparray("SHOW FRONTENDS").collect { it.Host }.unique()
+ assertTrue(!fe_hosts.isEmpty(), "No frontend found to copy UDF file to")
+ if (fe_hosts.size() == 1) {
+ def feAddress = java.net.InetAddress.getByName(fe_hosts[0].toString())
+ if (feAddress.isAnyLocalAddress() || feAddress.isLoopbackAddress() ||
+ java.net.NetworkInterface.getByInetAddress(feAddress) != null) {
+ logger.info("Only one local frontend, skip scp udf file")
+ return
+ }
+ }
+
+ fe_hosts.each { fe_host ->
+ sshExec("root", fe_host, "mkdir -p ${udf_file.parent}")
+ scpFiles("root", fe_host, udf_file.path, udf_file.path, false)
+ }
+ }
+
def check_fold_consistency = { test_sql ->
def re_fe = order_sql "select /*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql}"
def re_be = order_sql "select /*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql}"
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy
index dd360b353b8..fb33da6865d 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy
@@ -160,6 +160,13 @@ class SuiteContext implements Closeable {
Connection getConnectionByDbName(String dbName) {
def jdbcUrl = getJdbcUrl()
+ if (isTlsEnabled()) {
+ jdbcUrl = Config.buildTlsJdbcUrl(jdbcUrl,
+ config.otherConfigs.get("keyStorePath")?.toString(),
+ config.otherConfigs.get("keyStorePassword")?.toString(),
+ config.otherConfigs.get("trustStorePath")?.toString(),
+ config.otherConfigs.get("trustStorePassword")?.toString())
+ }
def jdbcConn = DriverManager.getConnection(jdbcUrl, config.jdbcUser, config.jdbcPassword)
try {
String sql = "CREATE DATABASE IF NOT EXISTS ${dbName}"
@@ -170,7 +177,9 @@ class SuiteContext implements Closeable {
} catch (Throwable t) {
throw new IllegalStateException("Create database failed, jdbcUrl: ${jdbcUrl}", t)
}
- def dbUrl = Config.buildUrlWithDb(jdbcUrl, dbName)
+ def dbUrl = isTlsEnabled()
+ ? Config.buildUrlWithDbImpl(jdbcUrl, dbName)
+ : Config.buildUrlWithDb(jdbcUrl, dbName)
log.info("connect to ${dbUrl}".toString())
return DriverManager.getConnection(dbUrl, config.jdbcUser, config.jdbcPassword)
}
@@ -362,6 +371,22 @@ class SuiteContext implements Closeable {
}
}
+ // Use this explicitly for Doris FE connections. Generic connect must also support external
+ // databases and TLS tests that intentionally supply their own (or no) certificates.
+ public <T> T connectToDoris(String user, String password, String url, Closure<T> actionSupplier) {
+ if (!url.startsWith("jdbc:mysql:")) {
+ throw new IllegalArgumentException("Doris FE connections require a MySQL JDBC URL")
+ }
+ if (isTlsEnabled()) {
+ url = Config.buildTlsJdbcUrl(url,
+ config.otherConfigs.get("keyStorePath")?.toString(),
+ config.otherConfigs.get("keyStorePassword")?.toString(),
+ config.otherConfigs.get("trustStorePath")?.toString(),
+ config.otherConfigs.get("trustStorePassword")?.toString())
+ }
+ return connect(user, password, url, actionSupplier)
+ }
+
public <T> T connect(String user, String password, String url, Closure<T> actionSupplier) {
def originConnection = threadLocalConn.get()
if ((config.otherConfigs.get("enableTLS")?.toString()?.equalsIgnoreCase("true")) ?: false) {
@@ -403,6 +428,10 @@ class SuiteContext implements Closeable {
}
}
+ private boolean isTlsEnabled() {
+ return (config.otherConfigs.get("enableTLS")?.toString()?.equalsIgnoreCase("true")) ?: false
+ }
+
Connection getMasterConnectionByDbName(String dbName) {
def result = JdbcUtils.executeToMapArray(getConnection(), "SHOW FRONTENDS")
def master = null
@@ -414,7 +443,16 @@ class SuiteContext implements Closeable {
}
if (master) {
log.info("master found: ${master.Host}:${master.HttpPort}")
- def url = Config.buildUrlWithDb(master.Host as String, master.QueryPort as Integer, dbName)
+ def url = isTlsEnabled()
+ ? Config.buildUrlWithDb(
+ master.Host as String,
+ master.QueryPort as Integer,
+ dbName,
+ config.otherConfigs.get("keyStorePath")?.toString(),
+ config.otherConfigs.get("keyStorePassword")?.toString(),
+ config.otherConfigs.get("trustStorePath")?.toString(),
+ config.otherConfigs.get("trustStorePassword")?.toString())
+ : Config.buildUrlWithDb(master.Host as String, master.QueryPort as Integer, dbName)
def username = config.jdbcUser
def password = config.jdbcPassword
@@ -436,7 +474,16 @@ class SuiteContext implements Closeable {
}
if (master) {
log.info("master found: ${master.Host}:${master.HttpPort}")
- def url = Config.buildUrlWithDb(master.Host as String, master.QueryPort as Integer, dbName)
+ def url = isTlsEnabled()
+ ? Config.buildUrlWithDb(
+ master.Host as String,
+ master.QueryPort as Integer,
+ dbName,
+ config.otherConfigs.get("keyStorePath")?.toString(),
+ config.otherConfigs.get("keyStorePassword")?.toString(),
+ config.otherConfigs.get("trustStorePath")?.toString(),
+ config.otherConfigs.get("trustStorePassword")?.toString())
+ : Config.buildUrlWithDb(master.Host as String, master.QueryPort as Integer, dbName)
ConnectionInfo connInfo = threadLocalConn.get()
def userName = null
def userPass = null
@@ -447,7 +494,7 @@ class SuiteContext implements Closeable {
userName = config.jdbcUser
userPass = config.jdbcPassword
}
- connectTo(url, connInfo.username, connInfo.password)
+ connectTo(url, userName, userPass)
log.info("Successfully reconnected to the master")
} else {
throw new Exception("No master found to reconnect")
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/DebugPoint.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/DebugPoint.groovy
index cbf4cda6f8d..58ca44abdf5 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/DebugPoint.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/DebugPoint.groovy
@@ -38,6 +38,11 @@ class DebugPoint {
this.suite = suite
}
+ private static String buildDebugPointUrl(String host, int httpPort, String path) {
+ String scheme = Http.enableTls ? "https" : "http"
+ return "${scheme}://${host}:${httpPort}${path}"
+ }
+
/* Enable debug point in regression
* Note: set BE config::enable_debug_points = true to take effect
* Parameters:
@@ -48,7 +53,7 @@ class DebugPoint {
* params: timeout, execute, or other customized input params
*/
static def enableDebugPoint(String host, int httpPort, NodeType type, String name, Map<String, String> params = null) {
- def url = 'http://' + host + ':' + httpPort + '/api/debug_point/add/' + name
+ def url = buildDebugPointUrl(host, httpPort, '/api/debug_point/add/' + name)
if (params != null && params.size() > 0) {
url += '?' + params.collect((k, v) -> k + '=' + v).join('&')
}
@@ -64,7 +69,7 @@ class DebugPoint {
* name: debug point name
*/
static def disableDebugPoint(String host, int httpPort, NodeType type, String name) {
- def url = 'http://' + host + ':' + httpPort + '/api/debug_point/remove/' + name
+ def url = buildDebugPointUrl(host, httpPort, '/api/debug_point/remove/' + name)
def result = Http.POST(url, null, true)
Http.checkHttpResult(result, type)
}
@@ -76,7 +81,7 @@ class DebugPoint {
* type: NodeType.BE or NodeType.FE
*/
static def clearDebugPoints(String host, int httpPort, NodeType type) {
- def url = 'http://' + host + ':' + httpPort + '/api/debug_point/clear'
+ def url = buildDebugPointUrl(host, httpPort, '/api/debug_point/clear')
def result = Http.POST(url, null, true)
Http.checkHttpResult(result, type)
}
@@ -197,4 +202,3 @@ class DebugPoint {
}
}
}
-
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Http.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Http.groovy
index 8ee33b5ecba..e902a345a66 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Http.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Http.groovy
@@ -66,6 +66,9 @@ class Http {
keyStorePath = keyPath
keyStorePassword = keyPassword
keyStoreType = keystoreType
+ if (!enableTls) {
+ return
+ }
initSSLContext()
}
@@ -110,13 +113,26 @@ class Http {
}
}
- static Object GET(url, isJson = false, printText = true) {
- if (enableTls) {
- url = url.replace("http://", "https://")
+ static String basicAuthorization(String user, String password) {
+ String credentials = "${user}:${password ?: ''}"
+ return 'Basic ' + credentials.getBytes('UTF-8').encodeBase64().toString()
+ }
+
+ static URLConnection openConnection(String url) {
+ if (enableTls && url.startsWith('http://')) {
+ url = 'https://' + url.substring('http://'.length())
}
- def conn = new URL(url).openConnection()
+ return new URL(url).openConnection()
+ }
+
+ static Object GET(url, isJson = false, printText = true) {
+ return GET(url, isJson, printText, 'root', '')
+ }
+
+ static Object GET(url, isJson, printText, String user, String password) {
+ def conn = openConnection(url)
conn.setRequestMethod('GET')
- conn.setRequestProperty('Authorization', 'Basic cm9vdDo=') //token for root
+ conn.setRequestProperty('Authorization', basicAuthorization(user, password))
def code = conn.responseCode
def text = conn.content.text
if (printText) {
@@ -135,12 +151,16 @@ class Http {
}
static Object POST(url, data = null, isJson = false) {
+ return POST(url, data, isJson, 'root', '')
+ }
+
+ static Object POST(url, data, isJson, String user, String password) {
if (enableTls) {
url = url.replace("http://", "https://")
}
def conn = new URL(url).openConnection()
conn.setRequestMethod('POST')
- conn.setRequestProperty('Authorization', 'Basic cm9vdDo=') //token for root
+ conn.setRequestProperty('Authorization', basicAuthorization(user, password))
if (data) {
if (isJson) {
conn.setRequestProperty('Content-Type', 'application/json')
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MultiClusterStateGuard.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MultiClusterStateGuard.groovy
new file mode 100644
index 00000000000..2594c43d02b
--- /dev/null
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MultiClusterStateGuard.groovy
@@ -0,0 +1,195 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.util
+
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
+
+/** Restores the dedicated multi-cluster test instance; never used for docker/upgrade tests. */
+class MultiClusterStateGuard {
+ Closure listClusters
+ Closure dropCluster
+ Closure addCluster
+ Closure readDefault
+ Closure setDefault
+ Closure awaitState
+ Closure restoreInstance = {}
+
+ /** Parse legacy IP:heartbeat:http:cloud-id entries with an optional brpc port. */
+ static List<Map> configuredBackends(String configuredBes) {
+ if (!configuredBes?.trim()) {
+ throw new IllegalArgumentException('Missing multiClusterBes configuration')
+ }
+ def nodes = []
+ configuredBes.split(',').eachWithIndex { entry, index ->
+ def parts = entry.trim().split(':', -1)
+ if (!(parts.length in [4, 5]) || parts.any { !it.trim() }) {
+ throw new IllegalArgumentException("Invalid multiClusterBes entry at index ${index}")
+ }
+ def ports = [parts[1], parts[2]] + (parts.length == 5 ? [parts[4]] : [])
+ if (ports.any { !it.isInteger() || it.toInteger() < 1 || it.toInteger() > 65535 }) {
+ throw new IllegalArgumentException("Invalid BE port at index ${index}")
+ }
+ nodes << [ip: parts[0], heartbeat_port: parts[1].toInteger(), http_port: parts[2].toInteger(),
+ cloud_unique_id: parts[3]]
+ }
+ // cloud_unique_id identifies a cloud deployment, not an individual BE process.
+ if (nodes.collect { [it.ip, it.heartbeat_port] }.toSet().size() != nodes.size()) {
+ throw new IllegalArgumentException('multiClusterBes contains duplicate BE endpoints')
+ }
+ return nodes
+ }
+
+ /** Shared fixture: the first two configured BEs each belong to one compute group. */
+ static List<Map> sharedBaseline(String configuredBes) {
+ def nodes = configuredBackends(configuredBes)
+ if (nodes.size() < 2) {
+ throw new IllegalArgumentException('The shared multi-cluster baseline requires at least two BEs')
+ }
+ return (0..1).collect { index ->
+ [type: 'COMPUTE', cluster_name: "regression_cluster_name${index}".toString(),
+ cluster_id: "regression_cluster_id${index}".toString(), cluster_status: 'NORMAL',
+ nodes: [nodes[index]]]
+ }
+ }
+
+ private static Object copy(Object value) {
+ return new JsonSlurper().parseText(JsonOutput.toJson(value))
+ }
+
+ // MS refreshes these timestamps when recreating a cluster or node.
+ static Object topology(Object value) {
+ if (value instanceof Map) {
+ def result = new TreeMap()
+ value.each { key, item ->
+ if (!(key in ['ctime', 'mtime'])) {
+ result[key] = topology(item)
+ }
+ }
+ return result
+ }
+ if (value instanceof List) {
+ return value.collect { topology(it) }.sort { a, b ->
+ JsonOutput.toJson(a) <=> JsonOutput.toJson(b)
+ }
+ }
+ return value
+ }
+
+ static boolean frontendMatches(List<Map> expected, List<Map> groups, List<Map> backends) {
+ if ((groups.collect { it.cluster } as Set) != (expected.collect { it.cluster_name } as Set)) {
+ return false
+ }
+ def expectedNodes = expected.collectMany { cluster ->
+ (cluster.nodes ?: []).collect { node ->
+ [cluster.cluster_id.toString(), (node.ip ?: node.host).toString(), node.heartbeat_port.toString(),
+ cluster.cluster_name.toString(), cluster.cluster_status.toString(), node.cloud_unique_id.toString()]
+ }
+ }.sort { a, b -> a.toString() <=> b.toString() }
+ // Backend.getCloudClusterStatus treats an absent status tag as NORMAL.
+ def actualNodes = backends.collect { backend ->
+ def tag = backend.Tag instanceof Map ? backend.Tag : new JsonSlurper().parseText(backend.Tag.toString())
+ [(tag.compute_group_id ?: tag.cloud_cluster_id)?.toString(), backend.Host.toString(),
+ backend.HeartbeatPort.toString(), (tag.compute_group_name ?: tag.cloud_cluster_name)?.toString(),
+ (tag.compute_group_status ?: tag.cloud_cluster_status ?: 'NORMAL').toString(), tag.cloud_unique_id?.toString()]
+ }.sort { a, b -> a.toString() <=> b.toString() }
+ return expectedNodes == actualNodes && backends.every {
+ it.Alive.toString().toBoolean() && !it.SystemDecommissioned?.toString()?.toBoolean()
+ }
+ }
+
+ private List<Map> computeClusters() {
+ List<Map> clusters = listClusters.call()
+ if (clusters.any { it.type == 'VIRTUAL' }) {
+ throw new IllegalStateException('Legacy multi-cluster state restoration does not support virtual clusters')
+ }
+ return clusters.findAll { it.type == 'COMPUTE' }
+ }
+
+ /** initializeBaseline keeps a successful load fixture, and rolls back a failed one. */
+ def run(boolean initializeBaseline, Closure body) {
+ List<Map> original = copy(computeClusters()) as List<Map>
+ if (original.any { it.cluster_status != 'NORMAL' ||
+ (it.nodes ?: []).any { n -> n.status && n.status != 'NODE_STATUS_RUNNING' } }) {
+ throw new IllegalStateException('Multi-cluster tests require normal clusters and running nodes')
+ }
+ String originalDefault = readDefault.call()
+ // A stale default cannot be restored with SET PROPERTY. Reject it before destructive tests.
+ if (originalDefault && !original.any { it.cluster_name == originalDefault }) {
+ throw new IllegalStateException("Initial default compute group does not exist: ${originalDefault}")
+ }
+ Throwable failure = null
+ try {
+ // Tests can replace the groups. New connections must not inherit a deleted group.
+ setDefault.call('')
+ awaitState.call(original, '')
+ return body.call()
+ } catch (Throwable t) {
+ failure = t
+ throw t
+ } finally {
+ def errors = []
+ def attempt = { Closure action ->
+ try {
+ action.call()
+ } catch (Throwable t) {
+ errors.add(t)
+ }
+ }
+ attempt { restoreInstance.call() }
+ List<Map> expected = original
+ String expectedDefault = originalDefault
+ attempt {
+ if (initializeBaseline && failure == null) {
+ // The load suite intentionally leaves its successful setup for subsequent suites.
+ expected = copy(computeClusters()) as List<Map>
+ expectedDefault = readDefault.call()
+ } else {
+ def expectedById = original.collectEntries { [(it.cluster_id): it] }
+ computeClusters().each { current ->
+ def saved = expectedById[current.cluster_id]
+ if (saved == null || topology(saved) != topology(current)) {
+ attempt { dropCluster.call(current) }
+ }
+ }
+ // Re-read after deletion; a failed drop must not prevent restoring other groups.
+ def currentById = computeClusters().collectEntries { [(it.cluster_id): it] }
+ original.each { saved ->
+ if (!currentById.containsKey(saved.cluster_id)) {
+ attempt { addCluster.call(copy(saved)) }
+ }
+ }
+ }
+ }
+ // Restore metadata and FE node maps before restoring the default property.
+ // Even if a wait fails, attempt the remaining cleanup and report every failure.
+ attempt { awaitState.call(expected, null) }
+ attempt { setDefault.call(expectedDefault) }
+ attempt { awaitState.call(expected, expectedDefault) }
+ if (!errors.isEmpty()) {
+ Throwable restoreFailure = new IllegalStateException('Failed to restore multi-cluster test state')
+ errors.each { restoreFailure.addSuppressed(it) }
+ if (failure != null) {
+ failure.addSuppressed(restoreFailure)
+ } else {
+ throw restoreFailure
+ }
+ }
+ }
+ }
+}
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MySqlClient.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MySqlClient.groovy
new file mode 100644
index 00000000000..e0de281b8f3
--- /dev/null
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MySqlClient.groovy
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.util
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.attribute.PosixFilePermissions
+
+class MySqlClient {
+ // MySQL option files interpret backslash escapes even inside quoted values.
+ private static String quoteOption(String value) {
+ String text = value ?: ''
+ if (text.indexOf(0) >= 0) {
+ throw new IllegalArgumentException('MySQL credentials must not contain NUL')
+ }
+ return '"' + text.replace('\\', '\\\\').replace('"', '\\"')
+ .replace('\n', '\\n').replace('\r', '\\r')
+ .replace('\t', '\\t').replace('\b', '\\b') + '"'
+ }
+
+ static Map execute(String user, String password, List<String> options, String sql,
+ File directory = null, String executable = 'mysql') {
+ def credentials = Files.createTempFile('doris-mysql-', '.cnf',
+ PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString('rw-------')))
+ Process process = null
+ try {
+ String config = "[client]\nuser=${quoteOption(user)}\npassword=${quoteOption(password)}\n"
+ Files.write(credentials, config.getBytes(StandardCharsets.UTF_8))
+ List<String> command = [executable, "--defaults-extra-file=${credentials}".toString()]
+ command.addAll(options.collect { it.toString() })
+ ProcessBuilder builder = new ProcessBuilder(command)
+ if (directory != null) {
+ builder.directory(directory)
+ }
+ process = builder.start()
+ StringBuilder stdout = new StringBuilder()
+ StringBuilder stderr = new StringBuilder()
+ Thread outputReader = process.consumeProcessOutputStream(stdout)
+ Thread errorReader = process.consumeProcessErrorStream(stderr)
+ process.outputStream.withWriter('UTF-8') { writer -> writer.write(sql) }
+ int exitCode = process.waitFor()
+ outputReader.join()
+ errorReader.join()
+ return [exitCode: exitCode, stdout: stdout.toString(), stderr: stderr.toString()]
+ } finally {
+ try {
+ if (process != null && process.isAlive()) {
+ process.destroyForcibly().waitFor()
+ }
+ } finally {
+ Files.deleteIfExists(credentials)
+ }
+ }
+ }
+}
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/RowsetMetaUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/RowsetMetaUtils.groovy
new file mode 100644
index 00000000000..26b48e23f48
--- /dev/null
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/RowsetMetaUtils.groovy
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.util
+
+import java.nio.charset.StandardCharsets
+
+class RowsetMetaUtils {
+ /** Only active rowsets count; a high max version must not hide a gap in the version chain. */
+ static boolean coversVersion(Map status, long version) {
+ if (version < 0) {
+ throw new IllegalArgumentException("negative target version: ${version}")
+ }
+ if (!(status.rowsets instanceof List)) {
+ throw new IllegalArgumentException("compaction response is missing rowsets: ${status}")
+ }
+ def ranges = status.rowsets.collect { rowset ->
+ def match = rowset.toString() =~ /^\[(\d+)-(\d+)\](?:\s.*)?$/
+ if (!match.matches()) {
+ throw new IllegalArgumentException("invalid compaction rowset: ${rowset}")
+ }
+ long start = match.group(1).toLong()
+ long end = match.group(2).toLong()
+ if (start > end) {
+ throw new IllegalArgumentException("invalid rowset version range: ${rowset}")
+ }
+ [start, end]
+ }.sort { a, b -> a[0] <=> b[0] }
+ long covered = -1
+ for (def range : ranges) {
+ if (range[0] > covered + 1) {
+ return false
+ }
+ covered = Math.max(covered, range[1] as long)
+ if (covered >= version) {
+ return true
+ }
+ }
+ return false
+ }
+
+ /** MS protobuf JSON encodes bytes as base64. Preserve one character per key byte. */
+ static Map decodeKeyBounds(Map meta) {
+ Map result = new LinkedHashMap(meta)
+ if (meta.segments_key_bounds instanceof List) {
+ result.segments_key_bounds = meta.segments_key_bounds.collect { bounds ->
+ Map decoded = new LinkedHashMap(bounds)
+ ['min_key', 'max_key'].each { key ->
+ if (!(bounds[key] instanceof String)) {
+ throw new IllegalArgumentException("key bounds are missing ${key}")
+ }
+ decoded[key] = new String(Base64.decoder.decode(bounds[key]), StandardCharsets.ISO_8859_1)
+ }
+ decoded
+ }
+ }
+ // These optional protobuf flags default to false when omitted from MS JSON.
+ ['segments_key_bounds_truncated', 'segments_key_bounds_aggregated'].each { key ->
+ if (!result.containsKey(key)) {
+ result[key] = false
+ }
+ }
+ return result
+ }
+}
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/WarmupMetricsUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/WarmupMetricsUtils.groovy
index 846ce4ac34b..e765c55a508 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/WarmupMetricsUtils.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/WarmupMetricsUtils.groovy
@@ -44,7 +44,7 @@ class WarmupMetricsUtils {
*/
static long getBrpcMetric(String ip, String port, String metricName) {
def url = "http://${ip}:${port}/brpc_metrics"
- def text = new URL(url).text
+ def text = readText(url)
def matcher = text =~ ~"${metricName}\\s+(\\d+)"
if (matcher.find()) {
return matcher[0][1] as long
@@ -53,7 +53,7 @@ class WarmupMetricsUtils {
}
static String getPrometheusMetrics(String ip, Object port) {
- return new URL("http://${ip}:${port}/metrics").text
+ return readText("http://${ip}:${port}/metrics")
}
static BigDecimal findPrometheusMetricValue(String metricsText, String metricName, Map labels) {
@@ -101,7 +101,7 @@ class WarmupMetricsUtils {
}
static void clearFileCache(String ip, String httpPort) {
- def response = new URL("http://${ip}:${httpPort}/api/file_cache?op=clear&sync=true").text
+ def response = readText("http://${ip}:${httpPort}/api/file_cache?op=clear&sync=true")
def json = new JsonSlurper().parseText(response)
if (json.status != "OK") {
throw new RuntimeException("Clear cache on ${ip}:${httpPort} failed: ${json.status}")
@@ -115,6 +115,96 @@ class WarmupMetricsUtils {
Thread.sleep(waitMs)
}
+ private static String backendIp(Object backend) {
+ return backend instanceof Map ? backend.ip.toString() : backend[1].toString()
+ }
+
+ private static String backendHttpPort(Object backend) {
+ return backend instanceof Map ? backend.httpPort.toString() : backend[4].toString()
+ }
+
+ private static String backendBrpcPort(Object backend) {
+ return backend instanceof Map ? backend.brpcPort.toString() : backend[5].toString()
+ }
+
+ private static String readText(String url) {
+ def connection = Http.openConnection(url)
+ connection.connectTimeout = 5000
+ connection.readTimeout = 60000
+ try {
+ return connection.inputStream.withCloseable { it.getText("UTF-8") }
+ } finally {
+ connection.disconnect()
+ }
+ }
+
+ static Map getBackendMetricValues(Collection backends, String metricName) {
+ Map values = [:]
+ for (be in backends) {
+ String key = "${backendIp(be)}:${backendBrpcPort(be)}"
+ values[key] = getBrpcMetric(backendIp(be), backendBrpcPort(be), metricName)
+ }
+ return values
+ }
+
+ static long getBackendMetricSum(Collection backends, String metricName) {
+ return getBackendMetricValues(backends, metricName).values().sum(0L) as long
+ }
+
+ static long waitForBackendMetricSum(Collection backends, String metricName,
+ Closure<Boolean> predicate, long timeoutMs = 60000,
+ String message = null) {
+ long deadline = System.currentTimeMillis() + timeoutMs
+ long value = 0L
+ while (System.currentTimeMillis() < deadline) {
+ value = getBackendMetricSum(backends, metricName)
+ if (predicate(value)) {
+ return value
+ }
+ logger.info("waiting ${metricName} sum, value=${value}")
+ Thread.sleep(1000)
+ }
+ throw new RuntimeException((message ?: "Metric ${metricName} sum did not satisfy predicate") +
+ " within ${timeoutMs}ms, last=${value}")
+ }
+
+ static long waitForBackendMetricSumStable(Collection backends, String metricName,
+ long stableMs = 10000, long timeoutMs = 60000) {
+ long deadline = System.currentTimeMillis() + timeoutMs
+ long lastValue = getBackendMetricSum(backends, metricName)
+ long stableSince = System.currentTimeMillis()
+ while (System.currentTimeMillis() < deadline) {
+ Thread.sleep(1000)
+ long currentValue = getBackendMetricSum(backends, metricName)
+ if (currentValue != lastValue) {
+ lastValue = currentValue
+ stableSince = System.currentTimeMillis()
+ } else if (System.currentTimeMillis() - stableSince >= stableMs) {
+ return currentValue
+ }
+ logger.info("waiting ${metricName} sum stable, value=${currentValue}")
+ }
+ throw new RuntimeException("Metric ${metricName} sum did not stay stable for ${stableMs}ms " +
+ "within ${timeoutMs}ms, last=${lastValue}")
+ }
+
+ static long waitForBrpcMetric(String ip, String port, String metricName,
+ Closure<Boolean> predicate, long timeoutMs = 60000,
+ String message = null) {
+ long deadline = System.currentTimeMillis() + timeoutMs
+ long value = 0L
+ while (System.currentTimeMillis() < deadline) {
+ value = getBrpcMetric(ip, port, metricName)
+ if (predicate(value)) {
+ return value
+ }
+ logger.info("waiting ${metricName} for ${ip}:${port}, value=${value}")
+ Thread.sleep(1000)
+ }
+ throw new RuntimeException((message ?: "Metric ${metricName} did not satisfy predicate") +
+ " within ${timeoutMs}ms, last=${value}")
+ }
+
static long sumProfileCounter(String profileText, String counterName) {
def matcher = profileText =~ ~"(?m)(?<![A-Za-z0-9_])${Pattern.quote(counterName)}\\s*:\\s*([0-9,]+)"
long sum = 0
diff --git a/regression-test/framework/src/test/groovy/org/apache/doris/regression/ConfigArrowFlightSqlUrlTest.groovy b/regression-test/framework/src/test/groovy/org/apache/doris/regression/ConfigArrowFlightSqlUrlTest.groovy
new file mode 100644
index 00000000000..e04fcaba42d
--- /dev/null
+++ b/regression-test/framework/src/test/groovy/org/apache/doris/regression/ConfigArrowFlightSqlUrlTest.groovy
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression
+
+import org.junit.jupiter.api.Test
+
+import static org.junit.jupiter.api.Assertions.assertEquals
+import static org.junit.jupiter.api.Assertions.assertFalse
+import static org.junit.jupiter.api.Assertions.assertTrue
+
+class ConfigArrowFlightSqlUrlTest {
+ @Test
+ void plaintextFlightUrlRemainsAvailableWhenTlsIsDisabled() {
+ Config config = new Config()
+ assertEquals("jdbc:arrow-flight-sql://localhost:8070/?useServerPrepStmts=false&useEncryption=false",
+ config.getArrowFlightSqlJdbcUrl("localhost", "8070"))
+ }
+
+ @Test
+ void mtlsFlightUrlUsesCaAndClientPemFiles() {
+ Config config = new Config()
+ config.otherConfigs.put("enableTLS", "true")
+ config.otherConfigs.put("trustCACert", "/test/ca.crt")
+ config.otherConfigs.put("trustCert", "/test/client.crt")
+ config.otherConfigs.put("trustCAKey", "/test/client.key")
+
+ String url = config.getArrowFlightSqlJdbcUrl("localhost", "8070", "test_db")
+ assertTrue(url.startsWith("jdbc:arrow-flight-sql://localhost:8070/catalog=test_db?"))
+ assertTrue(url.contains("useEncryption=true"))
+ assertTrue(url.contains("tlsRootCerts=/test/ca.crt"))
+ assertTrue(url.contains("clientCertificate=/test/client.crt"))
+ assertTrue(url.contains("clientKey=/test/client.key"))
+ assertFalse(url.contains("disableCertificateVerification=true"))
+ }
+
+ @Test
+ void flightUrlHonorsDisabledCertificateVerification() {
+ Config config = new Config()
+ config.otherConfigs.put("enableTLS", "true")
+ config.otherConfigs.put("tlsVerifyMode", "none")
+
+ String url = config.getArrowFlightSqlJdbcUrl("localhost", "8070")
+ assertTrue(url.contains("useEncryption=true"))
+ assertTrue(url.contains("disableCertificateVerification=true"))
+ }
+}
diff --git a/regression-test/framework/src/test/groovy/org/apache/doris/regression/ConfigRootPasswordTest.groovy b/regression-test/framework/src/test/groovy/org/apache/doris/regression/ConfigRootPasswordTest.groovy
new file mode 100644
index 00000000000..cd908606cba
--- /dev/null
+++ b/regression-test/framework/src/test/groovy/org/apache/doris/regression/ConfigRootPasswordTest.groovy
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression
+
+import org.junit.jupiter.api.Test
+
+import java.sql.Connection
+import java.sql.Driver
+import java.sql.DriverManager
+
+import static org.junit.jupiter.api.Assertions.*
+
+class ConfigRootPasswordTest {
+ @Test
+ void rootConnectionsOnlyReusePasswordsConfiguredForRoot() {
+ def calls = []
+ boolean closed = false
+ Connection connection = [close: { closed = true }] as Connection
+ Driver driver = [connect: { String url, Properties properties ->
+ if (url != 'jdbc:root-credentials:test') return null
+ calls.add(new HashMap(properties))
+ return connection
+ }] as Driver
+ DriverManager.registerDriver(driver)
+ try {
+ def cases = [
+ ['root', 'fake password \u5bc6\u78bc', 'fake password \u5bc6\u78bc'],
+ ['root', '', ''], ['root', null, ''], ['root', ' ', ' '],
+ ['test_user', 'other-user-password', ''], ['test_user', '', ''],
+ [null, 'other-user-password', '']
+ ]
+ cases.each { user, password, expected ->
+ Config config = new Config(jdbcUrl: 'jdbc:root-credentials:test',
+ jdbcUser: user, jdbcPassword: password)
+ assertEquals(expected, config.getRootPassword())
+ closed = false
+ config.getRootConnection().withCloseable { assertSame(connection, it) }
+ assertTrue(closed)
+ assertEquals('root', calls.last().user)
+ assertEquals(expected, calls.last().password)
+ }
+ } finally {
+ DriverManager.deregisterDriver(driver)
+ }
+ }
+}
diff --git a/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/CompactionPluginCompatibilityTest.groovy b/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/CompactionPluginCompatibilityTest.groovy
new file mode 100644
index 00000000000..c8106d9e7bf
--- /dev/null
+++ b/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/CompactionPluginCompatibilityTest.groovy
@@ -0,0 +1,154 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.suite
+
+import groovy.lang.Binding
+import groovy.lang.GroovyShell
+import groovy.lang.GroovySystem
+import groovy.json.JsonSlurper
+import org.apache.doris.regression.Config
+import org.junit.jupiter.api.AfterEach
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.io.TempDir
+import org.slf4j.LoggerFactory
+
+import java.nio.file.Path
+
+import static org.junit.jupiter.api.Assertions.assertEquals
+import static org.junit.jupiter.api.Assertions.assertNotNull
+import static org.junit.jupiter.api.Assertions.assertThrows
+import static org.junit.jupiter.api.Assertions.assertTrue
+
+class CompactionPluginCompatibilityTest {
+ @TempDir
+ Path tempDir
+
+ @AfterEach
+ void removePluginMethods() {
+ GroovySystem.metaClassRegistry.removeMetaClass(Suite)
+ }
+
+ @Test
+ void tabletStatusAndCompactionStatusSupportDefaultAndBoundedProbes() {
+ File plugin = [new File("../plugins/plugin_compaction.groovy"),
+ new File("regression-test/plugins/plugin_compaction.groovy")]
+ .find { it.isFile() }
+ assertNotNull(plugin, "plugin_compaction.groovy must be available to this test")
+
+ def logger = LoggerFactory.getLogger(CompactionPluginCompatibilityTest)
+ new GroovyShell(getClass().classLoader, new Binding(logger: logger)).evaluate(plugin)
+
+ def requests = []
+ Suite.metaClass.curl = { String method, String url, String body, Integer timeoutSec,
+ String user, String password, Integer maxRetries ->
+ requests.add([method, url, body, timeoutSec, user, password, maxRetries])
+ return [0, "{}", ""]
+ }
+
+ File suiteDir = tempDir.resolve("suites/compaction").toFile()
+ assertTrue(suiteDir.mkdirs())
+ File suiteFile = new File(suiteDir, "compaction_plugin_compatibility.groovy")
+ assertTrue(suiteFile.createNewFile())
+ Config config = new Config()
+ config.suitePath = tempDir.resolve("suites").toString()
+ config.dataPath = tempDir.resolve("data").toString()
+ config.realDataPath = tempDir.resolve("real-data").toString()
+ config.defaultDb = "regression_test"
+ config.feHttpUser = "http_user"
+ config.feHttpPassword = "http_password"
+ ScriptContext scriptContext = new ScriptContext(
+ suiteFile, null, null, config, Collections.emptyList(), { true })
+ SuiteCluster cluster = new SuiteCluster("compaction_plugin_compatibility", config)
+ SuiteContext context = new SuiteContext(
+ suiteFile, "compaction_plugin_compatibility", "p0", scriptContext,
+ cluster, null, null, config)
+ Suite suite = new Suite("compaction_plugin_compatibility", "p0", context, cluster)
+ assertEquals([0, "{}", ""], suite.be_show_tablet_status("127.0.0.1", "8040", "42"))
+ assertEquals([0, "{}", ""], suite.be_show_tablet_status("127.0.0.1", "8040", "42", 5, 1))
+ assertEquals([0, "{}", ""], suite.be_get_compaction_status("127.0.0.1", "8040", "42"))
+ assertEquals([0, "{}", ""], suite.be_get_compaction_status("127.0.0.1", "8040", "42", 5, 1))
+
+ assertEquals([
+ ["GET", "http://127.0.0.1:8040/api/compaction/show?tablet_id=42",
+ null, 10, "http_user", "http_password", 10],
+ ["GET", "http://127.0.0.1:8040/api/compaction/show?tablet_id=42",
+ null, 5, "http_user", "http_password", 1],
+ ["GET", "http://127.0.0.1:8040/api/compaction/run_status?tablet_id=42",
+ null, 10, "http_user", "http_password", 10],
+ ["GET", "http://127.0.0.1:8040/api/compaction/run_status?tablet_id=42",
+ null, 5, "http_user", "http_password", 1]
+ ], requests)
+
+ def defaultCalls = []
+ Suite.metaClass.curl = { String method, String url, String body, Integer timeoutSec,
+ String user, String password, Integer maxRetries ->
+ defaultCalls.add([method, url, body, timeoutSec, user, password, maxRetries])
+ return [0, "{}", ""]
+ }
+ assertEquals([0, "{}", ""], suite.be_get_overall_compaction_status("127.0.0.1", "8040"))
+ assertEquals([0, "{}", ""], suite.be_run_base_compaction("127.0.0.1", "8040", "42"))
+ assertEquals([0, "{}", ""], suite.be_run_cumulative_compaction("127.0.0.1", "8040", "42"))
+ assertEquals([0, "{}", ""], suite.be_run_full_compaction("127.0.0.1", "8040", "42"))
+ assertEquals([0, "{}", ""], suite.be_run_full_compaction_by_table_id("127.0.0.1", "8040", "7"))
+ assertEquals([
+ ["GET", "http://127.0.0.1:8040/api/compaction/run_status",
+ null, 10, "http_user", "http_password", 10],
+ ["POST", "http://127.0.0.1:8040/api/compaction/run?tablet_id=42&compact_type=base",
+ null, 10, "http_user", "http_password", 10],
+ ["POST", "http://127.0.0.1:8040/api/compaction/run?tablet_id=42&compact_type=cumulative",
+ null, 10, "http_user", "http_password", 10],
+ ["POST", "http://127.0.0.1:8040/api/compaction/run?tablet_id=42&compact_type=full",
+ null, 10, "http_user", "http_password", 10],
+ ["POST", "http://127.0.0.1:8040/api/compaction/run?table_id=7&compact_type=full",
+ null, 10, "http_user", "http_password", 10]
+ ], defaultCalls)
+
+ // This reproduces the plugin-internal three-argument call that failed in TeamCity 212706.
+ Suite.metaClass.getBackendIpHttpPort = { Map hosts, Map ports ->
+ hosts["1"] = "127.0.0.1"
+ ports["1"] = "8040"
+ }
+ Suite.metaClass.sql_return_maparray = { String ignored -> [[BackendId: "1", TabletId: "42"]] }
+ Suite.metaClass.sql = { String ignored -> [[null, '"disable_auto_compaction" = "true"']] }
+ Suite.metaClass.curl = { String method, String url, String body, Integer timeoutSec,
+ String user, String password, Integer maxRetries ->
+ throw new IllegalStateException("tablet probe reached")
+ }
+ IllegalStateException probe = assertThrows(IllegalStateException) {
+ suite.trigger_and_wait_compaction("t", "full")
+ }
+ assertEquals("tablet probe reached", probe.message)
+
+ Suite.metaClass.parseJson = { String value -> new JsonSlurper().parseText(value) }
+ int showCount = 0
+ Suite.metaClass.curl = { String method, String url, String body = null, Integer timeoutSec = 10,
+ String user = "", String password = "", Integer maxRetries = 10 ->
+ if (url.contains("/api/compaction/show?")) {
+ showCount++
+ return [0, '{"last full success time":"' + (showCount == 1 ? "old" : "new")
+ + '","last full failure time":"old"}', ""]
+ }
+ if (url.contains("/api/compaction/run_status?")) {
+ return [0, '{"status":"success","run_status":false}', ""]
+ }
+ return [0, '{"status":"success"}', ""]
+ }
+ suite.trigger_and_wait_compaction("t", "full")
+ assertTrue(showCount >= 2)
+ }
+}
diff --git a/regression-test/suites/show_p0/test_show_data.groovy b/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteSshExecTest.groovy
similarity index 61%
copy from regression-test/suites/show_p0/test_show_data.groovy
copy to regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteSshExecTest.groovy
index 20adc299d55..c7fe5e33f20 100644
--- a/regression-test/suites/show_p0/test_show_data.groovy
+++ b/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteSshExecTest.groovy
@@ -15,17 +15,17 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_show_data") {
- def result = sql """show data;"""
- logger.info("show data result:${result}");
- assertTrue(result.size() > 0);
+package org.apache.doris.regression.suite
- def jdbcUrlWithoutDbStr = (context.config.jdbcUrl).split(context.config.defaultDb)
- logger.info("jdbcUrlWithoutDbStr:${jdbcUrlWithoutDbStr}");
- def result2 = connect(context.config.jdbcUser, context.config.jdbcPassword, jdbcUrlWithoutDbStr[0]) {
- sql """show data;"""
- }
+import org.junit.jupiter.api.Test
+
+import static org.junit.jupiter.api.Assertions.assertEquals
- assertTrue(result2.size() > 0);
- assertTrue(result2[0].size() == 6);
-}
\ No newline at end of file
+class SuiteSshExecTest {
+ @Test
+ void sshCommandAcceptsNewRegressionHosts() {
+ assertEquals(
+ "ssh -o StrictHostKeyChecking=no root@172.17.0.1 'mkdir -p /tmp/udf'",
+ Suite.buildSshCommand("root", "172.17.0.1", "mkdir -p /tmp/udf"))
+ }
+}
diff --git a/regression-test/framework/src/test/groovy/org/apache/doris/regression/util/HttpAuthenticationTest.groovy b/regression-test/framework/src/test/groovy/org/apache/doris/regression/util/HttpAuthenticationTest.groovy
new file mode 100644
index 00000000000..a1542b24d13
--- /dev/null
+++ b/regression-test/framework/src/test/groovy/org/apache/doris/regression/util/HttpAuthenticationTest.groovy
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.util
+
+import com.sun.net.httpserver.HttpServer
+import groovy.json.JsonOutput
+import org.junit.jupiter.api.Test
+
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+
+import static org.junit.jupiter.api.Assertions.*
+
+class HttpAuthenticationTest {
+ @Test
+ void credentialsAreRequestScopedAndLegacyCallsStillUseEmptyRootPassword() {
+ def server = HttpServer.create(new InetSocketAddress('127.0.0.1', 0), 0)
+ def serverThreads = Executors.newCachedThreadPool()
+ def callers = Executors.newFixedThreadPool(6)
+ boolean originalTls = Http.enableTls
+ server.executor = serverThreads
+ server.createContext('/') { exchange ->
+ def response = [authorization: exchange.requestHeaders.getFirst('Authorization'),
+ body: exchange.requestBody.getText('UTF-8')]
+ byte[] bytes = JsonOutput.toJson(response).getBytes('UTF-8')
+ exchange.responseHeaders.set('Content-Type', 'application/json; charset=UTF-8')
+ exchange.sendResponseHeaders(200, bytes.length)
+ exchange.responseBody.withCloseable { it.write(bytes) }
+ }
+ try {
+ Http.enableTls = false
+ server.start()
+ String url = "http://127.0.0.1:${server.address.port}/"
+ def passwords = ['', null, 'test-password', '密碼 \\" :#; $()`\\\\\n']
+ def futures = (0..<24).collect { index ->
+ callers.submit({ ->
+ String user = "user${index}"
+ String password = passwords[index % passwords.size()]
+ def response = index % 2 == 0
+ ? Http.GET(url, true, false, user, password)
+ : Http.POST(url, [index: index], true, user, password)
+ String decoded = new String(Base64.decoder.decode(response.authorization.substring(6)), 'UTF-8')
+ assertEquals(user + ':' + (password ?: ''), decoded)
+ if (index % 2 != 0) assertEquals(JsonOutput.toJson([index: index]), response.body)
+ } as Runnable)
+ }
+ futures.each { it.get(10, TimeUnit.SECONDS) }
+ assertEquals('Basic cm9vdDo=', Http.GET(url, true, false).authorization)
+ assertEquals('Basic cm9vdDo=', Http.POST(url, null, true).authorization)
+ } finally {
+ Http.enableTls = originalTls
+ server.stop(0)
+ callers.shutdownNow()
+ serverThreads.shutdownNow()
+ }
+ }
+}
diff --git a/regression-test/framework/src/test/groovy/org/apache/doris/regression/util/MultiClusterStateGuardTest.groovy b/regression-test/framework/src/test/groovy/org/apache/doris/regression/util/MultiClusterStateGuardTest.groovy
new file mode 100644
index 00000000000..1cdc9a17d06
--- /dev/null
+++ b/regression-test/framework/src/test/groovy/org/apache/doris/regression/util/MultiClusterStateGuardTest.groovy
@@ -0,0 +1,256 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.util
+
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
+import org.junit.jupiter.api.Test
+
+import static org.junit.jupiter.api.Assertions.*
+
+class MultiClusterStateGuardTest {
+ private static def copy(value) {
+ new JsonSlurper().parseText(JsonOutput.toJson(value))
+ }
+
+ private static Map cluster(String id) {
+ [type: 'COMPUTE', cluster_id: id, cluster_name: "group_${id}".toString(),
+ cluster_status: 'NORMAL', public_endpoint: "public_${id}".toString(),
+ private_endpoint: "private_${id}".toString(), properties: [key: 'value'], ctime: 1,
+ nodes: [[cloud_unique_id: "be_${id}".toString(), ip: '127.0.0.1', heartbeat_port: 9050,
+ status: 'NODE_STATUS_RUNNING', ctime: 1],
+ [cloud_unique_id: "be2_${id}".toString(), ip: '127.0.0.2', heartbeat_port: 9051,
+ status: 'NODE_STATUS_RUNNING', ctime: 1]]]
+ }
+
+ private static Map fixture(List<Map> clusters = [cluster('a'), cluster('b')], String defaultGroup = 'group_a') {
+ def state = [clusters: copy(clusters), defaultGroup: defaultGroup, events: [],
+ failDrop: [], failAdd: [], failWait: false, instanceStatus: 'NORMAL']
+ state.guard = new MultiClusterStateGuard(
+ listClusters: { state.clusters },
+ restoreInstance: { state.events << 'instance'; state.instanceStatus = 'NORMAL' },
+ dropCluster: { c ->
+ state.events << "drop:${c.cluster_id}".toString()
+ if (c.cluster_id in state.failDrop) { throw new IOException('drop failed') }
+ state.clusters.removeAll { it.cluster_id == c.cluster_id }
+ },
+ addCluster: { c ->
+ state.events << "add:${c.cluster_id}".toString()
+ if (c.cluster_id in state.failAdd) { throw new IOException('add failed') }
+ c.ctime = 100
+ c.nodes.each { it.ctime = 100 }
+ state.clusters << c
+ },
+ readDefault: { state.defaultGroup },
+ setDefault: { saved -> state.events << 'default'; state.defaultGroup = saved },
+ awaitState: { expected, expectedDefault ->
+ state.events << "wait:${expectedDefault}".toString()
+ if (state.failWait) { throw new IOException('FE timeout') }
+ assertEquals(MultiClusterStateGuard.topology(expected), MultiClusterStateGuard.topology(state.clusters))
+ if (expectedDefault != null) { assertEquals(expectedDefault, state.defaultGroup) }
+ }
+ )
+ state
+ }
+
+ @Test
+ void sharedBaselineUsesFirstTwoConfiguredNodesWithoutSharingMutableSnapshots() {
+ String config = 'be0:9050:8040:id0,be1:9051:8041:id1,be2:9052:8042:id2'
+ def baseline = MultiClusterStateGuard.sharedBaseline(config)
+ assertEquals(['regression_cluster_name0', 'regression_cluster_name1'], baseline*.cluster_name)
+ assertEquals(['regression_cluster_id0', 'regression_cluster_id1'], baseline*.cluster_id)
+ assertEquals(['id0', 'id1'], baseline.collect { it.nodes[0].cloud_unique_id })
+ assertEquals(8041, baseline[1].nodes[0].http_port)
+ baseline[0].nodes.clear()
+ assertEquals(1, MultiClusterStateGuard.sharedBaseline(config)[0].nodes.size())
+ }
+
+ @Test
+ void acceptsOptionalBrpcPortAndSharedCloudId() {
+ def nodes = MultiClusterStateGuard.configuredBackends('be0:9050:8040:shared:8060,be1:9050:8040:shared')
+ assertEquals(['shared', 'shared'], nodes*.cloud_unique_id)
+ assertEquals(['be0', 'be1'], nodes*.ip)
+ assertEquals(2, MultiClusterStateGuard.sharedBaseline('be0:9050:8040:shared:8060,be1:9050:8040:shared:8060').size())
+ }
+
+ @Test
+ void initializerReplacesExistingFixtureAndRollsBackFailedReplacement() {
+ def state = fixture()
+ state.guard.run(true) {
+ assertEquals('', state.defaultGroup)
+ state.clusters = [cluster('canonical')]
+ state.defaultGroup = 'group_canonical'
+ }
+ assertEquals(['canonical'], state.clusters*.cluster_id)
+ assertEquals('group_canonical', state.defaultGroup)
+ state = fixture()
+ def original = copy(state.clusters)
+ assertThrows(IOException) {
+ state.guard.run(true) {
+ state.clusters = [cluster('canonical')]
+ state.defaultGroup = 'group_canonical'
+ throw new IOException('load failed')
+ }
+ }
+ assertEquals(MultiClusterStateGuard.topology(original), MultiClusterStateGuard.topology(state.clusters))
+ assertEquals('group_a', state.defaultGroup)
+ }
+
+ @Test
+ void sharedBaselineRejectsMissingMalformedOrDuplicateNodes() {
+ [null, '', 'be0:9050:8040:id0', 'be0:9050:8040:id0,invalid',
+ 'be0:0:8040:id0,be1:9050:8040:id1', 'be0:9050:8040:id0:invalid,be1:9050:8040:id0',
+ 'be0:9050:8040:id0,be0:9050:8041:id1'].each { config ->
+ assertThrows(IllegalArgumentException) { MultiClusterStateGuard.sharedBaseline(config) }
+ }
+ }
+
+ @Test
+ void successRestoresFullMetadataAndDefaultAfterTopology() {
+ def state = fixture()
+ def original = copy(state.clusters)
+ assertEquals(42, state.guard.run(false) {
+ state.clusters[0].nodes.remove(0)
+ state.clusters[0].properties.key = 'changed'
+ state.clusters << cluster('temporary')
+ state.defaultGroup = 'group_temporary'
+ state.instanceStatus = 'OVERDUE'
+ 42
+ })
+ assertEquals(MultiClusterStateGuard.topology(original), MultiClusterStateGuard.topology(state.clusters))
+ assertEquals('group_a', state.defaultGroup)
+ assertEquals('NORMAL', state.instanceStatus)
+ assertEquals(['default', 'wait:', 'instance', 'drop:a', 'drop:temporary', 'add:a', 'wait:null', 'default', 'wait:group_a'], state.events)
+ }
+
+ @Test
+ void failureAfterDeletingGroupsRestoresSnapshotAndRethrowsOriginal() {
+ def state = fixture()
+ def original = copy(state.clusters)
+ def failure = new AssertionError('original failure')
+ assertSame(failure, assertThrows(AssertionError) {
+ state.guard.run(false) {
+ state.clusters.clear()
+ state.clusters << cluster('temporary')
+ state.defaultGroup = 'group_temporary'
+ throw failure
+ }
+ })
+ assertEquals(MultiClusterStateGuard.topology(original), MultiClusterStateGuard.topology(state.clusters))
+ assertEquals('group_a', state.defaultGroup)
+ assertEquals(0, failure.suppressed.length)
+ }
+
+ @Test
+ void unchangedTopologyIsNotRecreated() {
+ def state = fixture()
+ state.guard.run(false) {
+ state.clusters.reverse(true)
+ state.clusters[0].ctime = 123
+ state.clusters[0].nodes.reverse(true)
+ }
+ assertEquals(['default', 'wait:', 'instance', 'wait:null', 'default', 'wait:group_a'], state.events)
+ }
+
+ @Test
+ void cleanupFailuresAreSuppressedAndOtherGroupsAreStillRestored() {
+ def state = fixture()
+ def failure = new AssertionError('query failed')
+ state.failDrop = ['temporary']
+ state.failAdd = ['a']
+ assertSame(failure, assertThrows(AssertionError) {
+ state.guard.run(false) {
+ state.failWait = true
+ state.clusters = [cluster('temporary')]
+ throw failure
+ }
+ })
+ assertTrue(state.events.contains('add:b'))
+ assertTrue(state.events.contains('default'))
+ assertEquals(1, failure.suppressed.length)
+ assertEquals(4, failure.suppressed[0].suppressed.length)
+ }
+
+ @Test
+ void cleanupFailureFailsAnOtherwiseSuccessfulSuite() {
+ def state = fixture()
+ def failure = assertThrows(IllegalStateException) { state.guard.run(false) { state.failWait = true; 42 } }
+ assertEquals(2, failure.suppressed.length)
+ assertTrue(state.events.contains('default'))
+ }
+
+ @Test
+ void bootstrapKeepsSuccessfulInitialSetupButRollsBackFailure() {
+ def state = fixture([], '')
+ state.guard.run(true) { state.clusters << cluster('a'); state.defaultGroup = 'group_a' }
+ assertEquals(['a'], state.clusters*.cluster_id)
+ assertEquals('group_a', state.defaultGroup)
+
+ state = fixture([], '')
+ assertThrows(IOException) {
+ state.guard.run(true) { state.clusters << cluster('a'); throw new IOException('load failed') }
+ }
+ assertTrue(state.clusters.isEmpty())
+ assertEquals('', state.defaultGroup)
+
+ state = fixture([], '')
+ state.guard.run(false) { state.clusters << cluster('a') }
+ assertTrue(state.clusters.isEmpty())
+ }
+
+ @Test
+ void rejectsInvalidInitialStateBeforeRunningBody() {
+ def suspended = cluster('a'); suspended.cluster_status = 'SUSPENDED'
+ def decommissioning = cluster('a'); decommissioning.nodes[0].status = 'NODE_STATUS_DECOMMISSIONING'
+ [fixture([cluster('a')], 'missing'), fixture([suspended]), fixture([decommissioning]),
+ fixture([[type: 'VIRTUAL']])].each { state ->
+ assertThrows(IllegalStateException) { state.guard.run(false) { fail('body must not run') } }
+ assertTrue(state.events.isEmpty())
+ }
+ }
+
+ @Test
+ void feMustSeeExactNodeMappingAndAllHeartbeats() {
+ def expected = [cluster('a')]
+ def groups = [[cluster: 'group_a']]
+ def nodes = [[Host: '127.0.0.1', HeartbeatPort: 9050, Alive: 'true', Tag: JsonOutput.toJson([cloud_cluster_id: 'a', cloud_cluster_name: 'group_a',
+ cloud_cluster_status: 'NORMAL', cloud_unique_id: 'be_a'])],
+ [Host: '127.0.0.2', HeartbeatPort: 9051, Alive: true, Tag: [compute_group_id: 'a',
+ compute_group_name: 'group_a', compute_group_status: 'NORMAL', cloud_unique_id: 'be2_a']]]
+ assertTrue(MultiClusterStateGuard.frontendMatches(expected, groups, nodes))
+ assertFalse(MultiClusterStateGuard.frontendMatches(expected, [], nodes))
+ assertFalse(MultiClusterStateGuard.frontendMatches(expected, groups, nodes.take(1)))
+ nodes[1].Tag.remove('compute_group_status')
+ assertTrue(MultiClusterStateGuard.frontendMatches(expected, groups, nodes))
+ nodes[1].Tag.compute_group_status = 'NORMAL'
+ nodes[1].Alive = false
+ assertFalse(MultiClusterStateGuard.frontendMatches(expected, groups, nodes))
+ nodes[1].Alive = true
+ nodes[1].HeartbeatPort = 9052
+ assertFalse(MultiClusterStateGuard.frontendMatches(expected, groups, nodes))
+ nodes[1].HeartbeatPort = 9051
+ nodes[1].Tag.compute_group_status = 'SUSPENDED'
+ assertFalse(MultiClusterStateGuard.frontendMatches(expected, groups, nodes))
+ nodes[1].Tag.compute_group_status = 'NORMAL'
+ nodes[1].SystemDecommissioned = true
+ assertFalse(MultiClusterStateGuard.frontendMatches(expected, groups, nodes))
+ nodes[1].SystemDecommissioned = false
+ nodes[1].Tag.compute_group_id = 'wrong'
+ assertFalse(MultiClusterStateGuard.frontendMatches(expected, groups, nodes))
+ }
+}
diff --git a/regression-test/framework/src/test/groovy/org/apache/doris/regression/util/MySqlClientTest.groovy b/regression-test/framework/src/test/groovy/org/apache/doris/regression/util/MySqlClientTest.groovy
new file mode 100644
index 00000000000..1444dc54ec7
--- /dev/null
+++ b/regression-test/framework/src/test/groovy/org/apache/doris/regression/util/MySqlClientTest.groovy
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.util
+
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.io.TempDir
+
+import java.nio.file.Files
+import java.nio.file.Path
+
+import static org.junit.jupiter.api.Assertions.*
+
+class MySqlClientTest {
+ @TempDir
+ Path tempDir
+
+ private File client(int exitCode) {
+ File script = tempDir.resolve('fake mysql client').toFile()
+ script.text = """#!/bin/sh
+file=\${1#--defaults-extra-file=}
+printf '%s\\n' "\$file"
+ls -l "\$file"
+printf '%s\\n' 'CONFIG'
+cat "\$file"
+printf '%s\\n' 'ARGS'
+printf '%s\\n' "\$@"
+printf '%s\\n' 'SQL'
+cat
+printf '%s\\n' 'DIR'
+pwd
+printf '%s' 'test diagnostic' >&2
+exit ${exitCode}
+"""
+ assertTrue(script.setExecutable(true))
+ return script
+ }
+
+ @Test
+ void passesOptionsAndSqlLiterallyAndCleansPrivateCredentials() {
+ File executable = client(0)
+ File directory = Files.createDirectory(tempDir.resolve('working directory')).toFile()
+ String sql = 'select "$HOME", "`id`", "\\n";\n'
+ ['', null, 'fake secret "\\\\#;\n\t密碼'].each { password ->
+ def result = MySqlClient.execute('root', password,
+ ['--ssl-mode=VERIFY_CA', '--ssl-ca=/path with spaces/ca.pem'], sql, directory, executable.path)
+ assertEquals(0, result.exitCode)
+ assertEquals('test diagnostic\n', result.stderr)
+ def lines = result.stdout.readLines()
+ assertFalse(new File(lines[0]).exists())
+ assertTrue(lines[1].startsWith('-rw-------'))
+ String args = result.stdout.split('ARGS\n', 2)[1].split('SQL\n', 2)[0]
+ assertEquals(["--defaults-extra-file=${lines[0]}".toString(), '--ssl-mode=VERIFY_CA',
+ '--ssl-ca=/path with spaces/ca.pem'], args.readLines())
+ assertTrue(result.stdout.contains('SQL\n' + sql))
+ assertTrue(result.stdout.endsWith('DIR\n' + directory.canonicalPath + '\n'))
+ String config = result.stdout.split('CONFIG\n', 2)[1].split('ARGS\n', 2)[0]
+ assertTrue(config.startsWith('[client]\nuser="root"\npassword="'))
+ if (!password) assertTrue(config.endsWith('password=""\n'))
+ else assertEquals(3, config.readLines().size())
+ }
+ }
+
+ @Test
+ void preservesNonRootUserAndFailureExitCodeAndStillCleansFile() {
+ def result = MySqlClient.execute('load_user', 'fake-secret', [], 'select 1;', null, client(7).path)
+ assertEquals(7, result.exitCode)
+ assertTrue(result.stdout.contains('user="load_user"'))
+ assertFalse(new File(result.stdout.readLines()[0]).exists())
+ }
+
+ @Test
+ void removesCredentialsWhenProcessCannotStart() {
+ def directory = new File(System.getProperty('java.io.tmpdir'))
+ def credentialFiles = { directory.listFiles().findAll { it.name.startsWith('doris-mysql-') }.collect { it.name }.toSet() }
+ def before = credentialFiles()
+ assertThrows(IOException) {
+ MySqlClient.execute('root', 'fake-secret', [], '', null, tempDir.resolve('missing-client').toString())
+ }
+ assertEquals(before, credentialFiles())
+ }
+}
diff --git a/regression-test/framework/src/test/groovy/org/apache/doris/regression/util/RowsetMetaUtilsTest.groovy b/regression-test/framework/src/test/groovy/org/apache/doris/regression/util/RowsetMetaUtilsTest.groovy
new file mode 100644
index 00000000000..baa3984591d
--- /dev/null
+++ b/regression-test/framework/src/test/groovy/org/apache/doris/regression/util/RowsetMetaUtilsTest.groovy
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.util
+
+import org.junit.jupiter.api.Test
+
+import java.nio.charset.StandardCharsets
+
+import static org.junit.jupiter.api.Assertions.*
+
+class RowsetMetaUtilsTest {
+ @Test
+ void mergedAndUnsortedRowsetsCoverCommittedVersion() {
+ assertTrue(RowsetMetaUtils.coversVersion(
+ [rowsets: ['[32-61] 3 DATA', '[0-1] 0 DATA', '[2-31] 2 DATA']], 61))
+ }
+
+ @Test
+ void maximumVersionCannotHideGapOrMissingInitialRowset() {
+ assertFalse(RowsetMetaUtils.coversVersion([rowsets: ['[0-1]', '[3-100]']], 11))
+ assertFalse(RowsetMetaUtils.coversVersion([rowsets: ['[2-100]']], 11))
+ assertFalse(RowsetMetaUtils.coversVersion([rowsets: ['[0-1]']], 2))
+ assertFalse(RowsetMetaUtils.coversVersion([rowsets: []], 2))
+ }
+
+ @Test
+ void gapAfterTargetDoesNotInvalidateTargetCoverage() {
+ assertTrue(RowsetMetaUtils.coversVersion([rowsets: ['[0-1]', '[2-6]', '[8-9]']], 6))
+ }
+
+ @Test
+ void malformedMetadataFailsInsteadOfTimingOut() {
+ assertThrows(IllegalArgumentException) { RowsetMetaUtils.coversVersion([:], 2) }
+ assertThrows(IllegalArgumentException) {
+ RowsetMetaUtils.coversVersion([rowsets: ['not a rowset']], 2)
+ }
+ assertThrows(IllegalArgumentException) {
+ RowsetMetaUtils.coversVersion([rowsets: ['[5-2]']], 2)
+ }
+ assertThrows(IllegalArgumentException) { RowsetMetaUtils.coversVersion([rowsets: []], -1) }
+ }
+
+ @Test
+ void keyBoundsPreserveAllBytesAndDoNotMutateInput() {
+ byte[] bytes = [0, 2, 127, -128, -1] as byte[]
+ String encoded = Base64.encoder.encodeToString(bytes)
+ def input = [segments_key_bounds: [[min_key: encoded, max_key: encoded]],
+ segments_key_bounds_truncated: true]
+ def decoded = RowsetMetaUtils.decodeKeyBounds(input)
+ assertArrayEquals(bytes, decoded.segments_key_bounds[0].min_key.getBytes(StandardCharsets.ISO_8859_1))
+ assertEquals(encoded, input.segments_key_bounds[0].min_key)
+ assertTrue(decoded.segments_key_bounds_truncated)
+ assertFalse(decoded.segments_key_bounds_aggregated)
+ assertFalse(input.containsKey('segments_key_bounds_aggregated'))
+ }
+
+ @Test
+ void missingOrInvalidKeyBoundsAreRejected() {
+ assertThrows(IllegalArgumentException) {
+ RowsetMetaUtils.decodeKeyBounds([segments_key_bounds: [[max_key: 'AA==']]])
+ }
+ assertThrows(IllegalArgumentException) {
+ RowsetMetaUtils.decodeKeyBounds([segments_key_bounds: [[min_key: '***', max_key: 'AA==']]])
+ }
+ }
+}
diff --git a/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy b/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy
index 66024db5649..9e514f98073 100644
--- a/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy
+++ b/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy
@@ -74,7 +74,7 @@ excludeDirectories = "000_the_start_sentinel_do_not_touch," + // keep this line
"variant_github_events_new_p2," +
"hdfs_vault_p2," +
"nereids_p0/hbo," +
- "cloud_p0/multi_cluster," +
+ "cloud_p0," +
"zzz_the_end_sentinel_do_not_touch"// keep this line as the last line
// for test csv with header
diff --git a/regression-test/plugins/cloud_multi_cluster_state.groovy b/regression-test/plugins/cloud_multi_cluster_state.groovy
new file mode 100644
index 00000000000..977410fc03e
--- /dev/null
+++ b/regression-test/plugins/cloud_multi_cluster_state.groovy
@@ -0,0 +1,136 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import groovy.json.JsonOutput
+import org.apache.doris.regression.Config
+import org.apache.doris.regression.suite.Suite
+import org.apache.doris.regression.util.MultiClusterStateGuard
+
+// Shared by the load fixture and cases that reuse or deliberately recreate its groups.
+Suite.metaClass.multiClusterBaseline = { ->
+ MultiClusterStateGuard.sharedBaseline(delegate.context.config.multiClusterBes)
+}
+
+Suite.metaClass.checkMultiClusterBaseline = { List<Map> baseline ->
+ def suite = delegate
+ suite.assertTrue(MultiClusterStateGuard.frontendMatches(baseline,
+ suite.sql_return_maparray('SHOW CLUSTERS'), suite.sql_return_maparray('SHOW BACKENDS')),
+ 'Expected shared regression_cluster_name0/1 groups with their configured BEs alive; ' +
+ 'run the multi_cluster_s3_load/load fixture first and check prior suite cleanup')
+}
+
+// Only for serial legacy suites on the dedicated multiClusterInstance.
+// Docker/VCG/upgrade suites have their own cluster lifecycle.
+Suite.metaClass.withRestoredMultiClusterState = { boolean initializeBaseline, Closure action ->
+ def suite = delegate
+ def config = suite.context.config
+ def nodes = MultiClusterStateGuard.configuredBackends(config.multiClusterBes)
+ if (!config.multiClusterInstance) {
+ throw new IllegalStateException('Missing dedicated multi-cluster test instance/BE configuration')
+ }
+ def request = { String operation, Map payload ->
+ def result
+ suite.httpTest {
+ printResponse false
+ endpoint config.metaServiceHttpAddress
+ uri "/MetaService/http/${operation}?token=${config.metaServiceToken}" +
+ "&instance_id=${java.net.URLEncoder.encode(config.multiClusterInstance, 'UTF-8')}"
+ body JsonOutput.toJson([instance_id: config.multiClusterInstance] + payload)
+ check { code, response ->
+ result = suite.parseJson(response)
+ suite.assertTrue(code == 200 && result.code == 'OK',
+ "Multi-cluster ${operation} failed: ${result.code}")
+ }
+ }
+ result
+ }
+ def readInstanceStatus = { request('get_instance', [:]).result.status }
+ if (readInstanceStatus() != 'NORMAL') {
+ throw new IllegalStateException('Multi-cluster tests require a normal dedicated instance')
+ }
+ def fetchClusters = { request('get_cluster', [cloud_unique_id: nodes[0].cloud_unique_id]).result.cluster }
+ def frontends = suite.sql_return_maparray('SHOW FRONTENDS')
+ if (frontends.isEmpty() || frontends.any { !it.Alive.toString().toBoolean() }) {
+ throw new IllegalStateException('All configured frontends must be alive before changing cluster state')
+ }
+ def readDefault = {
+ def rows = suite.sql_return_maparray('SHOW PROPERTY')
+ def row = rows.find { it.Key == 'default_cloud_cluster' }
+ if (row == null) {
+ throw new IllegalStateException('SHOW PROPERTY did not return default_cloud_cluster')
+ }
+ row.Value?.toString() ?: ''
+ }
+ def guard = new MultiClusterStateGuard(
+ listClusters: fetchClusters,
+ restoreInstance: {
+ if (readInstanceStatus() != 'NORMAL') {
+ request('set_instance_status', [op: 'SET_NORMAL'])
+ }
+ },
+ dropCluster: { saved ->
+ request('drop_cluster', [cluster: [type: 'COMPUTE',
+ cluster_name: saved.cluster_name, cluster_id: saved.cluster_id, nodes: []]])
+ },
+ addCluster: { saved -> request('add_cluster', [cluster: saved]) },
+ readDefault: readDefault,
+ setDefault: { String saved ->
+ def literal = saved.replace('\\', '\\\\').replace("'", "''")
+ suite.sql("SET PROPERTY 'default_cloud_cluster' = '${literal}'")
+ },
+ awaitState: { List<Map> expected, String expectedDefault ->
+ long deadline = System.nanoTime() + 120_000_000_000L
+ Throwable lastError = null
+ while (System.nanoTime() < deadline) {
+ try {
+ if (readInstanceStatus() != 'NORMAL') {
+ throw new IllegalStateException('MS instance status has not returned to normal')
+ }
+ def actual = fetchClusters().findAll { it.type == 'COMPUTE' }
+ if (MultiClusterStateGuard.topology(actual) != MultiClusterStateGuard.topology(expected)) {
+ throw new IllegalStateException('MS compute groups do not match the saved topology')
+ }
+ for (def frontend : frontends) {
+ boolean tls = config.otherConfigs.get('enableTLS')?.toString()?.toBoolean() ?: false
+ def url = tls ? Config.buildUrlWithDb(frontend.Host as String, frontend.QueryPort as int,
+ suite.context.dbName, config.otherConfigs.get('keyStorePath')?.toString(),
+ config.otherConfigs.get('keyStorePassword')?.toString(),
+ config.otherConfigs.get('trustStorePath')?.toString(),
+ config.otherConfigs.get('trustStorePassword')?.toString()) :
+ Config.buildUrlWithDb(frontend.Host as String, frontend.QueryPort as int, suite.context.dbName)
+ url += '&connectTimeout=5000&socketTimeout=5000'
+ suite.connect(config.jdbcUser, config.jdbcPassword, url) {
+ def visibleGroups = suite.sql_return_maparray('SHOW CLUSTERS')
+ def visibleNodes = suite.sql_return_maparray('SHOW BACKENDS')
+ if (!MultiClusterStateGuard.frontendMatches(expected, visibleGroups, visibleNodes)) {
+ throw new IllegalStateException("FE ${frontend.Host} has not restored groups/nodes/heartbeats")
+ }
+ if (expectedDefault != null && readDefault() != expectedDefault) {
+ throw new IllegalStateException("FE ${frontend.Host} has not restored the user default group")
+ }
+ }
+ }
+ return
+ } catch (Exception e) {
+ lastError = e
+ }
+ Thread.sleep(500)
+ }
+ throw new IllegalStateException('Timed out restoring multi-cluster state on MS and all frontends', lastError)
+ }
+ )
+ guard.run(initializeBaseline, action)
+}
diff --git a/regression-test/plugins/plugin_compaction.groovy b/regression-test/plugins/plugin_compaction.groovy
index 8cf6b1f0f87..b3e600ad958 100644
--- a/regression-test/plugins/plugin_compaction.groovy
+++ b/regression-test/plugins/plugin_compaction.groovy
@@ -19,45 +19,57 @@ import org.apache.doris.regression.suite.Suite
import java.util.concurrent.TimeUnit
import org.awaitility.Awaitility;
-Suite.metaClass.be_get_compaction_status{ String ip, String port, String tablet_id /* param */->
- return curl("GET", String.format("http://%s:%s/api/compaction/run_status?tablet_id=%s", ip, port, tablet_id))
+Suite.metaClass.be_get_compaction_status{ String ip, String port, String tablet_id,
+ Integer timeout_sec = 10, Integer max_retries = 10 /* param */->
+ return delegate.curl("GET", String.format("http://%s:%s/api/compaction/run_status?tablet_id=%s", ip, port, tablet_id),
+ null, timeout_sec, delegate.context.config.feHttpUser,
+ delegate.context.config.feHttpPassword, max_retries)
}
Suite.metaClass.be_get_overall_compaction_status{ String ip, String port /* param */->
- return curl("GET", String.format("http://%s:%s/api/compaction/run_status", ip, port))
+ return delegate.curl("GET", String.format("http://%s:%s/api/compaction/run_status", ip, port),
+ null, 10, delegate.context.config.feHttpUser, delegate.context.config.feHttpPassword, 10)
}
-Suite.metaClass.be_show_tablet_status{ String ip, String port, String tablet_id /* param */->
- return curl("GET", String.format("http://%s:%s/api/compaction/show?tablet_id=%s", ip, port, tablet_id))
+Suite.metaClass.be_show_tablet_status{ String ip, String port, String tablet_id,
+ Integer timeout_sec = 10, Integer max_retries = 10 /* param */->
+ return delegate.curl("GET", String.format("http://%s:%s/api/compaction/show?tablet_id=%s", ip, port, tablet_id),
+ null, timeout_sec, delegate.context.config.feHttpUser,
+ delegate.context.config.feHttpPassword, max_retries)
}
Suite.metaClass._be_run_compaction = { String ip, String port, String tablet_id, String compact_type ->
- return curl("POST", String.format("http://%s:%s/api/compaction/run?tablet_id=%s&compact_type=%s",
- ip, port, tablet_id, compact_type))
+ return delegate.curl("POST", String.format("http://%s:%s/api/compaction/run?tablet_id=%s&compact_type=%s",
+ ip, port, tablet_id, compact_type), null, 10, delegate.context.config.feHttpUser,
+ delegate.context.config.feHttpPassword, 10)
}
Suite.metaClass.be_run_base_compaction = { String ip, String port, String tablet_id /* param */->
- return _be_run_compaction(ip, port, tablet_id, "base")
+ return delegate._be_run_compaction(ip, port, tablet_id, "base")
}
logger.info("Added 'be_run_base_compaction' function to Suite")
Suite.metaClass.be_run_cumulative_compaction = { String ip, String port, String tablet_id /* param */->
- return _be_run_compaction(ip, port, tablet_id, "cumulative")
+ return delegate._be_run_compaction(ip, port, tablet_id, "cumulative")
}
logger.info("Added 'be_run_cumulative_compaction' function to Suite")
Suite.metaClass.be_run_full_compaction = { String ip, String port, String tablet_id /* param */->
- return _be_run_compaction(ip, port, tablet_id, "full")
+ return delegate._be_run_compaction(ip, port, tablet_id, "full")
}
Suite.metaClass.be_run_full_compaction_by_table_id = { String ip, String port, String table_id /* param */->
- return curl("POST", String.format("http://%s:%s/api/compaction/run?table_id=%s&compact_type=full", ip, port, table_id))
+ return delegate.curl("POST", String.format("http://%s:%s/api/compaction/run?table_id=%s&compact_type=full",
+ ip, port, table_id), null, 10, delegate.context.config.feHttpUser,
+ delegate.context.config.feHttpPassword, 10)
}
logger.info("Added 'be_run_full_compaction' function to Suite")
-Suite.metaClass.trigger_and_wait_compaction = { String table_name, String compaction_type, int timeout_seconds=300, String[] ignored_errors=[] ->
+Suite.metaClass.trigger_and_wait_compaction = { String table_name, String compaction_type, int timeout_seconds=300,
+ String[] ignored_errors=[], Collection tablet_ids=[],
+ String[] retryable_errors=[] ->
if (!(compaction_type in ["cumulative", "base", "full"])) {
throw new IllegalArgumentException("invalid compaction type: ${compaction_type}, supported types: cumulative, base, full")
}
@@ -66,6 +78,13 @@ Suite.metaClass.trigger_and_wait_compaction = { String table_name, String compac
def backendId_to_backendHttpPort = [:]
getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
def tablets = sql_return_maparray """show tablets from ${table_name}"""
+ if (!tablet_ids.isEmpty()) {
+ def requestedTabletIds = tablet_ids.collect { "${it}" }.toSet()
+ tablets = tablets.findAll { requestedTabletIds.contains("${it.TabletId}") }
+ def foundTabletIds = tablets.collect { "${it.TabletId}" }.toSet()
+ assert foundTabletIds == requestedTabletIds:
+ "Unable to find all requested tablets for ${table_name}, requested: ${requestedTabletIds}, found: ${foundTabletIds}"
+ }
def exit_code, stdout, stderr
def auto_compaction_disabled = sql("show create table ${table_name}")[0][1].contains('"disable_auto_compaction" = "true"')
@@ -76,7 +95,7 @@ Suite.metaClass.trigger_and_wait_compaction = { String table_name, String compac
for (tablet in tablets) {
def be_host = backendId_to_backendIP["${tablet.BackendId}"]
def be_port = backendId_to_backendHttpPort["${tablet.BackendId}"]
- (exit_code, stdout, stderr) = be_show_tablet_status(be_host, be_port, tablet.TabletId)
+ (exit_code, stdout, stderr) = delegate.be_show_tablet_status(be_host, be_port, tablet.TabletId)
assert exit_code == 0: "get tablet status failed, exit code: ${exit_code}, stdout: ${stdout}, stderr: ${stderr}"
def tabletStatus = parseJson(stdout.trim())
@@ -87,23 +106,38 @@ Suite.metaClass.trigger_and_wait_compaction = { String table_name, String compac
for (tablet in tablets) {
def be_host = backendId_to_backendIP["${tablet.BackendId}"]
def be_port = backendId_to_backendHttpPort["${tablet.BackendId}"]
- switch (compaction_type) {
- case "cumulative":
- (exit_code, stdout, stderr) = be_run_cumulative_compaction(be_host, be_port, tablet.TabletId)
- break
- case "base":
- (exit_code, stdout, stderr) = be_run_base_compaction(be_host, be_port, tablet.TabletId)
- break
- case "full":
- (exit_code, stdout, stderr) = be_run_full_compaction(be_host, be_port, tablet.TabletId)
- break
- }
- assert exit_code == 0: "trigger compaction failed, exit code: ${exit_code}, stdout: ${stdout}, stderr: ${stderr}"
- def trigger_status = parseJson(stdout.trim())
- if (trigger_status.status.toLowerCase() != "success") {
+ long triggerDeadline = System.currentTimeMillis() + timeout_seconds * 1000L
+ while (true) {
+ switch (compaction_type) {
+ case "cumulative":
+ (exit_code, stdout, stderr) = delegate.be_run_cumulative_compaction(be_host, be_port, tablet.TabletId)
+ break
+ case "base":
+ (exit_code, stdout, stderr) = delegate.be_run_base_compaction(be_host, be_port, tablet.TabletId)
+ break
+ case "full":
+ (exit_code, stdout, stderr) = delegate.be_run_full_compaction(be_host, be_port, tablet.TabletId)
+ break
+ }
+ assert exit_code == 0: "trigger compaction failed, exit code: ${exit_code}, stdout: ${stdout}, stderr: ${stderr}"
+ def trigger_status = parseJson(stdout.trim())
def status_lower = trigger_status.status.toLowerCase()
- if (status_lower == "already_exist") {
- triggered_tablets.add(tablet) // compaction already in queue, treat it as successfully triggered
+ if (status_lower == "success" || status_lower == "already_exist") {
+ triggered_tablets.add(tablet)
+ break
+ } else if (retryable_errors.any { error -> status_lower.contains(error.toLowerCase()) }) {
+ assert System.currentTimeMillis() < triggerDeadline:
+ "retry trigger compaction timeout, be host: ${be_host}, tablet id: ${tablet.TabletId}, status: ${trigger_status.status}"
+ logger.info("retry transient compaction trigger failure, be host: ${be_host}, tablet id: ${tablet.TabletId}, status: ${trigger_status.status}")
+ // The transient trigger failure updates the tablet's failure timestamp.
+ // Refresh the baseline so it cannot be mistaken for completion of the
+ // later successful asynchronous trigger.
+ (exit_code, stdout, stderr) = delegate.be_show_tablet_status(be_host, be_port, tablet.TabletId)
+ assert exit_code == 0:
+ "refresh tablet status failed, exit code: ${exit_code}, stdout: ${stdout}, stderr: ${stderr}"
+ be_tablet_compaction_status.put("${be_host}-${tablet.TabletId}", parseJson(stdout.trim()))
+ sleep(1000)
+ continue
} else if (!auto_compaction_disabled) {
// ignore the error if auto compaction enabled
} else if (status_lower.contains("e-2000") || status_lower.contains("e-2010")
@@ -119,8 +153,7 @@ Suite.metaClass.trigger_and_wait_compaction = { String table_name, String compac
} else {
throw new Exception("trigger compaction failed, be host: ${be_host}, tablet id: ${tablet.TabletId}, status: ${trigger_status.status}")
}
- } else {
- triggered_tablets.add(tablet)
+ break
}
}
@@ -136,24 +169,34 @@ Suite.metaClass.trigger_and_wait_compaction = { String table_name, String compac
return null
}
}
- Awaitility.await().atMost(timeout_seconds, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until(() -> {
+ // Parallel suites own their worker failures; do not capture uncaught exceptions from other suites.
+ Awaitility.await().dontCatchUncaughtExceptions().atMost(timeout_seconds, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until(() -> {
for (tablet in triggered_tablets) {
def be_host = backendId_to_backendIP["${tablet.BackendId}"]
def be_port = backendId_to_backendHttpPort["${tablet.BackendId}"]
- (exit_code, stdout, stderr) = be_get_compaction_status(be_host, be_port, tablet.TabletId)
- assert exit_code == 0: "get compaction status failed, exit code: ${exit_code}, stdout: ${stdout}, stderr: ${stderr}"
+ // Awaitility owns the retry loop. Keep each HTTP probe bounded so an
+ // inner curl retry cannot consume the entire compaction timeout.
+ (exit_code, stdout, stderr) = delegate.be_get_compaction_status(be_host, be_port, tablet.TabletId, 5, 1)
+ if (exit_code != 0) {
+ logger.warn("get compaction status failed, will retry, be host: ${be_host}, tablet id: ${tablet.TabletId}, exit code: ${exit_code}, stdout: ${stdout}, stderr: ${stderr}")
+ return false
+ }
def compactionStatus = parseJson(stdout.trim())
assert compactionStatus.status.toLowerCase() == "success": "compaction failed, be host: ${be_host}, tablet id: ${tablet.TabletId}, status: ${compactionStatus.status}"
// running is true means compaction is still running
running = compactionStatus.run_status
if (!is_time_series_compaction) {
- (exit_code, stdout, stderr) = be_show_tablet_status(be_host, be_port, tablet.TabletId)
- assert exit_code == 0: "get tablet status failed, exit code: ${exit_code}, stdout: ${stdout}, stderr: ${stderr}"
+ (exit_code, stdout, stderr) = delegate.be_show_tablet_status(be_host, be_port, tablet.TabletId, 5, 1)
+ if (exit_code != 0) {
+ logger.warn("get tablet status failed, will retry, be host: ${be_host}, tablet id: ${tablet.TabletId}, exit code: ${exit_code}, stdout: ${stdout}, stderr: ${stderr}")
+ return false
+ }
def tabletStatus = parseJson(stdout.trim())
def oldStatus = be_tablet_compaction_status.get("${be_host}-${tablet.TabletId}")
- // last compaction success/failure time isn't updated, indicates compaction is not started(so we treat it as running and wait)
+ // The HTTP trigger is asynchronous in cloud mode. run_status may already be
+ // false at the first poll, so also require a completion marker for this run.
def handedOffToBaseCompactionAfterDeleteVersion = false
def completedByBaseCompactionAfterDeleteVersion = false
if (compaction_type == "cumulative") {
@@ -163,9 +206,6 @@ Suite.metaClass.trigger_and_wait_compaction = { String table_name, String compac
def baseSuccessTimeChanged = oldStatus["last base success time"] != tabletStatus["last base success time"]
def cumulativeSuccessTimeChanged =
oldStatus["last cumulative success time"] != tabletStatus["last cumulative success time"]
- // E-2010 advances the cumulative point and lets base compaction handle delete-version rowsets.
- // In some timing windows, base success is already visible in the cached old status while
- // cumulative success advances later, so accept either success signal but not failure time alone.
handedOffToBaseCompactionAfterDeleteVersion = lastCumulativeStatus.contains("e-2010") &&
oldCumulativePoint != null && newCumulativePoint != null &&
newCumulativePoint > oldCumulativePoint
@@ -173,19 +213,21 @@ Suite.metaClass.trigger_and_wait_compaction = { String table_name, String compac
handedOffToBaseCompactionAfterDeleteVersion &&
(baseSuccessTimeChanged || cumulativeSuccessTimeChanged)
}
- def success_time_unchanged = (oldStatus["last ${compaction_type} success time"] == tabletStatus["last ${compaction_type} success time"])
- def failure_time_unchanged = (oldStatus["last ${compaction_type} failure time"] == tabletStatus["last ${compaction_type} failure time"])
- def currentCompactionTimestampChanged = !success_time_unchanged || !failure_time_unchanged
+ def successTimeUnchanged = oldStatus["last ${compaction_type} success time"] ==
+ tabletStatus["last ${compaction_type} success time"]
+ def failureTimeUnchanged = oldStatus["last ${compaction_type} failure time"] ==
+ tabletStatus["last ${compaction_type} failure time"]
+ def completionTimestampChanged = !successTimeUnchanged || !failureTimeUnchanged
def compactionFinished = completedByBaseCompactionAfterDeleteVersion ||
- (!handedOffToBaseCompactionAfterDeleteVersion && currentCompactionTimestampChanged)
+ (!handedOffToBaseCompactionAfterDeleteVersion && completionTimestampChanged)
running = running || !compactionFinished
if (running) {
logger.info("compaction is still running, be host: ${be_host}, tablet id: ${tablet.TabletId}, run status: ${compactionStatus.run_status}, old status: ${oldStatus}, new status: ${tabletStatus}")
return false
}
} else {
- // time series compaction sometimes doesn't update compaction success time
- // so we solely check run_status for it
+ // Time series compaction sometimes doesn't update compaction success time,
+ // so solely check run_status for it.
if (running) {
logger.info("compaction is still running, be host: ${be_host}, tablet id: ${tablet.TabletId}")
return false
diff --git a/regression-test/plugins/plugin_curl_requester.groovy b/regression-test/plugins/plugin_curl_requester.groovy
index a0a813fb800..e16945616db 100644
--- a/regression-test/plugins/plugin_curl_requester.groovy
+++ b/regression-test/plugins/plugin_curl_requester.groovy
@@ -146,7 +146,7 @@ Suite.metaClass.http_client = { String method, String url /* param */ ->
logger.info("Added 'http_client' function to Suite")
-Suite.metaClass.curl = { String method, String url, String body = null, Integer timeoutSec = 10, String user = "", String pwd = ""->
+Suite.metaClass.curl = { String method, String url, String body = null, Integer timeoutSec = 10, String user = "", String pwd = "", Integer maxRetries = 10 ->
Suite suite = delegate as Suite
if (method != "GET" && method != "POST") {
throw new Exception(String.format("invalid curl method: %s", method))
@@ -155,7 +155,6 @@ Suite.metaClass.curl = { String method, String url, String body = null, Integer
throw new Exception("invalid curl url, blank")
}
- Integer maxRetries = 10; // Maximum number of retries
Integer retryCount = 0; // Current retry count
Integer sleepTime = 5000; // Sleep time in milliseconds
diff --git a/regression-test/plugins/plugin_index_change.groovy b/regression-test/plugins/plugin_index_change.groovy
index eb47798f274..ad7edd1a12c 100644
--- a/regression-test/plugins/plugin_index_change.groovy
+++ b/regression-test/plugins/plugin_index_change.groovy
@@ -16,6 +16,7 @@
// under the License.
import org.apache.doris.regression.suite.Suite
+import org.apache.doris.regression.util.Http
import groovy.json.JsonSlurper
import java.util.regex.Matcher;
@@ -75,7 +76,7 @@ Suite.metaClass.wait_for_last_schema_change_finish = {table_name, OpTimeout ->
Suite.metaClass.http_get = { url ->
def dst = 'http://' + context.config.feHttpAddress
- def conn = new URL(dst + url).openConnection()
+ def conn = Http.openConnection(dst + url)
conn.setRequestMethod("GET")
def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
(context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
@@ -137,4 +138,4 @@ Suite.metaClass.check_bf_index_filter_rows = { sql, expectedRowsBfFiltered ->
log.info("filter number:{}", number)
assertEquals(expectedRowsBfFiltered, number)
}
-}
\ No newline at end of file
+}
diff --git a/regression-test/suites/account_p0/test_auth_show.groovy b/regression-test/suites/account_p0/test_auth_show.groovy
index d9d4f3fa7f1..80050aa06c8 100644
--- a/regression-test/suites/account_p0/test_auth_show.groovy
+++ b/regression-test/suites/account_p0/test_auth_show.groovy
@@ -45,7 +45,7 @@ suite("test_auth_show", "account") {
// With select priv for table, should be able to see db
sql """GRANT SELECT_PRIV ON ${dbName}.${tableName} TO ${user}"""
- def result1 = connect(user, "${pwd}", url) {
+ def result1 = connectToDoris(user, "${pwd}", url) {
sql """show databases like '${dbName}'"""
}
assertEquals(result1.size(), 1)
@@ -53,7 +53,7 @@ suite("test_auth_show", "account") {
// With show_view priv for table, should be able to see db
sql """GRANT SHOW_VIEW_PRIV ON ${dbName}.${tableName} TO ${user}"""
- def result2 = connect(user, "${pwd}", url) {
+ def result2 = connectToDoris(user, "${pwd}", url) {
sql """show databases like '${dbName}'"""
}
assertEquals(result2.size(), 1)
diff --git a/regression-test/suites/account_p0/test_grant_priv.groovy b/regression-test/suites/account_p0/test_grant_priv.groovy
index 21d26dd30d8..f23134bb214 100644
--- a/regression-test/suites/account_p0/test_grant_priv.groovy
+++ b/regression-test/suites/account_p0/test_grant_priv.groovy
@@ -40,7 +40,7 @@ suite("test_grant_priv") {
// test only have select_priv, can not grant to other user
sql """grant select_priv on ${dbName}.* to ${user1}"""
- connect(user1, "${pwd}", url) {
+ connectToDoris(user1, "${pwd}", url) {
try {
sql """grant select_priv on ${dbName}.* to ${user2}"""
Assert.fail("can not grant to other user");
@@ -51,7 +51,7 @@ suite("test_grant_priv") {
// test both have select_priv and grant_priv , can grant to other user
sql """grant grant_priv on ${dbName}.* to ${user1}"""
- connect(user1, "${pwd}", url) {
+ connectToDoris(user1, "${pwd}", url) {
try {
sql """grant select_priv on ${dbName}.* to ${user2}"""
} catch (Exception e) {
@@ -96,7 +96,7 @@ suite("test_grant_priv") {
sql """grant select_priv on *.*.* to role ${role2}"""
sql """grant '${role1}' to ${user1}"""
// test only have role1 can not grant
- connect(user1, "${pwd}", url) {
+ connectToDoris(user1, "${pwd}", url) {
test {
sql """grant select_priv on *.*.* to ${user2}"""
exception "denied"
@@ -105,7 +105,7 @@ suite("test_grant_priv") {
sql """revoke '${role1}' from ${user1}"""
sql """grant '${role2}' to ${user1}"""
// test only have role2 can not grant
- connect(user1, "${pwd}", url) {
+ connectToDoris(user1, "${pwd}", url) {
test {
sql """grant select_priv on *.*.* to ${user2}"""
exception "denied"
@@ -113,7 +113,7 @@ suite("test_grant_priv") {
}
// test both have role1 and role2 can grant to other
sql """grant '${role1}' to ${user1}"""
- connect(user1, "${pwd}", url) {
+ connectToDoris(user1, "${pwd}", url) {
sql """grant select_priv on *.*.* to ${user2}"""
}
sql """drop user if exists ${user1}"""
diff --git a/regression-test/suites/account_p0/test_grant_priv_resource.groovy b/regression-test/suites/account_p0/test_grant_priv_resource.groovy
index c8eea500a07..a9c10da6ef7 100644
--- a/regression-test/suites/account_p0/test_grant_priv_resource.groovy
+++ b/regression-test/suites/account_p0/test_grant_priv_resource.groovy
@@ -33,7 +33,7 @@ suite("test_grant_priv_resource") {
// test only have USAGE_PRIV, can not grant to other user
sql """grant USAGE_PRIV on RESOURCE ${resource1} to ${user1}"""
- connect(user1, "${pwd}", url) {
+ connectToDoris(user1, "${pwd}", url) {
try {
sql """grant USAGE_PRIV on RESOURCE ${resource1} to ${user2}"""
Assert.fail("can not grant to other user");
@@ -44,7 +44,7 @@ suite("test_grant_priv_resource") {
// test both have USAGE_PRIV and grant_priv , can grant to other user
sql """grant grant_priv on RESOURCE * to ${user1}"""
- connect(user1, "${pwd}", url) {
+ connectToDoris(user1, "${pwd}", url) {
try {
sql """grant USAGE_PRIV on RESOURCE ${resource1} to ${user2}"""
} catch (Exception e) {
diff --git a/regression-test/suites/account_p0/test_grant_priv_workload.groovy b/regression-test/suites/account_p0/test_grant_priv_workload.groovy
index 8fa9f901a3e..f0f831a55cb 100644
--- a/regression-test/suites/account_p0/test_grant_priv_workload.groovy
+++ b/regression-test/suites/account_p0/test_grant_priv_workload.groovy
@@ -45,7 +45,7 @@ suite("test_grant_priv_workload") {
// test only have USAGE_PRIV, can not grant to other user
sql """grant USAGE_PRIV on WORKLOAD GROUP ${workload1} to ${user1}"""
- connect(user1, "${pwd}", url) {
+ connectToDoris(user1, "${pwd}", url) {
try {
sql """grant USAGE_PRIV on WORKLOAD GROUP ${workload1} to ${user2}"""
Assert.fail("can not grant to other user");
@@ -56,7 +56,7 @@ suite("test_grant_priv_workload") {
// test both have USAGE_PRIV and grant_priv , can grant to other user
sql """grant grant_priv on WORKLOAD GROUP ${workload1} to ${user1}"""
- connect(user1, "${pwd}", url) {
+ connectToDoris(user1, "${pwd}", url) {
try {
sql """grant USAGE_PRIV on WORKLOAD GROUP ${workload1} to ${user2}"""
} catch (Exception e) {
diff --git a/regression-test/suites/account_p0/test_nereids_authentication.groovy b/regression-test/suites/account_p0/test_nereids_authentication.groovy
index 4b8c644495c..70648ee27b4 100644
--- a/regression-test/suites/account_p0/test_nereids_authentication.groovy
+++ b/regression-test/suites/account_p0/test_nereids_authentication.groovy
@@ -54,26 +54,26 @@ suite("test_nereids_authentication", "query") {
def tokens = context.config.jdbcUrl.split('/')
def url=tokens[0] + "//" + tokens[2] + "/" + dbName + "?"
- def result = connect(user, 'Doris_123456', url) {
+ def result = connectToDoris(user, 'Doris_123456', url) {
sql "SELECT * FROM ${tableName1}"
}
assertEquals(result.size(), 0)
- connect(user, 'Doris_123456', url) {
+ connectToDoris(user, 'Doris_123456', url) {
test {
sql "SELECT * FROM ${tableName2}"
exception "denied"
}
}
- connect(user, 'Doris_123456', url) {
+ connectToDoris(user, 'Doris_123456', url) {
test {
sql "SELECT count(*) FROM ${tableName2}"
exception "denied"
}
}
- connect(user, 'Doris_123456', url) {
+ connectToDoris(user, 'Doris_123456', url) {
test {
sql "SELECT * FROM ${tableName1}, ${tableName2} WHERE ${tableName1}.`key` = ${tableName2}.`key`"
exception "denied"
@@ -81,14 +81,14 @@ suite("test_nereids_authentication", "query") {
}
sql "GRANT SELECT_PRIV ON internal.${dbName}.${tableName2} TO ${user}"
- connect(user, 'Doris_123456', url) {
+ connectToDoris(user, 'Doris_123456', url) {
sql "SELECT * FROM ${tableName2}"
}
assertEquals(result.size(), 0)
- connect(user, 'Doris_123456', url) {
+ connectToDoris(user, 'Doris_123456', url) {
sql "SELECT count(*) FROM ${tableName2}"
}
- connect(user, 'Doris_123456', url) {
+ connectToDoris(user, 'Doris_123456', url) {
sql "SELECT * FROM ${tableName1}, ${tableName2} WHERE ${tableName1}.`key` = ${tableName2}.`key`"
}
assertEquals(result.size(), 0)
diff --git a/regression-test/suites/account_p0/test_nereids_row_policy.groovy b/regression-test/suites/account_p0/test_nereids_row_policy.groovy
index 4463a41c90b..0ed3e8dc9b5 100644
--- a/regression-test/suites/account_p0/test_nereids_row_policy.groovy
+++ b/regression-test/suites/account_p0/test_nereids_row_policy.groovy
@@ -32,10 +32,10 @@ suite("test_nereids_row_policy") {
}
def assertQueryResult = { size ->
- def result = connect(user, '123abc!@#', url) {
+ def result = connectToDoris(user, '123abc!@#', url) {
sql "SELECT * FROM ${tableName}"
}
- connect(user, '123abc!@#', url) {
+ connectToDoris(user, '123abc!@#', url) {
test {
sql "SELECT * FROM ${viewName}"
exception "does not have privilege for"
diff --git a/regression-test/suites/account_p0/test_set_password.groovy b/regression-test/suites/account_p0/test_set_password.groovy
index 65f08d5f40e..8ef1bbc2aa3 100644
--- a/regression-test/suites/account_p0/test_set_password.groovy
+++ b/regression-test/suites/account_p0/test_set_password.groovy
@@ -38,7 +38,7 @@ suite("test_set_password", "account") {
def tokens = context.config.jdbcUrl.split('/')
def url = tokens[0] + "//" + tokens[2] + "/" + "information_schema" + "?"
- connect(user, password1, url) {
+ connectToDoris(user, password1, url) {
def result = sql "SELECT 1"
assertEquals(1, result[0][0])
}
@@ -48,14 +48,14 @@ suite("test_set_password", "account") {
sql "SET PASSWORD FOR '${user}'@'%' = PASSWORD('${password2}')"
// verify login with new password
- connect(user, password2, url) {
+ connectToDoris(user, password2, url) {
def result = sql "SELECT 1"
assertEquals(1, result[0][0])
}
// verify old password no longer works
try {
- connect(user, password1, url) {
+ connectToDoris(user, password1, url) {
sql "SELECT 1"
}
assertTrue(false, "Old password should not work after SET PASSWORD")
diff --git a/regression-test/suites/account_p0/test_view_row_policy.groovy b/regression-test/suites/account_p0/test_view_row_policy.groovy
index 6a7b3d02109..45e5b70fcfa 100644
--- a/regression-test/suites/account_p0/test_view_row_policy.groovy
+++ b/regression-test/suites/account_p0/test_view_row_policy.groovy
@@ -65,7 +65,7 @@ suite("test_view_row_policy") {
}
// no policy
- connect(user, "${pwd}", url) {
+ connectToDoris(user, "${pwd}", url) {
order_qt_no_policy "SELECT * FROM ${viewName}"
}
@@ -74,7 +74,7 @@ suite("test_view_row_policy") {
CREATE ROW POLICY IF NOT EXISTS ${tablePolcyName} ON ${dbName}.${tableName}
AS RESTRICTIVE TO ${user} USING (k=1)
"""
- connect(user, "${pwd}", url) {
+ connectToDoris(user, "${pwd}", url) {
order_qt_table_policy "SELECT * FROM ${viewName}"
}
@@ -83,7 +83,7 @@ suite("test_view_row_policy") {
CREATE ROW POLICY IF NOT EXISTS ${viewPolcyName} ON ${dbName}.${viewName}
AS RESTRICTIVE TO ${user} USING (v=2)
"""
- connect(user, "${pwd}", url) {
+ connectToDoris(user, "${pwd}", url) {
order_qt_view_policy "SELECT * FROM ${viewName}"
}
diff --git a/regression-test/suites/ann_index_p0/ann_topn_small_candidate_fallback.groovy b/regression-test/suites/ann_index_p0/ann_topn_small_candidate_fallback.groovy
index b4f0e29085b..92e9fa56d0e 100644
--- a/regression-test/suites/ann_index_p0/ann_topn_small_candidate_fallback.groovy
+++ b/regression-test/suites/ann_index_p0/ann_topn_small_candidate_fallback.groovy
@@ -16,27 +16,16 @@
// under the License.
import groovy.json.JsonSlurper
+import org.apache.doris.regression.util.Http
def getProfileList = {
- def dst = "http://" + context.config.feHttpAddress
- def conn = new URL(dst + "/rest/v1/query_profile").openConnection()
- conn.setRequestMethod("GET")
- def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
- (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword))
- .getBytes("UTF-8"))
- conn.setRequestProperty("Authorization", "Basic ${encoding}")
- return conn.getInputStream().getText()
+ return Http.GET("http://${context.config.feHttpAddress}/rest/v1/query_profile", false, false,
+ context.config.feHttpUser, context.config.feHttpPassword)
}
def getProfile = { id ->
- def dst = "http://" + context.config.feHttpAddress
- def conn = new URL(dst + "/api/profile/text/?query_id=$id").openConnection()
- conn.setRequestMethod("GET")
- def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
- (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword))
- .getBytes("UTF-8"))
- conn.setRequestProperty("Authorization", "Basic ${encoding}")
- return conn.getInputStream().getText()
+ return Http.GET("http://${context.config.feHttpAddress}/api/profile/text/?query_id=$id", false, false,
+ context.config.feHttpUser, context.config.feHttpPassword)
}
def extractCounterValue = { String profileText, String counterName ->
diff --git a/regression-test/suites/arrow_flight_sql_p0/test_auth_remote_ip.groovy b/regression-test/suites/arrow_flight_sql_p0/test_auth_remote_ip.groovy
index 1d9f02260da..707dff1e1bf 100644
--- a/regression-test/suites/arrow_flight_sql_p0/test_auth_remote_ip.groovy
+++ b/regression-test/suites/arrow_flight_sql_p0/test_auth_remote_ip.groovy
@@ -57,8 +57,7 @@ suite("test_auth_remote_ip", "arrow_flight_sql") {
Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver")
String arrowFlightSqlHost = context.config.otherConfigs.get("extArrowFlightSqlHost")
String arrowFlightSqlPort = context.config.otherConfigs.get("extArrowFlightSqlPort")
- String arrowFlightSqlUrl = "jdbc:arrow-flight-sql://${arrowFlightSqlHost}:${arrowFlightSqlPort}" +
- "/?useServerPrepStmts=false&useSSL=false&useEncryption=false"
+ String arrowFlightSqlUrl = context.config.getArrowFlightSqlJdbcUrl(arrowFlightSqlHost, arrowFlightSqlPort)
Connection conn = DriverManager.getConnection(arrowFlightSqlUrl, user, password)
try {
diff --git a/regression-test/suites/audit/test_audit_log_queue_time.groovy b/regression-test/suites/audit/test_audit_log_queue_time.groovy
index e40d1688c42..e4cc20e9c52 100644
--- a/regression-test/suites/audit/test_audit_log_queue_time.groovy
+++ b/regression-test/suites/audit/test_audit_log_queue_time.groovy
@@ -17,6 +17,7 @@
suite("test_audit_log_queue_time", "nonConcurrent") {
// Check admin privilege
+ def originalAuditPlugin = sql("show global variables like 'enable_audit_plugin'")[0][1]
try {
sql "set global enable_audit_plugin = true"
} catch (Exception e) {
@@ -27,103 +28,148 @@ suite("test_audit_log_queue_time", "nonConcurrent") {
def tableName = "audit_queue_time_test"
def wgName = "test_queue_time_wg"
- def testMarker = UUID.randomUUID().toString().substring(0, 8)
+ def maxConcurrency = 1
+ def queueTimeoutMs = 30000
+ def guaranteedQueueTimeMs = 1000
+ def blockerSleepTime = 10
+ def markerPrefix = UUID.randomUUID().toString().substring(0, 8)
+ def blockerMarker = "${markerPrefix}_blocker"
+ def queuedMarker = "${markerPrefix}_queued"
+ def threads = []
+ def queryErrors = Collections.synchronizedList([])
- // Cleanup environment
- sql "drop table if exists ${tableName}"
- sql "drop workload group if exists ${wgName}"
+ def getQueueState = {
+ def row = sql("show workload groups").find { it[1].toString() == wgName }
+ if (row == null) {
+ return null
+ }
+ return [
+ running: row[row.size() - 2] as int,
+ waiting: row[row.size() - 1] as int
+ ]
+ }
- // Create test table
- sql """
- CREATE TABLE `${tableName}` (
- `id` bigint,
- `name` varchar(32)
- ) ENGINE=OLAP
- DUPLICATE KEY(`id`)
- DISTRIBUTED BY HASH(`id`) BUCKETS 1
- PROPERTIES ("replication_allocation" = "tag.location.default: 1")
- """
+ def waitForQueueState = { int expectedRunning, int expectedWaiting ->
+ def state = null
+ for (int i = 0; i < 100; i++) {
+ state = getQueueState()
+ if (state != null
+ && state.running == expectedRunning
+ && state.waiting == expectedWaiting) {
+ return
+ }
+ sleep(100)
+ }
+ throw new RuntimeException("workload group ${wgName} did not reach "
+ + "running=${expectedRunning}, waiting=${expectedWaiting}; last state=${state}")
+ }
+
+ try {
+ // Cleanup environment
+ sql "drop table if exists ${tableName}"
+ sql "drop workload group if exists ${wgName}"
+
+ // Create test table
+ sql """
+ CREATE TABLE `${tableName}` (
+ `id` bigint,
+ `name` varchar(32)
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+
+ sql "insert into ${tableName} values (1, 'test')"
+
+ // max_concurrency=1 ensures that the second query enters the queue.
+ sql """
+ create workload group ${wgName}
+ properties (
+ 'max_concurrency' = '${maxConcurrency}',
+ 'max_queue_size' = '10',
+ 'queue_timeout' = '${queueTimeoutMs}'
+ )
+ """
+
+ // Wait for workload group to take effect.
+ Thread.sleep(5000)
+
+ // Truncate audit_log for easier testing.
+ sql "truncate table __internal_schema.audit_log"
+
+ // Occupy the only running slot before submitting the query whose queue time is audited.
+ threads << Thread.start {
+ try {
+ sql "set workload_group=${wgName}"
+ sql """
+ select sleep(${blockerSleepTime}), '${blockerMarker}' as marker
+ from ${tableName} limit 1
+ """
+ } catch (Throwable t) {
+ queryErrors.add(t)
+ }
+ }
- sql "insert into ${tableName} values (1, 'test')"
+ waitForQueueState(maxConcurrency, 0)
- def maxConcurrency = 1
- // Create workload group: max_concurrency=1 ensures queries queue up
- sql """
- create workload group ${wgName}
- properties (
- 'max_concurrency' = '${maxConcurrency}',
- 'max_queue_size' = '10',
- 'queue_timeout' = '30000'
- )
- """
-
- // Wait for workload group to take effect
- Thread.sleep(5000)
-
- // Truncate audit_log for easier testing
- sql "truncate table __internal_schema.audit_log"
-
- // Submit concurrent queries with marker for later lookup
- def sqlSleepTime = 5
- def queuedSqlCnt = 1
- def threads = []
- for (int i = 0; i < maxConcurrency + queuedSqlCnt; i++) {
- def idx = i
threads << Thread.start {
try {
sql "set workload_group=${wgName}"
- // Use sleep function to simulate long query, ensuring subsequent queries need to queue
sql """
- select sleep(${sqlSleepTime}), '${testMarker}_${idx}' as marker
+ select id, '${queuedMarker}' as marker
from ${tableName} limit 1
"""
- } catch (Exception e) {
- log.warn("Query ${idx} failed: ${e.getMessage()}")
+ } catch (Throwable t) {
+ queryErrors.add(t)
}
}
- }
- // Wait for all queries to complete
- threads.each { it.join() }
-
- // Wait for audit log to flush
- Thread.sleep(5000)
- sql "call flush_audit_log()"
- Thread.sleep(5000)
-
- // Verify queue_time_ms column exists
- def schemaResult = sql "desc internal.__internal_schema.audit_log"
- def hasQueueTimeMs = schemaResult.any { it[0] == "queue_time_ms" }
- assertTrue(hasQueueTimeMs)
-
- // check result
- def retry = 10
- def query = """
- select query_id, queue_time_ms, stmt
- from __internal_schema.audit_log
- where stmt like '%${testMarker}%'
- and queue_time_ms > 0
- order by time
- """
- def auditResult = sql "${query}"
-
- while (auditResult.isEmpty()) {
- if (retry-- < 0) {
- throw new RuntimeException("It has retried a few but still failed, you need to check it")
+ waitForQueueState(maxConcurrency, 1)
+ sleep(guaranteedQueueTimeMs)
+ waitForQueueState(maxConcurrency, 1)
+
+ threads.each { it.join() }
+ assertTrue(queryErrors.isEmpty(), "query failures: ${queryErrors}")
+
+ // Verify queue_time_ms column exists.
+ def schemaResult = sql "desc internal.__internal_schema.audit_log"
+ def hasQueueTimeMs = schemaResult.any { it[0] == "queue_time_ms" }
+ assertTrue(hasQueueTimeMs)
+
+ // Match only the queued query. Excluding audit_log statements prevents the lookup
+ // queries themselves from matching the marker during retries.
+ def query = """
+ select query_id, queue_time_ms, stmt
+ from __internal_schema.audit_log
+ where stmt like '%${queuedMarker}%'
+ and stmt not like '%__internal_schema.audit_log%'
+ order by time
+ """
+ def auditResult = []
+ for (int retry = 0; retry < 10 && auditResult.isEmpty(); retry++) {
+ sql "call flush_audit_log()"
+ sleep(1000)
+ auditResult = sql "${query}"
}
- sql "call flush_audit_log()"
- sleep(3000)
- auditResult = sql "${query}"
- }
- auditResult.each { row ->
- assertTrue(row[1] >= sqlSleepTime * 1000)
+ assertFalse(auditResult.isEmpty(), "queued query was not found in audit log")
+ logger.info("Queued query audit result: ${auditResult}")
+ auditResult.each { row ->
+ def queueTimeMs = row[1] as long
+ assertTrue(queueTimeMs >= guaranteedQueueTimeMs,
+ "queue_time_ms ${queueTimeMs} is less than guaranteed wait ${guaranteedQueueTimeMs}")
+ assertTrue(queueTimeMs < queueTimeoutMs,
+ "queue_time_ms ${queueTimeMs} reached queue timeout ${queueTimeoutMs}")
+ }
+ } finally {
+ threads.each { thread ->
+ if (thread.isAlive()) {
+ thread.join(blockerSleepTime * 1000 + 5000)
+ }
+ }
+ sql "drop table if exists ${tableName}"
+ sql "drop workload group if exists ${wgName}"
+ sql "set global enable_audit_plugin = ${originalAuditPlugin}"
}
-
- assertTrue(auditResult.size() >= queuedSqlCnt)
-
- // Cleanup
- sql "drop table if exists ${tableName}"
- sql "drop workload group if exists ${wgName}"
- sql "set global enable_audit_plugin = false"
}
diff --git a/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy b/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy
index f09d28b167c..1845304d3b3 100644
--- a/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy
@@ -44,6 +44,7 @@ suite("test_dml_select_udf_auth","p0,auth_call") {
def jarPath = """${context.file.parent}/../javaudf_p0/jars/java-udf-case-jar-with-dependencies.jar"""
scp_udf_file_to_all_be(jarPath)
+ scp_udf_file_to_all_fe(jarPath)
log.info("Jar path: ${jarPath}".toString())
sql """ CREATE FUNCTION ${dbName}.${udfName}(string) RETURNS int PROPERTIES (
diff --git a/regression-test/suites/auth_call/test_dml_stream_load_be_auth_docker.groovy b/regression-test/suites/auth_call/test_dml_stream_load_be_auth_docker.groovy
index 46d06e5df81..10d0388ea0c 100644
--- a/regression-test/suites/auth_call/test_dml_stream_load_be_auth_docker.groovy
+++ b/regression-test/suites/auth_call/test_dml_stream_load_be_auth_docker.groovy
@@ -16,6 +16,27 @@
// under the License.
import org.apache.doris.regression.suite.ClusterOptions
+import java.util.concurrent.TimeUnit
+
+// Drain both pipes while curl runs. Reading process.text after waitForOrKill can read an
+// already closed stream and also risks blocking when the child fills a pipe.
+def executeCurl = { List command ->
+ def process = command.execute()
+ def stdout = new ByteArrayOutputStream()
+ def stderr = new ByteArrayOutputStream()
+ def stdoutThread = process.consumeProcessOutputStream(stdout)
+ def stderrThread = process.consumeProcessErrorStream(stderr)
+ boolean completed = process.waitFor(7200, TimeUnit.SECONDS)
+ if (!completed) {
+ process.destroyForcibly()
+ process.waitFor()
+ }
+ stdoutThread.join()
+ stderrThread.join()
+ assert completed : "curl timed out: ${stderr.toString('UTF-8')}"
+ assert process.exitValue() == 0 : "curl failed: ${stderr.toString('UTF-8')}"
+ return [stdout.toString('UTF-8').trim(), stderr.toString('UTF-8').trim()]
+}
suite("test_dml_stream_load_be_auth_off_docker", "docker,auth_call") {
def options = new ClusterOptions()
@@ -40,7 +61,7 @@ suite("test_dml_stream_load_be_auth_off_docker", "docker,auth_call") {
def parseCurlResult = { String out, String err ->
def lines = out.readLines()
- assertTrue("curl output should include http code, output=${out}, err=${err}", !lines.isEmpty())
+ assertTrue(!lines.isEmpty(), "curl output should include http code, output=${out}, err=${err}")
def httpCode = lines.last() as int
def body = lines.size() == 1 ? "" : lines[0..-2].join("\n")
return [httpCode, body, err]
@@ -63,10 +84,7 @@ suite("test_dml_stream_load_be_auth_off_docker", "docker,auth_call") {
"http://${endpoint}/api/${dbName}/${table}/_stream_load"
])
logger.info("stream load target: ${endpoint}, table: ${table}, label: ${label}, user: ${authUser}")
- def process = command.execute()
- process.waitForOrKill(7200000)
- def out = process.text.trim()
- def err = process.errorStream.text.trim()
+ def (out, err) = executeCurl(command)
logger.info("stream load out: ${out}, err: ${err}")
return parseCurlResult(out, err)
}
@@ -82,10 +100,7 @@ suite("test_dml_stream_load_be_auth_off_docker", "docker,auth_call") {
]
logger.info("stream load 2pc target: ${beHttpAddress}, table: ${table}, "
+ "txn operation: ${txnOperation}, user: ${authUser}")
- def process = command.execute()
- process.waitForOrKill(7200000)
- def out = process.text.trim()
- def err = process.errorStream.text.trim()
+ def (out, err) = executeCurl(command)
logger.info("stream load 2pc out: ${out}, err: ${err}")
return parseCurlResult(out, err)
}
@@ -93,7 +108,7 @@ suite("test_dml_stream_load_be_auth_off_docker", "docker,auth_call") {
def assertLoadFailed = { def result ->
assertEquals(200, result[0])
def json = parseJson(result[1])
- assertTrue("stream load should fail, body=${result[1]}", json.Status != "Success")
+ assertTrue(json.Status != "Success", "stream load should fail, body=${result[1]}")
}
def assertLoadSuccess = { def result ->
@@ -192,7 +207,7 @@ suite("test_dml_stream_load_be_auth_on_docker", "docker,auth_call") {
def parseCurlResult = { String out, String err ->
def lines = out.readLines()
- assertTrue("curl output should include http code, output=${out}, err=${err}", !lines.isEmpty())
+ assertTrue(!lines.isEmpty(), "curl output should include http code, output=${out}, err=${err}")
def httpCode = lines.last() as int
def body = lines.size() == 1 ? "" : lines[0..-2].join("\n")
return [httpCode, body, err]
@@ -215,10 +230,7 @@ suite("test_dml_stream_load_be_auth_on_docker", "docker,auth_call") {
"http://${endpoint}/api/${dbName}/${table}/_stream_load"
])
logger.info("stream load target: ${endpoint}, table: ${table}, label: ${label}, user: ${authUser}")
- def process = command.execute()
- process.waitForOrKill(7200000)
- def out = process.text.trim()
- def err = process.errorStream.text.trim()
+ def (out, err) = executeCurl(command)
logger.info("stream load out: ${out}, err: ${err}")
return parseCurlResult(out, err)
}
@@ -234,10 +246,7 @@ suite("test_dml_stream_load_be_auth_on_docker", "docker,auth_call") {
]
logger.info("stream load 2pc target: ${beHttpAddress}, table: ${table}, "
+ "txn operation: ${txnOperation}, user: ${authUser}")
- def process = command.execute()
- process.waitForOrKill(7200000)
- def out = process.text.trim()
- def err = process.errorStream.text.trim()
+ def (out, err) = executeCurl(command)
logger.info("stream load 2pc out: ${out}, err: ${err}")
return parseCurlResult(out, err)
}
@@ -245,7 +254,7 @@ suite("test_dml_stream_load_be_auth_on_docker", "docker,auth_call") {
def assertLoadFailed = { def result ->
assertEquals(200, result[0])
def json = parseJson(result[1])
- assertTrue("stream load should fail, body=${result[1]}", json.Status != "Success")
+ assertTrue(json.Status != "Success", "stream load should fail, body=${result[1]}")
}
def assertLoadSuccess = { def result ->
diff --git a/regression-test/suites/auth_p0/test_role_mapping_system_table.groovy b/regression-test/suites/auth_p0/test_role_mapping_system_table.groovy
index 284b533a352..95680a8df28 100644
--- a/regression-test/suites/auth_p0/test_role_mapping_system_table.groovy
+++ b/regression-test/suites/auth_p0/test_role_mapping_system_table.groovy
@@ -101,7 +101,7 @@ suite("test_role_mapping_system_table", "p0,auth") {
assertTrue(result[0][6] != null && result[0][6].length() > 0)
assertEquals(result[0][5], result[0][7])
- connect(user, "${pwd}", jdbcUrlWithoutSchema) {
+ connectToDoris(user, "${pwd}", jdbcUrlWithoutSchema) {
test {
sql """
SELECT NAME
diff --git a/regression-test/suites/cloud_p0/auth/test_set_default_cluster.groovy b/regression-test/suites/cloud_p0/auth/test_set_default_cluster.groovy
index ddcf6cd28f4..5193c3e5af6 100644
--- a/regression-test/suites/cloud_p0/auth/test_set_default_cluster.groovy
+++ b/regression-test/suites/cloud_p0/auth/test_set_default_cluster.groovy
@@ -42,8 +42,10 @@ suite("test_default_cluster", "docker") {
def user1 = "default_user1"
// admin role
def user2 = "default_user2"
- // domain user
- def user3 = "default_user3@'175.%'"
+ // The Docker client address depends on the network allocated to this run.
+ def clientHost = (sql "SELECT USER()")[0][0].toString().split('@').last().replace("'", "")
+ assertTrue(clientHost ==~ /[0-9A-Za-z.:_-]+/, "Unexpected Docker client host: ${clientHost}")
+ def user3 = "default_user3@'${clientHost}'"
sql """CREATE USER $user1 IDENTIFIED BY 'Cloud123456' DEFAULT ROLE 'admin'"""
sql """CREATE USER $user2 IDENTIFIED BY 'Cloud123456'"""
diff --git a/regression-test/suites/cloud_p0/balance/test_cross_cg_bvar_metrics.groovy b/regression-test/suites/cloud_p0/balance/test_cross_cg_bvar_metrics.groovy
index b21b347e84e..fdc7a2b4202 100644
--- a/regression-test/suites/cloud_p0/balance/test_cross_cg_bvar_metrics.groovy
+++ b/regression-test/suites/cloud_p0/balance/test_cross_cg_bvar_metrics.groovy
@@ -232,22 +232,26 @@ suite('test_cross_cg_bvar_metrics', 'docker') {
assert beA2 != null : "Could not identify BE_A2"
logger.info("T_bvar2: BE_A2 host={} brpcPort={}", beA2.Host, beA2.BrpcPort)
- // Wait until BE_A2 has both accepted tablets and registered same-CG candidates.
+ // Wait until this table has at least one tablet routed to BE_A2. TabletNum and the
+ // process-wide mapping gauge may become positive before FE updates this table's route.
+ def routedTabletIds = []
awaitUntil(120) {
- def latest = sql_return_maparray('show backends').find {
+ def routedTablets = sql_return_maparray("SHOW TABLETS FROM ${tbl}").findAll {
it.BackendId.toLong() == beA2.BackendId.toLong()
}
- latest != null && latest.TabletNum.toInteger() > 0
+ def candidateReady =
+ getBrpcMetric(beA2.Host, beA2.BrpcPort, "balance_tablet_be_mapping_size") > 0
+ routedTabletIds = routedTablets.collect { it.TabletId.toLong() }
+ logger.info("T_bvar2 readiness: BE_A2 routed tablets={}, candidateReady={}",
+ routedTabletIds, candidateReady)
+ !routedTabletIds.isEmpty() && candidateReady
}
- awaitUntil(120) {
- getBrpcMetric(beA2.Host, beA2.BrpcPort, "balance_tablet_be_mapping_size") > 0
- }
-
- // Give the rebalance warmup workflow a short window to submit and finish the skipped
- // warmup tasks before query traffic starts.
- sleep(2000)
sql "use @compute_cluster"
+ // Retries must reach BE_A2 after its tablet route becomes active. Otherwise the first
+ // result can be served repeatedly from FE SQL cache without exercising same-CG reads.
+ sql "SET enable_sql_cache = false"
+ sql "SET enable_query_cache = false"
def t2_sameBefore_A1 = getBrpcMetric(beA.Host, beA.BrpcPort, "peer_same_compute_group_read")
def t2_sameBefore_A2 = getBrpcMetric(beA2.Host, beA2.BrpcPort, "peer_same_compute_group_read")
@@ -255,7 +259,8 @@ suite('test_cross_cg_bvar_metrics', 'docker') {
// Run query iteratively; full-table scans should cover tablets rebalanced to BE_A2.
// Those scans see: (a) same-CG candidate (BE_A1) registered, (b) empty local cache
// after warmup timeout -> same-CG winner race with peer win -> peer_same_compute_group_read++.
- sql "SELECT * FROM ${tbl} ORDER BY k1"
+ def result = sql "SELECT * FROM ${tbl} ORDER BY k1"
+ assertEquals(6, result.size(), "T_bvar2 query should return all rows")
def sameAfterA1 = getBrpcMetric(beA.Host, beA.BrpcPort, "peer_same_compute_group_read")
def sameAfterA2 = getBrpcMetric(beA2.Host, beA2.BrpcPort, "peer_same_compute_group_read")
((sameAfterA1 - t2_sameBefore_A1) + (sameAfterA2 - t2_sameBefore_A2)) > 0
diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/read_write/sync_insert.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/read_write/sync_insert.groovy
index a80dab3105a..c653462c57d 100644
--- a/regression-test/suites/cloud_p0/cache/multi_cluster/read_write/sync_insert.groovy
+++ b/regression-test/suites/cloud_p0/cache/multi_cluster/read_write/sync_insert.groovy
@@ -15,148 +15,228 @@
// specific language governing permissions and limitations
// under the License.
-import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.WarmupMetricsUtils
-suite("sync_insert") {
- List<String> ipList = new ArrayList<>();
- List<String> hbPortList = new ArrayList<>()
- List<String> httpPortList = new ArrayList<>()
- List<String> brpcPortList = new ArrayList<>()
- List<String> beUniqueIdList = new ArrayList<>()
-
- String[] bes = context.config.multiClusterBes.split(',');
- println("the value is " + context.config.multiClusterBes);
- int num = 0
- for(String values : bes) {
- if (num++ == 2) break;
- println("the value is " + values);
- String[] beInfo = values.split(':');
- ipList.add(beInfo[0]);
- hbPortList.add(beInfo[1]);
- httpPortList.add(beInfo[2]);
- beUniqueIdList.add(beInfo[3]);
- brpcPortList.add(beInfo[4]);
+suite("sync_insert", "nonConcurrent") {
+ String sourceCluster = "regression_cluster_name0"
+ String targetCluster = "regression_cluster_name1"
+ String tableName = "test_dup_tab_basic_int_tab_nullable"
+ def values = [
+ [9,10,11,12], [9,10,11,12], [21,null,23,null], [1,2,3,4],
+ [1,2,3,4], [13,14,15,16], [13,21,22,16], [13,14,15,16],
+ [13,21,22,16], [17,18,19,20], [17,18,19,20], [null,21,null,23],
+ [22,null,24,25], [26,27,null,29], [5,6,7,8], [5,6,7,8]
+ ]
+ def backends = sql_return_maparray("SHOW BACKENDS")
+ def clusterBackends = [sourceCluster, targetCluster].collectEntries { clusterName ->
+ def members = backends.findAll { be ->
+ "${be.Alive}".equalsIgnoreCase("true") &&
+ parseJson(be.Tag.toString()).compute_group_name == clusterName
+ }.collect { be ->
+ [id: be.BackendId as Long, ip: be.Host.toString(),
+ httpPort: be.HttpPort.toString(), brpcPort: be.BrpcPort.toString()]
+ }
+ assertTrue(!members.isEmpty(), "No alive backends in ${clusterName}")
+ [(clusterName): members]
}
-
- println("the ip is " + ipList);
- println("the heartbeat port is " + hbPortList);
- println("the http port is " + httpPortList);
- println("the be unique id is " + beUniqueIdList);
- println("the brpc port is " + brpcPortList);
-
- def clearFileCache = { ip, port ->
- httpTest {
- endpoint ""
- uri ip + ":" + port + """/api/file_cache?op=clear&sync=true"""
- op "get"
- body ""
+ logger.info("Sync insert backends: ${clusterBackends}")
+ def tabletIds = []
+ def primaryBackends = [:]
+ def waitUntil = { String description, Closure condition ->
+ long deadline = System.currentTimeMillis() + 120000L
+ while (System.currentTimeMillis() < deadline) {
+ if (condition()) {
+ return
+ }
+ sleep(1000)
}
+ assertTrue(false, "Timeout after 120000ms: ${description}; see last state in log")
}
-
- def getMetricsMethod = { ip, port, check_func ->
+ def getGlobalBytes = {
+ [source: WarmupMetricsUtils.getBackendMetricSum(clusterBackends[sourceCluster], "file_cache_cache_size"),
+ target: WarmupMetricsUtils.getBackendMetricSum(clusterBackends[targetCluster], "file_cache_cache_size")]
+ }
+ def getCache = { String clusterName ->
+ sql "use @${clusterName}"
+ def beIds = clusterBackends[clusterName].collect { it.id }
+ def rows = sql """select tablet_id, be_id, lower(type), sum(size), count(*)
+ from information_schema.file_cache_info
+ where tablet_id in (${tabletIds.join(',')}) and be_id in (${beIds.join(',')})
+ group by tablet_id, be_id, lower(type)"""
+ rows.groupBy { "${it[0]}:${it[1]}".toString() }.collectEntries { key, entries ->
+ [(key): [normalBytes: entries.findAll { it[2]?.toString() == "normal" }.sum(0L) { it[3] as Long },
+ otherBlocks: entries.findAll { it[2]?.toString() != "normal" }.sum(0L) { it[4] as Long },
+ blocks: entries.sum(0L) { it[4] as Long }]]
+ }
+ }
+ def getHeader = { be, long tabletId ->
+ def meta = null
httpTest {
- endpoint ip + ":" + port
- uri "/brpc_metrics"
+ endpoint "${be.ip}:${be.httpPort}"
+ uri "/api/meta/header/${tabletId}"
op "get"
- check check_func
+ check { code, body ->
+ if ((code as int) == 200) {
+ meta = parseJson(body.toString())
+ }
+ }
}
+ meta
}
- def table1 = "test_dup_tab_basic_int_tab_nullable"
- sql """ drop table if exists ${table1} """
+ try {
+ sql "use @${sourceCluster}"
+ String dbName = sql("select database()")[0][0].toString()
+ sql "DROP TABLE IF EXISTS ${tableName} FORCE"
+ sql """CREATE TABLE ${tableName} (
+ siteid INT NULL, citycode INT NULL, userid INT NULL, pv INT NULL
+ ) ENGINE=OLAP DUPLICATE KEY(siteid)
+ DISTRIBUTED BY HASH(siteid) BUCKETS 1
+ PROPERTIES("disable_auto_compaction"="true", "file_cache_ttl_seconds"="0")"""
+ tabletIds = sql("show tablets from ${tableName}").collect { it[0] as Long }.unique()
+ assertTrue(tabletIds.size() == 1, "Expected one tablet: ${tabletIds}")
+ def initialPartitions = sql_return_maparray("show partitions from ${tableName}")
+ assertTrue(initialPartitions.size() == 1, "Expected one partition: ${initialPartitions}")
+ long initialVersion = initialPartitions[0].VisibleVersion as Long
- // clear all file cache on all BEs of all clusters
- clearFileCache.call(ipList[0], httpPortList[0]);
- clearFileCache.call(ipList[1], httpPortList[1]);
-
- sql """
-CREATE TABLE IF NOT EXISTS `${table1}` (
- `siteid` int(11) NULL COMMENT "",
- `citycode` int(11) NULL COMMENT "",
- `userid` int(11) NULL COMMENT "",
- `pv` int(11) NULL COMMENT ""
-) ENGINE=OLAP
-DUPLICATE KEY(`siteid`)
-COMMENT "OLAP"
-DISTRIBUTED BY HASH(`siteid`) BUCKETS 1
-"""
-
- long insertedSizeBytes = 795
- connect('root') {
- sql "use @regression_cluster_name0"
- sql "use regression_test_cloud_p0_cache_multi_cluster_read_write"
- sql """ set enable_multi_cluster_sync_load = true """
- // all the inserted values will result in insertedSizeBytes bytes of size
- sql """insert into test_dup_tab_basic_int_tab_nullable values
- (9,10,11,12),
- (9,10,11,12),
- (21,null,23,null),
- (1,2,3,4),
- (1,2,3,4),
- (13,14,15,16),
- (13,21,22,16),
- (13,14,15,16),
- (13,21,22,16),
- (17,18,19,20),
- (17,18,19,20),
- (null,21,null,23),
- (22,null,24,25),
- (26,27,null,29),
- (5,6,7,8),
- (5,6,7,8)
- """
- sql """ set enable_multi_cluster_sync_load = false """
- }
+ waitUntil("source and target primary tablet routes") {
+ [sourceCluster, targetCluster].each { clusterName ->
+ sql "use @${clusterName}"
+ def members = clusterBackends[clusterName].collectEntries { [(it.id): it] }
+ def routes = [:]
+ sql_return_maparray("show tablets from ${tableName}").each { tablet ->
+ long id = tablet.TabletId as Long
+ long primary = tablet.PrimaryBackendId as Long
+ if (id in tabletIds && members.containsKey(primary) && (tablet.BackendId as Long) == primary) {
+ routes[id] = members[primary]
+ }
+ }
+ primaryBackends[clusterName] = routes
+ }
+ logger.info("Sync insert primary routes: ${primaryBackends}")
+ [sourceCluster, targetCluster].every { primaryBackends[it].keySet() == tabletIds.toSet() }
+ }
+ def beforeClear = getGlobalBytes()
+ clusterBackends.values().flatten().each { be ->
+ WarmupMetricsUtils.clearFileCache(be.ip, be.httpPort)
+ }
+ def afterClear = getGlobalBytes()
+ logger.info("Sync insert clear: before=${beforeClear}, after=${afterClear}, " +
+ "source_minus_target_after_clear=${afterClear.source - afterClear.target}, " +
+ "source_change_after_clear=${afterClear.source - beforeClear.source}, " +
+ "target_change_after_clear=${afterClear.target - beforeClear.target}")
+ [sourceCluster, targetCluster].each { clusterName ->
+ def cache = getCache(clusterName)
+ assertTrue(cache.isEmpty(), "New test tablet must be cold in ${clusterName}: ${cache}")
+ }
- sleep(30000) // wait for download from remote
+ connect('root') {
+ sql "use @${sourceCluster}"
+ sql "use `${dbName.replace('`', '``')}`"
+ String previousSyncLoad = sql("select @@enable_multi_cluster_sync_load")[0][0].toString()
+ assertTrue(previousSyncLoad.toLowerCase() in ["true", "false", "1", "0"],
+ "Unexpected sync-load setting: ${previousSyncLoad}")
+ try {
+ sql "set enable_multi_cluster_sync_load = true"
+ def tuples = values.collect { row ->
+ "(" + row.collect { it == null ? "NULL" : it.toString() }.join(',') + ")"
+ }.join(',')
+ sql "INSERT INTO ${tableName} VALUES ${tuples}"
+ } finally {
+ sql "set enable_multi_cluster_sync_load = ${previousSyncLoad}"
+ }
+ }
- // get insert size of first cluster, may contain system tables
- long srcClusterSize = 0
- getMetricsMethod.call(ipList[0], brpcPortList[0]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("file_cache_cache_size")) {
- if (line.startsWith("#")) {
- continue
+ sql "use @${sourceCluster}"
+ long loadedVersion = -1L
+ waitUntil("INSERT visible version") {
+ def partitions = sql_return_maparray("show partitions from ${tableName}")
+ logger.info("Sync insert visible partition state: ${partitions}")
+ if (partitions.size() == 1) {
+ loadedVersion = partitions[0].VisibleVersion as Long
+ }
+ loadedVersion > initialVersion
+ }
+ def expectedBytes = [:]
+ waitUntil("source rowset metadata for the inserted rows") {
+ def ready = [:]
+ tabletIds.each { id ->
+ try {
+ def meta = getHeader(primaryBackends[sourceCluster][id], id)
+ if (meta == null || (meta.tablet_id as Long) != id) {
+ return
}
- def i = line.indexOf(' ')
- srcClusterSize = line.substring(i).toLong()
- flag = true
- break
+ // Cloud /api/meta/header omits rs_metas. This metadata scanner reads the
+ // source BE's rowset map without scanning data or filling target cache.
+ def rowsets = sql("""select rowset_id, start_version, end_version, rowset_num_rows,
+ data_disk_size, index_disk_size, num_segments
+ from information_schema.rowsets
+ where tablet_id = ${id} and backend_id = ${primaryBackends[sourceCluster][id].id}
+ and start_version > ${initialVersion} and end_version <= ${loadedVersion}""")
+ .collect { row ->
+ [rowset_id_v2: row[0], start_version: row[1] as Long, end_version: row[2] as Long,
+ num_rows: row[3] as Long, data_disk_size: row[4] as Long,
+ index_disk_size: row[5] as Long, num_segments: row[6] as Long]
+ }
+ logger.info("Inserted rowsets for ${id}: " + rowsets.collect {
+ [rowset: it.rowset_id_v2, start: it.start_version, end: it.end_version,
+ rows: it.num_rows, dataBytes: it.data_disk_size, indexBytes: it.index_disk_size,
+ segments: it.num_segments]
+ })
+ if (!rowsets.isEmpty() && rowsets.max { it.end_version as Long }.end_version as Long == loadedVersion &&
+ rowsets.sum(0L) { (it.num_rows ?: 0L) as Long } == values.size()) {
+ // This table has no inverted indexes. data_disk_size sums actual segment file sizes,
+ // including packed slices, without depending on a particular encoding or file format.
+ long bytes = rowsets.sum(0L) { (it.data_disk_size ?: 0L) as Long }
+ assertTrue(rowsets.every { ((it.index_disk_size ?: 0L) as Long) == 0L },
+ "Unexpected external index files for ${id}: ${rowsets}")
+ if (bytes > 0L) {
+ ready[id] = bytes
+ }
+ }
+ } catch (Exception e) {
+ logger.info("Waiting for inserted rowset metadata on ${id}: ${e.message}")
}
}
- assertTrue(flag)
- }
+ if (ready.keySet() == tabletIds.toSet()) {
+ expectedBytes = ready
+ return true
+ }
+ false
+ }
+ logger.info("Sync insert actual segment bytes: ${expectedBytes}")
- // get synced insert size of first cluster
- long dstClusterSize = 0
- getMetricsMethod.call(ipList[1], brpcPortList[1]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("file_cache_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- dstClusterSize = line.substring(i).toLong()
- flag = true
- break
+ waitUntil("source and target NORMAL cache to match actual inserted segment bytes") {
+ def source = getCache(sourceCluster)
+ def target = getCache(targetCluster)
+ logger.info("Sync insert cache: expected=${expectedBytes}, source=${source}, target=${target}, " +
+ "primary=${primaryBackends}")
+ // Inspect one complete copy on each cluster's primary BE. Do not add replicated copies:
+ // syncLoadForTablets can send the same tablet to multiple BEs in a target cluster.
+ [sourceCluster, targetCluster].every { clusterName ->
+ def cache = clusterName == sourceCluster ? source : target
+ tabletIds.every { id ->
+ String key = "${id}:${primaryBackends[clusterName][id].id}".toString()
+ def copy = cache[key]
+ copy != null && copy.blocks > 0L && copy.otherBlocks == 0L &&
+ copy.normalBytes == expectedBytes[id]
}
}
- assertTrue(flag)
+ }
+
+ // Query only after proving automatic download, so reads cannot mask a failed sync load.
+ connect('root') {
+ sql "use @${targetCluster}"
+ sql "use `${dbName.replace('`', '``')}`"
+ sql "set enable_sql_cache = false"
+ sql "set enable_query_cache = false"
+ def actual = sql("select siteid, citycode, userid, pv from ${tableName}")
+ .collect { row -> row.collect { it == null ? null : it as Long } }
+ def expected = values.collect { row -> row.collect { it == null ? null : it as Long } }
+ assertTrue(actual.countBy { it } == expected.countBy { it },
+ "Target data differs, including duplicate/NULL rows: actual=${actual}, expected=${expected}")
+ }
+ } finally {
+ sql "DROP TABLE IF EXISTS ${tableName} FORCE"
}
- // FIXME(gavin): this is a strong assertion, make it weaker and robuster
- assertEquals(insertedSizeBytes, dstClusterSize)
- org.junit.Assert.assertTrue("insertedSizeBytes ${insertedSizeBytes} <= dstClusterSize ${dstClusterSize}", insertedSizeBytes <= dstClusterSize)
- org.junit.Assert.assertTrue("insertedSizeBytes ${insertedSizeBytes} <= srcClusterSize ${dstClusterSize}", insertedSizeBytes <= srcClusterSize)
- org.junit.Assert.assertTrue("dstClusterSize ${insertedSizeBytes} <= srcClusterSize ${dstClusterSize}", dstClusterSize <= srcClusterSize)
- sql "drop table if exists ${table1}"
}
diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/read_write/test_multi_stale_rowset.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/read_write/test_multi_stale_rowset.groovy
index b2bc06ffd81..0b7b2e24430 100644
--- a/regression-test/suites/cloud_p0/cache/multi_cluster/read_write/test_multi_stale_rowset.groovy
+++ b/regression-test/suites/cloud_p0/cache/multi_cluster/read_write/test_multi_stale_rowset.groovy
@@ -17,7 +17,16 @@
import org.codehaus.groovy.runtime.IOGroovyMethods
-suite("test_multi_stale_rowset") {
+suite("test_multi_stale_rowset", "nonConcurrent") {
+ // This setting is static in FE; isolate it and restore it even when a cache assertion fails.
+ String originalSyncLoad = sql("select @@enable_multi_cluster_sync_load")[0][0].toString()
+ assertTrue(originalSyncLoad.toLowerCase() in ["true", "false", "1", "0"],
+ "Unexpected sync-load setting: ${originalSyncLoad}")
+ onFinish {
+ sql "set enable_multi_cluster_sync_load = ${originalSyncLoad}"
+ logger.info("Restored enable_multi_cluster_sync_load=${originalSyncLoad}")
+ }
+
def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="1200", "disable_auto_compaction"="true") """
List<String> ipList = new ArrayList<>();
List<String> hbPortList = new ArrayList<>()
@@ -138,6 +147,10 @@ suite("test_multi_stale_rowset") {
load_customer_once()
sleep(30000);
def backendIdToAfterLoadCacheSize = getCurCacheSize()
+ assertTrue(backendIdToAfterLoadCacheSize.get(beUniqueIdList[0]) > 0,
+ "Source backend should cache rowsets generated by load")
+ assertTrue(backendIdToAfterLoadCacheSize.get(beUniqueIdList[1]) > 0,
+ "Target backend should cache rowsets synchronized by multi-cluster sync load")
sql "use @regression_cluster_name0"
String[][] tablets = sql """ show tablets from ${tableName}; """
@@ -147,50 +160,17 @@ suite("test_multi_stale_rowset") {
sleep(90000);
def backendIdToAfterCompactionCacheSize = getCurCacheSize()
- assertTrue(backendIdToAfterCompactionCacheSize.get(beUniqueIdList[1]) < backendIdToAfterLoadCacheSize.get(beUniqueIdList[1]));
-
+ logger.info("File cache size after load: ${backendIdToAfterLoadCacheSize}, " +
+ "after compaction: ${backendIdToAfterCompactionCacheSize}")
+ assertTrue(backendIdToAfterCompactionCacheSize.get(beUniqueIdList[0]) > 0,
+ "Source backend should cache the compaction output rowsets")
+ assertTrue(backendIdToAfterCompactionCacheSize.get(beUniqueIdList[1]) <
+ backendIdToAfterLoadCacheSize.get(beUniqueIdList[1]),
+ "Target backend should recycle the stale rowset cache")
+
+ // Multi-cluster sync load only synchronizes rowsets generated by load. It does not
+ // synchronize subsequent compaction output rowsets, so the source and target cache
+ // sizes are not expected to remain equal after stale rowset compaction.
sql "use @regression_cluster_name1"
-
- long ttl_cache_size = 0
- getMetricsMethod.call(ipList[0], brpcPortList[0]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- ttl_cache_size = line.substring(i).toLong()
- flag = true
- break
- }
- }
- assertTrue(flag)
- }
-
- getMetricsMethod.call(ipList[1], brpcPortList[1]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- assertEquals(ttl_cache_size, line.substring(i).toLong())
- flag = true
- break
- }
- }
- assertTrue(flag)
- }
sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text
- sql """ set enable_multi_cluster_sync_load=false """
}
diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy
index 21a3773b262..9244e7f9e24 100644
--- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy
+++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy
@@ -15,177 +15,280 @@
// specific language governing permissions and limitations
// under the License.
-import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.WarmupMetricsUtils
-suite("test_warm_up_cluster_batch") {
- def getJobState = { jobId ->
- def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """
- return jobStateResult[0]
+suite("test_warm_up_cluster_batch", "nonConcurrent") {
+ // This setting is static in FE; isolate it and restore it even when a cache assertion fails.
+ String originalSyncLoad = sql("select @@enable_multi_cluster_sync_load")[0][0].toString()
+ assertTrue(originalSyncLoad.toLowerCase() in ["true", "false", "1", "0"],
+ "Unexpected sync-load setting: ${originalSyncLoad}")
+ onFinish {
+ sql "set enable_multi_cluster_sync_load = ${originalSyncLoad}"
+ logger.info("Restored enable_multi_cluster_sync_load=${originalSyncLoad}")
}
- def table = "customer"
+ // Only explicit warm-up should populate the target cache in this case.
+ sql "set enable_multi_cluster_sync_load = false"
+ logger.info("Explicit warmup: disabled automatic sync load, previous=${originalSyncLoad}")
- List<String> ipList = new ArrayList<>();
- List<String> hbPortList = new ArrayList<>()
- List<String> httpPortList = new ArrayList<>()
- List<String> brpcPortList = new ArrayList<>()
- List<String> beUniqueIdList = new ArrayList<>()
-
- String[] bes = context.config.multiClusterBes.split(',');
- println("the value is " + context.config.multiClusterBes);
- int num = 0
- for(String values : bes) {
- if (num++ == 2) break;
- println("the value is " + values);
- String[] beInfo = values.split(':');
- ipList.add(beInfo[0]);
- hbPortList.add(beInfo[1]);
- httpPortList.add(beInfo[2]);
- beUniqueIdList.add(beInfo[3]);
- brpcPortList.add(beInfo[4]);
- }
-
- println("the ip is " + ipList);
- println("the heartbeat port is " + hbPortList);
- println("the http port is " + httpPortList);
- println("the be unique id is " + beUniqueIdList);
- println("the brpc port is " + brpcPortList);
-
- sql new File("""${context.file.parent}/../ddl/${table}_delete.sql""").text
- sql new File("""${context.file.parent}/../ddl/supplier_delete.sql""").text
- // create table if not exists
- sql new File("""${context.file.parent}/../ddl/${table}.sql""").text
- sql new File("""${context.file.parent}/../ddl/supplier.sql""").text
-
- sql """ TRUNCATE TABLE __internal_schema.cloud_cache_hotspot; """
- sleep(30000)
-
- def s3BucketName = getS3BucketName()
- def s3WithProperties = """WITH S3 (
- |"AWS_ACCESS_KEY" = "${getS3AK()}",
- |"AWS_SECRET_KEY" = "${getS3SK()}",
- |"AWS_ENDPOINT" = "${getS3Endpoint()}",
- |"AWS_REGION" = "${getS3Region()}",
- |"provider" = "${getS3Provider()}")
- |PROPERTIES(
- |"exec_mem_limit" = "8589934592",
- |"load_parallelism" = "3")""".stripMargin()
-
-
-
- def clearFileCache = { ip, port ->
- httpTest {
- endpoint ""
- uri ip + ":" + port + """/api/file_cache?op=clear&sync=true"""
- op "get"
- body ""
- }
- }
-
- def getMetricsMethod = { ip, port, check_func ->
- httpTest {
- endpoint ip + ":" + port
- uri "/brpc_metrics"
- op "get"
- check check_func
+ String sourceCluster = "regression_cluster_name0"
+ String targetCluster = "regression_cluster_name1"
+ def tableNames = ["customer", "supplier"]
+ def dataQueries = [
+ customer: "select sum(C_CUSTKEY), sum(length(C_COMMENT)) from customer",
+ supplier: "select sum(s_suppkey), sum(length(s_address)) from supplier"
+ ]
+ def backends = sql_return_maparray("SHOW BACKENDS")
+ def clusterBackends = [sourceCluster, targetCluster].collectEntries { clusterName ->
+ def members = backends.findAll { be ->
+ "${be.Alive}".equalsIgnoreCase("true") &&
+ parseJson(be.Tag.toString()).compute_group_name == clusterName
+ }.collect { be ->
+ [id: be.BackendId as Long, ip: be.Host.toString(),
+ httpPort: be.HttpPort.toString(), brpcPort: be.BrpcPort.toString()]
}
+ assertTrue(!members.isEmpty(), "No alive backends in ${clusterName}")
+ [(clusterName): members]
}
+ logger.info("Batch warmup backends: ${clusterBackends}")
+ assertTrue(getFeConfig("enable_fetch_cluster_cache_hotspot").equalsIgnoreCase("true"),
+ "Cluster warmup requires the FE cache hotspot collector to be enabled at startup")
+ def hotspotInterval = getFeConfig("fetch_cluster_cache_hotspot_interval_ms")
- clearFileCache.call(ipList[0], httpPortList[0]);
- clearFileCache.call(ipList[1], httpPortList[1]);
-
- def load_customer_once = {
- def uniqueID = Math.abs(UUID.randomUUID().hashCode()).toString()
- def loadLabel = table + "_" + uniqueID
- // load data from cos
- def loadSql = new File("""${context.file.parent}/../ddl/${table}_load.sql""").text.replaceAll("\\\$\\{s3BucketName\\}", s3BucketName)
- loadSql = loadSql.replaceAll("\\\$\\{loadLabel\\}", loadLabel) + s3WithProperties
- sql loadSql
-
- // check load state
- while (true) {
- def stateResult = sql "show load where Label = '${loadLabel}'"
- def loadState = stateResult[stateResult.size() - 1][2].toString()
- if ("CANCELLED".equalsIgnoreCase(loadState)) {
- throw new IllegalStateException("load ${loadLabel} failed.")
- } else if ("FINISHED".equalsIgnoreCase(loadState)) {
- break
+ def waitUntil = { String description, long timeoutMs, Closure condition ->
+ long deadline = System.currentTimeMillis() + timeoutMs
+ while (System.currentTimeMillis() < deadline) {
+ if (condition()) {
+ return
}
- sleep(5000)
- }
- }
-
- sql "use @regression_cluster_name0"
- def waitJobDone = { jobId ->
- int retryTime = 120
- int i = 0
- for (; i < retryTime; i++) {
sleep(1000)
- def statuses = getJobState(jobId[0][0])
- if (statuses.any { it != null && it.equals("CANCELLED") }) {
- assertTrue(false);
- }
- if (statuses.any { it != null && it.equals("FINISHED") }) {
- break;
- }
- }
- if (i == retryTime) {
- sql "cancel warm up job where id = ${jobId[0][0]}"
- assertTrue(false);
}
+ assertTrue(false, "Timeout after ${timeoutMs}ms: ${description}")
+ }
+ def pendingLoads = [] as Set
+ def warmupJobs = [] as Set
+ def tableInfo = [:]
+ def getGlobalTtlBytes = {
+ [source: WarmupMetricsUtils.getBackendMetricSum(clusterBackends[sourceCluster], "ttl_cache_size"),
+ target: WarmupMetricsUtils.getBackendMetricSum(clusterBackends[targetCluster], "ttl_cache_size")]
}
- for (int k = 0; k < 10; k++) {
- load_customer_once()
- for (int i = 0; i < 10; i++) {
- sql "select count(*) from customer"
+ def getNormalCache = { String clusterName ->
+ sql "use @${clusterName}"
+ def beIds = clusterBackends[clusterName].collect { it.id }.join(',')
+ tableInfo.collectEntries { tableName, info ->
+ def rows = sql """select tablet_id, sum(size)
+ from information_schema.file_cache_info
+ where tablet_id in (${info.tabletIds.join(',')})
+ and be_id in (${beIds}) and lower(type) = 'normal'
+ group by tablet_id"""
+ def bytesByTablet = rows.collectEntries { [(it[0] as Long): it[1] as Long] }
+ [(tableName): [bytes: bytesByTablet.values().sum(0L) as Long,
+ complete: info.tabletIds.every { (bytesByTablet[it] ?: 0L) > 0L }]]
}
- if (k % 2 == 1) {
- sleep(40000)
- def jobId_ = sql "WARM UP CLUSTER regression_cluster_name1 WITH CLUSTER regression_cluster_name0"
- waitJobDone(jobId_)
- sleep(30000)
- long ttl_cache_size = 0
- getMetricsMethod.call(ipList[0], brpcPortList[0]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
+ }
+ def getHotspotCounts = {
+ sql "use @${sourceCluster}"
+ def rows = sql """select table_id, partition_id, sum(query_per_day)
+ from __internal_schema.cloud_cache_hotspot
+ where cluster_name = '${sourceCluster}'
+ and table_id in (${tableInfo.values().collect { it.id }.join(',')})
+ and insert_day = current_date()
+ group by table_id, partition_id"""
+ rows.collectEntries { [("${it[0]}:${it[1]}".toString()): it[2] as Long] }
+ }
+ def waitForTargetTablets = {
+ sql "use @${targetCluster}"
+ def targetBes = clusterBackends[targetCluster].collectEntries { [(it.id): it] }
+ def missing = []
+ waitUntil("target primary routing and tablet metadata; see missing tablet log", 120000L) {
+ missing = []
+ tableInfo.each { tableName, info ->
+ def ready = [] as Set
+ def tablets = sql_return_maparray("show tablets from ${tableName}")
+ tablets.each { tablet ->
+ long id = tablet.TabletId as Long
+ long primary = tablet.PrimaryBackendId as Long
+ if (info.tabletIds.contains(id) && targetBes.containsKey(primary) &&
+ (tablet.BackendId as Long) == primary) {
+ def be = targetBes[primary]
+ try {
+ // Load metadata only; querying the target table could mask a failed warmup.
+ httpTest {
+ endpoint "${be.ip}:${be.httpPort}"
+ uri "/api/meta/header/${id}"
+ op "get"
+ check { code, body ->
+ if ((code as int) == 200) {
+ def meta = parseJson(body.toString())
+ // Cloud header JSON omits rowsets; data readiness is
+ // established by the source scan and cache baseline.
+ if ((meta.tablet_id as Long) == id &&
+ (meta.ttl_seconds as Long) == 0L &&
+ meta.tablet_state == "PB_RUNNING" && meta.schema != null) {
+ ready.add(id)
+ }
+ }
+ }
}
- def i = line.indexOf(' ')
- ttl_cache_size = line.substring(i).toLong()
- flag = true
- break
+ } catch (Exception e) {
+ logger.info("Waiting for target tablet ${id} on BE ${primary}: ${e.message}")
}
}
- assertTrue(flag)
+ }
+ missing.addAll(info.tabletIds - ready)
}
+ logger.info("Target tablet readiness: missing=${missing}")
+ missing.isEmpty()
+ }
+ }
- getMetricsMethod.call(ipList[1], brpcPortList[1]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- assertEquals(ttl_cache_size, line.substring(i).toLong())
- flag = true
- break
+ // This case must exercise hotspot-based selection, not the all-partitions bypass.
+ setFeConfigTemporary([cloud_warm_up_force_all_partitions: false]) {
+ try {
+ sql "use @${sourceCluster}"
+ sql "set enable_sql_cache = false"
+ sql "set enable_query_cache = false"
+ // Only disabling SQL/query cache does not disable COUNT(*) constant rewriting.
+ // The SUM expressions above force scans of actual data columns on the source BE.
+ tableNames.each { tableName ->
+ sql "DROP TABLE IF EXISTS ${tableName} FORCE"
+ sql (new File("${context.file.parent}/../ddl/${tableName}.sql").text + """
+ PROPERTIES("disable_auto_compaction"="true", "file_cache_ttl_seconds"="0")""")
+ def tabletIds = sql("show tablets from ${tableName}").collect { it[0] as Long }.unique()
+ def partitionIds = sql("show partitions from ${tableName}").collect { it[0] as Long }.unique()
+ long tableId = getTableId(tableName)
+ assertTrue(tableId > 0L && !tabletIds.isEmpty() && !partitionIds.isEmpty(),
+ "Missing table/partition/tablet IDs for ${tableName}")
+ tableInfo[tableName] = [id: tableId, tabletIds: tabletIds, partitionIds: partitionIds]
+ }
+ logger.info("Batch warmup tables: ${tableInfo}")
+ sql "TRUNCATE TABLE __internal_schema.cloud_cache_hotspot"
+
+ def ttlBeforeClear = getGlobalTtlBytes()
+ clusterBackends.values().flatten().each { be ->
+ WarmupMetricsUtils.clearFileCache(be.ip, be.httpPort)
+ }
+ // Held blocks can survive sync=true. Keep their baseline for diagnosis only.
+ def ttlAfterClear = getGlobalTtlBytes()
+ logger.info("TTL cache clear: before=${ttlBeforeClear}, after=${ttlAfterClear}, " +
+ "source_minus_target_after_clear=${ttlAfterClear.source - ttlAfterClear.target}, " +
+ "source_change_after_clear=${ttlAfterClear.source - ttlBeforeClear.source}, " +
+ "target_change_after_clear=${ttlAfterClear.target - ttlBeforeClear.target}")
+
+ def s3BucketName = getS3BucketName()
+ def s3Properties = """WITH S3 (
+ "AWS_ACCESS_KEY" = "${getS3AK()}",
+ "AWS_SECRET_KEY" = "${getS3SK()}",
+ "AWS_ENDPOINT" = "${getS3Endpoint()}",
+ "AWS_REGION" = "${getS3Region()}",
+ "provider" = "${getS3Provider()}")
+ PROPERTIES("exec_mem_limit"="8589934592", "load_parallelism"="3")"""
+ def loadOnce = { String tableName ->
+ String label = "${tableName}_${UUID.randomUUID().toString().replace('-', '')}"
+ String loadSql = new File("${context.file.parent}/../ddl/${tableName}_load.sql").text
+ .replace('${s3BucketName}', s3BucketName).replace('${loadLabel}', label)
+ sql (loadSql + s3Properties)
+ pendingLoads.add(label)
+ waitUntil("load ${label} to finish", 300000L) {
+ def rows = sql "show load where Label = '${label}'"
+ if (rows.isEmpty()) {
+ return false
+ }
+ String state = rows.last()[2].toString()
+ assertTrue(!state.equalsIgnoreCase("CANCELLED"), "Load ${label} cancelled: ${rows.last()}")
+ if (state.equalsIgnoreCase("FINISHED")) {
+ pendingLoads.remove(label)
+ return true
+ }
+ false
+ }
+ }
+ def previousBytes = tableNames.collectEntries { [(it): 0L] }
+ for (int round = 0; round < 10; round++) {
+ sql "use @${sourceCluster}"
+ tableNames.each { loadOnce(it) }
+ if (round % 2 == 0) {
+ continue
+ }
+ int batch = (round + 1) / 2
+ def hotspotBeforeQueries = getHotspotCounts()
+ // One real scan per table is enough to advance the hotspot counters for this batch.
+ tableNames.each { tableName ->
+ def result = sql(dataQueries[tableName])
+ assertTrue(!result.isEmpty() && result[0].every { it != null },
+ "Data query should read nonempty ${tableName}")
+ }
+ // Require hotspot counters for these new table/partition IDs to advance beyond
+ // the pre-query snapshot, without assuming identical FE and BE timestamps.
+ waitUntil("batch ${batch} hotspot refresh (configured interval=${hotspotInterval}ms); " +
+ "run with a short FE startup hotspot interval", 180000L) {
+ def counts = getHotspotCounts()
+ logger.info("Batch ${batch} hotspot counts: before=${hotspotBeforeQueries}, current=${counts}")
+ tableInfo.values().every { info ->
+ info.partitionIds.every { partitionId ->
+ String key = "${info.id}:${partitionId}"
+ (counts[key] ?: 0L) > (hotspotBeforeQueries[key] ?: 0L)
}
}
- assertTrue(flag)
+ }
+ waitForTargetTablets()
+ def sourceCache = [:]
+ waitUntil("batch ${batch} source NORMAL cache to include new data", 120000L) {
+ sourceCache = getNormalCache(sourceCluster)
+ logger.info("Batch ${batch} source NORMAL cache=${sourceCache}, previous=${previousBytes}")
+ tableNames.every { sourceCache[it].complete && sourceCache[it].bytes > previousBytes[it] }
+ }
+ def job = sql "WARM UP CLUSTER ${targetCluster} WITH CLUSTER ${sourceCluster}"
+ def jobId = job[0][0]
+ warmupJobs.add(jobId)
+ waitUntil("warmup job ${jobId} to finish", 300000L) {
+ def rows = sql "SHOW WARM UP JOB WHERE ID = ${jobId}"
+ if (rows.isEmpty()) {
+ return false
+ }
+ def states = rows[0].collect { it?.toString() }
+ assertTrue(!states.contains("CANCELLED"), "Warmup ${jobId} cancelled: ${rows[0]}")
+ states.contains("FINISHED")
+ }
+ def targetCache = [:]
+ waitUntil("batch ${batch}: per-table NORMAL cache bytes must be positive and equal", 180000L) {
+ sourceCache = getNormalCache(sourceCluster)
+ targetCache = getNormalCache(targetCluster)
+ logger.info("Batch ${batch} NORMAL cache: source=${sourceCache}, target=${targetCache}")
+ tableNames.every { tableName ->
+ sourceCache[tableName].complete && targetCache[tableName].complete &&
+ sourceCache[tableName].bytes > previousBytes[tableName] &&
+ sourceCache[tableName].bytes == targetCache[tableName].bytes
+ }
+ }
+ previousBytes = sourceCache.collectEntries { name, cache -> [(name): cache.bytes] }
+ def currentTtl = getGlobalTtlBytes()
+ logger.info("Batch ${batch} global TTL=${currentTtl}, baseline_after_clear=${ttlAfterClear}, " +
+ "source_minus_target=${currentTtl.source - currentTtl.target}")
+ }
+ } finally {
+ warmupJobs.each { jobId ->
+ try {
+ def rows = sql "SHOW WARM UP JOB WHERE ID = ${jobId}"
+ if (!rows.isEmpty() && rows[0].any { it?.toString() in ["PENDING", "RUNNING"] }) {
+ sql "CANCEL WARM UP JOB WHERE ID = ${jobId}"
+ }
+ } catch (Exception e) {
+ logger.warn("Failed to clean up warmup ${jobId}: ${e.message}")
+ }
+ }
+ sql "use @${sourceCluster}"
+ pendingLoads.each { label ->
+ try {
+ sql "CANCEL LOAD WHERE LABEL = '${label}'"
+ } catch (Exception e) {
+ logger.warn("Failed to clean up load ${label}: ${e.message}")
+ }
+ }
+ try {
+ sql "DROP TABLE IF EXISTS customer FORCE"
+ } finally {
+ sql "DROP TABLE IF EXISTS supplier FORCE"
}
- sql "use @regression_cluster_name0"
}
}
- sql new File("""${context.file.parent}/../ddl/${table}_delete.sql""").text
- sql new File("""${context.file.parent}/../ddl/supplier_delete.sql""").text
-}
+}
diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_expired.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_expired.groovy
index 6114892039a..5cfef65570a 100644
--- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_expired.groovy
+++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_expired.groovy
@@ -16,6 +16,7 @@
// under the License.
import org.apache.doris.regression.suite.ClusterOptions
+import org.apache.doris.regression.util.NodeType
import groovy.json.JsonSlurper
// Covers a two-part bug on the event-driven warm-up path:
@@ -43,7 +44,11 @@ suite('test_warm_up_cluster_event_cancel_expired', 'docker') {
'file_cache_enter_disk_resource_limit_mode_percent=99',
'enable_evict_file_cache_in_advance=false',
'file_cache_background_monitor_interval_ms=1000',
+ // Expire cached replica locations before the post-cancel INSERTs so
+ // source BE must ask FE again and consume the CANCELLED status.
+ 'warmup_tablet_replica_info_cache_ttl_sec=5',
]
+ options.enableDebugPoints()
options.cloudMode = true
def clearFileCache = { ip, port ->
@@ -77,18 +82,22 @@ suite('test_warm_up_cluster_event_cancel_expired', 'docker') {
}
}
- def getSkippedRowsetSum = { cluster ->
+ def getMetricSum = { cluster, metric ->
def backends = sql """SHOW BACKENDS"""
def cluster_bes = backends.findAll {
it[19].contains("""\"compute_group_name\" : \"${cluster}\"""")
}
long sum = 0
for (be in cluster_bes) {
- sum += getBrpcMetrics(be[1], be[5], "file_cache_event_driven_warm_up_skipped_rowset_num")
+ sum += getBrpcMetrics(be[1], be[5], metric)
}
return sum
}
+ def getSkippedRowsetSum = { cluster ->
+ return getMetricSum(cluster, "file_cache_event_driven_warm_up_skipped_rowset_num")
+ }
+
docker(options) {
def clusterSrc = "warmup_source"
def clusterDst = "warmup_target"
@@ -124,27 +133,55 @@ suite('test_warm_up_cluster_event_cancel_expired', 'docker') {
}
sleep(5000)
- // 3. Cancel the job. After the fix, BE sees TStatus.CANCELLED on
- // the next get_replica_info and drops the job from
- // _tablet_replica_cache; before the fix (CANCELED typo) it
- // would keep the entry.
- sql """CANCEL WARM UP JOB WHERE ID = ${jobId}"""
+ // 3. Ignore the proactive CLEAR_JOB on source BE so this case
+ // exercises the passive cleanup path: after replica-info cache
+ // expiry, get_replica_info asks FE and receives TStatus.CANCELLED.
+ def sourceBes = sql("""SHOW BACKENDS""").findAll {
+ it[19].contains("""\"compute_group_name\" : \"${clusterSrc}\"""")
+ }
+ assertFalse(sourceBes.isEmpty(), "source cluster should have at least one BE")
+ def activeJobsBeforeCancel = getMetricSum(clusterSrc, "file_cache_warm_up_job_num")
+ assertTrue(activeJobsBeforeCancel > 0, "source BE should hold the event-driven job")
+ try {
+ for (be in sourceBes) {
+ GetDebugPoint().enableDebugPoint(be[1].toString(), be[4] as int, NodeType.BE,
+ "CloudWarmUpManager.set_event.ignore_all")
+ }
+ sql """CANCEL WARM UP JOB WHERE ID = ${jobId}"""
+ } finally {
+ for (be in sourceBes) {
+ GetDebugPoint().disableDebugPoint(be[1].toString(), be[4] as int, NodeType.BE,
+ "CloudWarmUpManager.set_event.ignore_all")
+ }
+ }
def st = sql """SHOW WARM UP JOB WHERE ID = ${jobId}"""
assertEquals("CANCELLED", st[0][3])
+ def activeJobsAfterCancel = getMetricSum(clusterSrc, "file_cache_warm_up_job_num")
+ assertEquals(activeJobsBeforeCancel, activeJobsAfterCancel,
+ "ignored CLEAR_JOB should leave the source BE cache intact")
+ sleep(6000)
- // 4. One more batch so BE actually sees the CANCELLED status
- // and (with the fix) purges its cache entry.
+ // 4. Waited past the 5s replica-info TTL after cancellation.
+ // The next batch makes source BE observe CANCELLED and purge the job.
for (int i = 0; i < 20; i++) {
sql """INSERT INTO t_exp VALUES (${100 + i}, 'y')"""
}
- sleep(5000)
+ def activeJobsAfterPassiveCleanup = getMetricSum(clusterSrc, "file_cache_warm_up_job_num")
+ for (int i = 0; i < 20 && activeJobsAfterPassiveCleanup >= activeJobsAfterCancel; i++) {
+ sleep(500)
+ activeJobsAfterPassiveCleanup = getMetricSum(clusterSrc, "file_cache_warm_up_job_num")
+ }
+ assertTrue(activeJobsAfterPassiveCleanup < activeJobsAfterCancel,
+ "source BE should remove the job after receiving TStatus.CANCELLED, " +
+ "before=${activeJobsAfterCancel} after=${activeJobsAfterPassiveCleanup}")
// 5. Baseline for the skipped-rowset counter. After BE has
// cleaned its cache, subsequent warm_up_rowset calls return
// early (empty replicas -> "skipping rowset") and bump this
- // counter on every commit. If the typo is unfixed the counter
- // stays flat because BE keeps calling FE.
- def skippedBaseline = getSkippedRowsetSum(clusterDst)
+ // counter on every commit.
+ // warm_up_rowset is invoked by the BE committing source rowsets, so
+ // this metric belongs to the source cluster rather than the target.
+ def skippedBaseline = getSkippedRowsetSum(clusterSrc)
logger.info("skipped_rowset baseline=${skippedBaseline}")
// 6. Wait past history_cloud_warm_up_job_keep_max_second plus one
@@ -160,13 +197,16 @@ suite('test_warm_up_cluster_event_cancel_expired', 'docker') {
logger.info("job ${jobId} removed from FE after ${i}s")
break
}
- } catch (Exception e) {
- if (!e.getMessage().contains("cloud warm up with job ${jobId} does not exist")) {
- throw e
+ } catch (Throwable t) {
+ def message = (t.getMessage() ?: t.toString()).toLowerCase()
+ if (message.contains("warm up") && message.contains("does not exist")) {
+ // FE may report an expired job as "does not exist" instead of an empty result.
+ // Both states mean the cancelled job has been removed from FE.
+ logger.info("job ${jobId} removed from FE after ${i}s: ${t.getMessage()}")
+ removed = true
+ break
}
- removed = true
- logger.info("job ${jobId} removed from FE after ${i}s: ${e.getMessage()}")
- break
+ throw t
}
sleep(1000)
}
@@ -197,11 +237,15 @@ suite('test_warm_up_cluster_event_cancel_expired', 'docker') {
"post-removal inserts should not be blocked by NPE sleeps, " +
"took ${elapsedMs}ms")
- // 8. On the fixed path every commit short-circuits through
- // g_file_cache_event_driven_warm_up_skipped_rowset_num.
- // We expect it to grow; on the buggy path it would be flat
- // since BE never stopped pursuing FE replicas.
- def skippedAfter = getSkippedRowsetSum(clusterDst)
+ // 8. After passive cleanup, source commits short-circuit through
+ // g_file_cache_event_driven_warm_up_skipped_rowset_num. The
+ // warm-up tasks are asynchronous, so allow a short observation
+ // window before checking that source-side traffic was counted.
+ def skippedAfter = getSkippedRowsetSum(clusterSrc)
+ for (int i = 0; i < 20 && skippedAfter <= skippedBaseline; i++) {
+ sleep(500)
+ skippedAfter = getSkippedRowsetSum(clusterSrc)
+ }
logger.info("skipped_rowset after=${skippedAfter}")
assertTrue(skippedAfter > skippedBaseline,
"BE should skip warm_up_rowset for tablets after cancel+expire, " +
diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy
index 19ca1958e2e..66e5b8c5019 100644
--- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy
+++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy
@@ -15,9 +15,21 @@
// specific language governing permissions and limitations
// under the License.
-import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.WarmupMetricsUtils
+
+suite("test_warm_up_compute_group", "nonConcurrent") {
+ // This setting is static in FE; isolate it and restore it even when a cache assertion fails.
+ String originalSyncLoad = sql("select @@enable_multi_cluster_sync_load")[0][0].toString()
+ assertTrue(originalSyncLoad.toLowerCase() in ["true", "false", "1", "0"],
+ "Unexpected sync-load setting: ${originalSyncLoad}")
+ onFinish {
+ sql "set enable_multi_cluster_sync_load = ${originalSyncLoad}"
+ logger.info("Restored enable_multi_cluster_sync_load=${originalSyncLoad}")
+ }
+ // Only explicit warm-up should populate the target cache in this case.
+ sql "set enable_multi_cluster_sync_load = false"
+ logger.info("Explicit warmup: disabled automatic sync load, previous=${originalSyncLoad}")
-suite("test_warm_up_compute_group") {
def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """
def getJobState = { jobId ->
def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """
@@ -25,31 +37,24 @@ suite("test_warm_up_compute_group") {
}
def table = "customer"
- List<String> ipList = new ArrayList<>();
- List<String> hbPortList = new ArrayList<>()
- List<String> httpPortList = new ArrayList<>()
- List<String> brpcPortList = new ArrayList<>()
- List<String> beUniqueIdList = new ArrayList<>()
-
- String[] bes = context.config.multiClusterBes.split(',');
- println("the value is " + context.config.multiClusterBes);
- int num = 0
- for(String values : bes) {
- if (num++ == 2) break;
- println("the value is " + values);
- String[] beInfo = values.split(':');
- ipList.add(beInfo[0]);
- hbPortList.add(beInfo[1]);
- httpPortList.add(beInfo[2]);
- beUniqueIdList.add(beInfo[3]);
- brpcPortList.add(beInfo[4]);
+ String sourceCluster = "regression_cluster_name0"
+ String targetCluster = "regression_cluster_name1"
+ def backends = sql_return_maparray("SHOW BACKENDS")
+ def clusterBackends = [sourceCluster, targetCluster].collectEntries { clusterName ->
+ def members = backends.findAll { be ->
+ "${be.Alive}".equalsIgnoreCase("true") &&
+ parseJson(be.Tag.toString()).compute_group_name == clusterName
+ }.collect { be ->
+ [id: be.BackendId as Long, ip: be.Host.toString(),
+ httpPort: be.HttpPort.toString(), brpcPort: be.BrpcPort.toString()]
+ }
+ assertTrue(!members.isEmpty(), "No alive backends in ${clusterName}")
+ [(clusterName): members]
}
-
- println("the ip is " + ipList);
- println("the heartbeat port is " + hbPortList);
- println("the http port is " + httpPortList);
- println("the be unique id is " + beUniqueIdList);
- println("the brpc port is " + brpcPortList);
+ logger.info("Warmup source=${sourceCluster}, target=${targetCluster}, backends=${clusterBackends}")
+ sql "use @${sourceCluster}"
+ sql "set enable_sql_cache = false"
+ sql "set enable_query_cache = false"
sql new File("""${context.file.parent}/../ddl/${table}_delete.sql""").text
sql new File("""${context.file.parent}/../ddl/supplier_delete.sql""").text
@@ -73,26 +78,46 @@ suite("test_warm_up_compute_group") {
- def clearFileCache = { ip, port ->
- httpTest {
- endpoint ""
- uri ip + ":" + port + """/api/file_cache?op=clear&sync=true"""
- op "get"
- body ""
- }
+ // Capture the IDs before DROP so the final check can still address this case's tablets.
+ def tableTabletIds = [table, "supplier"].collectEntries { tableName ->
+ def ids = sql("show tablets from ${tableName}").collect { it[0] as Long }.unique()
+ assertTrue(!ids.isEmpty(), "No tablets found for ${tableName}")
+ [(tableName): ids]
}
+ logger.info("Case table tablets: ${tableTabletIds}")
- def getMetricsMethod = { ip, port, check_func ->
- httpTest {
- endpoint ip + ":" + port
- uri "/brpc_metrics"
- op "get"
- check check_func
+ def getGlobalTtlBytes = {
+ [source: WarmupMetricsUtils.getBackendMetricSum(clusterBackends[sourceCluster], "ttl_cache_size"),
+ target: WarmupMetricsUtils.getBackendMetricSum(clusterBackends[targetCluster], "ttl_cache_size")]
+ }
+ def clearCacheAndLogTtl = { List<String> clustersToClear, String phase ->
+ def before = getGlobalTtlBytes()
+ clustersToClear.each { clusterName ->
+ clusterBackends[clusterName].each { be ->
+ WarmupMetricsUtils.clearFileCache(be.ip, be.httpPort)
+ }
}
+ // sync=true can leave referenced blocks pending recycle. Record the residual baseline;
+ // it is diagnostic data, not an assertion that either BE's entire cache must be empty.
+ def after = getGlobalTtlBytes()
+ logger.info("TTL cache clear phase=${phase}, bytes_before=${before}, bytes_after=${after}, " +
+ "source_minus_target_after_clear=${after.source - after.target}, " +
+ "source_change_after_clear=${after.source - before.source}, " +
+ "target_change_after_clear=${after.target - before.target}")
+ return after
}
-
- clearFileCache.call(ipList[0], httpPortList[0]);
- clearFileCache.call(ipList[1], httpPortList[1]);
+ def getTableTtlBytes = { String clusterName ->
+ sql "use @${clusterName}"
+ def backendIds = clusterBackends[clusterName].collect { it.id }.join(',')
+ tableTabletIds.collectEntries { tableName, ids ->
+ def rows = sql """select coalesce(sum(size), 0)
+ from information_schema.file_cache_info
+ where tablet_id in (${ids.join(',')})
+ and be_id in (${backendIds}) and lower(type) = 'ttl'"""
+ [(tableName): rows[0][0] as Long]
+ }
+ }
+ def initialTtlAfterClear = clearCacheAndLogTtl([sourceCluster, targetCluster], "before_load")
def load_customer_once = {
def uniqueID = Math.abs(UUID.randomUUID().hashCode()).toString()
@@ -172,47 +197,33 @@ suite("test_warm_up_compute_group") {
}
waitJobDone(jobId_)
- sleep(30000)
- long ttl_cache_size = 0
- getMetricsMethod.call(ipList[0], brpcPortList[0]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- ttl_cache_size = line.substring(i).toLong()
- flag = true
- break
- }
- }
- assertTrue(flag)
- }
-
- getMetricsMethod.call(ipList[1], brpcPortList[1]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- assertEquals(ttl_cache_size, line.substring(i).toLong())
- flag = true
- break
- }
- }
- assertTrue(flag)
+ // Cache metadata is published asynchronously. Compare each table independently so that
+ // missing bytes in one table cannot be hidden by extra bytes in the other table.
+ long cacheDeadline = System.currentTimeMillis() + 120000L
+ def sourceTableBytes = [:]
+ def targetTableBytes = [:]
+ boolean cacheMatches = false
+ while (System.currentTimeMillis() < cacheDeadline) {
+ sourceTableBytes = getTableTtlBytes(sourceCluster)
+ targetTableBytes = getTableTtlBytes(targetCluster)
+ logger.info("Case table TTL cache bytes: source=${sourceTableBytes}, target=${targetTableBytes}")
+ cacheMatches = tableTabletIds.keySet().every { tableName ->
+ sourceTableBytes[tableName] > 0L &&
+ sourceTableBytes[tableName] == targetTableBytes[tableName]
+ }
+ if (cacheMatches) {
+ break
+ }
+ sleep(1000)
}
+ def ttlAfterWarmup = getGlobalTtlBytes()
+ logger.info("Global TTL cache bytes after warmup=${ttlAfterWarmup}, " +
+ "source_minus_target=${ttlAfterWarmup.source - ttlAfterWarmup.target}, " +
+ "baseline_after_clear=${initialTtlAfterClear}, " +
+ "source_change_since_clear=${ttlAfterWarmup.source - initialTtlAfterClear.source}, " +
+ "target_change_since_clear=${ttlAfterWarmup.target - initialTtlAfterClear.target}")
+ assertTrue(cacheMatches, "Case table TTL cache bytes must be positive and match on both clusters: " +
+ "source=${sourceTableBytes}, target=${targetTableBytes}, tablets=${tableTabletIds}")
try {
sql "WARM UP COMPUTE GROUP regression_cluster_name1 WITH COMPUTE GROUP regression_cluster_name2"
@@ -238,27 +249,21 @@ suite("test_warm_up_compute_group") {
sql new File("""${context.file.parent}/../ddl/${table}_delete.sql""").text
sql new File("""${context.file.parent}/../ddl/supplier_delete.sql""").text
- clearFileCache.call(ipList[1], httpPortList[1]);
+ clearCacheAndLogTtl([targetCluster], "after_drop")
jobId_ = sql "WARM UP COMPUTE GROUP regression_cluster_name1 WITH COMPUTE GROUP regression_cluster_name0"
waitJobDone(jobId_)
- sleep(40000)
- getMetricsMethod.call(ipList[1], brpcPortList[1]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def j = line.indexOf(' ')
- assertEquals(0, line.substring(j).toLong())
- flag = true
- break
- }
- }
- assertTrue(flag)
+ long droppedCacheDeadline = System.currentTimeMillis() + 120000L
+ def droppedTableBytes = [:]
+ boolean droppedCacheCleared = false
+ while (System.currentTimeMillis() < droppedCacheDeadline) {
+ droppedTableBytes = getTableTtlBytes(targetCluster)
+ droppedCacheCleared = droppedTableBytes.values().every { it == 0L }
+ if (droppedCacheCleared) {
+ break
+ }
+ logger.info("Waiting for dropped table TTL cache to clear on ${targetCluster}: ${droppedTableBytes}")
+ sleep(1000)
}
+ assertTrue(droppedCacheCleared,
+ "Dropped tables must have no TTL cache on ${targetCluster}: ${droppedTableBytes}")
}
diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy
index fce44d37e36..c73c818a05c 100644
--- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy
+++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy
@@ -15,184 +15,271 @@
// specific language governing permissions and limitations
// under the License.
-import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.WarmupMetricsUtils
-suite("test_warm_up_partition") {
- def custoBeConfig = [
- enable_evict_file_cache_in_advance : false,
- file_cache_enter_disk_resource_limit_mode_percent : 99
- ]
- setBeConfigTemporary(custoBeConfig) {
- def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """
- def getJobState = { jobId ->
- def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """
- return jobStateResult[0]
+suite("test_warm_up_partition", "nonConcurrent") {
+ // This setting is static in FE; isolate it and restore it even when a cache assertion fails.
+ String originalSyncLoad = sql("select @@enable_multi_cluster_sync_load")[0][0].toString()
+ assertTrue(originalSyncLoad.toLowerCase() in ["true", "false", "1", "0"],
+ "Unexpected sync-load setting: ${originalSyncLoad}")
+ onFinish {
+ sql "set enable_multi_cluster_sync_load = ${originalSyncLoad}"
+ logger.info("Restored enable_multi_cluster_sync_load=${originalSyncLoad}")
}
+ // Only explicit warm-up should populate the target cache in this case.
+ sql "set enable_multi_cluster_sync_load = false"
+ logger.info("Explicit warmup: disabled automatic sync load, previous=${originalSyncLoad}")
- List<String> ipList = new ArrayList<>();
- List<String> hbPortList = new ArrayList<>()
- List<String> httpPortList = new ArrayList<>()
- List<String> brpcPortList = new ArrayList<>()
- List<String> beUniqueIdList = new ArrayList<>()
-
- String[] bes = context.config.multiClusterBes.split(',');
- println("the value is " + context.config.multiClusterBes);
- int num = 0
- for(String values : bes) {
- if (num++ == 2) break;
- println("the value is " + values);
- String[] beInfo = values.split(':');
- ipList.add(beInfo[0]);
- hbPortList.add(beInfo[1]);
- httpPortList.add(beInfo[2]);
- beUniqueIdList.add(beInfo[3]);
- brpcPortList.add(beInfo[4]);
+ String sourceCluster = "regression_cluster_name0"
+ String targetCluster = "regression_cluster_name1"
+ String tableName = "customer"
+ def partitionNames = ["p1", "p2", "p3"]
+ long ttlSeconds = 12000L
+ def partitionInfo = [:]
+ def warmupJobs = [] as Set
+ def pendingLoads = [] as Set
+ def backends = sql_return_maparray("SHOW BACKENDS")
+ def clusterBackends = [sourceCluster, targetCluster].collectEntries { clusterName ->
+ def members = backends.findAll { be ->
+ "${be.Alive}".equalsIgnoreCase("true") &&
+ parseJson(be.Tag.toString()).compute_group_name == clusterName
+ }.collect { be ->
+ [id: be.BackendId as Long, ip: be.Host.toString(),
+ httpPort: be.HttpPort.toString(), brpcPort: be.BrpcPort.toString()]
+ }
+ assertTrue(!members.isEmpty(), "No alive backends in ${clusterName}")
+ [(clusterName): members]
}
+ logger.info("Partition warmup backends: ${clusterBackends}")
- println("the ip is " + ipList);
- println("the heartbeat port is " + hbPortList);
- println("the http port is " + httpPortList);
- println("the be unique id is " + beUniqueIdList);
- println("the brpc port is " + brpcPortList);
-
- def s3BucketName = getS3BucketName()
- def s3WithProperties = """WITH S3 (
- |"AWS_ACCESS_KEY" = "${getS3AK()}",
- |"AWS_SECRET_KEY" = "${getS3SK()}",
- |"AWS_ENDPOINT" = "${getS3Endpoint()}",
- |"AWS_REGION" = "${getS3Region()}",
- |"provider" = "${getS3Provider()}")
- |PROPERTIES(
- |"exec_mem_limit" = "8589934592",
- |"load_parallelism" = "3")""".stripMargin()
-
-
-
- sql "use @regression_cluster_name0"
-
- def table = "customer"
- sql new File("""${context.file.parent}/../ddl/${table}_delete.sql""").text
- // create table if not exists
- sql (new File("""${context.file.parent}/../ddl/customer_with_partition.sql""").text + ttlProperties)
- sleep(10000)
-
- def load_customer_once = {
- def uniqueID = Math.abs(UUID.randomUUID().hashCode()).toString()
- def loadLabel = table + "_" + uniqueID
- // load data from cos
- def loadSql = new File("""${context.file.parent}/../ddl/${table}_load.sql""").text.replaceAll("\\\$\\{s3BucketName\\}", s3BucketName)
- loadSql = loadSql.replaceAll("\\\$\\{loadLabel\\}", loadLabel) + s3WithProperties
- sql loadSql
-
- // check load state
- while (true) {
- def stateResult = sql "show load where Label = '${loadLabel}'"
- def loadState = stateResult[stateResult.size() - 1][2].toString()
- if ("CANCELLED".equalsIgnoreCase(loadState)) {
- throw new IllegalStateException("load ${loadLabel} failed.")
- } else if ("FINISHED".equalsIgnoreCase(loadState)) {
- break
+ def waitUntil = { String description, long timeoutMs, Closure condition ->
+ long deadline = System.currentTimeMillis() + timeoutMs
+ while (System.currentTimeMillis() < deadline) {
+ if (condition()) {
+ return
}
- sleep(5000)
+ sleep(1000)
}
+ assertTrue(false, "Timeout after ${timeoutMs}ms: ${description}; see last state in log")
}
-
- def clearFileCache = { ip, port ->
- httpTest {
- endpoint ""
- uri ip + ":" + port + """/api/file_cache?op=clear&sync=true"""
- op "get"
- body ""
+ def getGlobalTtlBytes = {
+ [source: WarmupMetricsUtils.getBackendMetricSum(clusterBackends[sourceCluster], "ttl_cache_size"),
+ target: WarmupMetricsUtils.getBackendMetricSum(clusterBackends[targetCluster], "ttl_cache_size")]
+ }
+ def getPartitionCache = { String clusterName ->
+ sql "use @${clusterName}"
+ def tabletIds = partitionInfo.values().collectMany { it.tabletIds }
+ def beIds = clusterBackends[clusterName].collect { it.id }
+ // Read all cache types: a NORMAL block in an untouched partition is also unexpected.
+ def rows = sql """select tablet_id, lower(type), sum(size), count(*)
+ from information_schema.file_cache_info
+ where tablet_id in (${tabletIds.join(',')}) and be_id in (${beIds.join(',')})
+ group by tablet_id, lower(type)"""
+ partitionInfo.collectEntries { name, info ->
+ def partitionRows = rows.findAll { (it[0] as Long) in info.tabletIds }
+ def ttlRows = partitionRows.findAll { it[1]?.toString() == "ttl" }
+ def ttlByTablet = ttlRows.collectEntries { [(it[0] as Long): it[2] as Long] }
+ [(name): [ttlBytes: ttlByTablet.values().sum(0L) as Long, ttlByTablet: ttlByTablet,
+ complete: info.tabletIds.every { (ttlByTablet[it] ?: 0L) > 0L },
+ blocks: partitionRows.sum(0L) { it[3] as Long },
+ nonTtlBlocks: partitionRows.findAll { it[1]?.toString() != "ttl" }
+ .sum(0L) { it[3] as Long }]]
}
}
-
- def getMetricsMethod = { ip, port, check_func ->
- httpTest {
- endpoint ip + ":" + port
- uri "/brpc_metrics"
- op "get"
- check check_func
+ def waitForTargetTablets = {
+ sql "use @${targetCluster}"
+ def targetBes = clusterBackends[targetCluster].collectEntries { [(it.id): it] }
+ waitUntil("target primary routing and tablet metadata", 120000L) {
+ def missing = [:]
+ partitionInfo.each { name, info ->
+ def ready = [] as Set
+ def tablets = sql_return_maparray("show tablets from ${tableName} partition ${name}")
+ tablets.each { tablet ->
+ long id = tablet.TabletId as Long
+ long primary = tablet.PrimaryBackendId as Long
+ if (info.tabletIds.contains(id) && targetBes.containsKey(primary) &&
+ (tablet.BackendId as Long) == primary) {
+ def be = targetBes[primary]
+ try {
+ // Fetch metadata only; a target data query could populate the cache itself.
+ httpTest {
+ endpoint "${be.ip}:${be.httpPort}"
+ uri "/api/meta/header/${id}"
+ op "get"
+ check { code, body ->
+ if ((code as int) == 200) {
+ def meta = parseJson(body.toString())
+ // Cloud header JSON omits rowsets. The source baseline and
+ // per-partition warm-up assertions verify the loaded data.
+ if ((meta.tablet_id as Long) == id &&
+ (meta.ttl_seconds as Long) == ttlSeconds &&
+ meta.tablet_state == "PB_RUNNING" && meta.schema != null) {
+ ready.add(id)
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ logger.info("Waiting for ${name} tablet ${id} on BE ${primary}: ${e.message}")
+ }
+ }
+ }
+ missing[name] = [tablets: info.tabletIds - ready, loadedSourceVersion: info.version]
+ }
+ logger.info("Target tablet readiness: ${missing}")
+ missing.values().every { it.tablets.isEmpty() }
}
}
- clearFileCache.call(ipList[0], httpPortList[0]);
- clearFileCache.call(ipList[1], httpPortList[1]);
-
- load_customer_once()
+ setBeConfigTemporary([
+ enable_evict_file_cache_in_advance: false,
+ file_cache_enter_disk_resource_limit_mode_percent: 99
+ ]) {
+ try {
+ sql "use @${sourceCluster}"
+ sql "DROP TABLE IF EXISTS ${tableName} FORCE"
+ sql (new File("${context.file.parent}/../ddl/customer_with_partition.sql").text + """
+ PROPERTIES("file_cache_ttl_seconds"="${ttlSeconds}", "disable_auto_compaction"="true")""")
+ partitionNames.each { name ->
+ def ids = sql("show tablets from ${tableName} partition ${name}")
+ .collect { it[0] as Long }.unique()
+ assertTrue(!ids.isEmpty(), "Missing tablets for ${name}")
+ partitionInfo[name] = [tabletIds: ids]
+ }
- def jobId_ = sql "warm up cluster regression_cluster_name1 with table customer partition p1;"
- def waitJobDone = { jobId ->
- int retryTime = 120
- int i = 0
- for (; i < retryTime; i++) {
- sleep(1000)
- def statuses = getJobState(jobId[0][0])
- if (statuses.any { it != null && it.equals("CANCELLED") }) {
- assertTrue(false);
+ def ttlBeforeClear = getGlobalTtlBytes()
+ clusterBackends.values().flatten().each { be ->
+ WarmupMetricsUtils.clearFileCache(be.ip, be.httpPort)
}
- if (statuses.any { it != null && it.equals("FINISHED") }) {
- break;
+ // sync=true can leave held blocks from other tables. Keep global totals for diagnosis only.
+ def ttlAfterClear = getGlobalTtlBytes()
+ logger.info("TTL cache clear: before=${ttlBeforeClear}, after=${ttlAfterClear}, " +
+ "source_minus_target_after_clear=${ttlAfterClear.source - ttlAfterClear.target}, " +
+ "source_change_after_clear=${ttlAfterClear.source - ttlBeforeClear.source}, " +
+ "target_change_after_clear=${ttlAfterClear.target - ttlBeforeClear.target}")
+
+ String loadLabel = "customer_${UUID.randomUUID().toString().replace('-', '')}"
+ String loadSql = new File("${context.file.parent}/../ddl/customer_load.sql").text
+ .replace('${s3BucketName}', getS3BucketName()).replace('${loadLabel}', loadLabel)
+ sql (loadSql + """WITH S3 (
+ "AWS_ACCESS_KEY" = "${getS3AK()}", "AWS_SECRET_KEY" = "${getS3SK()}",
+ "AWS_ENDPOINT" = "${getS3Endpoint()}", "AWS_REGION" = "${getS3Region()}",
+ "provider" = "${getS3Provider()}")
+ PROPERTIES("exec_mem_limit"="8589934592", "load_parallelism"="3")""")
+ pendingLoads.add(loadLabel)
+ waitUntil("load ${loadLabel} to finish", 300000L) {
+ def rows = sql "show load where Label = '${loadLabel}'"
+ logger.info("Partition warmup load state: ${rows}")
+ if (rows.isEmpty()) {
+ return false
+ }
+ String state = rows.last()[2].toString()
+ assertTrue(!state.equalsIgnoreCase("CANCELLED"), "Load ${loadLabel} cancelled: ${rows.last()}")
+ if (state.equalsIgnoreCase("FINISHED")) {
+ pendingLoads.remove(loadLabel)
+ return true
+ }
+ false
}
- }
- if (i == retryTime) {
- sql "cancel warm up job where id = ${jobId[0][0]}"
- assertTrue(false);
- }
- }
- waitJobDone(jobId_)
- jobId_ = sql "warm up cluster regression_cluster_name1 with table customer partition p2;"
- waitJobDone(jobId_)
- jobId_ = sql "warm up cluster regression_cluster_name1 with table customer partition p3;"
- waitJobDone(jobId_)
+ def partitions = sql_return_maparray("show partitions from ${tableName}")
+ partitionNames.each { name ->
+ def partition = partitions.find { it.PartitionName.toString() == name }
+ assertTrue(partition != null && (partition.VisibleVersion as Long) > 1L,
+ "Missing loaded version for ${name}: ${partitions}")
+ partitionInfo[name].version = partition.VisibleVersion as Long
+ }
+ logger.info("Loaded partitions: ${partitionInfo}")
+ waitForTargetTablets()
- sleep(30000)
- long ttl_cache_size = 0
- getMetricsMethod.call(ipList[0], brpcPortList[0]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- ttl_cache_size = line.substring(i).toLong()
- flag = true
- break
+ def sourceBaseline = [:]
+ long stableSince = System.currentTimeMillis()
+ waitUntil("source partitions to have stable nonempty TTL cache", 120000L) {
+ def current = getPartitionCache(sourceCluster)
+ logger.info("Source partition cache before warmup: ${current}")
+ if (current != sourceBaseline) {
+ sourceBaseline = current
+ stableSince = System.currentTimeMillis()
}
+ current.values().every { it.complete && it.nonTtlBlocks == 0L } &&
+ System.currentTimeMillis() - stableSince >= 3000L
}
- assertTrue(flag)
- }
+ def targetBefore = getPartitionCache(targetCluster)
+ assertTrue(targetBefore.values().every { it.blocks == 0L },
+ "Target partitions must be cold before warmup: ${targetBefore}")
- getMetricsMethod.call(ipList[1], brpcPortList[1]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
+ def warmedPartitions = [] as Set
+ partitionNames.each { name ->
+ // Explicit partition warmup does not require hotspot queries or hotspot collection.
+ def job = sql "WARM UP CLUSTER ${targetCluster} WITH TABLE ${tableName} PARTITION ${name}"
+ assertTrue(!job.isEmpty(), "No warmup job returned for ${name}")
+ def jobId = job[0][0]
+ warmupJobs.add(jobId)
+ waitUntil("warmup ${name} job ${jobId} to finish", 120000L) {
+ def rows = sql "SHOW WARM UP JOB WHERE ID = ${jobId}"
+ logger.info("Partition ${name} warmup job ${jobId}: ${rows}")
+ if (rows.isEmpty()) {
+ return false
+ }
+ def states = rows[0].collect { it?.toString() }
+ assertTrue(!states.contains("CANCELLED"), "Warmup ${name} job ${jobId} cancelled: ${rows[0]}")
+ states.contains("FINISHED")
+ }
+ warmedPartitions.add(name)
+ waitUntil("after ${name}: partition TTL cache must match source and untouched partitions stay cold",
+ 120000L) {
+ def source = getPartitionCache(sourceCluster)
+ def target = getPartitionCache(targetCluster)
+ logger.info("After ${name} job ${jobId}: warmed=${warmedPartitions}, " +
+ "source=${source}, target=${target}, baseline=${sourceBaseline}")
+ // Fail immediately on scope/type violations; waiting must not hide unintended warmup.
+ partitionNames.each { partitionName ->
+ if (!warmedPartitions.contains(partitionName)) {
+ assertTrue(target[partitionName].blocks == 0L,
+ "Untouched partition ${partitionName} has cache after ${name}: ${target}")
+ } else {
+ assertTrue(target[partitionName].nonTtlBlocks == 0L,
+ "Expected only TTL cache for ${partitionName}: ${target[partitionName]}")
+ }
+ }
+ partitionNames.every { partitionName ->
+ def src = source[partitionName]
+ def dst = target[partitionName]
+ src.complete && src.nonTtlBlocks == 0L &&
+ src.ttlByTablet == sourceBaseline[partitionName].ttlByTablet &&
+ (!warmedPartitions.contains(partitionName) ||
+ (dst.complete && dst.ttlBytes == src.ttlBytes && dst.ttlByTablet == src.ttlByTablet))
}
- def i = line.indexOf(' ')
- assertEquals(ttl_cache_size, line.substring(i).toLong())
- flag = true
- break
}
}
- assertTrue(flag)
- }
-
- try {
- sql "warm up cluster regression_cluster_name1 with table customer partition p5;"
- assertTrue(false)
- } catch (Exception e) {
- assertTrue(true)
- }
- sql new File("""${context.file.parent}/../ddl/${table}_delete.sql""").text
+ sql "use @${targetCluster}"
+ test {
+ sql "WARM UP CLUSTER ${targetCluster} WITH TABLE ${tableName} PARTITION p5"
+ exception "The partition p5 doesn't exist"
+ }
+ } finally {
+ warmupJobs.each { jobId ->
+ try {
+ def rows = sql "SHOW WARM UP JOB WHERE ID = ${jobId}"
+ if (!rows.isEmpty() && rows[0].any { it?.toString() in ["PENDING", "RUNNING"] }) {
+ sql "CANCEL WARM UP JOB WHERE ID = ${jobId}"
+ }
+ } catch (Exception e) {
+ logger.warn("Failed to clean up warmup ${jobId}: ${e.message}")
+ }
+ }
+ // Keep DROP in finally so even a cleanup SQL failure does not leave the test table behind.
+ try {
+ sql "use @${sourceCluster}"
+ pendingLoads.each { label ->
+ try {
+ sql "CANCEL LOAD WHERE LABEL = '${label}'"
+ } catch (Exception e) {
+ logger.warn("Failed to clean up load ${label}: ${e.message}")
+ }
+ }
+ } finally {
+ sql "DROP TABLE IF EXISTS ${tableName} FORCE"
+ }
+ }
}
}
diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_same_table_multi_times.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_same_table_multi_times.groovy
index b774273f952..cc09dabff4e 100644
--- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_same_table_multi_times.groovy
+++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_same_table_multi_times.groovy
@@ -17,7 +17,19 @@
import org.codehaus.groovy.runtime.IOGroovyMethods
-suite("test_warm_up_same_table_multi_times") {
+suite("test_warm_up_same_table_multi_times", "nonConcurrent") {
+ // This setting is static in FE; isolate it and restore it even when a cache assertion fails.
+ String originalSyncLoad = sql("select @@enable_multi_cluster_sync_load")[0][0].toString()
+ assertTrue(originalSyncLoad.toLowerCase() in ["true", "false", "1", "0"],
+ "Unexpected sync-load setting: ${originalSyncLoad}")
+ onFinish {
+ sql "set enable_multi_cluster_sync_load = ${originalSyncLoad}"
+ logger.info("Restored enable_multi_cluster_sync_load=${originalSyncLoad}")
+ }
+ // Only explicit warm-up should populate the target cache in this case.
+ sql "set enable_multi_cluster_sync_load = false"
+ logger.info("Explicit warmup: disabled automatic sync load, previous=${originalSyncLoad}")
+
def custoBeConfig = [
enable_evict_file_cache_in_advance : false,
file_cache_enter_disk_resource_limit_mode_percent : 99
@@ -115,6 +127,26 @@ suite("test_warm_up_same_table_multi_times") {
}
}
+ def createdWarmUpJobIds = [] as Set
+ def recordWarmUpJob = { jobId ->
+ if (jobId != null && !jobId.isEmpty()) {
+ createdWarmUpJobIds.add(jobId[0][0])
+ }
+ }
+ def cleanupWarmUpJobs = {
+ createdWarmUpJobIds.each { jobId ->
+ try {
+ def statuses = getJobState(jobId)
+ if (statuses.any { it != null && (it.equals("PENDING") || it.equals("RUNNING")) }) {
+ sql "cancel warm up job where id = ${jobId}"
+ }
+ } catch (Exception e) {
+ logger.info("ignore warm up job cleanup failure, job id: ${jobId}, error: ${e.getMessage()}")
+ }
+ }
+ }
+
+ try {
clearFileCache.call();
sleep(30000)
@@ -124,11 +156,12 @@ suite("test_warm_up_same_table_multi_times") {
load_customer_once()
def jobId = sql "warm up cluster ${validCluster} with table customer;"
+ recordWarmUpJob(jobId)
try {
sql "warm up cluster ${validCluster} with table customer;"
- assertTrue(true) // dup warm up command can be send to fe queue now
- } catch (Exception e) {
assertTrue(false)
+ } catch (Exception e) {
+ assertTrue("${e.getMessage()}".contains("already has a pending job"))
}
int retryTime = 120
int j = 0
@@ -191,6 +224,7 @@ suite("test_warm_up_same_table_multi_times") {
// AGAIN!
jobId = sql "warm up cluster ${validCluster} with table customer;"
+ recordWarmUpJob(jobId)
retryTime = 120
j = 0
@@ -273,5 +307,8 @@ suite("test_warm_up_same_table_multi_times") {
long diff = skip_io_bytes_end - skip_io_bytes_start;
println("skip_io_bytes diff: " + diff);
assertTrue(diff > 1000);
+ } finally {
+ cleanupWarmUpJobs()
+ }
}
}
diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy
index 4e4979ab79c..0b5da4bcdd3 100644
--- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy
+++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy
@@ -15,9 +15,22 @@
// specific language governing permissions and limitations
// under the License.
-import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.Http
+import org.apache.doris.regression.util.WarmupMetricsUtils
+
+suite("test_warm_up_table", "nonConcurrent") {
+ // This setting is static in FE; isolate it and restore it even when a cache assertion fails.
+ String originalSyncLoad = sql("select @@enable_multi_cluster_sync_load")[0][0].toString()
+ assertTrue(originalSyncLoad.toLowerCase() in ["true", "false", "1", "0"],
+ "Unexpected sync-load setting: ${originalSyncLoad}")
+ onFinish {
+ sql "set enable_multi_cluster_sync_load = ${originalSyncLoad}"
+ logger.info("Restored enable_multi_cluster_sync_load=${originalSyncLoad}")
+ }
+ // Only explicit warm-up should populate the target cache in this case.
+ sql "set enable_multi_cluster_sync_load = false"
+ logger.info("Explicit warmup: disabled automatic sync load, previous=${originalSyncLoad}")
-suite("test_warm_up_table") {
def custoBeConfig = [
enable_evict_file_cache_in_advance : false,
file_cache_enter_disk_resource_limit_mode_percent : 99
@@ -113,17 +126,104 @@ suite("test_warm_up_table") {
}
}
- def getMetricsMethod = { ip, port, check_func ->
- httpTest {
- endpoint ip + ":" + port
- uri "/brpc_metrics"
- op "get"
- check check_func
+ def clusterBackends = ["regression_cluster_name0", "regression_cluster_name1"].collectEntries { name ->
+ def members = sql_return_maparray("SHOW BACKENDS").findAll { be ->
+ "${be.Alive}".equalsIgnoreCase("true") &&
+ parseJson(be.Tag.toString()).compute_group_name == name
+ }.collect { be ->
+ [id: be.BackendId as Long, ip: be.Host.toString(),
+ httpPort: be.HttpPort.toString(), brpcPort: be.BrpcPort.toString()]
+ }
+ assertTrue(!members.isEmpty(), "No alive backends in ${name}")
+ [(name): members]
+ }
+ def tabletIds = []
+ def waitUntil = { String stage, long timeoutMs, Closure ready ->
+ long deadline = System.currentTimeMillis() + timeoutMs
+ while (System.currentTimeMillis() < deadline) {
+ if (ready()) { return }
+ sleep(1000)
+ }
+ assertTrue(false, "Timeout waiting for ${stage}; see last cache state in log")
+ }
+ def getGlobalTtl = {
+ clusterBackends.collectEntries { name, members ->
+ [(name): WarmupMetricsUtils.getBackendMetricSum(members, "ttl_cache_size")]
+ }
+ }
+ def getTableCache = { String clusterName ->
+ sql "use @${clusterName}"
+ def beIds = clusterBackends[clusterName].collect { it.id }
+ def rows = sql """select tablet_id, lower(type), sum(size)
+ from information_schema.file_cache_info
+ where tablet_id in (${tabletIds.join(',')}) and be_id in (${beIds.join(',')})
+ group by tablet_id, lower(type)"""
+ def ttlByTablet = rows.findAll { it[1]?.toString() == "ttl" }
+ .collectEntries { [(it[0] as Long): it[2] as Long] }
+ [ttlByTablet: ttlByTablet, complete: tabletIds.every { (ttlByTablet[it] ?: 0L) > 0L },
+ nonTtlBytes: rows.findAll { it[1]?.toString() != "ttl" }.sum(0L) { it[2] as Long }]
+ }
+ def waitForTargetTablets = {
+ sql "use @regression_cluster_name1"
+ def targetBes = clusterBackends.regression_cluster_name1.collectEntries { [(it.id): it] }
+ waitUntil("target primary tablet metadata", 120000L) {
+ def ready = [] as Set
+ sql_return_maparray("show tablets from ${table}").each { tablet ->
+ long id = tablet.TabletId as Long
+ long primary = tablet.PrimaryBackendId as Long
+ if (id in tabletIds && targetBes.containsKey(primary) && (tablet.BackendId as Long) == primary) {
+ def be = targetBes[primary]
+ try {
+ // Metadata only: a business read here would populate the target cache.
+ def connection = Http.openConnection(
+ "http://${be.ip}:${be.httpPort}/api/meta/header/${id}")
+ connection.connectTimeout = 5000
+ connection.readTimeout = 5000
+ try {
+ def meta = connection.inputStream.withCloseable {
+ parseJson(it.getText("UTF-8"))
+ }
+ if ((meta.tablet_id as Long) == id && (meta.ttl_seconds as Long) == 12000L &&
+ meta.tablet_state == "PB_RUNNING" && meta.schema != null) {
+ ready.add(id)
+ }
+ } finally {
+ connection.disconnect()
+ }
+ } catch (Exception e) {
+ logger.info("Waiting for target tablet ${id} on BE ${primary}: ${e.message}")
+ }
+ }
+ }
+ logger.info("Target tablet metadata: missing=${tabletIds - ready}")
+ ready.size() == tabletIds.size()
}
}
- clearFileCache.call(ipList[0], httpPortList[0]);
- clearFileCache.call(ipList[1], httpPortList[1]);
+ def createdWarmUpJobIds = [] as Set
+ def recordWarmUpJob = { jobId ->
+ if (jobId != null && !jobId.isEmpty()) {
+ createdWarmUpJobIds.add(jobId[0][0])
+ }
+ }
+ def cleanupWarmUpJobs = {
+ createdWarmUpJobIds.each { jobId ->
+ try {
+ def statuses = getJobState(jobId)
+ if (statuses.any { it != null && (it.equals("PENDING") || it.equals("RUNNING")) }) {
+ sql "cancel warm up job where id = ${jobId}"
+ }
+ } catch (Exception e) {
+ logger.info("ignore warm up job cleanup failure, job id: ${jobId}, error: ${e.getMessage()}")
+ }
+ }
+ }
+
+ try {
+ def ttlBeforeClear = getGlobalTtl()
+ clusterBackends.values().flatten().each { be -> clearFileCache.call(be.ip, be.httpPort) }
+ def ttlAfterClear = getGlobalTtl()
+ logger.info("Single-table global TTL clear: before=${ttlBeforeClear}, after=${ttlAfterClear}")
sleep(30000)
load_customer_once()
@@ -140,12 +240,30 @@ suite("test_warm_up_table") {
load_customer_once()
load_customer_once()
+ tabletIds = sql_return_maparray("show tablets from ${table}")
+ .collect { it.TabletId as Long }.unique().sort()
+ assertTrue(!tabletIds.isEmpty(), "No tablets for ${table}")
+ logger.info("Single-table warmup tablets=${tabletIds}, backends=${clusterBackends}")
+ def sourceCache = null
+ def candidate = null
+ long stableSince = 0L
+ waitUntil("stable nonempty source TTL cache", 120000L) {
+ sourceCache = getTableCache("regression_cluster_name0")
+ boolean complete = sourceCache.complete && sourceCache.nonTtlBytes == 0L
+ if (!complete || sourceCache != candidate) { stableSince = System.currentTimeMillis() }
+ candidate = sourceCache
+ logger.info("Single-table source TTL baseline=${sourceCache}")
+ complete && System.currentTimeMillis() - stableSince >= 3000L
+ }
+ waitForTargetTablets()
+
def jobId = sql "warm up cluster regression_cluster_name1 with table customer;"
+ recordWarmUpJob(jobId)
try {
sql "warm up cluster regression_cluster_name1 with table customer;"
- assertTrue(true)
- } catch (Exception e) {
assertTrue(false)
+ } catch (Exception e) {
+ assertTrue("${e.getMessage()}".contains("already has a pending job"))
}
int retryTime = 120
int j = 0
@@ -166,46 +284,20 @@ suite("test_warm_up_table") {
def tablesString = getTablesFromShowCommand(jobId[0][0])
assertTrue(tablesString.any { it != null && it.contains("customer") })
- sleep(30000)
- long ttl_cache_size = 0
- getMetricsMethod.call(ipList[0], brpcPortList[0]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- ttl_cache_size = line.substring(i).toLong()
- flag = true
- break
- }
- }
- assertTrue(flag)
- }
-
- getMetricsMethod.call(ipList[1], brpcPortList[1]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- assertEquals(ttl_cache_size, line.substring(i).toLong())
- flag = true
- break
- }
- }
- assertTrue(flag)
+ // Global TTL includes residual blocks from other tables; assert this table on the target BEs.
+ try {
+ waitUntil("target per-tablet TTL cache to equal the nonzero source baseline", 180000L) {
+ def currentSource = getTableCache("regression_cluster_name0")
+ def targetCache = getTableCache("regression_cluster_name1")
+ logger.info("Single-table TTL cache: source=${currentSource}, target=${targetCache}")
+ currentSource == sourceCache && targetCache.complete && targetCache.nonTtlBytes == 0L &&
+ targetCache.ttlByTablet == sourceCache.ttlByTablet
+ }
+ } finally {
+ def ttlAfterWarmup = getGlobalTtl()
+ def delta = ttlAfterWarmup.collectEntries { name, bytes -> [(name): bytes - ttlAfterClear[name]] }
+ logger.info("Single-table global TTL after warmup=${ttlAfterWarmup}, " +
+ "baseline_after_clear=${ttlAfterClear}, delta_since_clear=${delta}")
}
try {
@@ -222,5 +314,9 @@ suite("test_warm_up_table") {
} catch (Exception e) {
assertTrue(true)
}
+ } finally {
+ cleanupWarmUpJobs()
+ sql "DROP TABLE IF EXISTS ${table} FORCE"
+ }
}
}
diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy
index be4521544fe..9a8e1281deb 100644
--- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy
+++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy
@@ -15,233 +15,325 @@
// specific language governing permissions and limitations
// under the License.
-import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.WarmupMetricsUtils
-suite("test_warm_up_tables") {
- def custoBeConfig = [
- enable_evict_file_cache_in_advance : false,
- file_cache_enter_disk_resource_limit_mode_percent : 99
- ]
-
- setBeConfigTemporary(custoBeConfig) {
- def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """
- def getJobState = { jobId ->
- def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """
- return jobStateResult[0]
- }
- def getTablesFromShowCommand = { jobId ->
- def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """
- return jobStateResult[0]
+suite("test_warm_up_tables", "nonConcurrent") {
+ // This setting is static in FE; isolate it and restore it even when a cache assertion fails.
+ String originalSyncLoad = sql("select @@enable_multi_cluster_sync_load")[0][0].toString()
+ assertTrue(originalSyncLoad.toLowerCase() in ["true", "false", "1", "0"],
+ "Unexpected sync-load setting: ${originalSyncLoad}")
+ onFinish {
+ sql "set enable_multi_cluster_sync_load = ${originalSyncLoad}"
+ logger.info("Restored enable_multi_cluster_sync_load=${originalSyncLoad}")
}
+ // Only explicit warm-up should populate the target cache in this case.
+ sql "set enable_multi_cluster_sync_load = false"
+ logger.info("Explicit warmup: disabled automatic sync load, previous=${originalSyncLoad}")
- List<String> ipList = new ArrayList<>();
- List<String> hbPortList = new ArrayList<>()
- List<String> httpPortList = new ArrayList<>()
- List<String> brpcPortList = new ArrayList<>()
- List<String> beUniqueIdList = new ArrayList<>()
-
- String[] bes = context.config.multiClusterBes.split(',');
- println("the value is " + context.config.multiClusterBes);
- int num = 0
- for(String values : bes) {
- if (num++ == 2) break;
- println("the value is " + values);
- String[] beInfo = values.split(':');
- ipList.add(beInfo[0]);
- hbPortList.add(beInfo[1]);
- httpPortList.add(beInfo[2]);
- beUniqueIdList.add(beInfo[3]);
- brpcPortList.add(beInfo[4]);
+ String sourceCluster = "regression_cluster_name0"
+ String targetCluster = "regression_cluster_name1"
+ def tableDefinitions = [customer: "customer_with_partition", supplier: "supplier"]
+ long ttlSeconds = 12000L
+ def partitionInfo = [
+ "customer.p1": [table: "customer", partition: "p1"],
+ "customer.p2": [table: "customer", partition: "p2"],
+ "customer.p3": [table: "customer", partition: "p3"],
+ "supplier": [table: "supplier", partition: null]
+ ]
+ def showTabletsSql = { info ->
+ "show tablets from ${info.table}" + (info.partition ? " partition ${info.partition}" : "")
}
-
- println("the ip is " + ipList);
- println("the heartbeat port is " + hbPortList);
- println("the http port is " + httpPortList);
- println("the be unique id is " + beUniqueIdList);
- println("the brpc port is " + brpcPortList);
-
- def s3BucketName = getS3BucketName()
- def s3WithProperties = """WITH S3 (
- |"AWS_ACCESS_KEY" = "${getS3AK()}",
- |"AWS_SECRET_KEY" = "${getS3SK()}",
- |"AWS_ENDPOINT" = "${getS3Endpoint()}",
- |"AWS_REGION" = "${getS3Region()}",
- |"provider" = "${getS3Provider()}")
- |PROPERTIES(
- |"exec_mem_limit" = "8589934592",
- |"load_parallelism" = "3")""".stripMargin()
-
-
- sql """ SET PROPERTY 'default_cloud_cluster' = "regression_cluster_name0"; """
- sql "use @regression_cluster_name0"
-
- def table = "customer"
- sql new File("""${context.file.parent}/../ddl/${table}_delete.sql""").text
- sql new File("""${context.file.parent}/../ddl/supplier_delete.sql""").text
- // create table if not exists
- sql (new File("""${context.file.parent}/../ddl/customer_with_partition.sql""").text + ttlProperties)
- sql (new File("""${context.file.parent}/../ddl/supplier.sql""").text + ttlProperties)
- sleep(10000)
-
- def load_customer_once = {
- def uniqueID = Math.abs(UUID.randomUUID().hashCode()).toString()
- def loadLabel = table + "_" + uniqueID
- // load data from cos
- def loadSql = new File("""${context.file.parent}/../ddl/${table}_load.sql""").text.replaceAll("\\\$\\{s3BucketName\\}", s3BucketName)
- loadSql = loadSql.replaceAll("\\\$\\{loadLabel\\}", loadLabel) + s3WithProperties
- sql loadSql
-
- // check load state
- while (true) {
- def stateResult = sql "show load where Label = '${loadLabel}'"
- def loadState = stateResult[stateResult.size() - 1][2].toString()
- if ("CANCELLED".equalsIgnoreCase(loadState)) {
- throw new IllegalStateException("load ${loadLabel} failed.")
- } else if ("FINISHED".equalsIgnoreCase(loadState)) {
- break
- }
- sleep(5000)
+ def warmupJobs = [] as Set
+ def pendingLoads = [] as Set
+ def backends = sql_return_maparray("SHOW BACKENDS")
+ def clusterBackends = [sourceCluster, targetCluster].collectEntries { clusterName ->
+ def members = backends.findAll { be ->
+ "${be.Alive}".equalsIgnoreCase("true") &&
+ parseJson(be.Tag.toString()).compute_group_name == clusterName
+ }.collect { be ->
+ [id: be.BackendId as Long, ip: be.Host.toString(),
+ httpPort: be.HttpPort.toString(), brpcPort: be.BrpcPort.toString()]
}
+ assertTrue(!members.isEmpty(), "No alive backends in ${clusterName}")
+ [(clusterName): members]
}
+ logger.info("Multi-table warmup backends: ${clusterBackends}")
- def load_supplier_once = {
- def uniqueID = Math.abs(UUID.randomUUID().hashCode()).toString()
- def loadLabel = "supplier_" + uniqueID
- // load data from cos
- def loadSql = new File("""${context.file.parent}/../ddl/supplier_load.sql""").text.replaceAll("\\\$\\{s3BucketName\\}", s3BucketName)
- loadSql = loadSql.replaceAll("\\\$\\{loadLabel\\}", loadLabel) + s3WithProperties
- sql loadSql
-
- // check load state
- while (true) {
- def stateResult = sql "show load where Label = '${loadLabel}'"
- def loadState = stateResult[stateResult.size() - 1][2].toString()
- if ("CANCELLED".equalsIgnoreCase(loadState)) {
- throw new IllegalStateException("load ${loadLabel} failed.")
- } else if ("FINISHED".equalsIgnoreCase(loadState)) {
- break
+ def waitUntil = { String description, long timeoutMs, Closure condition ->
+ long deadline = System.currentTimeMillis() + timeoutMs
+ while (System.currentTimeMillis() < deadline) {
+ if (condition()) {
+ return
}
- sleep(5000)
+ sleep(1000)
}
+ assertTrue(false, "Timeout after ${timeoutMs}ms: ${description}; see last state in log")
}
-
- def clearFileCache = { ip, port ->
- httpTest {
- endpoint ""
- uri ip + ":" + port + """/api/file_cache?op=clear&sync=true"""
- op "get"
- body ""
+ def getGlobalTtlBytes = {
+ [source: WarmupMetricsUtils.getBackendMetricSum(clusterBackends[sourceCluster], "ttl_cache_size"),
+ target: WarmupMetricsUtils.getBackendMetricSum(clusterBackends[targetCluster], "ttl_cache_size")]
+ }
+ def getPartitionCache = { String clusterName ->
+ sql "use @${clusterName}"
+ def tabletIds = partitionInfo.values().collectMany { it.tabletIds }
+ def beIds = clusterBackends[clusterName].collect { it.id }
+ // Read all cache types: a NORMAL block in an untouched partition is also unexpected.
+ def rows = sql """select tablet_id, lower(type), sum(size), count(*)
+ from information_schema.file_cache_info
+ where tablet_id in (${tabletIds.join(',')}) and be_id in (${beIds.join(',')})
+ group by tablet_id, lower(type)"""
+ partitionInfo.collectEntries { name, info ->
+ def partitionRows = rows.findAll { (it[0] as Long) in info.tabletIds }
+ def ttlRows = partitionRows.findAll { it[1]?.toString() == "ttl" }
+ def ttlByTablet = ttlRows.collectEntries { [(it[0] as Long): it[2] as Long] }
+ [(name): [ttlBytes: ttlByTablet.values().sum(0L) as Long, ttlByTablet: ttlByTablet,
+ bytesByType: partitionRows.groupBy { it[1]?.toString() }.collectEntries { type, entries ->
+ [(type): entries.sum(0L) { it[2] as Long }]
+ },
+ complete: info.tabletIds.every { (ttlByTablet[it] ?: 0L) > 0L },
+ blocks: partitionRows.sum(0L) { it[3] as Long },
+ nonTtlBlocks: partitionRows.findAll { it[1]?.toString() != "ttl" }
+ .sum(0L) { it[3] as Long }]]
}
}
-
- def getMetricsMethod = { ip, port, check_func ->
- httpTest {
- endpoint ip + ":" + port
- uri "/brpc_metrics"
- op "get"
- check check_func
+ def waitForTargetTablets = {
+ sql "use @${targetCluster}"
+ def targetBes = clusterBackends[targetCluster].collectEntries { [(it.id): it] }
+ waitUntil("target primary routing and tablet metadata", 120000L) {
+ def missing = [:]
+ partitionInfo.each { name, info ->
+ def ready = [] as Set
+ def tablets = sql_return_maparray(showTabletsSql(info))
+ tablets.each { tablet ->
+ long id = tablet.TabletId as Long
+ long primary = tablet.PrimaryBackendId as Long
+ if (info.tabletIds.contains(id) && targetBes.containsKey(primary) &&
+ (tablet.BackendId as Long) == primary) {
+ def be = targetBes[primary]
+ try {
+ // Fetch metadata only; a target data query could populate the cache itself.
+ httpTest {
+ endpoint "${be.ip}:${be.httpPort}"
+ uri "/api/meta/header/${id}"
+ op "get"
+ check { code, body ->
+ if ((code as int) == 200) {
+ def meta = parseJson(body.toString())
+ // Cloud header JSON omits rowsets. Source data and cache
+ // baselines are checked separately from target readiness.
+ if ((meta.tablet_id as Long) == id &&
+ (meta.ttl_seconds as Long) == ttlSeconds &&
+ meta.tablet_state == "PB_RUNNING" && meta.schema != null) {
+ ready.add(id)
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ logger.info("Waiting for ${name} tablet ${id} on BE ${primary}: ${e.message}")
+ }
+ }
+ }
+ missing[name] = [tablets: info.tabletIds - ready, loadedSourceVersion: info.version]
+ }
+ logger.info("Target tablet readiness: ${missing}")
+ missing.values().every { it.tablets.isEmpty() }
}
}
- clearFileCache.call(ipList[0], httpPortList[0]);
- clearFileCache.call(ipList[1], httpPortList[1]);
-
- load_customer_once()
- load_supplier_once()
-
- def jobId_ = sql "warm up cluster regression_cluster_name1 with table customer partition p1 and table customer partition p2;"
- def waitJobDone = { jobId ->
- int retryTime = 120
- int i = 0
- for (; i < retryTime; i++) {
- sleep(1000)
- def statuses = getJobState(jobId[0][0])
- if (statuses.any { it != null && it.equals("CANCELLED") }) {
- assertTrue(false);
- }
- if (statuses.any { it != null && it.equals("FINISHED") }) {
- break;
+ def clearCaches = { String phase, List members ->
+ def before = getGlobalTtlBytes()
+ members.each { be ->
+ httpTest {
+ endpoint "${be.ip}:${be.httpPort}"
+ uri "/api/file_cache?op=clear&sync=true"
+ op "get"
+ check { code, body ->
+ assertTrue((code as int) == 200, "Clear cache failed on BE ${be.id}: ${code}, ${body}")
+ def response = parseJson(body.toString())
+ assertTrue(response.status == "OK", "Clear cache failed on BE ${be.id}: ${body}")
+ logger.info("${phase} clear BE ${be.id}: ${body}")
+ }
}
}
- if (i == retryTime) {
- sql "cancel warm up job where id = ${jobId[0][0]}"
- assertTrue(false);
+ def after = getGlobalTtlBytes()
+ // Busy blocks belonging to other tables must not determine whether this case passes.
+ logger.info("${phase} global TTL: before=${before}, after=${after}, " +
+ "source_minus_target_after_clear=${after.source - after.target}, " +
+ "source_change_after_clear=${after.source - before.source}, " +
+ "target_change_after_clear=${after.target - before.target}")
+ }
+ def waitForTargetCold = { String phase ->
+ waitUntil("${phase}: target test-table cache cleanup incomplete", 120000L) {
+ def cache = getPartitionCache(targetCluster)
+ logger.info("${phase} target cache: ${cache}")
+ cache.values().every { it.blocks == 0L }
}
}
- waitJobDone(jobId_)
- jobId_ = sql "warm up cluster regression_cluster_name1 with table customer partition p3 and table supplier;"
- waitJobDone(jobId_);
-
- def tablesString = getTablesFromShowCommand(jobId_[0][0])
- assertTrue(tablesString.any { it != null && it.contains("customer") })
- assertTrue(tablesString.any { it != null && it.contains("supplier") })
+ setBeConfigTemporary([
+ enable_evict_file_cache_in_advance: false,
+ file_cache_enter_disk_resource_limit_mode_percent: 99
+ ]) {
+ try {
+ sql "use @${sourceCluster}"
+ String dbName = sql("select database()")[0][0].toString()
+ assertTrue(!dbName.isEmpty(), "Missing database for warmup Tables assertions")
+ tableDefinitions.each { tableName, ddlName ->
+ sql "DROP TABLE IF EXISTS ${tableName} FORCE"
+ sql (new File("${context.file.parent}/../ddl/${ddlName}.sql").text + """
+ PROPERTIES("file_cache_ttl_seconds"="${ttlSeconds}", "disable_auto_compaction"="true")""")
+ }
+ partitionInfo.each { name, info ->
+ def ids = sql(showTabletsSql(info)).collect { it[0] as Long }.unique()
+ assertTrue(!ids.isEmpty(), "Missing tablets for ${name}")
+ info.tabletIds = ids
+ }
+ clearCaches("initial", clusterBackends.values().flatten())
- sleep(30000)
- long ttl_cache_size = 0
- getMetricsMethod.call(ipList[0], brpcPortList[0]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
+ tableDefinitions.keySet().each { tableName ->
+ String label = "${tableName}_${UUID.randomUUID().toString().replace('-', '')}"
+ String loadSql = new File("${context.file.parent}/../ddl/${tableName}_load.sql").text
+ .replace('${s3BucketName}', getS3BucketName()).replace('${loadLabel}', label)
+ sql (loadSql + """WITH S3 (
+ "AWS_ACCESS_KEY" = "${getS3AK()}", "AWS_SECRET_KEY" = "${getS3SK()}",
+ "AWS_ENDPOINT" = "${getS3Endpoint()}", "AWS_REGION" = "${getS3Region()}",
+ "provider" = "${getS3Provider()}")
+ PROPERTIES("exec_mem_limit"="8589934592", "load_parallelism"="3")""")
+ pendingLoads.add(label)
+ waitUntil("load ${label} to finish", 300000L) {
+ def rows = sql "show load where Label = '${label}'"
+ logger.info("Multi-table warmup load state: ${rows}")
+ if (rows.isEmpty()) {
+ return false
}
- def i = line.indexOf(' ')
- ttl_cache_size = line.substring(i).toLong()
- flag = true
- break
+ String state = rows.last()[2].toString()
+ assertTrue(!state.equalsIgnoreCase("CANCELLED"), "Load ${label} cancelled: ${rows.last()}")
+ if (state.equalsIgnoreCase("FINISHED")) {
+ pendingLoads.remove(label)
+ return true
+ }
+ false
}
- }
- assertTrue(flag)
- }
-
- getMetricsMethod.call(ipList[1], brpcPortList[1]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
+ def partitions = sql_return_maparray("show partitions from ${tableName}")
+ partitionInfo.findAll { name, info -> info.table == tableName }.each { name, info ->
+ def matching = partitions.findAll {
+ info.partition == null || it.PartitionName.toString() == info.partition
}
- def i = line.indexOf(' ')
- assertEquals(ttl_cache_size, line.substring(i).toLong())
- flag = true
- break
+ assertTrue(matching.size() == 1 && (matching[0].VisibleVersion as Long) > 1L,
+ "Missing loaded version for ${name}: ${partitions}")
+ info.version = matching[0].VisibleVersion as Long
}
}
- assertTrue(flag)
- }
+ logger.info("Loaded table/partition tablets: ${partitionInfo}")
+ waitForTargetTablets()
- clearFileCache.call(ipList[1], httpPortList[1]);
- jobId_ = sql "warm up cluster regression_cluster_name1 with table customer and table supplier;"
- waitJobDone(jobId_);
+ def sourceBaseline = [:]
+ long stableSince = System.currentTimeMillis()
+ waitUntil("source tables to have stable nonempty TTL cache", 120000L) {
+ def current = getPartitionCache(sourceCluster)
+ logger.info("Source cache before warmup: ${current}")
+ if (current != sourceBaseline) {
+ sourceBaseline = current
+ stableSince = System.currentTimeMillis()
+ }
+ current.values().every { it.complete && it.nonTtlBlocks == 0L } &&
+ System.currentTimeMillis() - stableSince >= 3000L
+ }
+ waitForTargetCold("before first warmup")
- sleep(30000)
- getMetricsMethod.call(ipList[1], brpcPortList[1]) {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
+ def stages = [
+ [name: "customer_p1_p2",
+ clause: "TABLE customer PARTITION p1 AND TABLE customer PARTITION p2",
+ tables: ["customer.p1", "customer.p2"], warmed: ["customer.p1", "customer.p2"]],
+ [name: "customer_p3_and_supplier",
+ clause: "TABLE customer PARTITION p3 AND TABLE supplier",
+ tables: ["customer.p3", "supplier"], warmed: partitionInfo.keySet().toList()],
+ [name: "whole_tables", clause: "TABLE customer AND TABLE supplier",
+ tables: ["customer", "supplier"], warmed: partitionInfo.keySet().toList()]
+ ]
+ stages.each { stage ->
+ if (stage.name == "whole_tables") {
+ // Keep the source baseline, but require both target tables to become cold again.
+ clearCaches("before whole_tables", clusterBackends[targetCluster])
+ waitForTargetCold("before whole_tables")
+ }
+ sql "use @${targetCluster}"
+ // These explicit table/partition jobs do not depend on hotspot collection.
+ def job = sql "WARM UP CLUSTER ${targetCluster} WITH ${stage.clause}"
+ assertTrue(!job.isEmpty(), "No warmup job returned for ${stage.name}")
+ def jobId = job[0][0]
+ warmupJobs.add(jobId)
+ def expectedTables = stage.tables.collect { "${dbName}.${it}".toString() }.sort()
+ waitUntil("warmup ${stage.name} job ${jobId} to finish", 120000L) {
+ def rows = sql_return_maparray("SHOW WARM UP JOB WHERE ID = ${jobId}")
+ logger.info("${stage.name} warmup job ${jobId}: ${rows}")
+ if (rows.isEmpty()) {
+ return false
+ }
+ assertTrue(rows.size() == 1, "Unexpected warmup rows for ${jobId}: ${rows}")
+ def state = rows[0]
+ assertTrue(state.JobId.toString() == jobId.toString(), "Wrong warmup job: ${state}")
+ assertTrue(state.DstComputeGroup == targetCluster, "Wrong warmup target: ${state}")
+ assertTrue(state.Type == "TABLE", "Wrong warmup type: ${state}")
+ def actualTables = (state.Tables ?: "").toString().split(',').collect { it.trim() }.sort()
+ assertTrue(actualTables == expectedTables,
+ "Wrong Tables for ${stage.name}: expected=${expectedTables}, actual=${actualTables}")
+ assertTrue(state.Status != "CANCELLED", "Warmup ${stage.name} cancelled: ${state}")
+ state.Status == "FINISHED"
+ }
+ waitUntil("after ${stage.name}: per-tablet TTL must match source and untouched tables stay cold",
+ 120000L) {
+ def source = getPartitionCache(sourceCluster)
+ def target = getPartitionCache(targetCluster)
+ logger.info("After ${stage.name} job ${jobId}: source=${source}, target=${target}, " +
+ "baseline=${sourceBaseline}")
+ partitionInfo.keySet().each { name ->
+ if (!stage.warmed.contains(name)) {
+ assertTrue(target[name].blocks == 0L,
+ "Untouched ${name} has cache after ${stage.name}: ${target[name]}")
+ } else {
+ assertTrue(target[name].nonTtlBlocks == 0L,
+ "Expected only TTL cache for ${name}: ${target[name]}")
+ }
+ }
+ partitionInfo.keySet().every { name ->
+ def src = source[name]
+ def dst = target[name]
+ src.complete && src.nonTtlBlocks == 0L &&
+ src.ttlByTablet == sourceBaseline[name].ttlByTablet &&
+ (!stage.warmed.contains(name) ||
+ (dst.complete && dst.ttlBytes == src.ttlBytes && dst.ttlByTablet == src.ttlByTablet))
}
- def i = line.indexOf(' ')
- assertEquals(ttl_cache_size, line.substring(i).toLong())
- flag = true
- break
}
}
- assertTrue(flag)
- }
+ } finally {
+ warmupJobs.each { jobId ->
+ try {
+ def rows = sql_return_maparray("SHOW WARM UP JOB WHERE ID = ${jobId}")
+ if (!rows.isEmpty() && rows[0].Status in ["PENDING", "RUNNING"]) {
+ sql "CANCEL WARM UP JOB WHERE ID = ${jobId}"
+ }
+ } catch (Exception e) {
+ logger.warn("Failed to clean up warmup ${jobId}: ${e.message}")
+ }
+ }
+ try {
+ sql "use @${sourceCluster}"
+ pendingLoads.each { label ->
+ try {
+ sql "CANCEL LOAD WHERE LABEL = '${label}'"
+ } catch (Exception e) {
+ logger.warn("Failed to clean up load ${label}: ${e.message}")
+ }
+ }
+ } finally {
+ try {
+ sql "DROP TABLE IF EXISTS customer FORCE"
+ } finally {
+ sql "DROP TABLE IF EXISTS supplier FORCE"
+ }
+ }
+ }
}
}
diff --git a/regression-test/suites/cloud_p0/cache/test_clear_file_cache_on_load_failure.groovy b/regression-test/suites/cloud_p0/cache/test_clear_file_cache_on_load_failure.groovy
index 88d89eea0bf..a9705f6c03f 100644
--- a/regression-test/suites/cloud_p0/cache/test_clear_file_cache_on_load_failure.groovy
+++ b/regression-test/suites/cloud_p0/cache/test_clear_file_cache_on_load_failure.groovy
@@ -15,71 +15,38 @@
// specific language governing permissions and limitations
// under the License.
-import groovy.json.JsonSlurper
+import org.apache.doris.regression.util.WarmupMetricsUtils
suite("test_clear_file_cache_on_load_failure", "nonConcurrent") {
if (!isCloudMode()) {
return
}
+ String originalAutoAnalyze = sql("SHOW GLOBAL VARIABLES LIKE 'enable_auto_analyze'")[0][1].toString()
+ assertTrue(originalAutoAnalyze.toLowerCase() in ["true", "false", "1", "0"],
+ "Unexpected auto-analyze setting: ${originalAutoAnalyze}")
+ onFinish {
+ sql "SET GLOBAL enable_auto_analyze = ${originalAutoAnalyze}"
+ logger.info("Restored enable_auto_analyze=${originalAutoAnalyze}")
+ }
+
// Clear any existing debug points
GetDebugPoint().clearDebugPointsForAllFEs()
GetDebugPoint().clearDebugPointsForAllBEs()
- // Helper function to clear file cache on all backends
- def clearFileCache = { ip, port ->
- def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true"
- def response = new URL(url).text
- def json = new JsonSlurper().parseText(response)
- if (json.status != "OK") {
- throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}")
- }
- }
-
- def clearFileCacheOnAllBackends = {
- def backends = sql """SHOW BACKENDS"""
- for (be in backends) {
- def ip = be[1]
- def port = be[4]
- clearFileCache(ip, port)
+ def getCacheBackends = {
+ return (sql """SHOW BACKENDS""").collect { be ->
+ [ip: be[1], httpPort: be[4], brpcPort: be[5]]
}
- // Wait for async clear to complete
- sleep(5000)
}
- // Helper function to get brpc metrics
- def getBrpcMetrics = { ip, port, name ->
- def url = "http://${ip}:${port}/brpc_metrics"
- try {
- def metrics = new URL(url).text
- def matcher = metrics =~ ~"${name}\\s+(\\d+)"
- if (matcher.find()) {
- return matcher[0][1] as long
- }
- } catch (Exception e) {
- logger.warn("Failed to get brpc metrics from ${ip}:${port}: ${e.message}")
+ def requestFileCacheClearOnAllBackends = {
+ getCacheBackends().each { be ->
+ WarmupMetricsUtils.clearFileCache(be.ip.toString(), be.httpPort.toString())
}
- return 0L
- }
-
- // Helper function to get index queue cache size
- def getIndexQueueSize = { ip, port ->
- return getBrpcMetrics(ip, port, "file_cache_index_queue_cache_size")
}
- // Helper function to get total index queue cache size across all backends
- def getTotalIndexQueueSize = {
- def backends = sql """SHOW BACKENDS"""
- long totalSize = 0
- for (be in backends) {
- def ip = be[1]
- def brpcPort = be[5]
- def size = getIndexQueueSize(ip, brpcPort)
- totalSize += size
- logger.info("BE ${ip}:${brpcPort} index_queue_size = ${size}")
- }
- return totalSize
- }
+ def cacheSizeMetric = "file_cache_cache_size"
// Create test table with file cache enabled
def tableName = "test_load_failure_cache"
@@ -101,30 +68,37 @@ suite("test_clear_file_cache_on_load_failure", "nonConcurrent") {
// Disable auto analyze to avoid internal loads affecting cache size
sql """SET GLOBAL enable_auto_analyze = false"""
- // Clear file cache and wait for it to complete
- clearFileCacheOnAllBackends()
- sleep(3000)
+ requestFileCacheClearOnAllBackends()
- // Get initial cache size
- def initialCacheSize = getTotalIndexQueueSize()
+ // sync=true synchronously removes releasable blocks, but it does not guarantee that
+ // blocks still held by readers or writers have all been recycled. Use a stable metric
+ // snapshot as the baseline and verify cache-size increments instead of requiring zero.
+ def initialCacheSize = WarmupMetricsUtils.waitForBackendMetricSumStable(
+ getCacheBackends(),
+ cacheSizeMetric,
+ 10000,
+ 60000)
logger.info("Initial file cache size: ${initialCacheSize}")
// First, do a successful load to establish baseline
sql """INSERT INTO ${tableName} VALUES (1, 'test1', 100)"""
- // Wait for cache metrics to update
- sleep(5000)
- def afterSuccessfulLoadSize = getTotalIndexQueueSize()
+ def afterSuccessfulLoadSize = WarmupMetricsUtils.waitForBackendMetricSum(
+ getCacheBackends(),
+ cacheSizeMetric,
+ { value -> value > initialCacheSize },
+ 120000,
+ "Cache should increase after successful load. Initial: ${initialCacheSize}")
logger.info("Cache size after successful load: ${afterSuccessfulLoadSize}")
- assertTrue(afterSuccessfulLoadSize > initialCacheSize,
- "Cache should increase after successful load. Initial: ${initialCacheSize}, After: ${afterSuccessfulLoadSize}")
-
- // Clear cache again to reset
- clearFileCacheOnAllBackends()
- sleep(3000)
- def afterClearSize = getTotalIndexQueueSize()
- logger.info("Cache size after clear: ${afterClearSize}")
+ // The successful load may populate the cache asynchronously. Wait until it is stable,
+ // then use that value as the baseline for the failed-load cache-size increment.
+ afterSuccessfulLoadSize = WarmupMetricsUtils.waitForBackendMetricSumStable(
+ getCacheBackends(),
+ cacheSizeMetric,
+ 10000,
+ 60000)
+ logger.info("Stable cache size before failed load: ${afterSuccessfulLoadSize}")
// Enable debug point to make commit_rowset return error
GetDebugPoint().enableDebugPointForAllBEs("LoadChannel.add_batch.failed")
@@ -137,17 +111,22 @@ suite("test_clear_file_cache_on_load_failure", "nonConcurrent") {
}
// Wait for cleanup to complete and cache metrics to update
- sleep(5000)
+ def afterFailedLoadSize = WarmupMetricsUtils.waitForBackendMetricSumStable(
+ getCacheBackends(),
+ cacheSizeMetric,
+ 10000,
+ 60000)
// Get cache size after failed load
- def afterFailedLoadSize = getTotalIndexQueueSize()
logger.info("Cache size after failed load: ${afterFailedLoadSize}")
- // Verify cache size has not increased
- assertTrue(afterFailedLoadSize == afterClearSize,
+ // Pre-existing deleting blocks may be recycled during this interval, so a negative
+ // delta is valid. A failed load must not add cache bytes over the stable baseline.
+ def failedLoadCacheSizeDelta = afterFailedLoadSize - afterSuccessfulLoadSize
+ assertTrue(failedLoadCacheSizeDelta <= 0,
"Cache should not increase after failed load. " +
- "Before: ${afterClearSize}, After: ${afterFailedLoadSize}, " +
- "Difference: ${afterFailedLoadSize - afterClearSize}")
+ "Before: ${afterSuccessfulLoadSize}, After: ${afterFailedLoadSize}, " +
+ "Difference: ${failedLoadCacheSizeDelta}")
logger.info("Test passed: File cache was properly cleared after load failure")
} finally {
diff --git a/regression-test/suites/cloud_p0/cache/test_file_cache_info.groovy b/regression-test/suites/cloud_p0/cache/test_file_cache_info.groovy
index 7c8841f4544..26c4964bb72 100644
--- a/regression-test/suites/cloud_p0/cache/test_file_cache_info.groovy
+++ b/regression-test/suites/cloud_p0/cache/test_file_cache_info.groovy
@@ -15,120 +15,216 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_file_cache_info") {
- def custoBeConfig = [
- enable_evict_file_cache_in_advance : false,
- file_cache_enter_disk_resource_limit_mode_percent : 99
- ]
-
- setBeConfigTemporary(custoBeConfig) {
+import org.apache.doris.regression.util.Http
- String[][] backends = sql """ show backends """
- def backendSockets = []
- def backendIdToBackendIP = [:]
- def backendIdToBackendHttpPort = [:]
- for (String[] backend in backends) {
- if (backend[9].equals("true")) {
- backendIdToBackendIP.put(backend[0], backend[1])
- backendIdToBackendHttpPort.put(backend[0], backend[4])
+suite("test_file_cache_info", "nonConcurrent") {
+ def customBeConfig = [
+ enable_evict_file_cache_in_advance: false,
+ file_cache_enter_disk_resource_limit_mode_percent: 99
+ ]
+ setBeConfigTemporary(customBeConfig) {
+ String tableName = "test_file_cache_info_lifecycle"
+ def clusters = sql "SHOW CLUSTERS"
+ assertTrue(!clusters.isEmpty(), "No compute group found")
+ String clusterName = clusters[0][0].toString()
+ String dbName = sql("SELECT DATABASE()")[0][0].toString()
+ def backends = sql_return_maparray("SHOW BACKENDS").findAll { be ->
+ "${be.Alive}".equalsIgnoreCase("true") &&
+ parseJson(be.Tag.toString()).compute_group_name == clusterName
+ }.collectEntries { be -> [(be.BackendId as Long): be] }
+ assertTrue(!backends.isEmpty(), "No alive backends in ${clusterName}")
+ def backendIds = backends.keySet()
+ Long tabletId = null
+ def lastState = [:]
+ def clearResponses = [:]
+ def originalQueryCache = sql("select @@enable_sql_cache, @@enable_query_cache")[0].collect { value ->
+ String setting = value.toString().toLowerCase(Locale.ROOT)
+ assertTrue(setting in ["true", "false", "0", "1"], "Unexpected cache setting: ${value}")
+ setting
}
- }
- assertTrue(backendIdToBackendIP.size() > 0, "No alive backends found")
-
- backendIdToBackendIP.each { backendId, ip ->
- def socket = ip + ":" + backendIdToBackendHttpPort.get(backendId)
- backendSockets.add(socket)
- }
-
- sql "drop table IF EXISTS customer"
-
- sql """
- CREATE TABLE IF NOT EXISTS customer (
- `c_custkey` int NULL,
- `c_name` string NULL,
- `c_address` string NULL,
- `c_city` string NULL,
- `c_nation` string NULL,
- `c_region` string NULL,
- `c_phone` string NULL,
- `c_mktsegment` string NULL
- )
- DUPLICATE KEY(`c_custkey`)
- DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1
- PROPERTIES (
- "file_cache_ttl_seconds" = "3600"
- )
- """
-
- sql """
- insert into customer values
- (1, 'Customer#000000001', 'address1', 'city1', 'nation1', 'region1', 'phone1', 'segment1'),
- (2, 'Customer#000000002', 'address2', 'city2', 'nation2', 'region2', 'phone2', 'segment2'),
- (3, 'Customer#000000003', 'address3', 'city3', 'nation3', 'region3', 'phone3', 'segment3'),
- (4, 'Customer#000000004', 'address4', 'city4', 'nation4', 'region4', 'phone4', 'segment4'),
- (5, 'Customer#000000005', 'address5', 'city5', 'nation5', 'region5', 'phone5', 'segment5')
- """
- sql "sync"
-
- sql "select count(*) from customer"
-
- Thread.sleep(10000)
-
- def get_tablet_id = { String tbl_name ->
- def tablets = sql "show tablets from ${tbl_name}"
- assertEquals(tablets.size(), 1, "Should have exactly one tablet with BUCKETS=1")
- return tablets[0][0] as Long
- }
-
- def tablet_id = get_tablet_id("customer")
- println "Tablet ID: ${tablet_id}"
-
- def desc_cache_info = sql "desc information_schema.file_cache_info"
- assertTrue(desc_cache_info.size() > 0, "desc information_schema.file_cache_info should not be empty")
- assertEquals(desc_cache_info[0][0].toString().toUpperCase(), "HASH")
- assertEquals(desc_cache_info[1][0].toString().toUpperCase(), "OFFSET")
-
- def cache_info = sql "select * from information_schema.file_cache_info"
-
- assertTrue(cache_info.size() > 0, "file_cache_info should not be empty for tablet_id ${tablet_id}")
-
- println "First query - File cache info for tablet_id ${tablet_id}:"
- cache_info.each { row ->
- println " ${row}"
- }
-
- def clearResults = []
- backendSockets.each { socket ->
- httpTest {
- endpoint ""
- uri socket + "/api/file_cache?op=clear&sync=true"
- op "get"
- check {respCode, body ->
- assertEquals(respCode, 200, "clear local cache fail, maybe you can find something in respond: " + parseJson(body))
- clearResults.add(true)
+ def expectedRows = (1..5).collect { i ->
+ [i as Long, String.format('Customer#%09d', i), "address${i}".toString(),
+ "city${i}".toString(), "nation${i}".toString(), "region${i}".toString(),
+ "phone${i}".toString(), "segment${i}".toString()]
+ }
+ def readTable = {
+ sql "use @${clusterName}"
+ // Read and validate every column; result caches are disabled for both reads.
+ def rows = sql "select * from ${tableName} order by c_custkey"
+ assertTrue(rows.every { it.size() == 8 && it.every { value -> value != null } },
+ "Unexpected data result: ${rows}")
+ def normalized = rows.collect { row -> [row[0] as Long] + row.drop(1).collect { it.toString() } }
+ assertEquals(expectedRows, normalized)
+ }
+ def getCache = {
+ sql "use @${clusterName}"
+ def rows = sql """select be_id, cache_path, tablet_id, `hash`, `offset`, size, lower(type)
+ from information_schema.file_cache_info
+ where tablet_id = ${tabletId} and be_id in (${backendIds.join(',')})"""
+ def blocks = [:]
+ rows.each { row ->
+ def key = [row[0] as Long, row[1].toString(), row[2] as Long,
+ row[3].toString(), row[4] as Long]
+ assertTrue(backendIds.contains(key[0]) && key[2] == tabletId, "Unexpected cache owner: ${row}")
+ assertTrue(key[4] >= 0L && (row[5] as Long) > 0L, "Invalid cache range: ${row}")
+ assertTrue(!blocks.containsKey(key), "Duplicate cache block: ${key}")
+ blocks[key] = [size: row[5] as Long, type: row[6]?.toString()]
+ }
+ blocks
+ }
+ def summarizeCache = { Map blocks ->
+ blocks.groupBy { key, value -> [key[0], key[2]] }.collectEntries { owner, entries ->
+ [(owner): [block_count: entries.size(), bytes: entries.values().sum(0L) { it.size },
+ bytes_by_type: entries.values().groupBy { it.type }.collectEntries { type, values ->
+ [(type): values.sum(0L) { it.size }]
+ }]]
+ }
+ }
+ def waitForCache = { String phase, boolean expectPresent, long timeoutMs ->
+ long startedMs = System.currentTimeMillis()
+ long deadlineMs = startedMs + timeoutMs
+ long stableSince = 0L
+ long lastLogMs = 0L
+ def previous = null
+ while (System.currentTimeMillis() < deadlineMs) {
+ def blocks = getCache()
+ boolean matches = expectPresent ? !blocks.isEmpty() : blocks.isEmpty()
+ lastState = [phase: phase, elapsed_ms: System.currentTimeMillis() - startedMs,
+ cache_by_be_tablet: summarizeCache(blocks), blocks: blocks]
+ if (blocks != previous || System.currentTimeMillis() - lastLogMs >= 30000L) {
+ logger.info("file_cache_info lifecycle: cluster=${clusterName}, ${lastState}")
+ lastLogMs = System.currentTimeMillis()
+ }
+ if (!matches || blocks != previous) {
+ stableSince = System.currentTimeMillis()
+ }
+ previous = blocks
+ // A transient empty metadata snapshot is not a completed clear.
+ if (matches && System.currentTimeMillis() - stableSince >= 3000L) {
+ logger.info("file_cache_info phase complete: ${lastState}")
+ return blocks
+ }
+ sleep(1000)
+ }
+ assertTrue(false, "Timeout waiting for ${phase}: cluster=${clusterName}, tablet=${tabletId}, " +
+ "BEs=${backendIds}, clear_responses=${clearResponses}, last_state=${lastState}")
+ }
+ def waitForReaders = {
+ long deadlineMs = System.currentTimeMillis() + 60000L
+ long quietSince = System.currentTimeMillis()
+ while (System.currentTimeMillis() < deadlineMs) {
+ def active = sql_return_maparray("SHOW PROCESSLIST").findAll { row ->
+ row.Db?.toString() == dbName && row.Info?.toString()?.contains(tableName) &&
+ !"${row.Command}".equalsIgnoreCase("Sleep")
+ }
+ lastState = [phase: "reader_drain", active_queries: active]
+ if (!active.isEmpty()) {
+ quietSince = System.currentTimeMillis()
+ } else if (System.currentTimeMillis() - quietSince >= 3000L) {
+ return
+ }
+ sleep(1000)
+ }
+ assertTrue(false, "Timeout waiting for table queries to finish: ${lastState}")
+ }
+ def clearCache = { long beId ->
+ def be = backends[beId]
+ String url = "http://${be.Host}:${be.HttpPort}/api/file_cache?op=clear&sync=true"
+ long startedMs = System.currentTimeMillis()
+ clearResponses[beId] = [url: url]
+ logger.info("file_cache_info clear start: BE=${beId}, url=${url}")
+ def connection = Http.openConnection(url)
+ connection.setConnectTimeout(5000)
+ connection.setReadTimeout(60000)
+ try {
+ int status = connection.getResponseCode()
+ String body = (status == 200 ? connection.getInputStream() : connection.getErrorStream())
+ ?.withCloseable { it.getText("UTF-8") } ?: ""
+ clearResponses[beId] += [http_status: status, elapsed_ms: System.currentTimeMillis() - startedMs,
+ response: body]
+ logger.info("file_cache_info clear response: BE=${beId}, ${clearResponses[beId]}")
+ assertEquals(200, status, "Clear failed on BE ${beId}: ${body}")
+ def response = parseJson(body)
+ assertTrue("${response.status}".equalsIgnoreCase("OK"), "Clear failed on BE ${beId}: ${body}")
+ def counters = ["num_files_all", "num_cells_all", "num_cells_to_delete", "num_cells_wait_recycle"]
+ .collectEntries { name ->
+ def matcher = response.msg?.toString() =~ /\b${name}=(\d+)/
+ def values = []
+ while (matcher.find()) { values.add(matcher.group(1) as Long) }
+ [(name): values.isEmpty() ? null : values.sum(0L)]
+ }
+ clearResponses[beId].counters = counters
+ logger.info("file_cache_info clear counters: BE=${beId}, ${counters}")
+ } catch (Throwable t) {
+ clearResponses[beId].error = t.toString()
+ clearResponses[beId].elapsed_ms = System.currentTimeMillis() - startedMs
+ throw t
+ } finally {
+ connection.disconnect()
+ }
+ }
+ Throwable failure = null
+ try {
+ sql "use @${clusterName}"
+ sql "set enable_sql_cache = false"
+ sql "set enable_query_cache = false"
+ sql "drop table if exists ${tableName} force"
+ sql """CREATE TABLE ${tableName} (
+ c_custkey INT, c_name STRING, c_address STRING, c_city STRING,
+ c_nation STRING, c_region STRING, c_phone STRING, c_mktsegment STRING
+ ) DUPLICATE KEY(c_custkey)
+ DISTRIBUTED BY HASH(c_custkey) BUCKETS 1
+ PROPERTIES("file_cache_ttl_seconds"="3600", "disable_auto_compaction"="true")"""
+ def tablets = sql_return_maparray("show tablets from ${tableName}")
+ .collect { it.TabletId as Long }.unique()
+ assertEquals(1, tablets.size(), "Expected one tablet: ${tablets}")
+ tabletId = tablets[0]
+ String values = expectedRows.collect { row ->
+ "(${row[0]}, " + row.drop(1).collect { "'${it}'" }.join(", ") + ")"
+ }.join(", ")
+ sql "insert into ${tableName} values ${values}"
+ readTable()
+ def baseline = waitForCache("populated", true, 120000L)
+ waitForReaders()
+ // SQL quiescence does not prove that every BE holder has been released. The scoped
+ // empty-state check below remains the barrier for clear and async Meta Store deletion.
+ def cacheOwners = baseline.keySet().collect { it[0] }.unique().sort()
+ logger.info("file_cache_info before clear: cluster=${clusterName}, tablet=${tabletId}, " +
+ "cache_owners=${cacheOwners}, cache_by_be_tablet=${summarizeCache(baseline)}, blocks=${baseline}")
+ long clearStartedMs = System.currentTimeMillis()
+ cacheOwners.each { clearCache(it as Long) }
+ waitForCache("cleared", false, 600000L)
+ logger.info("file_cache_info clear verified: tablet=${tabletId}, " +
+ "clear_to_empty_ms=${System.currentTimeMillis() - clearStartedMs}, responses=${clearResponses}")
+
+ // Only now read the business table again. No repeated clear or forced release can
+ // bypass a stuck holder; persistent remnants must fail with the block details above.
+ long reloadStartedMs = System.currentTimeMillis()
+ readTable()
+ def reloaded = waitForCache("reloaded", true, 120000L)
+ // Readers may split a file into different blocks than the INSERT writer did.
+ logger.info("file_cache_info reload verified: tablet=${tabletId}, " +
+ "read_to_visible_ms=${System.currentTimeMillis() - reloadStartedMs}, " +
+ "cache_by_be_tablet=${summarizeCache(reloaded)}, blocks=${reloaded}")
+ } catch (Throwable t) {
+ failure = t
+ logger.warn("file_cache_info lifecycle failed: cluster=${clusterName}, tablet=${tabletId}, " +
+ "clear_responses=${clearResponses}, last_state=${lastState}, error=${t}")
+ throw t
+ } finally {
+ def cleanupErrors = []
+ try { sql "drop table if exists ${tableName} force" } catch (Throwable t) { cleanupErrors.add(t) }
+ [enable_sql_cache: originalQueryCache[0], enable_query_cache: originalQueryCache[1]].each { name, value ->
+ try { sql "set ${name} = ${value}" } catch (Throwable t) { cleanupErrors.add(t) }
+ }
+ if (!cleanupErrors.isEmpty()) {
+ if (failure != null) {
+ cleanupErrors.each { failure.addSuppressed(it) }
+ } else {
+ cleanupErrors.tail().each { cleanupErrors[0].addSuppressed(it) }
+ throw cleanupErrors[0]
+ }
}
}
- }
- assertEquals(clearResults.size(), backendSockets.size(), "Failed to clear cache on some backends")
-
- Thread.sleep(5000)
-
- def cache_info_after_clear = sql "select * from information_schema.file_cache_info where tablet_id = ${tablet_id}"
- assertEquals(cache_info_after_clear.size(), 0, "file_cache_info should be empty after clearing cache")
-
- println "After clearing cache - File cache info is empty as expected"
-
- sql "select * from customer"
-
- Thread.sleep(10000)
-
- def cache_info_reloaded = sql "select * from information_schema.file_cache_info where tablet_id = ${tablet_id}"
- assertTrue(cache_info_reloaded.size() > 0, "file_cache_info should not be empty after reloading data")
-
- println "After reloading data - File cache info for tablet_id ${tablet_id}:"
- cache_info_reloaded.each { row ->
- println " ${row}"
- }
-
}
}
diff --git a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_2.groovy b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_2.groovy
index cb804ae3b62..e652f2de6c1 100644
--- a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_2.groovy
+++ b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_2.groovy
@@ -15,9 +15,18 @@
// specific language governing permissions and limitations
// under the License.
-import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.WarmupMetricsUtils
-suite("alter_ttl_2") {
+suite("alter_ttl_2", "nonConcurrent") {
+ def originalQueryCache = sql("select @@enable_sql_cache, @@enable_query_cache")[0]
+ .collect { it.toString().toLowerCase(Locale.ROOT) }
+ assertTrue(originalQueryCache.every { it in ["true", "false", "1", "0"] })
+ onFinish {
+ sql "set enable_sql_cache = ${originalQueryCache[0]}"
+ sql "set enable_query_cache = ${originalQueryCache[1]}"
+ }
+ sql "set enable_sql_cache = false"
+ sql "set enable_query_cache = false"
def custoBeConfig = [
enable_evict_file_cache_in_advance : false,
file_cache_enter_disk_resource_limit_mode_percent : 99,
@@ -31,7 +40,9 @@ suite("alter_ttl_2") {
assertTrue(!clusters.isEmpty())
def validCluster = clusters[0][0]
sql """use @${validCluster};""";
- def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="300") """
+ long initialTtlSeconds = 3600L
+ long expiredTtlSeconds = 1L
+ def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="${initialTtlSeconds}") """
String[][] backends = sql """ show backends """
String backendId;
def backendIdToBackendIP = [:]
@@ -47,26 +58,11 @@ suite("alter_ttl_2") {
assertEquals(backendIdToBackendIP.size(), 1)
backendId = backendIdToBackendIP.keySet()[0]
- def url = backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendHttpPort.get(backendId) + """/api/file_cache?op=clear&sync=true"""
- logger.info(url)
- def clearFileCache = { check_func ->
- httpTest {
- endpoint ""
- uri url
- op "get"
- body ""
- check check_func
- }
- }
-
- def getMetricsMethod = { check_func ->
- httpTest {
- endpoint backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendBrpcPort.get(backendId)
- uri "/brpc_metrics"
- op "get"
- check check_func
- }
- }
+ def cacheBackend = [
+ ip: backendIdToBackendIP.get(backendId),
+ httpPort: backendIdToBackendHttpPort.get(backendId),
+ brpcPort: backendIdToBackendBrpcPort.get(backendId)
+ ]
def getTabletIds = { String tableName ->
def tablets = sql "show tablets from ${tableName}"
@@ -74,31 +70,57 @@ suite("alter_ttl_2") {
tablets.collect { it[0] as Long }
}
- def waitForFileCacheType = { List<Long> tabletIds, String expectedType, long timeoutMs = 180000L, long intervalMs = 2000L ->
- long start = System.currentTimeMillis()
- while (System.currentTimeMillis() - start < timeoutMs) {
- boolean allMatch = true
- for (Long tabletId in tabletIds) {
- def rows = sql "select type from information_schema.file_cache_info where tablet_id = ${tabletId}"
- if (rows.isEmpty()) {
- logger.warn("file_cache_info is empty for tablet ${tabletId} while waiting for ${expectedType}")
- allMatch = false
- break
- }
- def mismatch = rows.find { row -> !row[0]?.toString()?.equalsIgnoreCase(expectedType) }
- if (mismatch) {
- logger.info("tablet ${tabletId} has cache types ${rows.collect { it[0] }} while waiting for ${expectedType}")
- allMatch = false
- break
- }
- }
- if (allMatch) {
- logger.info("All file cache entries for tablets ${tabletIds} are ${expectedType}")
- return
+ def getCache = { List<Long> tabletIds ->
+ sql "use @${validCluster}"
+ def rows = sql """select be_id, cache_path, tablet_id, `hash`, `offset`, size, lower(type)
+ from information_schema.file_cache_info
+ where tablet_id in (${tabletIds.join(',')}) and be_id = ${backendId}"""
+ def blocks = [:]
+ rows.each { row ->
+ def key = [row[0] as Long, row[1].toString(), row[2] as Long,
+ row[3].toString(), row[4] as Long]
+ assertTrue(!blocks.containsKey(key), "Duplicate cache block: ${key}")
+ assertTrue((row[5] as Long) > 0L, "Invalid block size: ${row}")
+ blocks[key] = [size: row[5] as Long, type: row[6].toString()]
+ }
+ blocks
+ }
+ def sizes = { Map blocks -> blocks.collectEntries { key, value -> [(key): value.size] } }
+ def waitForFileCacheType = { List<Long> tabletIds, String expectedType, Map expectedBlocks = null ->
+ long deadline = System.currentTimeMillis() + 600000L
+ long stableSince = 0L
+ def previous = null
+ def lastState = [:]
+ while (System.currentTimeMillis() < deadline) {
+ def blocks = getCache(tabletIds)
+ def missing = tabletIds.findAll { id -> !blocks.keySet().any { it[2] == id } }
+ boolean ready = missing.isEmpty() && blocks.values().every { it.type == expectedType } &&
+ (expectedBlocks == null || sizes(blocks) == sizes(expectedBlocks))
+ lastState = [expected_type: expectedType, missing_tablets: missing,
+ by_type: blocks.values().groupBy { it.type }.collectEntries { type, values ->
+ [(type): [blocks: values.size(), bytes: values.sum(0L) { it.size }]]
+ }, missing_blocks: expectedBlocks == null ? [] : (expectedBlocks.keySet() - blocks.keySet()).take(10)]
+ if (blocks != previous) { logger.info("Scoped TTL transition: ${lastState}") }
+ if (!ready || blocks != previous) { stableSince = System.currentTimeMillis() }
+ previous = blocks
+ if (ready && System.currentTimeMillis() - stableSince >= 3000L) {
+ logger.info("Verified scoped cache: tablets=${tabletIds}, state=${lastState}")
+ return blocks
}
- sleep(intervalMs)
+ sleep(1000)
+ }
+ assertTrue(false, "Timeout waiting for scoped cache conversion on ${validCluster}/BE ${backendId}: ${lastState}")
+ }
+ def getGlobalMetrics = {
+ ["ttl_cache_size", "normal_queue_cache_size"].collectEntries { name ->
+ [(name): WarmupMetricsUtils.getBrpcMetric(cacheBackend.ip.toString(), cacheBackend.brpcPort.toString(), name)]
}
- assertTrue(false, "Timeout waiting for file_cache_info type ${expectedType} for tablets ${tabletIds}")
+ }
+ def scanTable = {
+ // Force a BE column scan and check fixture data before and after the cache transition.
+ def result = sql "select count(*), sum(C_CUSTKEY) from customer_ttl"
+ assertEquals(200L, result[0][0] as Long)
+ assertEquals(2020100L, result[0][1] as Long)
}
sql new File("""${context.file.parent}/../ddl/customer_ttl_delete.sql""").text
@@ -130,101 +152,30 @@ suite("alter_ttl_2") {
}
}
- clearFileCache.call() {
- respCode, body -> {}
- }
- sleep(30000)
- getMetricsMethod.call() {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag1 = false;
- for (String line in strs) {
- if (flag1) break;
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- logger.info("ttl_cache_size line before assert zero: " + line)
- assertEquals(line.substring(i).toLong(), 0)
- flag1 = true
- }
- }
- assertTrue(flag1)
- }
+ try {
+ def metricsBeforeClear = getGlobalMetrics()
+ WarmupMetricsUtils.clearFileCache(cacheBackend.ip.toString(), cacheBackend.httpPort.toString())
+ def metricsAfterClear = getGlobalMetrics()
+ // sync=true may leave held blocks from other tables. Global counters are diagnostic only.
+ logger.info("TTL clear metrics: before=${metricsBeforeClear}, after=${metricsAfterClear}")
- load_customer_ttl_once("customer_ttl")
- def tabletIds = getTabletIds.call("customer_ttl")
- waitForFileCacheType.call(tabletIds, "ttl", 60000L)
- sql """ select count(*) from customer_ttl """
- sleep(30000)
- long ttl_cache_size = 0
- getMetricsMethod.call() {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag1 = false;
- for (String line in strs) {
- if (flag1) break;
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- logger.info("ttl_cache_size line after load: " + line)
- ttl_cache_size = line.substring(i).toLong()
- flag1 = true
- }
- }
- assertTrue(flag1)
- }
- sql """ ALTER TABLE customer_ttl SET ("file_cache_ttl_seconds"="120") """
- sleep(80000)
- getMetricsMethod.call() {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag1 = false;
- for (String line in strs) {
- if (flag1) break;
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- logger.info("ttl_cache_size line after ttl update: " + line)
- assertEquals(line.substring(i).toLong(), ttl_cache_size)
- flag1 = true
- }
- }
- assertTrue(flag1)
- }
- // the first load data ttl is 300,so need wait for 200s until the ttl timeout
- sleep(200000)
- getMetricsMethod.call() {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag1 = false;
- for (String line in strs) {
- if (flag1) break;
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- assertEquals(line.substring(i).toLong(), 0)
- flag1 = true
- }
- }
- assertTrue(flag1)
- }
+ load_customer_ttl_once("customer_ttl")
+ def tabletIds = getTabletIds.call("customer_ttl")
+ scanTable()
+ def baseline = waitForFileCacheType(tabletIds, "ttl")
+ def metricsBeforeAlter = getGlobalMetrics()
+ logger.info("TTL before shortening: tablets=${tabletIds}, blocks=${baseline}, metrics=${metricsBeforeAlter}")
- waitForFileCacheType.call(tabletIds, "normal", 300000L)
+ // The stable TTL baseline above is already more than one second after tablet creation.
+ // Retain the positive -> positive transition; the original blocks must change type in place.
+ sql """ALTER TABLE customer_ttl SET ("file_cache_ttl_seconds"="${expiredTtlSeconds}")"""
+ waitForFileCacheType(tabletIds, "normal", baseline)
+ def metricsAfterAlter = getGlobalMetrics()
+ def delta = metricsAfterAlter.collectEntries { name, value -> [(name): value - metricsBeforeAlter[name]] }
+ logger.info("TTL after shortening: before=${metricsBeforeAlter}, after=${metricsAfterAlter}, delta=${delta}")
+ scanTable()
+ } finally {
+ sql "DROP TABLE IF EXISTS customer_ttl FORCE"
+ }
}
}
diff --git a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy
index 691994c36c5..c78a140ba47 100644
--- a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy
+++ b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy
@@ -15,205 +15,324 @@
// specific language governing permissions and limitations
// under the License.
-import org.codehaus.groovy.runtime.IOGroovyMethods
-
-suite("alter_ttl_4") {
- def custoBeConfig = [
- enable_evict_file_cache_in_advance : false,
- file_cache_enter_disk_resource_limit_mode_percent : 99
+suite("alter_ttl_4", "nonConcurrent") {
+ def customBeConfig = [
+ enable_evict_file_cache_in_advance: false,
+ file_cache_enter_disk_resource_limit_mode_percent: 99,
+ file_cache_background_ttl_gc_interval_ms: 1000,
+ file_cache_background_ttl_info_update_interval_ms: 1000,
+ file_cache_background_tablet_id_flush_interval_ms: 1000
]
-
- setBeConfigTemporary(custoBeConfig) {
- def clusters = sql " SHOW CLUSTERS; "
- assertTrue(!clusters.isEmpty())
- def validCluster = clusters[0][0]
- sql """use @${validCluster};""";
- def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="900") """
- String[][] backends = sql """ show backends """
- String backendId;
- def backendIdToBackendIP = [:]
- def backendIdToBackendHttpPort = [:]
- def backendIdToBackendBrpcPort = [:]
- for (String[] backend in backends) {
- if (backend[9].equals("true") && backend[19].contains("${validCluster}")) {
- backendIdToBackendIP.put(backend[0], backend[1])
- backendIdToBackendHttpPort.put(backend[0], backend[4])
- backendIdToBackendBrpcPort.put(backend[0], backend[5])
+ setBeConfigTemporary(customBeConfig) {
+ String targetTable = "alter_ttl_4_target"
+ String pressureTable = "alter_ttl_4_pressure"
+ long initialTtl = 3600L
+ long rowsPerLoad = 15000000L // TPCH SF100 customer.tbl
+ long maxPressureRows = 64L * rowsPerLoad
+ def clusters = sql "SHOW CLUSTERS"
+ assertTrue(!clusters.isEmpty())
+ String clusterName = clusters[0][0].toString()
+ def backends = sql_return_maparray("SHOW BACKENDS").findAll { be ->
+ "${be.Alive}".equalsIgnoreCase("true") &&
+ parseJson(be.Tag.toString()).compute_group_name == clusterName
+ }.collectEntries { be -> [(be.BackendId as Long): be] }
+ assertTrue(!backends.isEmpty(), "No alive backends in ${clusterName}")
+ def backendIds = backends.keySet()
+ def tabletIds = []
+ def pendingLoads = [] as Set
+ def lastState = [:]
+ def cacheDifference = [:]
+ def originalQueryCache = sql("select @@enable_sql_cache, @@enable_query_cache")[0].collect { value ->
+ String setting = value.toString().toLowerCase(Locale.ROOT)
+ assertTrue(setting in ["true", "false", "0", "1"], "Unexpected cache setting: ${value}")
+ setting
}
- }
- assertEquals(backendIdToBackendIP.size(), 1)
-
- backendId = backendIdToBackendIP.keySet()[0]
- def url = backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendHttpPort.get(backendId) + """/api/file_cache?op=clear&sync=true"""
- logger.info(url)
- def clearFileCache = { check_func ->
- httpTest {
- endpoint ""
- uri url
- op "get"
- body ""
- check check_func
+ def waitUntil = { String description, long timeoutMs, Closure ready ->
+ long deadline = System.currentTimeMillis() + timeoutMs
+ while (System.currentTimeMillis() < deadline) {
+ if (ready()) {
+ return
+ }
+ sleep(1000)
+ }
+ assertTrue(false, "Timeout after ${timeoutMs}ms: ${description}; last_state=${lastState}")
}
- }
-
- def getMetricsMethod = { check_func ->
- httpTest {
- endpoint backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendBrpcPort.get(backendId)
- uri "/brpc_metrics"
- op "get"
- check check_func
+ def scanTarget = {
+ sql "use @${clusterName}"
+ // Read every data column, including strings; neither COUNT metadata nor LIMIT can satisfy this scan.
+ def rows = sql """select count(*), sum(C_CUSTKEY), sum(length(C_NAME)), sum(length(C_ADDRESS)),
+ sum(C_NATIONKEY), sum(length(C_PHONE)), sum(C_ACCTBAL),
+ sum(length(C_MKTSEGMENT)), sum(length(C_COMMENT)) from ${targetTable}"""
+ assertTrue(rows.size() == 1 && rows[0].size() == 9 && rows[0].every { it != null },
+ "Unexpected data result: ${rows}")
+ assertEquals(rowsPerLoad, rows[0][0] as Long)
+ rows[0].collect { new BigDecimal(it.toString()).stripTrailingZeros() }
}
- }
-
- def s3BucketName = getS3BucketName()
- def s3WithProperties = """WITH S3 (
- |"AWS_ACCESS_KEY" = "${getS3AK()}",
- |"AWS_SECRET_KEY" = "${getS3SK()}",
- |"AWS_ENDPOINT" = "${getS3Endpoint()}",
- |"AWS_REGION" = "${getS3Region()}",
- |"provider" = "${getS3Provider()}")
- |PROPERTIES(
- |"exec_mem_limit" = "8589934592",
- |"load_parallelism" = "3")""".stripMargin()
-
-
- sql new File("""${context.file.parent}/../ddl/customer_ttl_delete.sql""").text
- sql new File("""${context.file.parent}/../ddl/customer_delete.sql""").text
- def load_customer_ttl_once = { String table ->
- def uniqueID = Math.abs(UUID.randomUUID().hashCode()).toString()
- // def table = "customer"
- // create table if not exists
- sql (new File("""${context.file.parent}/../ddl/${table}.sql""").text + ttlProperties)
- sql """ alter table ${table} set ("disable_auto_compaction" = "true") """ // no influence from compaction
- def loadLabel = table + "_" + uniqueID
- // load data from cos
- def loadSql = new File("""${context.file.parent}/../ddl/${table}_load.sql""").text.replaceAll("\\\$\\{s3BucketName\\}", s3BucketName)
- loadSql = loadSql.replaceAll("\\\$\\{loadLabel\\}", loadLabel) + s3WithProperties
- sql loadSql
-
- // check load state
- while (true) {
- def stateResult = sql "show load where Label = '${loadLabel}'"
- def loadState = stateResult[stateResult.size() - 1][2].toString()
- if ("CANCELLED".equalsIgnoreCase(loadState)) {
- throw new IllegalStateException("load ${loadLabel} failed.")
- } else if ("FINISHED".equalsIgnoreCase(loadState)) {
- break
- }
- sleep(5000)
+ def getCache = {
+ sql "use @${clusterName}"
+ def rows = sql """select be_id, cache_path, tablet_id, `hash`, `offset`, size, lower(type)
+ from information_schema.file_cache_info
+ where tablet_id in (${tabletIds.join(',')}) and be_id in (${backendIds.join(',')})"""
+ def blocks = [:]
+ rows.each { row ->
+ def key = [row[0] as Long, row[1].toString(), row[2] as Long,
+ row[3].toString(), row[4] as Long]
+ assertTrue(backendIds.contains(key[0]) && tabletIds.contains(key[2]), "Unexpected cache owner: ${row}")
+ assertTrue(key[4] >= 0L && (row[5] as Long) > 0L, "Invalid cache range: ${row}")
+ assertTrue(!blocks.containsKey(key), "Duplicate cache block: ${key}")
+ blocks[key] = [size: row[5] as Long, type: row[6]?.toString()]
+ }
+ blocks
}
- }
-
- def load_customer_once = { String table ->
- def uniqueID = Math.abs(UUID.randomUUID().hashCode()).toString()
- // def table = "customer"
- // create table if not exists
- sql new File("""${context.file.parent}/../ddl/${table}.sql""").text
- def loadLabel = table + "_" + uniqueID
- // load data from cos
- def loadSql = new File("""${context.file.parent}/../ddl/${table}_load.sql""").text.replaceAll("\\\$\\{s3BucketName\\}", s3BucketName)
- loadSql = loadSql.replaceAll("\\\$\\{loadLabel\\}", loadLabel) + s3WithProperties
- sql loadSql
-
- // check load state
- while (true) {
- def stateResult = sql "show load where Label = '${loadLabel}'"
- def loadState = stateResult[stateResult.size() - 1][2].toString()
- if ("CANCELLED".equalsIgnoreCase(loadState)) {
- throw new IllegalStateException("load ${loadLabel} failed.")
- } else if ("FINISHED".equalsIgnoreCase(loadState)) {
- break
- }
- sleep(5000)
+ def blockSizes = { Map blocks -> blocks.collectEntries { key, value -> [(key): value.size] } }
+ def sameBlocks = { Map expected, Map actual ->
+ cacheDifference = [missing: (expected.keySet() - actual.keySet()).take(10),
+ unexpected: (actual.keySet() - expected.keySet()).take(10),
+ changed_sizes: actual.findAll { key, value -> value.size != expected[key]?.size }.take(10)]
+ blockSizes(expected) == blockSizes(actual)
}
- }
-
- clearFileCache.call() {
- respCode, body -> {}
- }
- sleep(30000)
+ def cacheSummary = { Map blocks ->
+ blocks.groupBy { key, value -> [key[0], key[2], value.type] }
+ .collectEntries { key, entries -> [(key): entries.values().sum(0L) { it.size }] }
+ }
+ def coverage = { Map blocks ->
+ blocks.groupBy { key, value -> key[0..3] }.collectEntries { file, entries ->
+ def ranges = []
+ entries.collect { key, value -> [key[4], key[4] + value.size] }.sort { a, b -> a[0] <=> b[0] }
+ .each { range ->
+ assertTrue(ranges.isEmpty() || range[0] >= ranges.last()[1],
+ "Overlapping cache ranges: file=${file}, ranges=${ranges}, next=${range}")
+ if (!ranges.isEmpty() && range[0] == ranges.last()[1]) {
+ ranges.last()[1] = range[1]
+ } else {
+ ranges.add(range)
+ }
+ }
+ [(file): ranges]
+ }
+ }
+ def missingCoverage = { Map expected, Map actual ->
+ expected.findAll { file, ranges ->
+ !ranges.every { range ->
+ actual[file]?.any { present -> present[0] <= range[0] && present[1] >= range[1] }
+ }
+ }
+ }
+ def waitForCache = { String stage, String type, Closure matches ->
+ def previous = null
+ def result = [:]
+ long stableSince = 0L
+ waitUntil(stage, 600000L) {
+ result = getCache()
+ def missingTablets = tabletIds.findAll { id -> !result.keySet().any { it[2] == id } }
+ boolean matching = matches(result)
+ boolean complete = missingTablets.isEmpty() &&
+ result.values().every { it.type == type } && matching
+ lastState = [stage: stage, cache_by_be_tablet_type: cacheSummary(result),
+ missing_tablets: missingTablets, differences: cacheDifference]
+ if (result != previous) {
+ logger.info("${lastState}")
+ }
+ if (!complete || result != previous) {
+ stableSince = System.currentTimeMillis()
+ }
+ previous = result
+ complete && System.currentTimeMillis() - stableSince >= 3000L
+ }
+ result
+ }
+ def metricNames = ["file_cache_ttl_cache_size", "file_cache_capacity", "file_cache_evict_by_self_lru_normal"]
+ def getMetrics = {
+ // Per-BE diagnostic snapshots; residual cache from other tables is not a failure.
+ backends.collectEntries { id, be ->
+ def values = metricNames.collectEntries { [(it): null] }
+ try {
+ String text = new URL("http://${be.Host}:${be.BrpcPort}/brpc_metrics")
+ .getText(connectTimeout: 5000, readTimeout: 5000)
+ text.readLines().each { line ->
+ def fields = line.trim().split(/\s+/)
+ if (fields.size() == 2 && !fields[0].startsWith("#")) {
+ metricNames.each { name ->
+ if (fields[0] == name || fields[0].endsWith("_${name}")) {
+ values[name] = (values[name] ?: 0L) + (fields[1] as Long)
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ logger.warn("Cannot read diagnostic metrics for BE ${id}: ${e.message}")
+ }
+ [(id): values]
+ }
+ }
+ def logMetricDelta = { String stage, Map before, Map after ->
+ def delta = backendIds.collectEntries { id ->
+ [(id): metricNames.collectEntries { name ->
+ [(name): before[id][name] != null && after[id][name] != null ?
+ after[id][name] - before[id][name] : null]
+ }]
+ }
+ logger.info("${stage} metrics: before=${before}, after=${after}, delta_by_be=${delta}")
+ }
+ def loadTable = { String table, long timeoutMs ->
+ long deadline = System.currentTimeMillis() + timeoutMs
+ String label = "${table}_${UUID.randomUUID().toString().replace('-', '')}"
+ pendingLoads.add(label)
+ String statement = new File("${context.file.parent}/../ddl/customer_ttl_load.sql").text
+ .replace('${loadLabel}', label).replace('${s3BucketName}', getS3BucketName())
+ .replace('INTO TABLE customer_ttl', "INTO TABLE ${table}")
+ sql(statement + """WITH S3 (
+ "AWS_ACCESS_KEY"="${getS3AK()}", "AWS_SECRET_KEY"="${getS3SK()}",
+ "AWS_ENDPOINT"="${getS3Endpoint()}", "AWS_REGION"="${getS3Region()}",
+ "provider"="${getS3Provider()}")
+ PROPERTIES("exec_mem_limit"="8589934592", "load_parallelism"="3")""")
+ waitUntil("load ${label}", Math.max(0L, deadline - System.currentTimeMillis())) {
+ def rows = sql "show load where Label = '${label}'"
+ lastState = [label: label, state: rows.isEmpty() ? "NOT_VISIBLE" : rows.last()[2]]
+ if (rows.isEmpty()) {
+ return false
+ }
+ String state = rows.last()[2].toString()
+ assertTrue(!state.equalsIgnoreCase("CANCELLED"), "Load cancelled: ${label}, details=${rows.last()}")
+ if (state.equalsIgnoreCase("FINISHED")) {
+ pendingLoads.remove(label)
+ return true
+ }
+ false
+ }
+ }
+ Throwable failure = null
+ try {
+ sql "use @${clusterName}"
+ sql "set enable_sql_cache = false"
+ sql "set enable_query_cache = false"
+ [(targetTable): initialTtl, (pressureTable): 0L].each { table, ttl ->
+ sql "drop table if exists ${table} force"
+ String ddl = new File("${context.file.parent}/../ddl/customer_ttl.sql").text
+ .replace('customer_ttl', table)
+ sql(ddl + """ PROPERTIES("file_cache_ttl_seconds"="${ttl}", "disable_auto_compaction"="true")""")
+ }
+ tabletIds = sql_return_maparray("show tablets from ${targetTable}")
+ .collect { it.TabletId as Long }.unique().sort()
+ assertTrue(tabletIds.size() == 32, "Expected 32 target tablets: ${tabletIds}")
+ loadTable(targetTable, 300000L)
+ def dataBaseline = scanTarget()
- load_customer_ttl_once("customer_ttl")
- sql """ select count(*) from customer_ttl """
- sleep(30000)
- long custom_ttl_size = 0
- getMetricsMethod.call() {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag2 = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
+ def creationTimes = [:]
+ // The full scan above verifies data; cloud header JSON does not include rowsets.
+ waitUntil("target tablet TTL metadata", 600000L) {
+ creationTimes = [:]
+ sql_return_maparray("show tablets from ${targetTable}").each { tablet ->
+ long id = tablet.TabletId as Long
+ long primary = tablet.PrimaryBackendId as Long
+ if (!(id in tabletIds) || !backends.containsKey(primary) || (tablet.BackendId as Long) != primary) {
+ return
+ }
+ def be = backends[primary]
+ try {
+ httpTest {
+ endpoint "${be.Host}:${be.HttpPort}"
+ uri "/api/meta/header/${id}"
+ op "get"
+ check { code, body ->
+ if ((code as int) == 200) {
+ def meta = parseJson(body.toString())
+ if ((meta.tablet_id as Long) == id && (meta.creation_time as Long) > 0L &&
+ (meta.ttl_seconds as Long) == initialTtl &&
+ meta.tablet_state == "PB_RUNNING" && meta.schema != null) {
+ creationTimes[id] = meta.creation_time as Long
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ logger.info("Waiting for metadata: tablet=${id}, BE=${primary}, ${e.message}")
}
- def i = line.indexOf(' ')
- custom_ttl_size = line.substring(i).toLong()
- flag2 = true
}
+ lastState = [missing_tablets: tabletIds - creationTimes.keySet()]
+ creationTimes.size() == tabletIds.size()
}
- assertTrue(flag2)
- }
- sleep(60000)
- // one customer table would take about 1.3GB, the total cache size is 20GB
- // the following would take 20G all
- // evict customer_ttl
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
- load_customer_once("customer")
+ def ttlBaseline = waitForCache("initial TTL cache", "ttl") { true }
+ def baselineCoverage = coverage(ttlBaseline)
+ waitUntil("tablet creation time plus one second to have passed", 60000L) {
+ long now = sql("select unix_timestamp()")[0][0] as Long
+ assertTrue(now < creationTimes.values().min() + initialTtl, "Initial TTL already expired")
+ now > creationTimes.values().max() + 1L
+ }
+ logger.info("TTL baseline: creation_times=${creationTimes}, metrics=${getMetrics()}")
- // some datas in s3 and will download them
- sql """ select C_CUSTKEY from customer_ttl order by C_CUSTKEY limit 1"""
- sql """ select C_NAME from customer_ttl order by C_NAME limit 1"""
- sql """ select C_ADDRESS from customer_ttl order by C_ADDRESS limit 1"""
- sql """ select C_NATIONKEY from customer_ttl order by C_NATIONKEY limit 1"""
- sql """ select C_PHONE from customer_ttl order by C_PHONE limit 1 """
- sql """ select C_ACCTBAL from customer_ttl order by C_ACCTBAL limit 1"""
- sql """ select C_MKTSEGMENT from customer_ttl order by C_MKTSEGMENT limit 1"""
- sql """ select C_COMMENT from customer_ttl order by C_COMMENT limit 1"""
+ sql "alter table ${targetTable} set (\"file_cache_ttl_seconds\"=\"1\")"
+ waitForCache("expired TTL blocks to become NORMAL", "normal") { blocks ->
+ sameBlocks(ttlBaseline, blocks)
+ }
- sql """ ALTER TABLE customer_ttl SET ("file_cache_ttl_seconds"="3600") """
- // wait for fetching new tablet meta in BE
- sleep(60000)
- // wait for updating file cache metrics
- sleep(30000)
- getMetricsMethod.call() {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag1 = false;
- for (String line in strs) {
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
+ long pressureDeadline = System.currentTimeMillis() + 1200000L
+ long pressureRows = 0L
+ def metricsBeforePressure = getMetrics()
+ def evictedFiles = [:]
+ while (evictedFiles.isEmpty() && pressureRows < maxPressureRows &&
+ System.currentTimeMillis() < pressureDeadline) {
+ loadTable(pressureTable, Math.min(300000L, pressureDeadline - System.currentTimeMillis()))
+ pressureRows += rowsPerLoad
+ // Poll cache metadata between loads without touching target data or clearing caches.
+ long checkDeadline = Math.min(pressureDeadline, System.currentTimeMillis() + 5000L)
+ while (evictedFiles.isEmpty() && System.currentTimeMillis() < checkDeadline) {
+ def current = getCache()
+ assertTrue(current.values().every { it.type == "normal" },
+ "Expired target cache changed type during pressure: ${cacheSummary(current)}")
+ evictedFiles = missingCoverage(baselineCoverage, coverage(current))
+ if (evictedFiles.isEmpty()) {
+ sleep(1000)
}
- def i = line.indexOf(' ')
- assertEquals(custom_ttl_size, line.substring(i).toLong())
- flag1 = true
}
+ logger.info("Cache pressure: imported_rows=${pressureRows}, missing_files=${evictedFiles.keySet()}")
}
- assertTrue(flag1)
- }
+ def metricsAfterPressure = getMetrics()
+ logMetricDelta("Pressure", metricsBeforePressure, metricsAfterPressure)
+ assertTrue(!evictedFiles.isEmpty(), "No target cache eviction before pressure limit: " +
+ "imported_rows=${pressureRows}/${maxPressureRows}, metrics=${metricsAfterPressure}")
+
+ assertEquals(dataBaseline, scanTarget())
+ // Re-reading an evicted file may split blocks differently. Compare merged byte ranges.
+ def reloaded = waitForCache("reloaded NORMAL cache coverage", "normal") { blocks ->
+ def current = coverage(blocks)
+ cacheDifference = [missing: missingCoverage(baselineCoverage, current).take(10),
+ extra: missingCoverage(current, baselineCoverage).take(10)]
+ current == baselineCoverage
+ }
+ def metricsBeforeExtension = getMetrics()
+ long now = sql("select unix_timestamp()")[0][0] as Long
+ long extendedTtl = Math.max(initialTtl, now - (creationTimes.values().min() as Long) + 1200L)
+ sql "alter table ${targetTable} set (\"file_cache_ttl_seconds\"=\"${extendedTtl}\")"
+ // Keep positive -> positive: routing through TTL=0 would hide expired-TTL renewal bugs.
+ // No target reads after ALTER until the background conversion has been verified.
+ waitForCache("renewed TTL on existing NORMAL blocks (1 -> ${extendedTtl})", "ttl") { blocks ->
+ sameBlocks(reloaded, blocks)
+ }
+ logMetricDelta("Extended TTL=${extendedTtl}", metricsBeforeExtension, getMetrics())
+ assertEquals(dataBaseline, scanTarget())
+ } catch (Throwable t) {
+ failure = t
+ throw t
+ } finally {
+ def cleanupErrors = []
+ pendingLoads.each { label ->
+ try { sql "cancel load where label = '${label}'" } catch (Throwable t) { cleanupErrors.add(t) }
+ }
+ [targetTable, pressureTable].each { table ->
+ try { sql "drop table if exists ${table} force" } catch (Throwable t) { cleanupErrors.add(t) }
+ }
+ [enable_sql_cache: originalQueryCache[0], enable_query_cache: originalQueryCache[1]].each { name, value ->
+ try { sql "set ${name} = ${value}" } catch (Throwable t) { cleanupErrors.add(t) }
+ }
+ if (!cleanupErrors.isEmpty()) {
+ if (failure != null) {
+ cleanupErrors.each { failure.addSuppressed(it) }
+ } else {
+ cleanupErrors.tail().each { cleanupErrors[0].addSuppressed(it) }
+ throw cleanupErrors[0]
+ }
+ }
+ }
}
}
diff --git a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_random.groovy b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_random.groovy
index 3aa0a3a2c58..c875757807c 100644
--- a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_random.groovy
+++ b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_random.groovy
@@ -91,20 +91,27 @@ suite("test_ttl_random") {
load_customer_once("customer_ttl")
- Random random = new Random()
+ long maxFileCacheTtlSeconds = Long.MAX_VALUE / 2L
+ long randomSeed = 20260902L
+ Random random = new Random(randomSeed)
+ long lastAlterTtlSeconds = 180L
+ logger.info("Use deterministic random seed ${randomSeed} for TTL ALTER values")
for (int j = 0; j < 40; j++) {
- long number = random.nextLong() % 10l;
+ int number = j % 10
if (number < 5) {
sql """ select * from customer_ttl limit 10"""
} else if (number >= 5 && number < 8) {
load_customer_once("customer_ttl")
} else {
- def alterTtlSeconds = random.nextLong()
- if (alterTtlSeconds < 0) {
- alterTtlSeconds *= -1
- }
- sql """ ALTER TABLE customer_ttl SET ("file_cache_ttl_seconds"="${alterTtlSeconds}") """
+ lastAlterTtlSeconds = Math.floorMod(
+ random.nextLong(), maxFileCacheTtlSeconds + 1L)
+ sql """ ALTER TABLE customer_ttl SET ("file_cache_ttl_seconds"="${lastAlterTtlSeconds}") """
sleep(40000)
}
}
+
+ def showCreateTable = sql "show create table customer_ttl"
+ assertTrue(showCreateTable[0][1].toString().contains(
+ "\"file_cache_ttl_seconds\" = \"${lastAlterTtlSeconds}\""),
+ "file_cache_ttl_seconds should converge to the last ALTER value ${lastAlterTtlSeconds}")
}
diff --git a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_seconds.groovy b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_seconds.groovy
deleted file mode 100644
index 491576e5601..00000000000
--- a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_seconds.groovy
+++ /dev/null
@@ -1,170 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import org.codehaus.groovy.runtime.IOGroovyMethods
-
-suite("test_ttl_seconds") {
- def custoBeConfig = [
- enable_evict_file_cache_in_advance : false,
- file_cache_enter_disk_resource_limit_mode_percent : 99,
- file_cache_background_ttl_gc_interval_ms : 1000,
- file_cache_background_ttl_info_update_interval_ms : 1000,
- file_cache_background_tablet_id_flush_interval_ms : 1000
- ]
-
- setBeConfigTemporary(custoBeConfig) {
- sql "set global enable_auto_analyze = false"
- sql "set global enable_audit_plugin = false"
- def clusters = sql " SHOW CLUSTERS; "
- assertTrue(!clusters.isEmpty())
- def validCluster = clusters[0][0]
- sql """use @${validCluster};""";
- def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="5") """
- String[][] backends = sql """ show backends """
- String backendId;
- def backendIdToBackendIP = [:]
- def backendIdToBackendHttpPort = [:]
- def backendIdToBackendBrpcPort = [:]
- for (String[] backend in backends) {
- if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) {
- backendIdToBackendIP.put(backend[0], backend[1])
- backendIdToBackendHttpPort.put(backend[0], backend[4])
- backendIdToBackendBrpcPort.put(backend[0], backend[5])
- }
- }
- assertEquals(backendIdToBackendIP.size(), 1)
-
- backendId = backendIdToBackendIP.keySet()[0]
- def url = backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendHttpPort.get(backendId) + """/api/file_cache?op=clear&sync=true"""
- logger.info(url)
- def clearFileCache = { check_func ->
- httpTest {
- endpoint ""
- uri url
- op "get"
- body ""
- check check_func
- }
- }
-
- sql new File("""${context.file.parent}/../ddl/customer_ttl_delete.sql""").text
- def load_customer_once = { String table ->
- sql (new File("""${context.file.parent}/../ddl/${table}.sql""").text + ttlProperties)
- sql """ alter table ${table} set ("disable_auto_compaction" = "true") """ // no influence from compaction
- def totalRows = 200
- def batchSize = 100
- def commentSuffix = ' ' + ('X' * 50)
- for (int offset = 0; offset < totalRows; offset += batchSize) {
- def sb = new StringBuilder()
- int batchEnd = Math.min(totalRows, offset + batchSize)
- for (int idx = offset; idx < batchEnd; idx++) {
- def customerId = 10001 + idx
- def customerName = String.format('Customer#%09d', customerId)
- sb.append("""INSERT INTO ${table} VALUES (
- ${customerId},
- '${customerName}',
- 'Address Line 1',
- 15,
- '123-456-7890',
- 12345.67,
- 'AUTOMOBILE',
- 'This is a test comment for the customer.${commentSuffix}'
- );
- """)
- }
- sql sb.toString()
- }
- }
-
- def getMetricsMethod = { check_func ->
- httpTest {
- endpoint backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendBrpcPort.get(backendId)
- uri "/brpc_metrics"
- op "get"
- check check_func
- }
- }
-
- def getTabletIds = { String tableName ->
- def tablets = sql "show tablets from ${tableName}"
- assertTrue(tablets.size() > 0, "No tablets found for table ${tableName}")
- tablets.collect { it[0] as Long }
- }
-
- def waitForFileCacheType = { List<Long> tabletIds, String expectedType, long timeoutMs = 60000L, long intervalMs = 1000L ->
- long start = System.currentTimeMillis()
- while (System.currentTimeMillis() - start < timeoutMs) {
- boolean allMatch = true
- for (Long tabletId in tabletIds) {
- def rows = sql "select type from information_schema.file_cache_info where tablet_id = ${tabletId}"
- if (rows.isEmpty()) {
- logger.warn("file_cache_info is empty for tablet ${tabletId} while waiting for ${expectedType}")
- allMatch = false
- break
- }
- def mismatch = rows.find { row -> !row[0]?.toString()?.equalsIgnoreCase(expectedType) }
- if (mismatch) {
- logger.info("tablet ${tabletId} has cache types ${rows.collect { it[0] }} while waiting for ${expectedType}")
- allMatch = false
- break
- }
- }
- if (allMatch) {
- logger.info("All file cache entries for tablets ${tabletIds} are ${expectedType}")
- return
- }
- sleep(intervalMs)
- }
- assertTrue(false, "Timeout waiting for file_cache_info type ${expectedType} for tablets ${tabletIds}")
- }
-
- clearFileCache.call() {
- respCode, body -> {}
- }
- sleep(30000)
-
- load_customer_once("customer_ttl")
- def tabletIds = getTabletIds.call("customer_ttl")
- // No wait for the "ttl" type here. The TTL deadline is the tablet creation time plus
- // file_cache_ttl_seconds, and with a 5s ttl the load itself outlives it, so most of the
- // data is written straight into the normal queue and the table never has all of its
- // blocks in the TTL queue at once.
- sleep(30000) // 30s
- getMetricsMethod.call() {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag1 = false;
- for (String line in strs) {
- if (flag1) break;
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- logger.info("ttl_cache_size after load: " + line)
- assertEquals(line.substring(i).toLong(), 0)
- flag1 = true
- }
- }
- assertTrue(flag1)
- }
-
- waitForFileCacheType.call(tabletIds, "normal", 60000L)
- }
-}
diff --git a/regression-test/suites/cloud_p0/cache/ttl/st04_alter_ttl_n_to_0_runtime.groovy b/regression-test/suites/cloud_p0/cache/ttl/st04_alter_ttl_n_to_0_runtime.groovy
index ffc864e0e53..824cb8d7dd0 100644
--- a/regression-test/suites/cloud_p0/cache/ttl/st04_alter_ttl_n_to_0_runtime.groovy
+++ b/regression-test/suites/cloud_p0/cache/ttl/st04_alter_ttl_n_to_0_runtime.groovy
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-suite("st04_alter_ttl_n_to_0_runtime") {
+suite("st04_alter_ttl_n_to_0_runtime", "nonConcurrent") {
def customBeConfig = [
enable_evict_file_cache_in_advance : false,
file_cache_enter_disk_resource_limit_mode_percent : 99,
@@ -27,119 +27,160 @@ suite("st04_alter_ttl_n_to_0_runtime") {
setBeConfigTemporary(customBeConfig) {
def clusters = sql "SHOW CLUSTERS"
assertTrue(!clusters.isEmpty())
- def validCluster = clusters[0][0]
- sql """use @${validCluster};"""
-
+ String clusterName = clusters[0][0].toString()
+ def backends = sql_return_maparray("SHOW BACKENDS").findAll { be ->
+ "${be.Alive}".equalsIgnoreCase("true") &&
+ parseJson(be.Tag.toString()).compute_group_name == clusterName
+ }.collectEntries { be -> [(be.BackendId as Long): be] }
+ assertTrue(!backends.isEmpty(), "No alive backends in ${clusterName}")
+ def backendIds = backends.keySet()
String tableName = "st04_ttl_n_to_0_tpl"
- def ddl = new File("""${context.file.parent}/../ddl/st04_alter_ttl_n_to_0_runtime.sql""").text
- .replace("\${TABLE_NAME}", tableName)
- sql ddl
-
- String[][] backends = sql """show backends"""
- def backendIdToBackendIP = [:]
- def backendIdToBackendHttpPort = [:]
- def backendIdToBackendBrpcPort = [:]
- for (String[] backend in backends) {
- if (backend[9].equals("true") && backend[19].contains("${validCluster}")) {
- backendIdToBackendIP.put(backend[0], backend[1])
- backendIdToBackendHttpPort.put(backend[0], backend[4])
- backendIdToBackendBrpcPort.put(backend[0], backend[5])
- }
+ String dataQuery = "select count(*), sum(k1), sum(length(v1)) from ${tableName}"
+ def tabletIds = []
+ def originalQueryCache = sql("select @@enable_sql_cache, @@enable_query_cache")[0].collect { value ->
+ String setting = value.toString().toLowerCase(Locale.ROOT)
+ assertTrue(setting in ["true", "false", "0", "1"], "Unexpected cache setting: ${value}")
+ setting
}
- assertEquals(backendIdToBackendIP.size(), 1)
- def backendId = backendIdToBackendIP.keySet()[0]
- def clearUrl = backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendHttpPort.get(backendId) + "/api/file_cache?op=clear&sync=true"
- httpTest {
- endpoint ""
- uri clearUrl
- op "get"
- body ""
- check { respCode, body ->
- assertEquals("${respCode}".toString(), "200")
+ def getCache = {
+ sql "use @${clusterName}"
+ def rows = sql """select be_id, cache_path, tablet_id, `hash`, `offset`, size, lower(type)
+ from information_schema.file_cache_info
+ where tablet_id in (${tabletIds.join(',')}) and be_id in (${backendIds.join(',')})"""
+ def blocks = [:]
+ rows.each { row ->
+ def key = [row[0] as Long, row[1].toString(), row[2] as Long,
+ row[3].toString(), row[4] as Long]
+ assertTrue(backendIds.contains(key[0]) && tabletIds.contains(key[2]),
+ "Cache outside ${clusterName}/${tabletIds}: ${row}")
+ assertTrue(!blocks.containsKey(key), "Duplicate cache block: ${key}")
+ blocks[key] = [size: row[5] as Long, type: row[6]?.toString()]
}
+ blocks
}
-
- def getTabletIds = { String tbl ->
- def tablets = sql """show tablets from ${tbl}"""
- assertTrue(tablets.size() > 0, "No tablets found for table ${tbl}")
- tablets.collect { it[0] as Long }
+ def summarizeCache = { Map blocks ->
+ blocks.groupBy { key, value -> [key[0], key[2]] }.collectEntries { key, entries ->
+ [(key): [blocks: entries.size(),
+ bytesByType: entries.values().groupBy { it.type }.collectEntries { type, values ->
+ [(type): values.sum(0L) { it.size }]
+ }]]
+ }
}
-
- def waitForFileCacheType = { List<Long> tabletIds, String expectedType, long timeoutMs = 600000L, long intervalMs = 2000L ->
- long start = System.currentTimeMillis()
- while (System.currentTimeMillis() - start < timeoutMs) {
- boolean allMatch = true
- for (Long tabletId in tabletIds) {
- def rows = sql """select type from information_schema.file_cache_info where tablet_id = ${tabletId}"""
- if (rows.isEmpty()) {
- allMatch = false
- break
- }
- def mismatch = rows.find { row -> !row[0]?.toString()?.equalsIgnoreCase(expectedType) }
- if (mismatch) {
- allMatch = false
- break
- }
+ def waitForCache = { String description, Closure ready ->
+ // Updating the interval cannot interrupt an already sleeping background thread.
+ long deadline = System.currentTimeMillis() + 600000L
+ def blocks = [:]
+ def previous = null
+ while (System.currentTimeMillis() < deadline) {
+ blocks = getCache()
+ if (blocks != previous) {
+ logger.info("${description}: cluster=${clusterName}, " +
+ "cache_by_be_tablet=${summarizeCache(blocks)}, blocks=${blocks}")
+ previous = blocks
}
- if (allMatch) {
- return
+ if (ready(blocks)) {
+ return blocks
}
- sleep(intervalMs)
+ sleep(2000)
}
- assertTrue(false, "Timeout waiting for ${expectedType}, tablets=${tabletIds}")
+ assertTrue(false, "Timeout after 600000ms waiting for ${description}: " +
+ "cluster=${clusterName}, tablets=${tabletIds}, BEs=${backendIds}, blocks=${blocks}")
}
-
- def waitTtlCacheSizeZero = { long timeoutMs = 120000L, long intervalMs = 2000L ->
- long start = System.currentTimeMillis()
- while (System.currentTimeMillis() - start < timeoutMs) {
- long ttlCacheSize = -1L
- httpTest {
- endpoint backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendBrpcPort.get(backendId)
- uri "/brpc_metrics"
- op "get"
- check { respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def lines = out.split('\n')
- for (String line in lines) {
- if (line.startsWith("#")) {
- continue
- }
- if (line.contains("ttl_cache_size")) {
- def idx = line.indexOf(' ')
- ttlCacheSize = line.substring(idx).trim().toLong()
- break
- }
+ def getGlobalTtlBytes = {
+ // Diagnostic only: other tables and busy blocks may keep the BE's TTL queue nonempty.
+ backends.collectEntries { id, be ->
+ Long bytes = null
+ try {
+ String metrics = new URL("http://${be.Host}:${be.BrpcPort}/brpc_metrics")
+ .getText(connectTimeout: 5000, readTimeout: 5000)
+ def values = metrics.readLines().findResults { line ->
+ def fields = line.trim().split(/\s+/)
+ if (fields.size() == 2 && (fields[0] == "file_cache_ttl_cache_size" ||
+ fields[0].endsWith("_file_cache_ttl_cache_size"))) {
+ return fields[1] as Long
}
+ null
}
+ if (!values.isEmpty()) {
+ bytes = values.sum(0L)
+ }
+ } catch (Exception e) {
+ logger.warn("Cannot read diagnostic TTL metric for BE ${id}: ${e.message}")
}
- if (ttlCacheSize == 0L) {
- return
- }
- sleep(intervalMs)
+ [(id): bytes]
}
- assertTrue(false, "Timeout waiting ttl_cache_size = 0")
}
- def insertBatch = { int start, int end ->
- def values = (start..<end).collect { i -> "(${i}, 'value_${i}')" }.join(",")
- sql """insert into ${tableName} values ${values}"""
- }
- insertBatch(0, 200)
- insertBatch(200, 400)
-
- qt_sql """select count(*) from ${tableName} where v1 like 'value_%'"""
- sleep(5000)
+ try {
+ sql "use @${clusterName}"
+ sql "set enable_sql_cache = false"
+ sql "set enable_query_cache = false"
+ def ddl = new File("${context.file.parent}/../ddl/st04_alter_ttl_n_to_0_runtime.sql").text
+ .replace('${TABLE_NAME}', tableName)
+ sql ddl
+ tabletIds = sql_return_maparray("show tablets from ${tableName}")
+ .collect { it.TabletId as Long }.unique().sort()
+ assertTrue(tabletIds.size() == 8, "Expected 8 tablets: ${tabletIds}")
- def tabletIds = getTabletIds.call(tableName)
- waitForFileCacheType.call(tabletIds, "ttl")
+ // Fresh tablet IDs isolate this case without clearing other tables' caches.
+ [0, 200].each { start ->
+ def values = (start..<(start + 200)).collect { i -> "(${i}, 'value_${i}')" }.join(",")
+ sql "insert into ${tableName} values ${values}"
+ }
+ qt_before_alter dataQuery
- // ST-04 未覆盖点模板:运行期 ALTER N->0 后,缓存类型应从 ttl 转为 normal
- sql """alter table ${tableName} set ("file_cache_ttl_seconds"="0")"""
- waitForFileCacheType.call(tabletIds, "normal")
- waitTtlCacheSizeZero.call()
+ def candidate = null
+ long stableSince = 0L
+ def baseline = waitForCache("stable TTL baseline") { blocks ->
+ boolean allTtl = tabletIds.every { id -> blocks.keySet().any { it[2] == id } } &&
+ blocks.values().every { it.size > 0L && it.type == "ttl" }
+ if (!allTtl || blocks != candidate) {
+ candidate = allTtl ? blocks : null
+ stableSince = System.currentTimeMillis()
+ return false
+ }
+ System.currentTimeMillis() - stableSince >= 3000L
+ }
+ def globalBefore = getGlobalTtlBytes()
+ logger.info("TTL before ALTER: global_by_be=${globalBefore}, " +
+ "case_by_be_tablet=${summarizeCache(baseline)}")
+ sql "alter table ${tableName} set (\"file_cache_ttl_seconds\"=\"0\")"
- sql """drop table if exists ${tableName}"""
+ // Do not read table data here: verify the background conversion of existing blocks.
+ def lastDifference = null
+ def converted = waitForCache("existing TTL blocks to become NORMAL") { blocks ->
+ def difference = [missing: baseline.keySet() - blocks.keySet(),
+ unexpected: blocks.keySet() - baseline.keySet(),
+ changed: blocks.findAll { key, value ->
+ value.type != "normal" || value.size != baseline[key]?.size
+ }]
+ if (difference != lastDifference) {
+ logger.info("TTL conversion differences: ${difference}")
+ lastDifference = difference
+ }
+ difference.values().every { it.isEmpty() }
+ }
+ // Exact block equality also ensures every tablet retains its nonzero baseline bytes.
+ def globalAfter = getGlobalTtlBytes()
+ def globalDelta = backendIds.collectEntries { id ->
+ [(id): globalBefore[id] != null && globalAfter[id] != null ?
+ globalAfter[id] - globalBefore[id] : null]
+ }
+ logger.info("TTL after ALTER: global_by_be=${globalAfter}, global_delta_by_be=${globalDelta}, " +
+ "case_by_be_tablet=${summarizeCache(converted)}")
+ qt_after_alter dataQuery
+ } finally {
+ try {
+ sql "use @${clusterName}"
+ sql "drop table if exists ${tableName} force"
+ } finally {
+ try {
+ sql "set enable_sql_cache = ${originalQueryCache[0]}"
+ } finally {
+ sql "set enable_query_cache = ${originalQueryCache[1]}"
+ }
+ }
+ }
}
}
diff --git a/regression-test/suites/cloud_p0/cache/ttl/st06_warmup_ttl_type_assert.groovy b/regression-test/suites/cloud_p0/cache/ttl/st06_warmup_ttl_type_assert.groovy
index 0d8e55c9da5..fbc90987acc 100644
--- a/regression-test/suites/cloud_p0/cache/ttl/st06_warmup_ttl_type_assert.groovy
+++ b/regression-test/suites/cloud_p0/cache/ttl/st06_warmup_ttl_type_assert.groovy
@@ -15,7 +15,19 @@
// specific language governing permissions and limitations
// under the License.
-suite("st06_warmup_ttl_type_assert") {
+suite("st06_warmup_ttl_type_assert", "nonConcurrent") {
+ // This setting is static in FE; isolate it and restore it even when a cache assertion fails.
+ String originalSyncLoad = sql("select @@enable_multi_cluster_sync_load")[0][0].toString()
+ assertTrue(originalSyncLoad.toLowerCase() in ["true", "false", "1", "0"],
+ "Unexpected sync-load setting: ${originalSyncLoad}")
+ onFinish {
+ sql "set enable_multi_cluster_sync_load = ${originalSyncLoad}"
+ logger.info("Restored enable_multi_cluster_sync_load=${originalSyncLoad}")
+ }
+ // Only explicit warm-up should populate the target cache in this case.
+ sql "set enable_multi_cluster_sync_load = false"
+ logger.info("Explicit warmup: disabled automatic sync load, previous=${originalSyncLoad}")
+
def customBeConfig = [
enable_evict_file_cache_in_advance : false,
file_cache_enter_disk_resource_limit_mode_percent : 99
@@ -32,95 +44,154 @@ suite("st06_warmup_ttl_type_assert") {
def targetCluster = clusters[1][0]
String tableName = "st06_warmup_ttl_tpl"
- sql """use @${sourceCluster};"""
- def ddl = new File("""${context.file.parent}/../ddl/st06_warmup_ttl_type_assert.sql""").text
- .replace("\${TABLE_NAME}", tableName)
- sql ddl
-
- def values = (0..<200).collect { i -> "(${i}, 'warmup_tpl_${i}')" }.join(",")
- sql """insert into ${tableName} values ${values}"""
- qt_source_preheat """select count(*) from ${tableName}"""
-
- def sourceTablets = sql """show tablets from ${tableName}"""
- assertTrue(sourceTablets.size() > 0, "No tablets found for table ${tableName} in source cluster ${sourceCluster}")
- def sourceTabletIds = sourceTablets.collect { it[0] as Long }
-
- // ST-06 部分覆盖点模板:显式断言 warmup 后目标集群缓存类型为 ttl
- def jobIdRows = sql """warm up cluster ${targetCluster} with table ${tableName};"""
- assertTrue(!jobIdRows.isEmpty())
- def jobId = jobIdRows[0][0]
-
- def waitWarmUpJobFinished = { Object id, long timeoutMs = 600000L, long intervalMs = 5000L ->
- long start = System.currentTimeMillis()
- while (System.currentTimeMillis() - start < timeoutMs) {
- def stateRows = sql """SHOW WARM UP JOB WHERE ID = ${id}"""
- if (stateRows.isEmpty()) {
- sleep(intervalMs)
- continue
+ // Read both data columns on BE; count(*) may be evaluated entirely in FE.
+ sql "set enable_sql_cache = false"
+ sql "set enable_query_cache = false"
+ String dataQuery = "select sum(k1), sum(length(v1)) from ${tableName}"
+
+ def getClusterBackends = { String clusterName ->
+ sql_return_maparray("SHOW BACKENDS").findAll { be ->
+ "${be.Alive}".equalsIgnoreCase("true") &&
+ parseJson(be.Tag.toString()).compute_group_name == clusterName
+ }.collectEntries { be -> [(be.BackendId as Long): be] }
+ }
+
+ def waitForTargetTablets = { List<Long> tabletIds ->
+ sql "use @${targetCluster}"
+ long deadline = System.currentTimeMillis() + 120000L
+ def missing = tabletIds
+ while (System.currentTimeMillis() < deadline) {
+ def backends = getClusterBackends.call(targetCluster)
+ def tablets = sql_return_maparray("show tablets from ${tableName}")
+ def readyIds = []
+ for (def tablet : tablets) {
+ long tabletId = tablet.TabletId as Long
+ long primaryBeId = tablet.PrimaryBackendId as Long
+ // BackendId alone can be computed on demand before the primary route exists.
+ if (!tabletIds.contains(tabletId) || !backends.containsKey(primaryBeId) ||
+ (tablet.BackendId as Long) != primaryBeId) {
+ continue
+ }
+ def be = backends[primaryBeId]
+ boolean ready = false
+ try {
+ // Fetch only tablet metadata, leaving the target data cache cold for warm-up.
+ httpTest {
+ endpoint "${be.Host}:${be.HttpPort}"
+ uri "/api/meta/header/${tabletId}"
+ op "get"
+ check { respCode, body ->
+ if ((respCode as int) == 200) {
+ def meta = parseJson(body.toString())
+ // Cloud header JSON omits rs_metas. Check tablet identity/state here;
+ // the source scan and post-warm-up cache checks establish data readiness.
+ ready = (meta.tablet_id as Long) == tabletId &&
+ (meta.ttl_seconds as Long) == 3600L &&
+ meta.tablet_state == "PB_RUNNING" && meta.schema != null
+ }
+ }
+ }
+ } catch (Exception e) {
+ logger.info("Waiting for tablet ${tabletId} on ${targetCluster}, " +
+ "BE ${primaryBeId}: ${e.message}")
+ }
+ if (ready) {
+ readyIds.add(tabletId)
+ }
}
- def state = stateRows[0][3].toString()
- if ("FINISHED".equalsIgnoreCase(state)) {
+ missing = tabletIds - readyIds
+ if (missing.isEmpty()) {
+ logger.info("Target tablet metadata is ready: cluster=${targetCluster}, tablets=${tabletIds}")
return
}
- if ("CANCELLED".equalsIgnoreCase(state) || "FAILED".equalsIgnoreCase(state)) {
- assertTrue(false, "Warm up job failed, id=${id}, state=${state}")
- }
- sleep(intervalMs)
+ logger.info("Waiting for target tablet metadata: cluster=${targetCluster}, missing=${missing}")
+ sleep(1000)
}
- assertTrue(false, "Timeout waiting warm up job finished, id=${id}")
+ assertTrue(false, "Timeout waiting for target tablet metadata: cluster=${targetCluster}, missing=${missing}")
}
- waitWarmUpJobFinished.call(jobId)
-
- sql """use @${targetCluster};"""
- qt_target_query """select count(*) from ${tableName}"""
- def targetTablets = sql """show tablets from ${tableName}"""
- assertTrue(targetTablets.size() > 0, "No tablets found for table ${tableName} in target cluster ${targetCluster}")
- def targetTabletIds = targetTablets.collect { it[0] as Long }
- assertTrue(sourceTabletIds.size() == targetTabletIds.size(),
- "Tablet size mismatch between source and target, source=${sourceTabletIds.size()}, target=${targetTabletIds.size()}")
-
- def waitForFileCacheType = { List<Long> sourceIds, List<Long> targetIds, String expectedType, long timeoutMs = 600000L, long intervalMs = 2000L ->
- logger.info("waitForFileCacheType, sourceIds=${sourceIds.toString()}, targetIds=${targetIds.toString()}, expectedType=${expectedType}")
- assertTrue(sourceIds.size() == targetIds.size(),
- "Tablet size mismatch before waiting file cache type, source=${sourceIds.size()}, target=${targetIds.size()}")
- long start = System.currentTimeMillis()
- while (System.currentTimeMillis() - start < timeoutMs) {
- int sourceMatched = 0
- int targetMatched = 0
- for (Long sourceTabletId in sourceIds) {
- def sourceTabletIdStr = sql """select * from information_schema.file_cache_info where tablet_id = ${sourceTabletId}"""
- logger.info("[source tablet] tablet_id=${sourceTabletId}, tablet_cache_info=${sourceTabletIdStr.toString()}")
- def rows = sql """select type from information_schema.file_cache_info where tablet_id = ${sourceTabletId}"""
- if (!rows.isEmpty()) {
- def mismatch = rows.find { row -> !row[0]?.toString()?.equalsIgnoreCase(expectedType) }
- if (!mismatch) {
- sourceMatched++
- }
- }
- }
- for (Long targetTabletId in targetIds) {
- def targetTabletIdStr = sql """select * from information_schema.file_cache_info where tablet_id = ${targetTabletId}"""
- logger.info("[target tablet] tablet_id=${targetTabletId}, tablet_cache_info=${targetTabletIdStr.toString()}")
- def rows = sql """select type from information_schema.file_cache_info where tablet_id = ${targetTabletId}"""
- if (!rows.isEmpty()) {
- def mismatch = rows.find { row -> !row[0]?.toString()?.equalsIgnoreCase(expectedType) }
- if (!mismatch) {
- targetMatched++
- }
- }
+
+ def waitForFileCacheType = { String clusterName, List<Long> tabletIds ->
+ sql "use @${clusterName}"
+ def backendIds = getClusterBackends.call(clusterName).keySet()
+ assertTrue(!backendIds.isEmpty(), "No alive backends in ${clusterName}")
+ long deadline = System.currentTimeMillis() + 600000L
+ def rows = []
+ def missing = tabletIds
+ while (System.currentTimeMillis() < deadline) {
+ rows = sql """select tablet_id, type, be_id from information_schema.file_cache_info
+ where tablet_id in (${tabletIds.join(',')})"""
+ assertTrue(rows.every { backendIds.contains(it[2] as Long) },
+ "Cache query ran outside ${clusterName}: expected BEs=${backendIds}, rows=${rows}")
+ missing = tabletIds.findAll { tabletId ->
+ def tabletRows = rows.findAll { (it[0] as Long) == tabletId }
+ tabletRows.isEmpty() || tabletRows.any { !it[1]?.toString()?.equalsIgnoreCase("ttl") }
}
-
- if (sourceMatched == sourceIds.size() && targetMatched == targetIds.size()) {
+ if (missing.isEmpty()) {
return
}
- sleep(intervalMs)
+ logger.info("Waiting for TTL cache: cluster=${clusterName}, missing=${missing}, rows=${rows}")
+ sleep(2000)
}
- assertTrue(false, "Timeout waiting for ${expectedType}, sourceTablets=${sourceIds}, targetTablets=${targetIds}")
+ assertTrue(false, "Timeout waiting for TTL cache: cluster=${clusterName}, missing=${missing}, rows=${rows}")
}
- waitForFileCacheType.call(sourceTabletIds, targetTabletIds, "ttl")
- // cleanup
- sql """use @${sourceCluster};"""
- sql """drop table if exists ${tableName}"""
+ try {
+ sql """use @${sourceCluster};"""
+ def ddl = new File("""${context.file.parent}/../ddl/st06_warmup_ttl_type_assert.sql""").text
+ .replace("\${TABLE_NAME}", tableName)
+ sql ddl
+
+ def values = (0..<200).collect { i -> "(${i}, 'warmup_tpl_${i}')" }.join(",")
+ sql """insert into ${tableName} values ${values}"""
+ qt_source_preheat dataQuery
+
+ def sourceTablets = sql """show tablets from ${tableName}"""
+ assertTrue(sourceTablets.size() > 0, "No tablets found for table ${tableName} in source cluster ${sourceCluster}")
+ def sourceTabletIds = sourceTablets.collect { it[0] as Long }
+
+ waitForTargetTablets.call(sourceTabletIds)
+
+ // ST-06 部分覆盖点模板:显式断言 warmup 后目标集群缓存类型为 ttl
+ def jobIdRows = sql """warm up cluster ${targetCluster} with table ${tableName};"""
+ assertTrue(!jobIdRows.isEmpty())
+ def jobId = jobIdRows[0][0]
+
+ def waitWarmUpJobFinished = { Object id, long timeoutMs = 600000L, long intervalMs = 5000L ->
+ long start = System.currentTimeMillis()
+ while (System.currentTimeMillis() - start < timeoutMs) {
+ def stateRows = sql """SHOW WARM UP JOB WHERE ID = ${id}"""
+ if (stateRows.isEmpty()) {
+ sleep(intervalMs)
+ continue
+ }
+ def state = stateRows[0][3].toString()
+ if ("FINISHED".equalsIgnoreCase(state)) {
+ return
+ }
+ if ("CANCELLED".equalsIgnoreCase(state) || "FAILED".equalsIgnoreCase(state)) {
+ assertTrue(false, "Warm up job failed, id=${id}, state=${state}")
+ }
+ sleep(intervalMs)
+ }
+ assertTrue(false, "Timeout waiting warm up job finished, id=${id}")
+ }
+ waitWarmUpJobFinished.call(jobId)
+
+ sql """use @${targetCluster};"""
+ def targetTablets = sql """show tablets from ${tableName}"""
+ assertTrue(targetTablets.size() > 0, "No tablets found for table ${tableName} in target cluster ${targetCluster}")
+ def targetTabletIds = targetTablets.collect { it[0] as Long }
+ assertTrue(sourceTabletIds.toSet() == targetTabletIds.toSet(),
+ "Tablet IDs differ between source and target, source=${sourceTabletIds}, target=${targetTabletIds}")
+
+ // Check warm-up before reading target data, so a query cannot populate a missing cache.
+ waitForFileCacheType.call(sourceCluster, sourceTabletIds)
+ waitForFileCacheType.call(targetCluster, targetTabletIds)
+ sql """use @${targetCluster};"""
+ qt_target_query dataQuery
+ } finally {
+ sql """use @${sourceCluster};"""
+ sql """drop table if exists ${tableName}"""
+ }
}
}
diff --git a/regression-test/suites/cloud_p0/cache/ttl/test_show_ttl.groovy b/regression-test/suites/cloud_p0/cache/ttl/test_show_ttl.groovy
new file mode 100644
index 00000000000..91ca261fe5b
--- /dev/null
+++ b/regression-test/suites/cloud_p0/cache/ttl/test_show_ttl.groovy
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_show_ttl", "nonConcurrent") {
+ String tableName = "test_show_ttl_properties"
+ def checkTtlProperty = { long expectedTtl ->
+ def rows = sql "SHOW CREATE TABLE ${tableName}"
+ assertEquals(1, rows.size())
+ assertEquals(tableName, rows[0][0].toString())
+ String createTable = rows[0][1].toString()
+ def matcher = createTable =~ /"file_cache_ttl_seconds"\s*=\s*"(\d+)"/
+ assertTrue(matcher.find(), "Missing file_cache_ttl_seconds in SHOW CREATE TABLE: ${createTable}")
+ long actualTtl = matcher.group(1).toLong()
+ assertFalse(matcher.find(), "Duplicate file_cache_ttl_seconds in SHOW CREATE TABLE: ${createTable}")
+ logger.info("SHOW CREATE TABLE TTL property: table=${tableName}, expected=${expectedTtl}, actual=${actualTtl}")
+ assertEquals(expectedTtl, actualTtl)
+ }
+
+ try {
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ CREATE TABLE ${tableName} (
+ C_CUSTKEY INTEGER NOT NULL,
+ C_NAME VARCHAR(25) NOT NULL,
+ C_ADDRESS VARCHAR(40) NOT NULL,
+ C_NATIONKEY INTEGER NOT NULL,
+ C_PHONE CHAR(15) NOT NULL,
+ C_ACCTBAL DECIMAL(15,2) NOT NULL,
+ C_MKTSEGMENT CHAR(10) NOT NULL,
+ C_COMMENT VARCHAR(117) NOT NULL
+ )
+ DUPLICATE KEY(C_CUSTKEY, C_NAME)
+ DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 32
+ PROPERTIES("file_cache_ttl_seconds"="300")
+ """
+ // Other SHOW CREATE TABLE properties depend on cluster defaults.
+ checkTtlProperty(300L)
+ sql "ALTER TABLE ${tableName} SET (\"file_cache_ttl_seconds\"=\"0\")"
+ checkTtlProperty(0L)
+ } finally {
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ }
+}
diff --git a/regression-test/suites/cloud_p0/cache/ttl/test_show_ttl.sql b/regression-test/suites/cloud_p0/cache/ttl/test_show_ttl.sql
deleted file mode 100644
index ce7c5c1986c..00000000000
--- a/regression-test/suites/cloud_p0/cache/ttl/test_show_ttl.sql
+++ /dev/null
@@ -1,40 +0,0 @@
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
-
-DROP TABLE IF EXISTS customer_ttl;
-
-CREATE TABLE IF NOT EXISTS customer_ttl (
- C_CUSTKEY INTEGER NOT NULL,
- C_NAME VARCHAR(25) NOT NULL,
- C_ADDRESS VARCHAR(40) NOT NULL,
- C_NATIONKEY INTEGER NOT NULL,
- C_PHONE CHAR(15) NOT NULL,
- C_ACCTBAL DECIMAL(15,2) NOT NULL,
- C_MKTSEGMENT CHAR(10) NOT NULL,
- C_COMMENT VARCHAR(117) NOT NULL
-)
-DUPLICATE KEY(C_CUSTKEY, C_NAME)
-DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 32
-PROPERTIES(
- "file_cache_ttl_seconds"="300"
-);
-
-show create table customer_ttl;
-
-alter table customer_ttl set ("file_cache_ttl_seconds"="0");
-
-show create table customer_ttl;
diff --git a/regression-test/suites/cloud_p0/cache/ttl/test_ttl.groovy b/regression-test/suites/cloud_p0/cache/ttl/test_ttl.groovy
index 09ee37aef07..2deb8eac632 100644
--- a/regression-test/suites/cloud_p0/cache/ttl/test_ttl.groovy
+++ b/regression-test/suites/cloud_p0/cache/ttl/test_ttl.groovy
@@ -15,190 +15,295 @@
// specific language governing permissions and limitations
// under the License.
-import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.Http
-suite("test_ttl") {
- def custoBeConfig = [
- enable_evict_file_cache_in_advance : false,
- file_cache_enter_disk_resource_limit_mode_percent : 99,
- file_cache_background_ttl_gc_interval_ms : 1000,
- file_cache_background_ttl_info_update_interval_ms : 1000,
- file_cache_background_tablet_id_flush_interval_ms : 1000
+suite("test_ttl", "nonConcurrent") {
+ def customBeConfig = [
+ enable_evict_file_cache_in_advance: false,
+ file_cache_enter_disk_resource_limit_mode_percent: 99,
+ file_cache_background_ttl_gc_interval_ms: 1000,
+ file_cache_background_ttl_info_update_interval_ms: 1000,
+ file_cache_background_tablet_id_flush_interval_ms: 1000
]
-
- setBeConfigTemporary(custoBeConfig) {
- def clusters = sql " SHOW CLUSTERS; "
- assertTrue(!clusters.isEmpty())
- def validCluster = clusters[0][0]
- sql """use @${validCluster};""";
- def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="180") """
- String[][] backends = sql """ show backends """
- String backendId;
- def backendIdToBackendIP = [:]
- def backendIdToBackendHttpPort = [:]
- def backendIdToBackendBrpcPort = [:]
- for (String[] backend in backends) {
- if (backend[9].equals("true") && backend[19].contains("${validCluster}")) {
- backendIdToBackendIP.put(backend[0], backend[1])
- backendIdToBackendHttpPort.put(backend[0], backend[4])
- backendIdToBackendBrpcPort.put(backend[0], backend[5])
+ setBeConfigTemporary(customBeConfig) {
+ String tableName = "test_ttl_natural_expiration"
+ long ttlSeconds = 300L
+ int bucketCount = 4
+ def clusters = sql "SHOW CLUSTERS"
+ assertTrue(!clusters.isEmpty())
+ String clusterName = clusters[0][0].toString()
+ def backends = sql_return_maparray("SHOW BACKENDS").findAll { be ->
+ "${be.Alive}".equalsIgnoreCase("true") &&
+ parseJson(be.Tag.toString()).compute_group_name == clusterName
+ }.collectEntries { be -> [(be.BackendId as Long): be] }
+ assertTrue(!backends.isEmpty(), "No alive backends in ${clusterName}")
+ def backendIds = backends.keySet()
+ def tabletIds = []
+ def metadata = [:]
+ def lastState = [:]
+ def metricsBaseline = null
+ def originalQueryCache = sql("select @@enable_sql_cache, @@enable_query_cache")[0].collect { value ->
+ String setting = value.toString().toLowerCase(Locale.ROOT)
+ assertTrue(setting in ["true", "false", "0", "1"], "Unexpected cache setting: ${value}")
+ setting
}
- }
- assertEquals(backendIdToBackendIP.size(), 1)
-
- backendId = backendIdToBackendIP.keySet()[0]
- def url = backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendHttpPort.get(backendId) + """/api/file_cache?op=clear&sync=true"""
- logger.info(url)
- def clearFileCache = { check_func ->
- try {
- httpTest {
- endpoint ""
- uri url
- op "get"
- body ""
- check check_func
+ def serverTime = { sql("select unix_timestamp()")[0][0] as Long }
+ def waitUntil = { String stage, long deadlineMs, Closure ready ->
+ while (System.currentTimeMillis() < deadlineMs) {
+ if (ready()) {
+ return
+ }
+ sleep(1000)
}
- } catch (Exception e) {
- logger.error("Failed to clear file cache: ${e.message}")
- throw e
+ assertTrue(false, "Timeout waiting for ${stage}: cluster=${clusterName}, " +
+ "tablets=${tabletIds}, BEs=${backendIds}, last_state=${lastState}")
}
- }
-
- sql new File("""${context.file.parent}/../ddl/customer_ttl_delete.sql""").text
- def load_customer_once = { String table ->
- try {
- sql (new File("""${context.file.parent}/../ddl/${table}.sql""").text + ttlProperties)
- sql """ alter table ${table} set ("disable_auto_compaction" = "true") """ // no influence from compaction
- def totalRows = 200
- def batchSize = 100
- def commentSuffix = ' ' + ('X' * 50)
- for (int offset = 0; offset < totalRows; offset += batchSize) {
- def sb = new StringBuilder()
- int batchEnd = Math.min(totalRows, offset + batchSize)
- for (int idx = offset; idx < batchEnd; idx++) {
- def customerId = 10001 + idx
- def customerName = String.format('Customer#%09d', customerId)
- sb.append("""INSERT INTO ${table} VALUES (
- ${customerId},
- '${customerName}',
- 'Address Line 1',
- 15,
- '123-456-7890',
- 12345.67,
- 'AUTOMOBILE',
- 'This is a test comment for the customer.${commentSuffix}'
- );
- """)
- }
- sql sb.toString()
+ def getCache = {
+ sql "use @${clusterName}"
+ def blocks = [:]
+ def rows = sql """select be_id, cache_path, tablet_id, `hash`, `offset`, size, lower(type)
+ from information_schema.file_cache_info
+ where tablet_id in (${tabletIds.join(',')}) and be_id in (${backendIds.join(',')})"""
+ rows.each { row ->
+ def key = [row[0] as Long, row[1].toString(), row[2] as Long,
+ row[3].toString(), row[4] as Long]
+ assertTrue(backendIds.contains(key[0]) && tabletIds.contains(key[2]), "Unexpected cache owner: ${row}")
+ assertTrue(key[4] >= 0L && (row[5] as Long) > 0L, "Invalid cache range: ${row}")
+ assertTrue(!blocks.containsKey(key), "Duplicate cache block: ${key}")
+ blocks[key] = [size: row[5] as Long, type: row[6]?.toString()]
}
- } catch (Exception e) {
- logger.error("Failed to load customer data: ${e.message}")
- throw e
+ blocks
}
- }
-
- def getMetricsMethod = { check_func ->
- try {
- httpTest {
- endpoint backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendBrpcPort.get(backendId)
- uri "/brpc_metrics"
- op "get"
- check check_func
+ def blockSizes = { Map blocks -> blocks.collectEntries { key, value -> [(key): value.size] } }
+ def summarizeCache = { Map blocks ->
+ blocks.groupBy { key, value -> [key[0], key[2]] }.collectEntries { owner, entries ->
+ [(owner): [block_count: entries.size(), bytes: entries.values().sum(0L) { it.size },
+ by_type: (["ttl", "normal"] + entries.values().collect { it.type }).unique().collectEntries { type ->
+ def values = entries.values().findAll { it.type == type }
+ [(type): [block_count: values.size(), bytes: values.sum(0L) { it.size }]]
+ }]]
}
- } catch (Exception e) {
- logger.error("Failed to get metrics: ${e.message}")
- throw e
}
- }
-
- def getTabletIds = { String tableName ->
- def tablets = sql "show tablets from ${tableName}"
- assertTrue(tablets.size() > 0, "No tablets found for table ${tableName}")
- tablets.collect { it[0] as Long }
- }
-
- def waitForFileCacheType = { List<Long> tabletIds, String expectedType, long timeoutMs = 60000L, long intervalMs = 2000L ->
- long start = System.currentTimeMillis()
- while (System.currentTimeMillis() - start < timeoutMs) {
- boolean allMatch = true
- for (Long tabletId in tabletIds) {
- def rows = sql "select type from information_schema.file_cache_info where tablet_id = ${tabletId}"
- if (rows.isEmpty()) {
- logger.warn("file_cache_info is empty for tablet ${tabletId} while waiting for ${expectedType}")
- allMatch = false
- break
+ def metricNames = ["file_cache_ttl_cache_size", "file_cache_normal_queue_cache_size",
+ "file_cache_ttl_cache_lru_queue_size", "file_cache_ttl_cache_lru_queue_element_count",
+ "file_cache_normal_queue_element_count", "file_cache_ttl_cache_evict_size",
+ "file_cache_normal_queue_evict_size", "file_cache_ttl_mgr_tablet_id_set_size"]
+ def getMetrics = {
+ // These include other tables and update asynchronously. Only scoped cache data is asserted.
+ backends.collectEntries { id, be ->
+ def values = metricNames.collectEntries { [(it): null] }
+ try {
+ String text = new URL("http://${be.Host}:${be.BrpcPort}/brpc_metrics")
+ .getText(connectTimeout: 5000, readTimeout: 5000)
+ text.readLines().each { line ->
+ def fields = line.trim().split(/\s+/)
+ if (fields.size() == 2 && !fields[0].startsWith("#")) {
+ metricNames.each { name ->
+ if (fields[0] == name || fields[0].endsWith("_${name}")) {
+ values[name] = (values[name] ?: 0L) + (fields[1] as Long)
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ logger.warn("Cannot read diagnostic metrics for BE ${id}: ${e.message}")
}
- def mismatches = rows.findAll { row -> !row[0]?.toString()?.equalsIgnoreCase(expectedType) }
- if (!mismatches.isEmpty()) {
- logger.info("tablet ${tabletId} has cache types ${rows.collect { it[0] }} while waiting for ${expectedType}")
- allMatch = false
- break
+ def missing = values.findAll { name, value -> value == null }.keySet()
+ if (!missing.isEmpty()) {
+ logger.warn("Unavailable diagnostic metrics: BE=${id}, names=${missing}")
}
+ [(id): values]
}
- if (allMatch) {
- logger.info("All file cache entries for tablets ${tabletIds} are ${expectedType}")
- return
+ }
+ def logMetrics = { String stage ->
+ def current = getMetrics()
+ def delta = backendIds.collectEntries { id ->
+ [(id): metricNames.collectEntries { name ->
+ [(name): metricsBaseline != null && metricsBaseline[id][name] != null && current[id][name] != null ?
+ current[id][name] - metricsBaseline[id][name] : null]
+ }]
}
- sleep(intervalMs)
+ logger.info("TTL metrics: stage=${stage}, cluster=${clusterName}, by_be=${current}, " +
+ "delta_from_ttl_baseline=${delta}")
+ current
}
- assertTrue(false, "Timeout waiting for file_cache_info type ${expectedType} for tablets ${tabletIds}")
- }
-
- clearFileCache.call() {
- respCode, body -> {}
- }
- sleep(10000)
+ String address = "Address Line 1"
+ String phone = "123-456-7890"
+ String segment = "AUTOMOBILE"
+ String comment = "This is a test comment for the customer. " + ("X" * 50)
+ def customerIds = (10001..10200).toList()
+ def customerNames = customerIds.collect { String.format('Customer#%09d', it) }
+ def normalize = { List values -> values.collect { new BigDecimal(it.toString()).stripTrailingZeros() } }
+ def expectedData = normalize([customerIds.size(), customerIds.sum(0L), customerNames.sum(0L) { it.length() },
+ 200L * address.length(), 200L * 15L, 200L * phone.length(),
+ 200G * 12345.67G, 200L * segment.length(), 200L * comment.length()])
+ def scanTable = {
+ // Read every column; result caches and metadata-only COUNT cannot satisfy this query.
+ def rows = sql """select count(*), sum(C_CUSTKEY), sum(length(C_NAME)), sum(length(C_ADDRESS)),
+ sum(C_NATIONKEY), sum(length(rtrim(C_PHONE))), sum(C_ACCTBAL),
+ sum(length(rtrim(C_MKTSEGMENT))), sum(length(C_COMMENT)) from ${tableName}"""
+ assertTrue(rows.size() == 1 && rows[0].size() == 9 && rows[0].every { it != null },
+ "Unexpected data result: ${rows}")
+ assertEquals(expectedData, normalize(rows[0]))
+ }
+ Throwable failure = null
+ try {
+ sql "use @${clusterName}"
+ sql "set enable_sql_cache = false"
+ sql "set enable_query_cache = false"
+ logMetrics("before_create")
+ sql "drop table if exists ${tableName} force"
+ long prepareStartedMs = System.currentTimeMillis()
+ // Leave a minute for observing TTL before expiry, even on ASAN and with delayed metadata flushes.
+ long prepareDeadlineMs = prepareStartedMs + (ttlSeconds - 60L) * 1000L
+ String ddl = new File("${context.file.parent}/../ddl/customer_ttl.sql").text
+ .replace('customer_ttl', tableName).replace('BUCKETS 32', "BUCKETS ${bucketCount}")
+ sql(ddl + """ PROPERTIES("file_cache_ttl_seconds"="${ttlSeconds}", "disable_auto_compaction"="true")""")
+ tabletIds = sql_return_maparray("show tablets from ${tableName}")
+ .collect { it.TabletId as Long }.unique().sort()
+ assertTrue(tabletIds.size() == bucketCount, "Expected ${bucketCount} tablets: ${tabletIds}")
+ String values = customerIds.withIndex().collect { id, i ->
+ "(${id}, '${customerNames[i]}', '${address}', 15, '${phone}', 12345.67, '${segment}', '${comment}')"
+ }.join(",")
+ long insertStartedMs = System.currentTimeMillis()
+ sql "insert into ${tableName} values ${values}"
+ logger.info("TTL fixture: rows=200, tablets=${tabletIds}, insert_elapsed_ms=${System.currentTimeMillis() - insertStartedMs}")
+ scanTable()
- def tabletIds = []
- load_customer_once("customer_ttl")
- sleep(10000)
- tabletIds = getTabletIds.call("customer_ttl")
- waitForFileCacheType.call(tabletIds, "ttl")
- getMetricsMethod.call() {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag1 = false;
- long ttl_cache_size = 0;
- for (String line in strs) {
- if (flag1) break;
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
+ // The full scan above verifies data; cloud header JSON does not include rowsets.
+ waitUntil("tablet TTL metadata", prepareDeadlineMs) {
+ metadata = [:]
+ sql_return_maparray("show tablets from ${tableName}").each { tablet ->
+ long id = tablet.TabletId as Long
+ long primary = tablet.PrimaryBackendId as Long
+ if (!(id in tabletIds) || !backends.containsKey(primary) || (tablet.BackendId as Long) != primary) {
+ return
+ }
+ def be = backends[primary]
+ try {
+ def connection = Http.openConnection(
+ "http://${be.Host}:${be.HttpPort}/api/meta/header/${id}")
+ connection.connectTimeout = 5000
+ connection.readTimeout = 5000
+ try {
+ def meta = connection.inputStream.withCloseable {
+ parseJson(it.getText("UTF-8"))
+ }
+ if ((meta.tablet_id as Long) == id && (meta.creation_time as Long) > 0L &&
+ (meta.ttl_seconds as Long) == ttlSeconds &&
+ meta.tablet_state == "PB_RUNNING" && meta.schema != null) {
+ metadata[id] = [be_id: primary, creation_time: meta.creation_time as Long,
+ expires_at: (meta.creation_time as Long) + ttlSeconds]
+ }
+ } finally {
+ connection.disconnect()
+ }
+ } catch (Exception e) {
+ logger.info("Waiting for loaded metadata: tablet=${id}, BE=${primary}, ${e.message}")
}
- def i = line.indexOf(' ')
- ttl_cache_size = line.substring(i).toLong()
- logger.info("ttl_cache_size (initial) line: " + line)
- flag1 = true
}
+ lastState = [missing_tablets: tabletIds - metadata.keySet(), metadata: metadata]
+ metadata.size() == tabletIds.size()
}
- assertTrue(flag1)
- assertTrue(ttl_cache_size > 10737)
- }
- sleep(180000)
- getMetricsMethod.call() {
- respCode, body ->
- assertEquals("${respCode}".toString(), "200")
- String out = "${body}".toString()
- def strs = out.split('\n')
- Boolean flag1 = false;
- for (String line in strs) {
- if (flag1) break;
- if (line.contains("ttl_cache_size")) {
- if (line.startsWith("#")) {
- continue
- }
- def i = line.indexOf(' ')
- logger.info("ttl_cache_size line before assert zero: " + line)
- assertEquals(line.substring(i).toLong(), 0)
- flag1 = true
+ long now = serverTime()
+ assertTrue(metadata.values().every { it.creation_time <= now }, "Future tablet creation time: ${metadata}, now=${now}")
+ long earliestExpiry = metadata.values().min { it.expires_at }.expires_at
+ long latestExpiry = metadata.values().max { it.expires_at }.expires_at
+ logger.info("TTL deadlines: ttl_seconds=${ttlSeconds}, server_now=${now}, by_tablet=${metadata}")
+ def baseline = [:]
+ def candidate = null
+ long stableSince = 0L
+ waitUntil("stable TTL baseline before expiry", prepareDeadlineMs) {
+ baseline = getCache()
+ now = serverTime()
+ assertTrue(now + 30L < earliestExpiry, "Preparation exhausted TTL window: now=${now}, metadata=${metadata}")
+ def missing = tabletIds.findAll { id -> !baseline.keySet().any { it[2] == id } }
+ boolean allTtl = missing.isEmpty() && baseline.values().every { it.type == "ttl" }
+ lastState = [missing_tablets: missing, cache_by_be_tablet: summarizeCache(baseline), seconds_to_expiry: earliestExpiry - now]
+ if (baseline != candidate) {
+ logger.info("TTL baseline progress: ${lastState}")
+ }
+ if (!allTtl || baseline != candidate) {
+ stableSince = System.currentTimeMillis()
}
+ candidate = baseline
+ allTtl && System.currentTimeMillis() - stableSince >= 3000L
}
- assertTrue(flag1)
- }
+ logger.info("TTL baseline: preparation_elapsed_ms=${System.currentTimeMillis() - prepareStartedMs}, " +
+ "cache_by_be_tablet=${summarizeCache(baseline)}, blocks=${baseline}")
+ metricsBaseline = logMetrics("ttl_baseline")
- waitForFileCacheType.call(tabletIds, "normal")
+ // Keep TTL unchanged: this case covers natural expiry, separately from ALTER TTL cases.
+ // A new interval does not wake a thread already sleeping for the old 180-second interval.
+ now = serverTime()
+ long conversionDeadlineMs = System.currentTimeMillis() + Math.max(0L, latestExpiry + 1L - now) * 1000L + 600000L
+ def firstNormalAt = [:]
+ def converted = [:]
+ def previous = null
+ long lastLogMs = 0L
+ stableSince = 0L
+ waitUntil("natural TTL expiration", conversionDeadlineMs) {
+ // Do not read business data until the original blocks have converted completely.
+ converted = getCache()
+ now = serverTime() // Read time after the snapshot to avoid a boundary crossing during the query.
+ def differences = [missing: (baseline.keySet() - converted.keySet()).take(10),
+ unexpected: (converted.keySet() - baseline.keySet()).take(10),
+ changed_sizes: converted.findAll { key, value -> value.size != baseline[key]?.size }.take(10)]
+ converted.each { key, value ->
+ // FE sets creation_time; BE checks its own clock. Allow two seconds of clock skew.
+ assertTrue(value.type != "normal" || now + 2L >= metadata[key[2]].expires_at,
+ "Cache converted before expiry: block=${key}, now=${now}, metadata=${metadata[key[2]]}")
+ }
+ tabletIds.each { id ->
+ def expected = baseline.findAll { key, value -> key[2] == id }
+ def actual = converted.findAll { key, value -> key[2] == id }
+ if (!firstNormalAt.containsKey(id) && blockSizes(expected) == blockSizes(actual) &&
+ actual.values().every { it.type == "normal" }) {
+ firstNormalAt[id] = now
+ }
+ }
+ boolean complete = blockSizes(baseline) == blockSizes(converted) &&
+ converted.values().every { it.type == "normal" } && now > latestExpiry
+ lastState = [server_now: now, seconds_to_last_expiry: latestExpiry - now,
+ cache_by_be_tablet: summarizeCache(converted), differences: differences,
+ tablets_not_yet_normal: tabletIds - firstNormalAt.keySet()]
+ if (converted != previous || System.currentTimeMillis() - lastLogMs >= 30000L) {
+ logger.info("TTL expiration progress: ${lastState}")
+ lastLogMs = System.currentTimeMillis()
+ }
+ if (!complete || converted != previous) {
+ stableSince = System.currentTimeMillis()
+ }
+ previous = converted
+ complete && System.currentTimeMillis() - stableSince >= 3000L
+ }
+ def timing = metadata.collectEntries { id, meta ->
+ [(id): meta + [first_normal_observed_at: firstNormalAt[id],
+ observed_conversion_lag_seconds: firstNormalAt[id] - meta.expires_at]]
+ }
+ logger.info("TTL natural expiration verified: by_tablet=${timing}, " +
+ "before=${summarizeCache(baseline)}, after=${summarizeCache(converted)}")
+ logMetrics("normal_after_expiry")
+ scanTable()
+ } catch (Throwable t) {
+ failure = t
+ logger.warn("TTL case failed: cluster=${clusterName}, metadata=${metadata}, last_state=${lastState}")
+ try { logMetrics("failure") } catch (Throwable diagnosticError) { t.addSuppressed(diagnosticError) }
+ throw t
+ } finally {
+ def cleanupErrors = []
+ try { sql "drop table if exists ${tableName} force" } catch (Throwable t) { cleanupErrors.add(t) }
+ [enable_sql_cache: originalQueryCache[0], enable_query_cache: originalQueryCache[1]].each { name, value ->
+ try { sql "set ${name} = ${value}" } catch (Throwable t) { cleanupErrors.add(t) }
+ }
+ if (!cleanupErrors.isEmpty()) {
+ if (failure != null) {
+ cleanupErrors.each { failure.addSuppressed(it) }
+ } else {
+ cleanupErrors.tail().each { cleanupErrors[0].addSuppressed(it) }
+ throw cleanupErrors[0]
+ }
+ }
+ }
}
}
diff --git a/regression-test/suites/cloud_p0/cache/warm_up/test_file_cache_warmup_read_metrics_docker.groovy b/regression-test/suites/cloud_p0/cache/warm_up/test_file_cache_warmup_read_metrics_docker.groovy
index e2fc6d6fe65..8ef6eed35d7 100644
--- a/regression-test/suites/cloud_p0/cache/warm_up/test_file_cache_warmup_read_metrics_docker.groovy
+++ b/regression-test/suites/cloud_p0/cache/warm_up/test_file_cache_warmup_read_metrics_docker.groovy
@@ -60,8 +60,8 @@ suite('test_file_cache_warmup_read_metrics_docker', 'docker') {
'DORIS_CLOUD_AK=minioadmin',
'DORIS_CLOUD_SK=minioadmin',
"DORIS_CLOUD_BUCKET=${minioBucket}",
- "DORIS_CLOUD_ENDPOINT=host.docker.internal:${minioPort}",
- "DORIS_CLOUD_EXTERNAL_ENDPOINT=host.docker.internal:${minioPort}",
+ "DORIS_CLOUD_ENDPOINT=http://host.docker.internal:${minioPort}",
+ "DORIS_CLOUD_EXTERNAL_ENDPOINT=http://host.docker.internal:${minioPort}",
'DORIS_CLOUD_REGION=us-east-1',
'DORIS_CLOUD_PROVIDER=S3',
]
diff --git a/regression-test/suites/cloud_p0/cache/warm_up/test_warmup_table_docker.groovy b/regression-test/suites/cloud_p0/cache/warm_up/test_warmup_table_docker.groovy
index e93560b389e..14c02d48f8b 100644
--- a/regression-test/suites/cloud_p0/cache/warm_up/test_warmup_table_docker.groovy
+++ b/regression-test/suites/cloud_p0/cache/warm_up/test_warmup_table_docker.groovy
@@ -27,7 +27,7 @@ suite('test_warmup_table_docker', 'docker') {
options.beConfigs += [
'file_cache_enter_disk_resource_limit_mode_percent=99',
'enable_evict_file_cache_in_advance=false',
- 'enable_only_warm_up_idx=true',
+ 'file_cache_enable_only_warm_up_idx=true',
'file_cache_background_gc_interval_ms=10',
'file_cache_background_monitor_interval_ms=100',
'file_cache_background_block_lru_update_interval_ms=100',
@@ -189,8 +189,8 @@ suite('test_warmup_table_docker', 'docker') {
def cache_size_after_clear = getClusterFileCacheSize(clusterName)
assertEquals(0, cache_size_after_clear)
- // Set enable_only_warm_up_idx = true
- updateBeConf(clusterName, "enable_only_warm_up_idx", "true")
+ // Set file_cache_enable_only_warm_up_idx = true
+ updateBeConf(clusterName, "file_cache_enable_only_warm_up_idx", "true")
// Trigger warm up
def jobId = sql "WARM UP CLUSTER ${clusterName} WITH TABLE ${testTable}"
diff --git a/regression-test/suites/cloud_p0/compaction/test_cloud_alter_disable_auto_compaction.groovy b/regression-test/suites/cloud_p0/compaction/test_cloud_alter_disable_auto_compaction.groovy
index 799cc05810b..65d78d48ed7 100644
--- a/regression-test/suites/cloud_p0/compaction/test_cloud_alter_disable_auto_compaction.groovy
+++ b/regression-test/suites/cloud_p0/compaction/test_cloud_alter_disable_auto_compaction.groovy
@@ -28,8 +28,10 @@ suite("test_cloud_alter_disable_auto_compaction", "p0,docker") {
'cloud_cluster_check_interval_second=1',
]
options.beConfigs += [
- // Short sync interval for faster property propagation
- 'tablet_sync_interval_s=5',
+ // A tablet becomes eligible for meta sync after this interval.
+ 'tablet_sync_interval_s=1',
+ // The background sync thread must also run frequently enough to propagate ALTER properties.
+ 'schedule_sync_tablets_interval_s=1',
// Trigger cumulative compaction after 5 rowsets
'cumulative_compaction_min_deltas=5',
// Enable file cache (required for sync_meta to work)
@@ -116,7 +118,7 @@ suite("test_cloud_alter_disable_auto_compaction", "p0,docker") {
assertTrue(result[0][1].contains('"disable_auto_compaction" = "true"'))
// Wait for sync_meta to propagate the change to BE
- // tablet_sync_interval_s is set to 5 seconds, so wait a bit longer
+ // Both tablet staleness and background scheduling intervals are set to 1 second.
logger.info("Waiting for sync_meta to propagate disable_auto_compaction=true to BE...")
sleep(10000)
diff --git a/regression-test/suites/cloud_p0/multi_cluster/async_copy_into/async_load.groovy b/regression-test/suites/cloud_p0/multi_cluster/async_copy_into/async_load.groovy
index 0e346d0389a..89cb6124e1c 100644
--- a/regression-test/suites/cloud_p0/multi_cluster/async_copy_into/async_load.groovy
+++ b/regression-test/suites/cloud_p0/multi_cluster/async_copy_into/async_load.groovy
@@ -18,6 +18,7 @@
import groovy.json.JsonOutput
suite("async_load") {
+ withRestoredMultiClusterState(false) {
def externalStageName = "regression_test_tpch"
def prefix = "tpch/sf1"
@@ -51,7 +52,7 @@ suite("async_load") {
}
}
}
- wait_cluster_change()
+ sleep(20000)
List<List<Object>> result = sql "show clusters"
assertTrue(result.size() == 0);
@@ -60,7 +61,7 @@ suite("async_load") {
"regression_cluster_name0", "regression_cluster_id0");
add_cluster.call(beUniqueIdList[1], ipList[1], hbPortList[1],
"regression_cluster_name1", "regression_cluster_id1");
- wait_cluster_change()
+ sleep(20000)
result = sql "show clusters"
assertEquals(result.size(), 2);
@@ -97,7 +98,7 @@ suite("async_load") {
'prefix' = 'regression' ,
'ak' = '${getS3AK()}' ,
'sk' = '${getS3SK()}' ,
- 'provider' = '${getProvider()}',
+ 'provider' = '${getS3Provider()}',
'access_type' = 'aksk',
'default.file.column_separator' = "|"
);
@@ -109,7 +110,7 @@ suite("async_load") {
assertTrue(result.size() == 1)
assertTrue(result[0].size() == 8)
//assertTrue(result[0][1].equals("FINISHED"))
- id = result[0][0]
+ def id = result[0][0]
while(true) {
result = sql "show copy where id = \'${id}\'"
logger.info("copy result: " + result)
@@ -272,4 +273,5 @@ suite("async_load") {
assertTrue(before_cluster0_query_scan_rows == after_cluster0_query_scan_rows)
assertTrue(before_cluster1_query_scan_rows < after_cluster1_query_scan_rows)
+ }
}
diff --git a/regression-test/suites/cloud_p0/multi_cluster/copy_into/sync_laod.groovy b/regression-test/suites/cloud_p0/multi_cluster/copy_into/sync_laod.groovy
index 72c84017185..32c7344e838 100644
--- a/regression-test/suites/cloud_p0/multi_cluster/copy_into/sync_laod.groovy
+++ b/regression-test/suites/cloud_p0/multi_cluster/copy_into/sync_laod.groovy
@@ -16,8 +16,25 @@
// under the License.
import groovy.json.JsonOutput
+import org.apache.doris.regression.action.ProfileAction
suite("sync_load") {
+ withRestoredMultiClusterState(false) {
+
+ def queryFrontend = sql_return_maparray('SHOW FRONTENDS').find {
+ it.CurrentConnected?.toString()?.equalsIgnoreCase('Yes')
+ }
+ assertNotNull(queryFrontend, 'Cannot locate the FE executing profile queries')
+ def profileAction = new ProfileAction(context,
+ new InetSocketAddress(queryFrontend.Host.toString(), queryFrontend.HttpPort.toString().toInteger()))
+ def checkQueryProfile = { query, pattern ->
+ // Match this execution, even when the profile list normalizes or truncates the SQL text.
+ def marker = "multi_cluster_${UUID.randomUUID().toString().replace('-', '')}"
+ sql 'sync'
+ sql "/* ${marker} */ ${query}"
+ profileAction.getProfileBySql(marker, [pattern.toString()], 60000L, 500L)
+ }
+
def externalStageName = "regression_test_tpch"
def prefix = "tpch/sf1"
@@ -25,7 +42,6 @@ suite("sync_load") {
List<String> hbPortList = new ArrayList<>()
List<String> httpPortList = new ArrayList<>()
List<String> beUniqueIdList = new ArrayList<>()
- List<String> bePortList = new ArrayList<>()
String[] bes = context.config.multiClusterBes.split(',');
println("the value is " + context.config.multiClusterBes);
@@ -51,7 +67,7 @@ suite("sync_load") {
}
}
}
- wait_cluster_change()
+ sleep(20000)
List<List<Object>> result = sql "show clusters"
assertTrue(result.size() == 0);
@@ -60,7 +76,7 @@ suite("sync_load") {
"regression_cluster_name0", "regression_cluster_id0");
add_cluster.call(beUniqueIdList[1], ipList[1], hbPortList[1],
"regression_cluster_name1", "regression_cluster_id1");
- wait_cluster_change()
+ sleep(20000)
result = sql "show clusters"
assertEquals(result.size(), 2);
@@ -98,7 +114,7 @@ suite("sync_load") {
'prefix' = 'regression' ,
'ak' = '${getS3AK()}' ,
'sk' = '${getS3SK()}' ,
- 'provider' = '${getProvider()}',
+ 'provider' = '${getS3Provider()}',
'access_type' = 'aksk',
'default.file.column_separator' = "|"
);
@@ -205,21 +221,18 @@ suite("sync_load") {
// sql new File("""${context.file.parent}/ddl/${sql}.sql""").text
//}
- // fill bePortList
- for (int i = 0; i < ipList.size(); ++i) {
- result = sql """show backends"""
- for (row : result) {
- println row
- println row[2]
- if (ipList[i] == row[1] && hbPortList[i] == row[2]) {
- bePortList.add(row[3]);
- }
+ // QeProcessorImpl attaches execution profiles using Backend.getHeartbeatAddress().
+ def profileBackends = sql_return_maparray('SHOW BACKENDS')
+ def profileHeartbeatPorts = (0..1).collect { index ->
+ def backend = profileBackends.find {
+ it.Host == ipList[index] && it.HeartbeatPort.toString() == hbPortList[index]
}
+ assertNotNull(backend, "Missing profile backend ${ipList[index]}:${hbPortList[index]}")
+ backend.HeartbeatPort.toString()
}
- assertEquals(bePortList.size(), 2);
// q01
- sql """
+ def q01 = """
SELECT
l_returnflag,
l_linestatus,
@@ -243,11 +256,10 @@ suite("sync_load") {
l_linestatus
"""
- def set = [ipList[0] + ":" +bePortList[0]] as Set
- checkProfile.call(set)
+ checkQueryProfile(q01, "hostname:${ipList[0]}, port:${profileHeartbeatPorts[0]}")
// q02
- sql """
+ def q02 = """
SELECT
s_acctbal,
s_name,
@@ -290,10 +302,10 @@ suite("sync_load") {
p_partkey
LIMIT 100
"""
- checkProfile.call(set)
+ checkQueryProfile(q02, "hostname:${ipList[0]}, port:${profileHeartbeatPorts[0]}")
// q03
- sql """
+ def q03 = """
SELECT
l_orderkey,
sum(l_extendedprice * (1 - l_discount)) AS revenue,
@@ -318,10 +330,10 @@ suite("sync_load") {
o_orderdate
LIMIT 10
"""
- checkProfile.call(set)
+ checkQueryProfile(q03, "hostname:${ipList[0]}, port:${profileHeartbeatPorts[0]}")
// q04
- sql """
+ def q04 = """
SELECT
o_orderpriority,
count(*) AS order_count
@@ -341,10 +353,10 @@ suite("sync_load") {
ORDER BY
o_orderpriority
"""
- checkProfile.call(set)
+ checkQueryProfile(q04, "hostname:${ipList[0]}, port:${profileHeartbeatPorts[0]}")
// q05
- sql """
+ def q05 = """
SELECT
n_name,
sum(l_extendedprice * (1 - l_discount)) AS revenue
@@ -370,10 +382,10 @@ suite("sync_load") {
ORDER BY
revenue DESC
"""
- checkProfile.call(set)
+ checkQueryProfile(q05, "hostname:${ipList[0]}, port:${profileHeartbeatPorts[0]}")
// q06
- sql """
+ def q06 = """
SELECT sum(l_extendedprice * l_discount) AS revenue
FROM
lineitem
@@ -383,10 +395,10 @@ suite("sync_load") {
AND l_discount BETWEEN 0.06 - 0.01 AND .06 + 0.01
AND l_quantity < 24
"""
- checkProfile.call(set)
+ checkQueryProfile(q06, "hostname:${ipList[0]}, port:${profileHeartbeatPorts[0]}")
// q07
- sql """
+ def q07 = """
SELECT
supp_nation,
cust_nation,
@@ -426,13 +438,12 @@ suite("sync_load") {
cust_nation,
l_year
"""
- checkProfile.call(set)
+ checkQueryProfile(q07, "hostname:${ipList[0]}, port:${profileHeartbeatPorts[0]}")
sql "use @regression_cluster_name1"
- set = [ipList[1] + ":" +bePortList[1]] as Set
// q08
- sql """
+ def q08 = """
SELECT
o_year,
sum(CASE
@@ -471,10 +482,10 @@ suite("sync_load") {
ORDER BY
o_year
"""
- checkProfile.call(set)
+ checkQueryProfile(q08, "hostname:${ipList[1]}, port:${profileHeartbeatPorts[1]}")
// q09
- sql """
+ def q09 = """
SELECT
nation,
o_year,
@@ -507,10 +518,10 @@ suite("sync_load") {
nation,
o_year DESC
"""
- checkProfile.call(set)
+ checkQueryProfile(q09, "hostname:${ipList[1]}, port:${profileHeartbeatPorts[1]}")
// q10
- sql """
+ def q10 = """
SELECT
c_custkey,
c_name,
@@ -544,10 +555,10 @@ suite("sync_load") {
revenue DESC
LIMIT 20
"""
- checkProfile.call(set)
+ checkQueryProfile(q10, "hostname:${ipList[1]}, port:${profileHeartbeatPorts[1]}")
// q11
- sql """
+ def q11 = """
SELECT
ps_partkey,
sum(ps_supplycost * ps_availqty) AS value
@@ -576,10 +587,10 @@ suite("sync_load") {
ORDER BY
value DESC
"""
- checkProfile.call(set)
+ checkQueryProfile(q11, "hostname:${ipList[1]}, port:${profileHeartbeatPorts[1]}")
// q12
- sql """
+ def q12 = """
SELECT
l_shipmode,
sum(CASE
@@ -609,10 +620,10 @@ suite("sync_load") {
ORDER BY
l_shipmode
"""
- checkProfile.call(set)
+ checkQueryProfile(q12, "hostname:${ipList[1]}, port:${profileHeartbeatPorts[1]}")
// q13
- sql """
+ def q13 = """
SELECT
c_count,
count(*) AS custdist
@@ -634,10 +645,10 @@ suite("sync_load") {
custdist DESC,
c_count DESC
"""
- checkProfile.call(set)
+ checkQueryProfile(q13, "hostname:${ipList[1]}, port:${profileHeartbeatPorts[1]}")
// q14
- sql """
+ def q14 = """
SELECT 100.00 * sum(CASE
WHEN p_type LIKE 'PROMO%'
THEN l_extendedprice * (1 - l_discount)
@@ -651,10 +662,10 @@ suite("sync_load") {
AND l_shipdate >= DATE '1995-09-01'
AND l_shipdate < DATE '1995-09-01' + INTERVAL '1' MONTH
"""
- checkProfile.call(set)
+ checkQueryProfile(q14, "hostname:${ipList[1]}, port:${profileHeartbeatPorts[1]}")
// q15
- sql """
+ def q15 = """
SELECT
s_suppkey,
s_name,
@@ -674,10 +685,10 @@ suite("sync_load") {
ORDER BY
s_suppkey;
"""
- checkProfile.call(set)
+ checkQueryProfile(q15, "hostname:${ipList[1]}, port:${profileHeartbeatPorts[1]}")
// q16
- sql """
+ def q16 = """
SELECT
p_brand,
p_type,
@@ -708,10 +719,10 @@ suite("sync_load") {
p_type,
p_size
"""
- checkProfile.call(set)
+ checkQueryProfile(q16, "hostname:${ipList[1]}, port:${profileHeartbeatPorts[1]}")
// q17
- sql """
+ def q17 = """
SELECT sum(l_extendedprice) / 7.0 AS avg_yearly
FROM
lineitem,
@@ -728,10 +739,10 @@ suite("sync_load") {
l_partkey = p_partkey
)
"""
- checkProfile.call(set)
+ checkQueryProfile(q17, "hostname:${ipList[1]}, port:${profileHeartbeatPorts[1]}")
// q18
- sql """
+ def q18 = """
SELECT
c_name,
c_custkey,
@@ -766,10 +777,10 @@ suite("sync_load") {
o_orderdate
LIMIT 100
"""
- checkProfile.call(set)
+ checkQueryProfile(q18, "hostname:${ipList[1]}, port:${profileHeartbeatPorts[1]}")
// q19
- sql """
+ def q19 = """
SELECT sum(l_extendedprice * (1 - l_discount)) AS revenue
FROM
lineitem,
@@ -805,10 +816,10 @@ suite("sync_load") {
AND l_shipinstruct = 'DELIVER IN PERSON'
)
"""
- checkProfile.call(set)
+ checkQueryProfile(q19, "hostname:${ipList[1]}, port:${profileHeartbeatPorts[1]}")
// q20
- sql """
+ def q20 = """
SELECT
s_name,
s_address
@@ -842,5 +853,6 @@ suite("sync_load") {
AND n_name = 'CANADA'
ORDER BY s_name
"""
- checkProfile.call(set)
+ checkQueryProfile(q20, "hostname:${ipList[1]}, port:${profileHeartbeatPorts[1]}")
+ }
}
diff --git a/regression-test/suites/cloud_p0/multi_cluster/default_cluster.groovy b/regression-test/suites/cloud_p0/multi_cluster/default_cluster.groovy
index 7580ca13a8b..5d7f1368c4b 100644
--- a/regression-test/suites/cloud_p0/multi_cluster/default_cluster.groovy
+++ b/regression-test/suites/cloud_p0/multi_cluster/default_cluster.groovy
@@ -18,6 +18,7 @@
import groovy.json.JsonOutput
suite("default_cluster") {
+ withRestoredMultiClusterState(false) {
def token = context.config.metaServiceToken
def instance_id = context.config.multiClusterInstance
@@ -27,9 +28,9 @@ suite("default_cluster") {
List<String> beUniqueIdList = new ArrayList<>()
String[] bes = context.config.multiClusterBes.split(',');
- println("the value is " + context.config.multiClusterBes);
+ logger.info("the value is " + context.config.multiClusterBes);
for(String values : bes) {
- println("the value is " + values);
+ logger.info("the value is " + values);
String[] beInfo = values.split(':');
ipList.add(beInfo[0]);
hbPortList.add(beInfo[1]);
@@ -37,10 +38,10 @@ suite("default_cluster") {
beUniqueIdList.add(beInfo[3]);
}
- println("the ip is " + ipList);
- println("the heartbeat port is " + hbPortList);
- println("the http port is " + httpPortList);
- println("the be unique id is " + beUniqueIdList);
+ logger.info("the ip is " + ipList);
+ logger.info("the heartbeat port is " + hbPortList);
+ logger.info("the http port is " + httpPortList);
+ logger.info("the be unique id is " + beUniqueIdList);
for (unique_id : beUniqueIdList) {
resp = get_cluster.call(unique_id);
@@ -50,7 +51,7 @@ suite("default_cluster") {
}
}
}
- wait_cluster_change()
+ sleep(20000)
List<List<Object>> result = sql "show clusters"
assertTrue(result.size() == 0);
@@ -59,22 +60,22 @@ suite("default_cluster") {
"regression_cluster_name2", "regression_cluster_id2");
add_cluster.call(beUniqueIdList[1], ipList[1], hbPortList[1],
"regression_cluster_name3", "regression_cluster_id3");
- wait_cluster_change()
+ sleep(20000)
result = sql "show clusters"
assertTrue(result.size() == 2);
for (row : result) {
- log.info("show cluster row: ${row}".toString())
+ logger.info("show cluster row: ${row}".toString())
}
sql "SET PROPERTY 'default_cloud_cluster' = 'regression_cluster_name3'"
context.reconnectFe()
def before_cluster0_flush = get_be_metric(ipList[0], httpPortList[0], "memtable_flush_total");
- log.info("before_cluster0_flush : ${before_cluster0_flush}".toString())
+ logger.info("before_cluster0_flush : ${before_cluster0_flush}".toString())
def before_cluster1_flush = get_be_metric(ipList[1], httpPortList[1], "memtable_flush_total");
- log.info("before_cluster1_flush : ${before_cluster1_flush}".toString())
+ logger.info("before_cluster1_flush : ${before_cluster1_flush}".toString())
sql """
@@ -92,17 +93,17 @@ suite("default_cluster") {
"""
def after_cluster0_flush = get_be_metric(ipList[0], httpPortList[0], "memtable_flush_total");
- log.info("after_cluster0_flush : ${after_cluster0_flush}".toString())
+ logger.info("after_cluster0_flush : ${after_cluster0_flush}".toString())
def after_cluster1_flush = get_be_metric(ipList[1], httpPortList[1], "memtable_flush_total");
- log.info("after_cluster1_flush : ${after_cluster1_flush}".toString())
+ logger.info("after_cluster1_flush : ${after_cluster1_flush}".toString())
assertTrue(before_cluster0_flush == after_cluster0_flush)
assertTrue(before_cluster1_flush < after_cluster1_flush)
result = sql "show clusters"
for (row : result) {
- println row
+ logger.info("row: ${row.toString()}")
if(row[0] == "regression_cluster_name2") {
assertTrue(row[1].toString().toLowerCase() == "false")
}
@@ -115,7 +116,7 @@ suite("default_cluster") {
sql "use @regression_cluster_name2"
result = sql "show clusters"
for (row : result) {
- println row
+ logger.info("row: ${row.toString()}")
if(row[0] == "regression_cluster_name2") {
assertTrue(row[1].toString().toLowerCase() == "true")
}
@@ -128,7 +129,7 @@ suite("default_cluster") {
context.reconnectFe()
result = sql "show clusters"
for (row : result) {
- println row
+ logger.info("row: ${row.toString()}")
if(row[0] == "regression_cluster_name2") {
assertTrue(row[1].toString().toLowerCase() == "true")
}
@@ -140,7 +141,7 @@ suite("default_cluster") {
sql "use @regression_cluster_name3"
result = sql "show clusters"
for (row : result) {
- println row
+ logger.info("row: ${row.toString()}")
if(row[0] == "regression_cluster_name2") {
assertTrue(row[1].toString().toLowerCase() == "false")
}
@@ -160,7 +161,7 @@ suite("default_cluster") {
result = sql "show property"
for (row : result) {
- println row
+ logger.info("row: ${row.toString()}")
if(row[0] == "default_cloud_cluster") {
assertTrue(row[1].toString().isEmpty())
}
@@ -169,4 +170,5 @@ suite("default_cluster") {
sql """
drop table if exists test_table
"""
+ }
}
diff --git a/regression-test/suites/cloud_p0/multi_cluster/multi_cluster_s3_load/load.groovy b/regression-test/suites/cloud_p0/multi_cluster/multi_cluster_s3_load/load.groovy
index 20108615679..aeed4a1c95b 100644
--- a/regression-test/suites/cloud_p0/multi_cluster/multi_cluster_s3_load/load.groovy
+++ b/regression-test/suites/cloud_p0/multi_cluster/multi_cluster_s3_load/load.groovy
@@ -27,52 +27,33 @@
// sed -nr 's/.*tables: (.*)$/\1/gp' /path/to/*.sql | sed -nr 's/,/\n/gp' | sort | uniq
import groovy.json.JsonOutput
+import org.apache.doris.regression.util.MultiClusterStateGuard
suite("load") {
- List<String> ipList = new ArrayList<>()
- List<String> hbPortList = new ArrayList<>()
- List<String> httpPortList = new ArrayList<>()
- List<String> beUniqueIdList = new ArrayList<>()
-
- String[] bes = context.config.multiClusterBes.split(',');
- println("the value is " + context.config.multiClusterBes);
- for(String values : bes) {
- println("the value is " + values);
- String[] beInfo = values.split(':');
- ipList.add(beInfo[0]);
- hbPortList.add(beInfo[1]);
- httpPortList.add(beInfo[2]);
- beUniqueIdList.add(beInfo[3]);
- }
-
- println("the ip is " + ipList);
- println("the heartbeat port is " + hbPortList);
- println("the http port is " + httpPortList);
- println("the be unique id is " + beUniqueIdList);
-
- for (unique_id : beUniqueIdList) {
- resp = get_cluster.call(unique_id);
- for (cluster : resp) {
- if (cluster.type == "COMPUTE") {
- drop_cluster.call(cluster.cluster_name, cluster.cluster_id);
- }
+ withRestoredMultiClusterState(true) {
+ def baseline = multiClusterBaseline()
+ def ipList = baseline.collect { it.nodes[0].ip }
+ def httpPortList = baseline.collect { it.nodes[0].http_port.toString() }
+
+ // Establish the shared baseline after a previous suite or run left different group names.
+ def existing = get_cluster.call(baseline[0].nodes[0].cloud_unique_id)
+ .findAll { it.type == 'COMPUTE' }
+ if (!MultiClusterStateGuard.frontendMatches(baseline,
+ sql_return_maparray('SHOW CLUSTERS'), sql_return_maparray('SHOW BACKENDS'))) {
+ existing.each { group -> drop_cluster.call(group.cluster_name, group.cluster_id) }
+ sleep(20000)
+ baseline.each { group ->
+ def node = group.nodes[0]
+ add_cluster.call(node.cloud_unique_id, node.ip, node.heartbeat_port.toString(),
+ group.cluster_name, group.cluster_id)
}
+ sleep(20000)
}
- wait_cluster_change()
-
- List<List<Object>> result = sql "show clusters"
- assertTrue(result.size() == 0);
+ checkMultiClusterBaseline(baseline)
+ sql "SET PROPERTY 'default_cloud_cluster' = '${baseline[0].cluster_name}'"
+ List<List<Object>> result
- add_cluster.call(beUniqueIdList[0], ipList[0], hbPortList[0],
- "regression_cluster_name0", "regression_cluster_id0");
- add_cluster.call(beUniqueIdList[1], ipList[1], hbPortList[1],
- "regression_cluster_name1", "regression_cluster_id1");
- wait_cluster_change()
-
- result = sql "show clusters"
- assertEquals(result.size(), 2);
-
... 12426 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 31/35: branch-4.1: [fix](jsonb) Stop truncating JSON text at a U+0000 inside a string #68244 (#68735)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 98e5b46adf691d942c12484161cfebf6f69c4561
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Thu Oct 8 09:33:20 2026 +0800
branch-4.1: [fix](jsonb) Stop truncating JSON text at a U+0000 inside a string #68244 (#68735)
Cherry-picked from #68244
Co-authored-by: Jerry Hu <hu...@selectdb.com>
---
be/src/util/jsonb_utils.h | 9 +++-
be/test/util/jsonb_parser_simd_test.cpp | 37 +++++++++++++
.../table_function/test_json_text_embedded_nul.out | 19 +++++++
.../test_json_text_embedded_nul.groovy | 62 ++++++++++++++++++++++
4 files changed, 125 insertions(+), 2 deletions(-)
diff --git a/be/src/util/jsonb_utils.h b/be/src/util/jsonb_utils.h
index abf05b4d311..05b2ab5ad08 100644
--- a/be/src/util/jsonb_utils.h
+++ b/be/src/util/jsonb_utils.h
@@ -107,8 +107,11 @@ private:
break;
}
case JsonbType::T_String: {
+ // getBlobLen() is the stored payload length. length() would drop a
+ // trailing NUL that belongs to the value, so it cannot be used here;
+ // the writer stores strings by their exact length and never pads.
string_to_json(val->unpack<JsonbStringVal>()->getBlob(),
- val->unpack<JsonbStringVal>()->length());
+ val->unpack<JsonbStringVal>()->getBlobLen());
break;
}
case JsonbType::T_Binary: {
@@ -162,7 +165,9 @@ private:
return;
}
char char_buffer[16];
- for (const char* ptr = str; ptr != str + len && *ptr; ++ptr) {
+ // A JSON string may legally contain U+0000, so the loop must be bounded by
+ // the length only; the NUL itself is escaped as \u0000 by the default branch.
+ for (const char* ptr = str; ptr != str + len; ++ptr) {
if ((unsigned char)*ptr > 31 && *ptr != '\"' && *ptr != '\\') {
os_.put(*ptr);
} else {
diff --git a/be/test/util/jsonb_parser_simd_test.cpp b/be/test/util/jsonb_parser_simd_test.cpp
index 53a666955ba..8f5b418a193 100644
--- a/be/test/util/jsonb_parser_simd_test.cpp
+++ b/be/test/util/jsonb_parser_simd_test.cpp
@@ -400,4 +400,41 @@ TEST_F(JsonbParserTest, ParseJsonWithToLongKey) {
EXPECT_FALSE(st.ok());
std::cout << st.msg() << std::endl;
}
+
+TEST_F(JsonbParserTest, ParseJsonWithEscapedNulInString) {
+ std::string_view json_with_nul = R"({"key":"a\u0000b"})";
+ std::string_view expected_json_with_nul = R"({"key":"a\u0000b"})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul), Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithEscapedNulInNestedArray) {
+ std::string_view json_with_nul = R"({"key":["a\u0000b","c"]})";
+ std::string_view expected_json_with_nul = R"({"key":["a\u0000b","c"]})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul), Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithTrailingNulInString) {
+ std::string_view json_with_nul = R"({"key":"a\u0000"})";
+ std::string_view expected_json_with_nul = R"({"key":"a\u0000"})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul), Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithOnlyNulInString) {
+ std::string_view json_with_nul = R"({"key":"\u0000"})";
+ std::string_view expected_json_with_nul = R"({"key":"\u0000"})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul), Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithTrailingNulInNestedArray) {
+ std::string_view json_with_nul = R"({"key":["a\u0000","\u0000"]})";
+ std::string_view expected_json_with_nul = R"({"key":["a\u0000","\u0000"]})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul), Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithEscapedNulInKey) {
+ std::string_view json_with_nul = R"({"a\u0000b":1})";
+ std::string_view expected_json_with_nul = R"({"a\u0000b":1})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul), Status::OK());
+}
+
} // namespace doris
\ No newline at end of file
diff --git a/regression-test/data/query_p0/sql_functions/table_function/test_json_text_embedded_nul.out b/regression-test/data/query_p0/sql_functions/table_function/test_json_text_embedded_nul.out
new file mode 100644
index 00000000000..a3c8fb2de52
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/table_function/test_json_text_embedded_nul.out
@@ -0,0 +1,19 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !json_each_text_nested_array --
+x 12 ["a\\u0000b"]
+
+-- !json_each_text_nested_object --
+x 16 {"y":"a\\u0000b"}
+
+-- !json_each_text_trailing_nul --
+x 20 ["a\\u0000","\\u0000"]
+
+-- !explode_json_array_string_direct --
+3 610062
+
+-- !explode_json_array_string_nested --
+12 ["a\\u0000b"]
+
+-- !explode_json_array_string_trailing_nul --
+11 ["a\\u0000"]
+
diff --git a/regression-test/suites/query_p0/sql_functions/table_function/test_json_text_embedded_nul.groovy b/regression-test/suites/query_p0/sql_functions/table_function/test_json_text_embedded_nul.groovy
new file mode 100644
index 00000000000..7e1e3beeea7
--- /dev/null
+++ b/regression-test/suites/query_p0/sql_functions/table_function/test_json_text_embedded_nul.groovy
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_json_text_embedded_nul") {
+ // unhex('5C') is a backslash, so every JSON literal below really contains
+ // \u0000, the legal escape for U+0000 inside a JSON string. Rendering such
+ // a nested value back to text must not stop at the NUL.
+
+ qt_json_each_text_nested_array '''
+ SELECT k, length(v), v
+ FROM (SELECT 1) d
+ LATERAL VIEW json_each_text(concat('{"x":["a', unhex('5C'), 'u0000b"]}')) t AS k, v
+ '''
+
+ qt_json_each_text_nested_object '''
+ SELECT k, length(v), v
+ FROM (SELECT 1) d
+ LATERAL VIEW json_each_text(concat('{"x":{"y":"a', unhex('5C'), 'u0000b"}}')) t AS k, v
+ '''
+
+ // A NUL at the very end of a string used to be trimmed by the stored-length
+ // helper, so keep a dedicated case for it.
+ qt_json_each_text_trailing_nul '''
+ SELECT k, length(v), v
+ FROM (SELECT 1) d
+ LATERAL VIEW json_each_text(concat('{"x":["a', unhex('5C'), 'u0000","', unhex('5C'), 'u0000"]}')) t AS k, v
+ '''
+
+ // A direct string element is copied by length, so it keeps the raw NUL byte;
+ // compare it through hex() to keep the expected output printable.
+ qt_explode_json_array_string_direct '''
+ SELECT length(c), hex(c)
+ FROM (SELECT 1) d
+ LATERAL VIEW explode_json_array_string(concat('["a', unhex('5C'), 'u0000b"]')) t AS c
+ '''
+
+ qt_explode_json_array_string_nested '''
+ SELECT length(c), c
+ FROM (SELECT 1) d
+ LATERAL VIEW explode_json_array_string(concat('[["a', unhex('5C'), 'u0000b"]]')) t AS c
+ '''
+
+ qt_explode_json_array_string_trailing_nul '''
+ SELECT length(c), c
+ FROM (SELECT 1) d
+ LATERAL VIEW explode_json_array_string(concat('[["a', unhex('5C'), 'u0000"]]')) t AS c
+ '''
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/35: [fix](regression) Wait for final Iceberg Variant STRING profile counters (#68597)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e3e6cdbfb7136a89f0203bc39a91ffed07ca5f93
Author: Gabriel <li...@selectdb.com>
AuthorDate: Tue Sep 29 14:23:37 2026 +0800
[fix](regression) Wait for final Iceberg Variant STRING profile counters (#68597)
### What problem does this PR solve?
Related PR: #66941
`test_iceberg_variant_read` can fail its STRING leaf projection
assertion after the SQL result has already passed the 4,096-row check.
The profile fetch only requires counter names to be present, which does
not ensure their final values have arrived.
In the observed failure, the profile HTTP request started at
`03:16:04.601`, overlapping the final BE report processed at
`03:16:04.604`. The archived profile subsequently contained
`VariantLeafProjectionRowGroupColumns = 1`, `VariantDirectLeafRows =
8192`, and zero residual/full projection and reconstruction counters.
Reuse the suite's bounded positive-counter wait for the STRING query.
Allow the helper to also require the counters expected to remain zero,
preserving every existing assertion and presence check. Permanently zero
positive counters still time out with counter diagnostics; unexpected
reconstruction still fails.
This targets `branch-4.1`, where the affected STRING assertion exists.
### Release note
None
### Check List (For Author)
- Test
- [x] Manual test: ran a Groovy 4 harness that executes the suite's
actual parsing/wait closures and STRING assertions against deterministic
profile snapshots. The delayed-counter sequence reproduced the original
assertion failure before the fix and passed afterward. Also verified an
already-ready profile, a permanent-zero timeout, and rejection of
nonzero reconstruction. This is a focused replay, not an end-to-end
cluster run.
- [x] Manual test: full Groovy suite syntax check and `git diff --check`
passed.
- Full external regression was not run locally; CI validation is
pending.
- Behavior changed:
- [x] No production behavior change; only regression profile
synchronization changes.
- Does this need documentation?
- [x] No.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../iceberg/test_iceberg_variant_read.groovy | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_read.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_read.groovy
index d1d02d12247..b0190b28813 100644
--- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_read.groovy
+++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_read.groovy
@@ -1220,14 +1220,16 @@ public class AppendVariantEqualityDelete {
}
return sum
}
- def getProfileByToken = { String token, List<String> positiveCounters = [] ->
- String lastProfile = profileAction.getProfileBySql(token, positiveCounters)
+ def getProfileByToken = { String token, List<String> positiveCounters = [],
+ List<String> requiredCounters = [] ->
+ List<String> requiredContents = positiveCounters + requiredCounters
+ String lastProfile = profileAction.getProfileBySql(token, requiredContents)
if (positiveCounters.every { String counter -> counterSum(lastProfile, counter) > 0 }) {
return lastProfile
}
try {
return profileAction.waitProfile({
- lastProfile = profileAction.getProfileBySql(token, positiveCounters)
+ lastProfile = profileAction.getProfileBySql(token, requiredContents)
return positiveCounters.every {
String counter -> counterSum(lastProfile, counter) > 0
} ? lastProfile : ""
@@ -1831,11 +1833,12 @@ public class AppendVariantEqualityDelete {
assertEquals(1, stringLeafRows.size())
assertEquals(4096L, ((Number) stringLeafRows[0][1]).longValue(),
"The pinned shredded STRING fixture did not return all padding rows")
- String stringLeafProfile = profileAction.getProfileBySql(stringLeafToken,
- ["VariantLeafProjectionRowGroupColumns",
- "VariantResidualProjectionRowGroupColumns",
- "VariantFullProjectionRowGroupColumns",
- "VariantDirectLeafRows", "VariantReconstructedRows"]).toString()
+ // Counter names can arrive before the final BE report fills in their values. Wait for the
+ // positive counters while still requiring the zero-valued counters used by the assertions.
+ String stringLeafProfile = getProfileByToken(stringLeafToken,
+ ["VariantLeafProjectionRowGroupColumns", "VariantDirectLeafRows"],
+ ["VariantResidualProjectionRowGroupColumns",
+ "VariantFullProjectionRowGroupColumns", "VariantReconstructedRows"]).toString()
assertTrue(counterSum(stringLeafProfile, "VariantLeafProjectionRowGroupColumns") > 0,
"A STRING leaf predicate did not retain its physical row-group leaf projection")
assertEquals(0L, counterSum(stringLeafProfile,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 07/35: add temp dir in testutil (#68638)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d3f3f52ecaa1d2dbae3b59efa4229c6cfa1afb18
Author: yiguolei <gu...@selectdb.com>
AuthorDate: Tue Sep 29 17:39:12 2026 +0800
add temp dir in testutil (#68638)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/test/testutil/scoped_temp_dir.h | 60 ++++++++++++++++++++++++++++++++++++++
1 file changed, 60 insertions(+)
diff --git a/be/test/testutil/scoped_temp_dir.h b/be/test/testutil/scoped_temp_dir.h
new file mode 100644
index 00000000000..0c5e02a95ed
--- /dev/null
+++ b/be/test/testutil/scoped_temp_dir.h
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <unistd.h>
+
+#include <cerrno>
+#include <filesystem>
+#include <string>
+#include <string_view>
+#include <system_error>
+#include <utility>
+
+namespace doris::test {
+
+class ScopedTempDirectory {
+public:
+ explicit ScopedTempDirectory(std::string_view prefix) {
+ std::string path_template =
+ (std::filesystem::temp_directory_path() / (std::string(prefix) + "_XXXXXX"))
+ .string();
+ if (::mkdtemp(path_template.data()) == nullptr) {
+ const int error = errno;
+ throw std::filesystem::filesystem_error(
+ "cannot create temporary directory", path_template,
+ std::error_code(error, std::generic_category()));
+ }
+ _path = std::move(path_template);
+ }
+
+ ~ScopedTempDirectory() {
+ std::error_code error;
+ std::filesystem::remove_all(_path, error);
+ }
+
+ ScopedTempDirectory(const ScopedTempDirectory&) = delete;
+ ScopedTempDirectory& operator=(const ScopedTempDirectory&) = delete;
+
+ const std::filesystem::path& path() const { return _path; }
+
+private:
+ std::filesystem::path _path;
+};
+
+} // namespace doris::test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 21/35: branch-4.1:[feat](paimon) support paimon-rust reader on BE (#66227)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e98db0594f7a0ed47e5aaafc53a05dc3b222a349
Author: Fangyuan Deng <98...@qq.com>
AuthorDate: Wed Sep 30 21:08:37 2026 +0800
branch-4.1:[feat](paimon) support paimon-rust reader on BE (#66227)
### What problem does this PR solve?
Related PR: #65883 (for master , current closed)
**this pr add paimon rust reader both in scanner v1 and v2.
the scanner v1 code is just for legacy, will be removed later.**
Problem Summary:
Adds a third BE-side reader for Paimon FORMAT_JNI splits, backed by the
`paimon-rust` C bindings (`libpaimon_c.a`, statically linked). Rust
Reader is gated by the session variable
`enable_paimon_rust_reader` (default `false`).
Compared with the existing readers:
- **vs paimon-jni**: no JVM round-trip on the hot path — the split is
deserialized in-process and rows arrive via the Arrow C Data Interface,
which shortens the scanner critical section and removes JNI copy cost.
**Architecture**
FE (`PaimonScanNode`): when `enable_paimon_rust_reader` is on and the
split
is a `DataSplit`, encode it with paimon's native binary serialization
(`PaimonUtil.encodeDataSplitToString`), stamp the range with
`TPaimonReaderType.PAIMON_RUST`, and attach `db_name` / `table_name` /
`paimon_table` so BE can open the table through a catalog.
BE (`FileScanner::_get_next_reader`): dispatches on `reader_type`. On
`PAIMON_RUST` it instantiates `PaimonRustReader`, which:
1. `paimon_catalog_create({warehouse, metastore=filesystem, ...})`
- warehouse is derived from the table root path by stripping
`/<db>.db/<table>` (paimon FS convention).
2. `paimon_identifier_new(db, tbl)` → `paimon_catalog_get_table`
3. `paimon_table_new_read_builder` → projection (case-insensitive) →
optional filter from `PaimonRustPredicateConverter` (translates Doris
push-down conjuncts into paimon-rust predicates, best-effort so any
non-representable conjunct is silently dropped and re-applied by the
engine).
4. `paimon_plan_from_split_bytes(split_bytes)` — one BE scanner reads
exactly its FE-planned split (no re-planning).
5. `paimon_table_read_to_arrow` → streamed arrow record batches → block.
Partition columns are back-filled in `on_after_read_block` from
FE-provided `columns_from_path*`, mirroring `PaimonCppReader`. OSS/S3
hadoop-style credential keys (`fs.oss.*` / `fs.s3a.*`) are remapped to
`AWS_ACCESS_KEY` / `AWS_ENDPOINT` / ... so paimon-rust's FileIO can
authenticate.
**Thirdparty**
- Introduces `paimon-rust-2fb5e49` (built with Rust 1.91.0, feature
`paimon/storage-hdfs`, `--locked`).
- Auto-installs `cbindgen` and regenerates the C header from the Rust
extern "C" surface via a locally-written `cbindgen.toml`
(`language=C, include_guard=PAIMON_C_H, pragma_once=true,
cpp_compat=true`),
so the shipped `paimon.h` always matches the built `.a`.
### Release note
Add `enable_paimon_rust_reader` session variable. When enabled, Paimon
non-native reads on BE use the paimon-rust reader instead of paimon-jni
/
paimon-cpp.
### Check List (For Author)
- Test
- [x] Unit Test
- [x] Manual test (add detailed scripts or steps below)
1. `CREATE CATALOG paimon_fs PROPERTIES ("type"="paimon",
"paimon.catalog.type"="filesystem",
"warehouse"="file:///media/ssd1/test_paimon/warehouse/");` when using
"warehouse"="hdfs://.../warehouse/", add HADOOP_CONF_DIR=xx in be.conf
2. `SET enable_paimon_rust_reader=true; SET force_jni_scanner=1`
3. `SELECT` from a partitioned/DV paimon table; results must match
`enable_paimon_rust_reader=false and force_jni_scanner=0`.
4. `SELECT count(*)` — should hit table-level row count push-down.
5. `WHERE a=1 AND b>2` — BE INFO log should show `paimon-rust predicate
pushdown: applied`.
- [x] Regression test
- [ ] No need to test or manual test.
- Behavior changed:
- [x] No. Guarded by `enable_paimon_rust_reader` (default `false`); when
off, all paths are unchanged.
- Does this need documentation?
- [ ] No.
- [x] Yes. <!-- Add doris-website PR link here for the new session
variable -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---------
Co-authored-by: dengfangyuan <de...@kuaishou.com>
Co-authored-by: Claude Opus 5 (1M context) <no...@anthropic.com>
Co-authored-by: Gabriel <ga...@gmail.com>
---
be/CMakeLists.txt | 3 +-
be/cmake/thirdparty.cmake | 7 +
be/src/agent/heartbeat_server.cpp | 2 +
be/src/exec/scan/file_scanner.cpp | 19 +
be/src/exec/scan/file_scanner_v2.cpp | 15 +-
be/src/format_v2/table/paimon_reader.cpp | 62 +-
be/src/format_v2/table/paimon_reader.h | 18 +-
.../table/paimon_rust_predicate_converter.cpp | 840 +++++++++++
.../table/paimon_rust_predicate_converter.h | 161 ++
.../format_v2/table/paimon_rust_table_reader.cpp | 924 ++++++++++++
be/src/format_v2/table/paimon_rust_table_reader.h | 188 +++
be/test/exec/scan/file_scanner_v2_test.cpp | 42 +
.../test_data/paimon_rust_batch_size/README.md | 32 +
...-3b6dc957-0664-4ec4-946c-9535d228957a-0.parquet | Bin 0 -> 772 bytes
.../paimon_rust_batch_size/schema/schema-0 | 22 +
.../test_data/paimon_rust_batch_size/split.bin | Bin 0 -> 557 bytes
be/test/format_v2/table/paimon_reader_test.cpp | 167 ++-
.../table/paimon_rust_predicate_converter_test.cpp | 674 +++++++++
.../table/paimon_rust_table_reader_test.cpp | 712 +++++++++
build.sh | 17 +-
.../doris/common/profile/ExecutionProfile.java | 69 +-
.../org/apache/doris/common/profile/Profile.java | 68 +
.../doris/common/profile/SummaryProfile.java | 10 +
.../paimon/DorisToPaimonTypeVisitor.java | 17 +-
.../doris/datasource/paimon/PaimonScanParams.java | 59 +
.../apache/doris/datasource/paimon/PaimonUtil.java | 33 +
.../source/PaimonRustReaderCapabilities.java | 282 ++++
.../datasource/paimon/source/PaimonScanNode.java | 683 ++++++++-
.../java/org/apache/doris/qe/SessionVariable.java | 19 +
.../qe/runtime/PipelineExecutionTaskBuilder.java | 9 +-
.../main/java/org/apache/doris/system/Backend.java | 11 +
.../org/apache/doris/system/BackendHbResponse.java | 10 +
.../java/org/apache/doris/system/HeartbeatMgr.java | 5 +-
.../common/profile/ProfileCompletionStateTest.java | 217 +++
.../doris/common/profile/ProfileManagerTest.java | 29 +
.../paimon/DorisToPaimonTypeVisitorTest.java | 78 +
.../source/PaimonRustBackendCapabilityTest.java | 118 ++
.../source/PaimonRustReaderCompatibilityTest.java | 496 +++++++
.../source/PaimonRustReaderEvolutionTest.java | 184 +++
.../source/PaimonRustReaderLegacySchemaTest.java | 101 ++
.../paimon/source/PaimonScanNodeTest.java | 1545 +++++++++++++++++++-
.../runtime/PipelineExecutionTaskBuilderTest.java | 96 ++
.../system/BackendPaimonRustCapabilityTest.java | 70 +
gensrc/thrift/HeartbeatService.thrift | 1 +
gensrc/thrift/PaloInternalService.thrift | 3 +
gensrc/thrift/PlanNodes.thrift | 7 +
.../test_paimon_rust_reader_cast_predicates.groovy | 192 +++
.../test_paimon_rust_reader_compatibility.groovy | 308 ++++
.../test_paimon_rust_reader_eq_for_null.groovy | 555 +++++++
...test_paimon_rust_reader_schema_evolution.groovy | 134 ++
.../paimon/test_paimon_rust_reader_v2.groovy | 146 ++
thirdparty/build-thirdparty.sh | 182 ++-
thirdparty/test/paimon-build-test.py | 279 ++++
thirdparty/vars.sh | 7 +
54 files changed, 9840 insertions(+), 88 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 0da7cdd2b0f..47e69d3907e 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -744,7 +744,8 @@ if (GLIBC_COMPATIBILITY)
glibc-compatibility-explicit
glibc-compatibility
-lm
- lance_c)
+ lance_c
+ paimon_c)
endif()
if (NOT OS_MACOSX)
diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
index 7f518a8cb5e..b67c8e9d0aa 100644
--- a/be/cmake/thirdparty.cmake
+++ b/be/cmake/thirdparty.cmake
@@ -117,6 +117,13 @@ add_thirdparty(lance_c LIB64 NOTADD)
if (NOT GLIBC_COMPATIBILITY)
list(APPEND COMMON_THIRDPARTY m lance_c)
endif()
+# libpaimon_c.a (built from paimon-rust) brings in Rust compiler_builtins that
+# would otherwise steal libm symbols. Place libm before it so the final linker
+# resolves C math symbols from the system library first.
+add_thirdparty(paimon_c LIB64 NOTADD)
+if (NOT GLIBC_COMPATIBILITY)
+ list(APPEND COMMON_THIRDPARTY m paimon_c)
+endif()
add_thirdparty(brpc LIB64)
add_thirdparty(rocksdb)
add_thirdparty(cyrus-sasl LIBNAME "lib/libsasl2.a")
diff --git a/be/src/agent/heartbeat_server.cpp b/be/src/agent/heartbeat_server.cpp
index 441218f469c..5cf8c7df01d 100644
--- a/be/src/agent/heartbeat_server.cpp
+++ b/be/src/agent/heartbeat_server.cpp
@@ -95,6 +95,8 @@ void HeartbeatServer::heartbeat(THeartbeatResult& heartbeat_result,
heartbeat_result.backend_info.__set_fragment_last_active_time(
get_fragment_last_active_time());
heartbeat_result.backend_info.__set_be_mem(MemInfo::physical_mem());
+ // Reader enum support cannot be inferred from a release version during rolling upgrades.
+ heartbeat_result.backend_info.__set_supports_paimon_rust_reader(true);
}
watch.stop();
if (watch.elapsed_time() > 1000L * 1000L * 1000L) {
diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp
index 34cf0928553..ee5203f16f0 100644
--- a/be/src/exec/scan/file_scanner.cpp
+++ b/be/src/exec/scan/file_scanner.cpp
@@ -1117,6 +1117,25 @@ Status FileScanner::_get_next_reader() {
_cur_reader = std::move(mc_reader);
} else if (range.__isset.table_format_params &&
range.table_format_params.table_format_type == "paimon") {
+ const auto& paimon_params = range.table_format_params.paimon_params;
+ if (paimon_params.__isset.reader_type) {
+ switch (paimon_params.reader_type) {
+ case TPaimonReaderType::PAIMON_JNI:
+ break;
+ case TPaimonReaderType::PAIMON_RUST:
+ return Status::NotSupported(
+ "paimon rust reader requires enable_file_scanner_v2=true; the V1 "
+ "FileScanner no longer supports PAIMON_RUST splits");
+ case TPaimonReaderType::PAIMON_NATIVE:
+ return Status::InternalError(
+ "invalid PAIMON_NATIVE reader_type for paimon FORMAT_JNI split, "
+ "possibly caused by FE/BE protocol mismatch");
+ default:
+ return Status::InternalError(
+ "unknown paimon reader_type for paimon FORMAT_JNI split, possibly "
+ "caused by FE/BE protocol mismatch");
+ }
+ }
_cur_reader = PaimonJniReader::create_unique(_file_slot_descs, _state, _profile,
range, _params);
init_status = ((PaimonJniReader*)(_cur_reader.get()))->init_reader();
diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp
index 3f824b57fc1..ab055f61dc3 100644
--- a/be/src/exec/scan/file_scanner_v2.cpp
+++ b/be/src/exec/scan/file_scanner_v2.cpp
@@ -150,8 +150,19 @@ bool is_supported_jni_table_format(const TFileRangeDesc& range) {
}
const auto& params = range.table_format_params.paimon_params;
if (params.__isset.reader_type) {
- return params.reader_type == TPaimonReaderType::PAIMON_JNI &&
- params.__isset.paimon_split;
+ if (params.reader_type == TPaimonReaderType::PAIMON_JNI) {
+ return params.__isset.paimon_split;
+ }
+ if (params.reader_type == TPaimonReaderType::PAIMON_RUST) {
+ // The rust leaf reader inside PaimonHybridReader consumes the logical DataSplit
+ // directly. It needs the schema-json pipeline fields; a missing one fails fast
+ // in PaimonRustTableReader::prepare_split with a precise error.
+ return params.__isset.paimon_split && params.__isset.paimon_table_schema_json &&
+ !params.paimon_table_schema_json.empty();
+ }
+ // V2 cannot pass a logical DataSplit through a raw native child without silently
+ // dropping its multi-file semantics, so PAIMON_CPP must remain on the V1 fallback.
+ return false;
}
if (params.__isset.paimon_split) {
// Before reader_type was added, an encoded split unambiguously selected the Java
diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp
index 9a724716c35..2ec6bde53f2 100644
--- a/be/src/format_v2/table/paimon_reader.cpp
+++ b/be/src/format_v2/table/paimon_reader.cpp
@@ -35,6 +35,7 @@
#include "format/table/paimon_reader.h"
#include "format_v2/column_mapper.h"
#include "format_v2/jni/paimon_jni_reader.h"
+#include "format_v2/table/paimon_rust_table_reader.h"
#include "format_v2/table/schema_history_util.h"
#include "gen_cpp/PlanNodes_types.h"
@@ -326,15 +327,18 @@ Status PaimonHybridReader::prepare_split(const format::SplitReadOptions& options
// timer around the first native or JNI child and double-count that initialization.
RETURN_IF_ERROR(_ensure_current_split_reader(options));
DORIS_CHECK(_current_split_reader != nullptr);
- if (!_is_jni_split(options.current_range)) {
- auto native_options = options;
- // Legacy FE plans wrap native files in FORMAT_JNI; normalize the child contract so the
- // physical reader does not overwrite its recovered Parquet/ORC format with that wrapper.
- RETURN_IF_ERROR(
- _to_file_format(options.current_range, &native_options.current_split_format));
- return _current_split_reader->prepare_split(native_options);
- }
- return _current_split_reader->prepare_split(options);
+ // Rust splits (like JNI splits) stay on the serialized DataSplit path. They must not go
+ // through the native format normalization below: FE sets paimon_params.file_format on
+ // serialized splits too, so FORMAT_JNI would be rewritten to PARQUET and dispatched to the
+ // native child.
+ if (_is_jni_split(options.current_range) || _is_rust_split(options.current_range)) {
+ return _current_split_reader->prepare_split(options);
+ }
+ auto native_options = options;
+ // Legacy FE plans wrap native files in FORMAT_JNI; normalize the child contract so the
+ // physical reader does not overwrite its recovered Parquet/ORC format with that wrapper.
+ RETURN_IF_ERROR(_to_file_format(options.current_range, &native_options.current_split_format));
+ return _current_split_reader->prepare_split(native_options);
}
Status PaimonHybridReader::build_physical_splits(const FileScanSplit& source_split,
@@ -391,6 +395,12 @@ Status PaimonHybridReader::close() {
close_status = std::move(status);
}
}
+ if (_rust_reader != nullptr) {
+ auto status = _rust_reader->close();
+ if (!status.ok() && close_status.ok()) {
+ close_status = std::move(status);
+ }
+ }
_current_split_reader = nullptr;
return close_status;
}
@@ -403,17 +413,36 @@ void PaimonHybridReader::set_batch_size(size_t batch_size) {
if (_jni_reader != nullptr) {
_jni_reader->set_batch_size(_batch_size);
}
+ if (_rust_reader != nullptr) {
+ _rust_reader->set_batch_size(_batch_size);
+ }
}
int64_t PaimonHybridReader::condition_cache_hit_count() const {
- // Both children survive split switches, so the wrapper must publish their cumulative totals;
+ // Child readers survive split switches, so the wrapper must publish their cumulative totals;
// returning only the active child would make FileScannerV2's monotonic delta go backwards.
return (_native_reader == nullptr ? 0 : _native_reader->condition_cache_hit_count()) +
- (_jni_reader == nullptr ? 0 : _jni_reader->condition_cache_hit_count());
+ (_jni_reader == nullptr ? 0 : _jni_reader->condition_cache_hit_count()) +
+ (_rust_reader == nullptr ? 0 : _rust_reader->condition_cache_hit_count());
}
Status PaimonHybridReader::_ensure_current_split_reader(const format::SplitReadOptions& options) {
- if (_is_jni_split(options.current_range)) {
+ if (_is_rust_split(options.current_range)) {
+ DCHECK(options.current_split_format == format::FileFormat::JNI);
+ if (_rust_reader == nullptr) {
+#ifdef BE_TEST
+ if (_test_rust_reader_factory) {
+ _rust_reader = _test_rust_reader_factory();
+ } else {
+ _rust_reader = std::make_unique<format::paimon::PaimonRustTableReader>();
+ }
+#else
+ _rust_reader = std::make_unique<format::paimon::PaimonRustTableReader>();
+#endif
+ RETURN_IF_ERROR(_init_child_reader(_rust_reader.get(), format::FileFormat::JNI));
+ }
+ _current_split_reader = _rust_reader.get();
+ } else if (_is_jni_split(options.current_range)) {
DCHECK(options.current_split_format == format::FileFormat::JNI);
if (_jni_reader == nullptr) {
#ifdef BE_TEST
@@ -500,6 +529,15 @@ bool PaimonHybridReader::_is_jni_split(const TFileRangeDesc& range) {
(!params.__isset.reader_type || params.reader_type == TPaimonReaderType::PAIMON_JNI);
}
+bool PaimonHybridReader::_is_rust_split(const TFileRangeDesc& range) {
+ if (!range.__isset.table_format_params || !range.table_format_params.__isset.paimon_params) {
+ return false;
+ }
+ const auto& params = range.table_format_params.paimon_params;
+ return params.__isset.paimon_split && params.__isset.reader_type &&
+ params.reader_type == TPaimonReaderType::PAIMON_RUST;
+}
+
Status PaimonHybridReader::_to_file_format(const TFileRangeDesc& range,
format::FileFormat* file_format) {
DORIS_CHECK(file_format != nullptr);
diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h
index 25c6eb75b61..f0ac448828b 100644
--- a/be/src/format_v2/table/paimon_reader.h
+++ b/be/src/format_v2/table/paimon_reader.h
@@ -89,6 +89,7 @@ public:
#ifdef BE_TEST
static bool TEST_is_jni_split(const TFileRangeDesc& range) { return _is_jni_split(range); }
+ static bool TEST_is_rust_split(const TFileRangeDesc& range) { return _is_rust_split(range); }
static Status TEST_to_file_format(const TFileRangeDesc& range,
format::FileFormat* file_format) {
return _to_file_format(range, file_format);
@@ -96,19 +97,25 @@ public:
void TEST_install_batch_size_children() {
_native_reader = std::make_unique<format::TableReader>();
_jni_reader = std::make_unique<format::TableReader>();
+ _rust_reader = std::make_unique<format::TableReader>();
}
- std::pair<size_t, size_t> TEST_child_batch_sizes() const {
- return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()};
+ std::tuple<size_t, size_t, size_t> TEST_child_batch_sizes() const {
+ return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size(),
+ _rust_reader->TEST_batch_size()};
}
- void TEST_set_child_condition_cache_hits(int64_t native_hits, int64_t jni_hits) {
+ void TEST_set_child_condition_cache_hits(int64_t native_hits, int64_t jni_hits,
+ int64_t rust_hits) {
_native_reader->TEST_set_condition_cache_hit_count(native_hits);
_jni_reader->TEST_set_condition_cache_hit_count(jni_hits);
+ _rust_reader->TEST_set_condition_cache_hit_count(rust_hits);
}
void TEST_set_child_reader_factories(
std::function<std::unique_ptr<format::TableReader>()> native_factory,
- std::function<std::unique_ptr<format::TableReader>()> jni_factory) {
+ std::function<std::unique_ptr<format::TableReader>()> jni_factory,
+ std::function<std::unique_ptr<format::TableReader>()> rust_factory) {
_test_native_reader_factory = std::move(native_factory);
_test_jni_reader_factory = std::move(jni_factory);
+ _test_rust_reader_factory = std::move(rust_factory);
}
#endif
@@ -117,14 +124,17 @@ private:
Status _init_child_reader(format::TableReader* reader, format::FileFormat file_format);
Status _clone_conjuncts(VExprContextSPtrs* conjuncts) const;
static bool _is_jni_split(const TFileRangeDesc& range);
+ static bool _is_rust_split(const TFileRangeDesc& range);
static Status _to_file_format(const TFileRangeDesc& range, format::FileFormat* file_format);
std::unique_ptr<format::TableReader> _native_reader; // handle parquet/orc native splits
std::unique_ptr<format::TableReader> _jni_reader; // handle serialized JNI splits
+ std::unique_ptr<format::TableReader> _rust_reader; // handle serialized rust splits
format::TableReader* _current_split_reader = nullptr;
#ifdef BE_TEST
std::function<std::unique_ptr<format::TableReader>()> _test_native_reader_factory;
std::function<std::unique_ptr<format::TableReader>()> _test_jni_reader_factory;
+ std::function<std::unique_ptr<format::TableReader>()> _test_rust_reader_factory;
#endif
};
diff --git a/be/src/format_v2/table/paimon_rust_predicate_converter.cpp b/be/src/format_v2/table/paimon_rust_predicate_converter.cpp
new file mode 100644
index 00000000000..8b1e280399e
--- /dev/null
+++ b/be/src/format_v2/table/paimon_rust_predicate_converter.cpp
@@ -0,0 +1,840 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format_v2/table/paimon_rust_predicate_converter.h"
+
+#include <algorithm>
+#include <cctype>
+#include <limits>
+#include <memory>
+#include <utility>
+
+#include "common/logging.h"
+#include "core/column/column_const.h"
+#include "core/column/column_nullable.h"
+#include "core/data_type/data_type.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/field.h"
+#include "core/types.h"
+#include "core/value/decimalv2_value.h"
+#include "core/value/timestamptz_value.h"
+#include "core/value/vdatetime_value.h"
+#include "exprs/runtime_filter_expr.h"
+#include "exprs/vcompound_pred.h"
+#include "exprs/vdirect_in_predicate.h"
+#include "exprs/vectorized_fn_call.h"
+#include "exprs/vexpr.h"
+#include "exprs/vin_predicate.h"
+#include "exprs/vliteral.h"
+#include "exprs/vslot_ref.h"
+
+namespace doris {
+
+namespace {
+// paimon_datum tags (see paimon.h / bindings/c/src/table.rs::datum_from_c).
+constexpr int32_t kTagBool = 0;
+constexpr int32_t kTagTinyInt = 1;
+constexpr int32_t kTagSmallInt = 2;
+constexpr int32_t kTagInt = 3;
+constexpr int32_t kTagLong = 4;
+constexpr int32_t kTagDouble = 6;
+constexpr int32_t kTagString = 7;
+constexpr int32_t kTagDate = 8;
+constexpr int32_t kTagTimestamp = 10;
+constexpr int32_t kTagDecimal = 12;
+constexpr int32_t kTagBytes = 13;
+
+// paimon decimal precision ceiling (paimon::Decimal::MAX_PRECISION).
+constexpr int32_t kPaimonDecimalMaxPrecision = 38;
+
+// RAII for an owned paimon_predicate*. and/or/not consume their inputs, so we
+// release() before handing pointers to them.
+struct predicate_deleter {
+ void operator()(paimon_predicate* p) const {
+ if (p) {
+ paimon_predicate_free(p);
+ }
+ }
+};
+using predicate_ptr = std::unique_ptr<paimon_predicate, predicate_deleter>;
+
+// RAII for an owned paimon_error*.
+struct error_deleter {
+ void operator()(paimon_error* p) const {
+ if (p) {
+ paimon_error_free(p);
+ }
+ }
+};
+using error_ptr = std::unique_ptr<paimon_error, error_deleter>;
+
+// Render a paimon_error into a string. Takes ownership of `err` via RAII so it
+// is freed on every return path. Safe to call with nullptr.
+std::string consume_predicate_error(paimon_error* err) {
+ error_ptr owned(err);
+ if (!owned) {
+ return "unknown error";
+ }
+ std::string msg;
+ if (owned->message.data != nullptr && owned->message.len > 0) {
+ msg.assign(reinterpret_cast<const char*>(owned->message.data), owned->message.len);
+ }
+ return "code=" + std::to_string(owned->code) + ", msg=" + msg;
+}
+} // namespace
+
+PaimonRustPredicateConverter::PaimonRustPredicateConverter(
+ const std::vector<std::string>& column_names, const std::vector<DataTypePtr>& column_types,
+ const paimon_table* table)
+ : _table(table) {
+ DORIS_CHECK(column_names.size() == column_types.size());
+ _columns_by_name.reserve(column_names.size());
+ for (size_t i = 0; i < column_names.size(); ++i) {
+ _columns_by_name.emplace(_normalize_name(column_names[i]),
+ std::make_pair(column_names[i], column_types[i]));
+ }
+ // Paimon TIMESTAMP (wall clock) is stored as epoch-millis-of-the-wall-time
+ // and the DateTimeV2 serde decodes timezone-naive arrow values in UTC, so
+ // timestamp literals convert wall->epoch in UTC. utc_time_zone() needs no
+ // tzdata lookup, so the conversion cannot silently fall back to a
+ // machine-local zone.
+ _utc_tz = cctz::utc_time_zone();
+}
+
+paimon_predicate* PaimonRustPredicateConverter::build(const VExprContextSPtrs& conjuncts) {
+ _converted_conjuncts = 0;
+ _converted_runtime_filters = 0;
+ if (_table == nullptr) {
+ return nullptr;
+ }
+ predicate_ptr result;
+ for (const auto& conjunct : conjuncts) {
+ if (!conjunct || !conjunct->root()) {
+ continue;
+ }
+ auto root = conjunct->root();
+ const bool is_runtime_filter = root->is_rf_wrapper();
+ if (root->is_rf_wrapper()) {
+ if (auto impl = root->get_impl()) {
+ // A null-aware runtime filter (an EQ_FOR_NULL join) must stay
+ // residual: its wrapper execution restores NULL probe rows to
+ // true (RuntimeFilterExpr::change_null_to_true), while the
+ // unwrapped impl — rebuilt as an ordinary IN predicate through
+ // VDirectInPredicate::get_slot_in_expr — treats NULL as
+ // not-in-set and would prune the NULL probes permanently
+ // before the join sees them. Keep the wrapper itself: it fails
+ // every dispatch below, so the conjunct stays in the residual,
+ // and it is safe to execute on selected rows, so later
+ // conjuncts keep pushing. The lance pushdown declines
+ // is_null_aware() filters for the same reason.
+ // is_null_aware() is concrete on RuntimeFilterExpr (the only
+ // class whose is_rf_wrapper() is true), so the dynamic_cast
+ // never fails in practice; the null guard keeps the unwrap for
+ // any future wrapper shape.
+ auto* rf_wrapper = dynamic_cast<RuntimeFilterExpr*>(root.get());
+ if (rf_wrapper == nullptr || !rf_wrapper->is_null_aware()) {
+ root = impl;
+ }
+ }
+ }
+ // Preserve a safe prefix of the conjunct order: a later pushed
+ // predicate (e.g. an arrived IN runtime filter) could otherwise prune
+ // rows on which an earlier error-preserving conjunct —
+ // assert_true(...), a failing cast, ... — must still raise. The v1
+ // partition-pruning path (FileScanner::_init_runtime_filter_partition_
+ // prune_ctxs) stops at is_safe_to_execute_on_selected_rows() for the
+ // same reason, so a convertible predicate after an unsafe conjunct
+ // must not be pushed. Safe conjuncts that cannot be converted keep
+ // the old skip: they cannot raise, so pruning rows before they are
+ // evaluated as the residual never loses an error.
+ if (!_is_safe_to_push(root)) {
+ break;
+ }
+ predicate_ptr pred(_convert_expr(root));
+ if (!pred) {
+ continue;
+ }
+ ++_converted_conjuncts;
+ _converted_runtime_filters += is_runtime_filter;
+ if (!result) {
+ result = std::move(pred);
+ } else {
+ // and consumes both inputs regardless of success.
+ result.reset(paimon_predicate_and(result.release(), pred.release()));
+ if (!result) {
+ _converted_conjuncts = 0;
+ _converted_runtime_filters = 0;
+ return nullptr;
+ }
+ }
+ }
+ return result.release();
+}
+
+bool PaimonRustPredicateConverter::_is_safe_to_push(const VExprSPtr& expr) {
+ if (expr->is_safe_to_execute_on_selected_rows()) {
+ return true;
+ }
+ // VectorizedFnCall::is_safe_to_execute_on_selected_rows() admits a fixed
+ // whitelist that does not include `like`, so a like conjunct would always
+ // stop the safe prefix here and never reach _convert_like. A like call
+ // whose children are themselves safe is just as total as the whitelisted
+ // comparisons: it only compares strings, and its regex is built by escaping
+ // the pattern operand (FunctionLike::convert_like_pattern), so no scanned
+ // value can make it raise — a malformed pattern or escape operand fails
+ // at open() on every execution, regardless of which rows survive. Admit
+ // it here rather than widening the generic whitelist, which gates shared
+ // BE pushdown paths outside this reader's scope.
+ auto* fn = dynamic_cast<VectorizedFnCall*>(expr.get());
+ if (fn == nullptr || _normalize_name(fn->function_name()) != "like") {
+ return false;
+ }
+ for (uint16_t i = 0; i < fn->get_num_children(); ++i) {
+ if (!fn->get_child(i)->is_safe_to_execute_on_selected_rows()) {
+ return false;
+ }
+ }
+ return true;
+}
+
+paimon_predicate* PaimonRustPredicateConverter::_convert_expr(const VExprSPtr& expr) {
+ if (!expr) {
+ return nullptr;
+ }
+
+ // Casts are not unwrapped anywhere (predicate root included): a cast node
+ // fails every dispatch below and the conjunct stays in the Doris residual,
+ // mirroring the FE converter, which keeps casted expressions unconverted.
+ if (auto* direct_in = dynamic_cast<VDirectInPredicate*>(expr.get())) {
+ VExprSPtr in_expr;
+ if (direct_in->get_slot_in_expr(in_expr)) {
+ return _convert_in(in_expr);
+ }
+ return nullptr;
+ }
+
+ if (dynamic_cast<VInPredicate*>(expr.get()) != nullptr) {
+ return _convert_in(expr);
+ }
+
+ switch (expr->op()) {
+ case TExprOpcode::COMPOUND_AND:
+ case TExprOpcode::COMPOUND_OR:
+ return _convert_compound(expr);
+ case TExprOpcode::COMPOUND_NOT:
+ return nullptr;
+ case TExprOpcode::EQ:
+ case TExprOpcode::EQ_FOR_NULL:
+ case TExprOpcode::NE:
+ case TExprOpcode::GE:
+ case TExprOpcode::GT:
+ case TExprOpcode::LE:
+ case TExprOpcode::LT:
+ return _convert_binary(expr);
+ default:
+ break;
+ }
+
+ if (auto* fn = dynamic_cast<VectorizedFnCall*>(expr.get())) {
+ auto fn_name = _normalize_name(fn->function_name());
+ if (fn_name == "is_null_pred" || fn_name == "is_not_null_pred") {
+ return _convert_is_null(expr, fn_name);
+ }
+ if (fn_name == "like") {
+ return _convert_like(expr);
+ }
+ }
+
+ return nullptr;
+}
+
+paimon_predicate* PaimonRustPredicateConverter::_convert_compound(const VExprSPtr& expr) {
+ if (!expr || expr->get_num_children() != 2) {
+ return nullptr;
+ }
+ predicate_ptr left(_convert_expr(expr->get_child(0)));
+ if (!left) {
+ return nullptr;
+ }
+ predicate_ptr right(_convert_expr(expr->get_child(1)));
+ if (!right) {
+ return nullptr;
+ }
+
+ if (expr->op() == TExprOpcode::COMPOUND_AND) {
+ return paimon_predicate_and(left.release(), right.release());
+ }
+ if (expr->op() == TExprOpcode::COMPOUND_OR) {
+ return paimon_predicate_or(left.release(), right.release());
+ }
+ return nullptr;
+}
+
+paimon_predicate* PaimonRustPredicateConverter::_convert_in(const VExprSPtr& expr) {
+ auto* in_pred = dynamic_cast<VInPredicate*>(expr.get());
+ // Runtime-generated IN sets can exceed VExpr's 16-bit child count. A wrapped
+ // count would push only a prefix and permanently discard valid probe rows.
+ if (!in_pred || expr->children().size() < 2 ||
+ expr->children().size() > std::numeric_limits<uint16_t>::max()) {
+ return nullptr;
+ }
+ auto field_meta = _resolve_field(expr->get_child(0));
+ if (!field_meta) {
+ return nullptr;
+ }
+
+ const auto num_values = expr->get_num_children() - 1;
+ // Reserve up front so the backing strings never reallocate: each datum's
+ // str_data points into storages[i], which must stay stable.
+ std::vector<std::string> storages;
+ std::vector<paimon_datum> datums;
+ storages.reserve(num_values);
+ datums.reserve(num_values);
+ for (uint16_t i = 1; i < expr->get_num_children(); ++i) {
+ // Casted list values are rejected by _convert_literal (the same rule
+ // as the binary RHS): with debug_skip_fold_constant the cast reaches
+ // the BE un-folded, and unwrapping it would filter on the pre-cast
+ // value — in `amount IN (CAST(1.24 AS DECIMAL(10,1)))` Doris keeps
+ // the 1.2 rows while the unwrapped 1.24 push removes them. Rejecting
+ // the value rejects the whole predicate; the residual applies the
+ // cast correctly.
+ auto holder = _convert_literal(expr->get_child(i), field_meta->type);
+ if (!holder) {
+ return nullptr;
+ }
+ storages.emplace_back(std::move(holder->storage));
+ paimon_datum datum = holder->datum;
+ _bind_datum_storage(&datum, storages.back());
+ datums.emplace_back(datum);
+ }
+
+ if (datums.empty()) {
+ return nullptr;
+ }
+ if (in_pred->is_not_in()) {
+ return _take(paimon_predicate_is_not_in(_table, field_meta->column.c_str(), datums.data(),
+ datums.size()));
+ }
+ return _take(paimon_predicate_is_in(_table, field_meta->column.c_str(), datums.data(),
+ datums.size()));
+}
+
+paimon_predicate* PaimonRustPredicateConverter::_convert_binary(const VExprSPtr& expr) {
+ if (!expr || expr->get_num_children() != 2) {
+ return nullptr;
+ }
+ auto field_meta = _resolve_field(expr->get_child(0));
+ if (!field_meta) {
+ return nullptr;
+ }
+ const char* column = field_meta->column.c_str();
+
+ // Convert the RHS first so EQ_FOR_NULL (<=>) only converts when the RHS is
+ // a convertible literal, mirroring the FE converter, which rejects a
+ // non-literal RHS. A column-to-column `a <=> b` must therefore stay in the
+ // Doris residual: it has no single-column rust predicate, and pushing
+ // `a IS NULL` would wrongly discard rows like (1, 1) — rows dropped by the
+ // pushed filter cannot be recovered by the residual conjunct.
+ auto holder = _convert_literal(expr->get_child(1), field_meta->type);
+ if (!holder) {
+ return nullptr;
+ }
+
+ if (expr->op() == TExprOpcode::EQ_FOR_NULL) {
+ return _take(paimon_predicate_is_null(_table, column));
+ }
+
+ // `holder` is a local, so its storage stays put for the duration of the call.
+ _bind_datum_storage(&holder->datum, holder->storage);
+ const paimon_datum& datum = holder->datum;
+
+ switch (expr->op()) {
+ case TExprOpcode::EQ:
+ return _take(paimon_predicate_equal(_table, column, datum));
+ case TExprOpcode::NE:
+ return _take(paimon_predicate_not_equal(_table, column, datum));
+ case TExprOpcode::GE:
+ return _take(paimon_predicate_greater_or_equal(_table, column, datum));
+ case TExprOpcode::GT:
+ return _take(paimon_predicate_greater_than(_table, column, datum));
+ case TExprOpcode::LE:
+ return _take(paimon_predicate_less_or_equal(_table, column, datum));
+ case TExprOpcode::LT:
+ return _take(paimon_predicate_less_than(_table, column, datum));
+ default:
+ break;
+ }
+ return nullptr;
+}
+
+paimon_predicate* PaimonRustPredicateConverter::_convert_is_null(const VExprSPtr& expr,
+ const std::string& fn_name) {
+ if (!expr || expr->get_num_children() != 1) {
+ return nullptr;
+ }
+ auto field_meta = _resolve_field(expr->get_child(0));
+ if (!field_meta) {
+ return nullptr;
+ }
+ if (fn_name == "is_not_null_pred") {
+ return _take(paimon_predicate_is_not_null(_table, field_meta->column.c_str()));
+ }
+ return _take(paimon_predicate_is_null(_table, field_meta->column.c_str()));
+}
+
+paimon_predicate* PaimonRustPredicateConverter::_convert_like(const VExprSPtr& expr) {
+ if (!expr || expr->get_num_children() < 2) {
+ return nullptr;
+ }
+ auto field_meta = _resolve_field(expr->get_child(0));
+ if (!field_meta || !_is_string_type(field_meta->type->get_primitive_type())) {
+ return nullptr;
+ }
+
+ auto pattern_opt = _extract_string_literal(expr->get_child(1));
+ if (!pattern_opt) {
+ return nullptr;
+ }
+ const std::string& pattern = *pattern_opt;
+
+ // Doris's 3-arg like(col, pattern, escape) form: the pinned rust
+ // predicate only supports the backslash escape (its builder rejects any
+ // other escape character), so only the default-escape shape converts.
+ // The 2-arg form always carries the Doris/SQL default '\'.
+ if (expr->get_num_children() >= 3) {
+ auto escape_opt = _extract_string_literal(expr->get_child(2));
+ if (!escape_opt || escape_opt->size() != 1 || (*escape_opt)[0] != '\\') {
+ return nullptr;
+ }
+ }
+
+ // The pinned rust like implements SQL LIKE (% = any run, _ = one char,
+ // \X = literal X for any X — arrow's like kernel), and its builder
+ // optimizes `prefix%` / `%suffix` / `%mid%` shapes internally, so every
+ // pattern without a backslash matches Doris semantics exactly. The one
+ // divergence is the escape handling for a backslash before an ordinary
+ // character: Doris keeps both characters (only \%, \_ and \\ are
+ // escapes; `a\qb` matches a backslash followed by q b) while rust
+ // consumes the backslash (`a\qb` matches aqb) — pushing such a pattern
+ // would wrongly prune the Doris-matching rows before the residual can
+ // see them. Reject exactly the divergent shapes: every backslash must
+ // precede %, _ or \ (identical literal semantics on both sides) or end
+ // the pattern (literal backslash on both sides); anything else stays in
+ // the residual.
+ for (size_t i = 0; i < pattern.size(); ++i) {
+ if (pattern[i] == '\\') {
+ if (i + 1 >= pattern.size()) {
+ continue; // trailing backslash: literal on both sides
+ }
+ char next = pattern[i + 1];
+ if (next != '%' && next != '_' && next != '\\') {
+ return nullptr;
+ }
+ ++i;
+ }
+ }
+
+ paimon_datum datum {};
+ datum.tag = kTagString;
+ // `pattern` outlives the build call (it is a const ref into the caller's
+ // optional), so the datum may point straight at it.
+ _bind_datum_storage(&datum, pattern);
+ return _take(paimon_predicate_like(_table, field_meta->column.c_str(), datum, '\\'));
+}
+
+std::optional<PaimonRustPredicateConverter::FieldMeta> PaimonRustPredicateConverter::_resolve_field(
+ const VExprSPtr& expr) const {
+ if (!expr) {
+ return std::nullopt;
+ }
+ // Mirror the FE converter's convertDorisExprToSlotRef: a casted column is
+ // rejected, never unwrapped. Stripping a lossy cast changes which rows match
+ // — for a DECIMAL(10,2) column, CAST(amount AS DECIMAL(10,1)) = 1.2 keeps
+ // the row 1.24 while the unwrapped `amount = 1.2` prunes it — and rows
+ // pruned by the pushed filter cannot be recovered by the Doris residual.
+ // The conjunct stays in the residual instead.
+ auto* slot_ref = dynamic_cast<VSlotRef*>(expr.get());
+ if (!slot_ref) {
+ return std::nullopt;
+ }
+ // FileScannerV2 rewrites conjunct VSlotRefs to table global indices, so slot_id
+ // is a position, not a slot id; resolve by the carried column name against the
+ // projected-column registry instead of the desc table.
+ auto it = _columns_by_name.find(_normalize_name(slot_ref->column_name()));
+ if (it == _columns_by_name.end()) {
+ return std::nullopt;
+ }
+ const auto& [column, type] = it->second;
+ if (!_is_supported_slot_type(type->get_primitive_type())) {
+ return std::nullopt;
+ }
+ return FieldMeta {column, type};
+}
+
+std::optional<PaimonRustPredicateConverter::DatumHolder>
+PaimonRustPredicateConverter::_convert_literal(const VExprSPtr& expr,
+ const DataTypePtr& column_type) const {
+ // A casted literal is rejected, never unwrapped: the cast is not executed
+ // here (with constant folding disabled — debug_skip_fold_constant — it
+ // reaches the BE un-folded), so unwrapping would push the pre-cast value
+ // while the Doris residual compares against the cast result. A
+ // scale-reducing cast makes the two disagree — `amount =
+ // CAST(1.24 AS DECIMAL(10,1))` keeps a stored 1.20 row in Doris (1.24
+ // rounds down to 1.2) but the unwrapped `amount = 1.24` push prunes it, and
+ // rows pruned by the pushed filter cannot be recovered by the residual.
+ // The conjunct stays in the residual, where the cast is evaluated exactly.
+ if (expr->node_type() == TExprNodeType::CAST_EXPR) {
+ return std::nullopt;
+ }
+ auto* literal = dynamic_cast<VLiteral*>(expr.get());
+ if (!literal) {
+ return std::nullopt;
+ }
+
+ auto literal_type = remove_nullable(literal->get_data_type());
+ PrimitiveType literal_primitive = literal_type->get_primitive_type();
+ PrimitiveType slot_primitive = column_type->get_primitive_type();
+
+ ColumnPtr col = literal->get_column_ptr()->convert_to_full_column_if_const();
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(*col)) {
+ if (nullable->is_null_at(0)) {
+ return std::nullopt;
+ }
+ col = nullable->get_nested_column_ptr();
+ }
+
+ Field field;
+ col->get(0, field);
+
+ DatumHolder holder;
+ paimon_datum& datum = holder.datum;
+
+ switch (slot_primitive) {
+ case TYPE_BOOLEAN: {
+ if (literal_primitive != TYPE_BOOLEAN) {
+ return std::nullopt;
+ }
+ datum.tag = kTagBool;
+ datum.int_val = static_cast<bool>(field.get<TYPE_BOOLEAN>()) ? 1 : 0;
+ return holder;
+ }
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT: {
+ if (!_is_integer_type(literal_primitive)) {
+ return std::nullopt;
+ }
+ int64_t value = 0;
+ switch (literal_primitive) {
+ case TYPE_TINYINT:
+ value = field.get<TYPE_TINYINT>();
+ break;
+ case TYPE_SMALLINT:
+ value = field.get<TYPE_SMALLINT>();
+ break;
+ case TYPE_INT:
+ value = field.get<TYPE_INT>();
+ break;
+ case TYPE_BIGINT:
+ value = field.get<TYPE_BIGINT>();
+ break;
+ default:
+ return std::nullopt;
+ }
+ datum.int_val = value;
+ switch (slot_primitive) {
+ case TYPE_TINYINT:
+ datum.tag = kTagTinyInt;
+ break;
+ case TYPE_SMALLINT:
+ datum.tag = kTagSmallInt;
+ break;
+ case TYPE_INT:
+ datum.tag = kTagInt;
+ break;
+ default:
+ datum.tag = kTagLong;
+ break;
+ }
+ return holder;
+ }
+ case TYPE_DOUBLE: {
+ if (literal_primitive != TYPE_DOUBLE && literal_primitive != TYPE_FLOAT) {
+ return std::nullopt;
+ }
+ datum.tag = kTagDouble;
+ datum.double_val = literal_primitive == TYPE_FLOAT
+ ? static_cast<double>(field.get<TYPE_FLOAT>())
+ : field.get<TYPE_DOUBLE>();
+ return holder;
+ }
+ case TYPE_DATE:
+ case TYPE_DATEV2: {
+ if (!_is_date_type(literal_primitive)) {
+ return std::nullopt;
+ }
+ int64_t seconds = 0;
+ if (literal_primitive == TYPE_DATE) {
+ const auto& dt = field.get<TYPE_DATE>();
+ if (!dt.is_valid_date()) {
+ return std::nullopt;
+ }
+ dt.unix_timestamp(&seconds, _utc_tz);
+ } else {
+ const auto& dt = field.get<TYPE_DATEV2>();
+ if (!dt.is_valid_date()) {
+ return std::nullopt;
+ }
+ dt.unix_timestamp(&seconds, _utc_tz);
+ }
+ datum.tag = kTagDate;
+ datum.int_val = _seconds_to_days(seconds);
+ return holder;
+ }
+ case TYPE_DATETIME:
+ case TYPE_DATETIMEV2: {
+ if (!_is_datetime_type(literal_primitive)) {
+ return std::nullopt;
+ }
+ datum.tag = kTagTimestamp;
+ if (literal_primitive == TYPE_DATETIME) {
+ const auto& dt = field.get<TYPE_DATETIME>();
+ if (!dt.is_valid_date()) {
+ return std::nullopt;
+ }
+ int64_t seconds = 0;
+ dt.unix_timestamp(&seconds, _utc_tz);
+ // No sub-second part in datetime (v1): millis only, nanos stays 0.
+ datum.int_val = seconds * 1000;
+ } else {
+ const auto& dt = field.get<TYPE_DATETIMEV2>();
+ if (!dt.is_valid_date()) {
+ return std::nullopt;
+ }
+ // ts is (seconds since epoch, the microsecond-of-second part);
+ // split it into the rust timestamp's (millis, nanos): truncating
+ // the sub-millisecond remainder would make an equality/IN predicate
+ // more selective than the original conjunct (e.g. .123456 pushed as
+ // .123000), wrongly discarding matching rows. DATETIMEV2 carries at
+ // most microseconds, and micros % 1000 * 1000 <= 999000 fits the
+ // nanos field, so the value is always representable exactly.
+ std::pair<int64_t, int64_t> ts;
+ dt.unix_timestamp(&ts, _utc_tz);
+ datum.int_val = ts.first * 1000 + ts.second / 1000;
+ datum.int_val2 = (ts.second % 1000) * 1000;
+ }
+ return holder;
+ }
+ case TYPE_VARCHAR:
+ case TYPE_STRING: {
+ if (!_is_string_type(literal_primitive)) {
+ return std::nullopt;
+ }
+ const auto& value = field.get<TYPE_STRING>();
+ datum.tag = kTagString;
+ holder.storage.assign(value.data(), value.size());
+ return holder;
+ }
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256: {
+ if (!_is_decimal_type(literal_primitive)) {
+ return std::nullopt;
+ }
+ int32_t precision = static_cast<int32_t>(literal_type->get_precision());
+ int32_t scale = static_cast<int32_t>(literal_type->get_scale());
+ if (precision <= 0 || precision > kPaimonDecimalMaxPrecision) {
+ return std::nullopt;
+ }
+
+ __int128 value = 0;
+ switch (literal_primitive) {
+ case TYPE_DECIMALV2:
+ value = field.get<TYPE_DECIMALV2>().value();
+ break;
+ case TYPE_DECIMAL32:
+ value = field.get<TYPE_DECIMAL32>().value;
+ break;
+ case TYPE_DECIMAL64:
+ value = field.get<TYPE_DECIMAL64>().value;
+ break;
+ case TYPE_DECIMAL128I:
+ value = field.get<TYPE_DECIMAL128I>().value;
+ break;
+ default:
+ return std::nullopt;
+ }
+ datum.tag = kTagDecimal;
+ // rust reassembles as ((int_val2 as i128) << 64) | (int_val as u64 as i128):
+ // int_val holds the low 64 bits, int_val2 the (sign-extended) high 64 bits.
+ datum.int_val = static_cast<int64_t>(static_cast<uint64_t>(value));
+ datum.int_val2 = static_cast<int64_t>(value >> 64);
+ datum.uint_val = static_cast<uint32_t>(precision);
+ datum.uint_val2 = static_cast<uint32_t>(scale);
+ return holder;
+ }
+ default:
+ break;
+ }
+ return std::nullopt;
+}
+
+std::optional<std::string> PaimonRustPredicateConverter::_extract_string_literal(
+ const VExprSPtr& expr) const {
+ // Same rule as _convert_literal: a bare string literal converts, a casted
+ // one is rejected — the cast is not executed here, so the pushed pattern
+ // must come from the literal's own value. A truncating cast (e.g. to
+ // CHAR(k)) would otherwise push a different prefix than the residual
+ // matches. Cast trees stay in the residual.
+ if (expr->node_type() == TExprNodeType::CAST_EXPR) {
+ return std::nullopt;
+ }
+ auto* literal = dynamic_cast<VLiteral*>(expr.get());
+ if (!literal) {
+ return std::nullopt;
+ }
+ auto literal_type = remove_nullable(literal->get_data_type());
+ if (!_is_string_type(literal_type->get_primitive_type())) {
+ return std::nullopt;
+ }
+
+ ColumnPtr col = literal->get_column_ptr()->convert_to_full_column_if_const();
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(*col)) {
+ if (nullable->is_null_at(0)) {
+ return std::nullopt;
+ }
+ col = nullable->get_nested_column_ptr();
+ }
+ Field field;
+ col->get(0, field);
+ const auto& value = field.get<TYPE_STRING>();
+ return std::string(value.data(), value.size());
+}
+
+paimon_predicate* PaimonRustPredicateConverter::_take(paimon_result_predicate result) {
+ if (result.error != nullptr) {
+ // A single leaf failing to build is not fatal: that conjunct is simply
+ // dropped from the pushed-down filter and the engine still re-applies it.
+ // Log at WARNING so it is visible without verbose logging enabled.
+ LOG(WARNING) << "paimon-rust build predicate failed: "
+ << consume_predicate_error(result.error);
+ return nullptr;
+ }
+ return result.predicate;
+}
+
+void PaimonRustPredicateConverter::_bind_datum_storage(paimon_datum* datum,
+ const std::string& storage) {
+ if (datum->tag == kTagString || datum->tag == kTagBytes) {
+ datum->str_data = reinterpret_cast<const uint8_t*>(storage.data());
+ datum->str_len = storage.size();
+ }
+}
+
+std::string PaimonRustPredicateConverter::_normalize_name(std::string_view name) {
+ std::string out(name);
+ std::transform(out.begin(), out.end(), out.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ return out;
+}
+
+int32_t PaimonRustPredicateConverter::_seconds_to_days(int64_t seconds) {
+ static constexpr int64_t kSecondsPerDay = 24 * 60 * 60;
+ int64_t days = seconds / kSecondsPerDay;
+ if (seconds < 0 && seconds % kSecondsPerDay != 0) {
+ --days;
+ }
+ return static_cast<int32_t>(days);
+}
+
+bool PaimonRustPredicateConverter::_is_integer_type(PrimitiveType type) {
+ switch (type) {
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool PaimonRustPredicateConverter::_is_string_type(PrimitiveType type) {
+ return type == TYPE_CHAR || type == TYPE_VARCHAR || type == TYPE_STRING;
+}
+
+bool PaimonRustPredicateConverter::_is_decimal_type(PrimitiveType type) {
+ switch (type) {
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool PaimonRustPredicateConverter::_is_date_type(PrimitiveType type) {
+ return type == TYPE_DATE || type == TYPE_DATEV2;
+}
+
+bool PaimonRustPredicateConverter::_is_datetime_type(PrimitiveType type) {
+ return type == TYPE_DATETIME || type == TYPE_DATETIMEV2;
+}
+
+bool PaimonRustPredicateConverter::_is_supported_slot_type(PrimitiveType type) {
+ switch (type) {
+ case TYPE_BOOLEAN:
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ case TYPE_STRING:
+ case TYPE_DATE:
+ case TYPE_DATEV2:
+ return true;
+ case TYPE_VARCHAR:
+ case TYPE_DATETIME:
+ case TYPE_DATETIMEV2:
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ // Rust filters files/statistics before reconciling them to the current
+ // schema. The slot cannot prove that historical VARCHAR bounds/decimal
+ // scales or source timestamp precision match Doris. A current-domain IN
+ // member could reject a row that truncation/rescaling would make equal.
+ // Keep these predicates residual until per-file equivalence is known.
+ return false;
+ case TYPE_DOUBLE:
+ case TYPE_FLOAT:
+ // Rust uses IEEE partial ordering; Doris orders NaN above finite values
+ // and equal to itself. Pruned NaN rows cannot be restored by a residual.
+ return false;
+ case TYPE_CHAR:
+ default:
+ return false;
+ }
+}
+
+} // namespace doris
diff --git a/be/src/format_v2/table/paimon_rust_predicate_converter.h b/be/src/format_v2/table/paimon_rust_predicate_converter.h
new file mode 100644
index 00000000000..76c76037b51
--- /dev/null
+++ b/be/src/format_v2/table/paimon_rust_predicate_converter.h
@@ -0,0 +1,161 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "cctz/time_zone.h"
+#include "core/data_type/data_type.h"
+#include "core/data_type/define_primitive_type.h"
+#include "exprs/vexpr_fwd.h"
+
+// paimon-rust C bindings. This header (and the paimon_* C API) is only pulled
+// into the rust-specific translation units (the converter, the rust reader and
+// its test); it is intentionally kept out of file_scanner and other generic BE
+// code, mirroring how paimon-cpp's headers are kept isolated.
+extern "C" {
+#include "paimon_rust/paimon.h"
+}
+
+namespace doris {} // namespace doris
+
+namespace doris {
+
+// Converts Doris push-down conjuncts into a paimon-rust filter predicate.
+//
+// This mirrors PaimonPredicateConverter (the paimon-cpp variant) but targets the
+// paimon-rust C bindings. The key API differences drive the design:
+// * paimon-cpp builds predicates standalone via PredicateBuilder using a field
+// index + paimon::FieldType + paimon::Literal.
+// * paimon-rust builds predicates from a live paimon_table handle and a column
+// name (`paimon_predicate_equal(table, column, datum)`), resolving the field
+// index and type from the table schema itself, and takes typed values as a
+// flat tagged `paimon_datum`.
+//
+// Because the predicate functions need the opened table, the converter is
+// constructed with the table handle and must run after the table is open. The
+// returned `paimon_predicate*` is owned by the caller, who must either transfer
+// it to `paimon_read_builder_with_filter` (which consumes it) or release it via
+// `paimon_predicate_free`.
+//
+// Conversion preserves a safe prefix: conjuncts that are not safe to execute
+// on selected rows (assert_true(...), failing casts — anything that must keep
+// raising on rows a later pushed predicate could prune) stop the pushdown, so
+// no conjunct after them is pushed either; safe conjuncts that cannot be
+// represented are skipped (the engine still re-applies the full conjunct list
+// to the scanned block), so a partial filter only ever prunes a superset and
+// never changes results.
+class PaimonRustPredicateConverter {
+public:
+ // Resolve fields by column name from the supplied registry. FileScannerV2
+ // rewrites conjunct VSlotRefs to table global indices (positions, not slot
+ // ids), so a desc-table lookup must not run; the rewritten refs carry the
+ // original column name and data type instead. `column_names` should list only
+ // the columns the reader actually reads (e.g. exclude partition keys); the
+ // caller guarantees names and types are parallel.
+ PaimonRustPredicateConverter(const std::vector<std::string>& column_names,
+ const std::vector<DataTypePtr>& column_types,
+ const paimon_table* table);
+
+ // Builds an owned predicate AND-combining every convertible conjunct, or
+ // nullptr when nothing can be pushed down.
+ paimon_predicate* build(const VExprContextSPtrs& conjuncts);
+
+ // Counts top-level conjuncts in the returned filter, not its internal nodes.
+ size_t converted_conjuncts() const { return _converted_conjuncts; }
+ size_t converted_runtime_filters() const { return _converted_runtime_filters; }
+
+private:
+ struct FieldMeta {
+ // Original file column name passed verbatim to the paimon_predicate_*
+ // functions (rust resolves the field index/type from the schema by name).
+ std::string column;
+ // Column type used for literal conversion.
+ DataTypePtr type;
+ };
+
+ // A datum plus its backing byte storage. String/Bytes datums hold a borrowed
+ // pointer into `storage`; the storage must outlive the predicate build call,
+ // so callers keep the holder alive and (re)bind str_data after final placement.
+ struct DatumHolder {
+ // The `{}` value-initializes every paimon_datum field to zero. Conversion
+ // code relies on this: each branch only sets the fields its tag uses (e.g.
+ // a Timestamp sets int_val but leaves int_val2/nanos at 0), so the rest
+ // must already be zeroed. Do not drop the `{}`.
+ paimon_datum datum {};
+ std::string storage;
+ };
+
+ // Each returns an owned paimon_predicate* (transfer or free) or nullptr.
+ paimon_predicate* _convert_expr(const VExprSPtr& expr);
+ paimon_predicate* _convert_compound(const VExprSPtr& expr);
+ paimon_predicate* _convert_in(const VExprSPtr& expr);
+ paimon_predicate* _convert_binary(const VExprSPtr& expr);
+ paimon_predicate* _convert_is_null(const VExprSPtr& expr, const std::string& fn_name);
+ paimon_predicate* _convert_like(const VExprSPtr& expr);
+
+ std::optional<FieldMeta> _resolve_field(const VExprSPtr& expr) const;
+ // Whether the conjunct may join the pushed prefix: total over its input
+ // domain, on top of the generic is_safe_to_execute_on_selected_rows gate.
+ static bool _is_safe_to_push(const VExprSPtr& expr);
+ std::optional<DatumHolder> _convert_literal(const VExprSPtr& expr,
+ const DataTypePtr& column_type) const;
+ std::optional<std::string> _extract_string_literal(const VExprSPtr& expr) const;
+
+ // Consumes a paimon_result_predicate: frees the error (logging it) and
+ // returns the predicate, or nullptr on error.
+ static paimon_predicate* _take(paimon_result_predicate result);
+ // Point a String/Bytes datum's str_data/str_len at the given stable storage.
+ static void _bind_datum_storage(paimon_datum* datum, const std::string& storage);
+
+ static std::string _normalize_name(std::string_view name);
+ static int32_t _seconds_to_days(int64_t seconds);
+ static bool _is_integer_type(PrimitiveType type);
+ static bool _is_string_type(PrimitiveType type);
+ static bool _is_decimal_type(PrimitiveType type);
+ static bool _is_date_type(PrimitiveType type);
+ static bool _is_datetime_type(PrimitiveType type);
+ // Only push types whose file and Doris comparison domains are equivalent
+ // without per-file schema bounds, scale, or timestamp precision metadata.
+ static bool _is_supported_slot_type(PrimitiveType type);
+
+ std::unordered_map<std::string, std::pair<std::string, DataTypePtr>> _columns_by_name;
+ const paimon_table* _table = nullptr;
+ cctz::time_zone _utc_tz;
+ size_t _converted_conjuncts = 0;
+ size_t _converted_runtime_filters = 0;
+
+#ifdef BE_TEST
+ // Exposed for unit tests so they can inspect the converted paimon_datum
+ // (tag / int_val / int_val2) without a live predicate build. Declared
+ // here, after DatumHolder, because the return type is not in the
+ // complete-class context and cannot reference a member declared later.
+ std::optional<DatumHolder> TEST_convert_literal(const VExprSPtr& expr,
+ const DataTypePtr& column_type) const {
+ return _convert_literal(expr, column_type);
+ }
+#endif
+};
+
+} // namespace doris
diff --git a/be/src/format_v2/table/paimon_rust_table_reader.cpp b/be/src/format_v2/table/paimon_rust_table_reader.cpp
new file mode 100644
index 00000000000..731508957bb
--- /dev/null
+++ b/be/src/format_v2/table/paimon_rust_table_reader.cpp
@@ -0,0 +1,924 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format_v2/table/paimon_rust_table_reader.h"
+
+#include <algorithm>
+#include <utility>
+
+#include "arrow/c/abi.h"
+#include "arrow/c/bridge.h"
+#include "arrow/record_batch.h"
+#include "arrow/result.h"
+#include "common/logging.h"
+#include "core/assert_cast.h"
+#include "core/block/block.h"
+#include "core/block/column_with_type_and_name.h"
+#include "core/column/column_const.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_string.h"
+#include "exprs/vexpr_context.h"
+#include "exprs/vliteral.h"
+#include "format_v2/column_mapper.h"
+#include "format_v2/table/paimon_rust_predicate_converter.h"
+#include "runtime/descriptors.h"
+#include "runtime/file_scan_profile.h"
+#include "runtime/runtime_state.h"
+#include "util/string_util.h"
+#include "util/timezone_utils.h"
+#include "util/url_coding.h"
+
+extern "C" {
+#include "paimon_rust/paimon.h"
+}
+
+namespace doris::format::paimon {
+
+namespace {
+constexpr const char* VALUE_KIND_FIELD = "_VALUE_KIND";
+
+// ---------------------------------------------------------------------------
+// RAII wrappers over the paimon-rust C handles. Each handle is an opaque
+// pointer owned by Rust and released by a matching paimon_*_free function.
+// ---------------------------------------------------------------------------
+#define PAIMON_OWNED(type, freefn) \
+ struct type##_deleter { \
+ void operator()(paimon_##type* p) const { \
+ if (p) { \
+ freefn(p); \
+ } \
+ } \
+ }; \
+ using type##_ptr = std::unique_ptr<paimon_##type, type##_deleter>
+
+PAIMON_OWNED(table, paimon_table_free);
+PAIMON_OWNED(read_builder, paimon_read_builder_free);
+PAIMON_OWNED(plan, paimon_plan_free);
+PAIMON_OWNED(table_read, paimon_table_read_free);
+PAIMON_OWNED(record_batch_reader, paimon_record_batch_reader_free);
+PAIMON_OWNED(error, paimon_error_free);
+
+#undef PAIMON_OWNED
+
+// One Arrow batch (schema + array containers). Owning it requires a two-step
+// teardown that the unique_ptr deleters above can't express: first invoke the
+// Arrow C Data Interface `release` callback on each struct (hands buffers back
+// to the producer), then free the container structs via paimon_arrow_batch_free.
+class ArrowBatch {
+public:
+ explicit ArrowBatch(paimon_arrow_batch batch) : batch_(batch) {}
+ ~ArrowBatch() {
+ auto* schema = static_cast<ArrowSchema*>(batch_.schema);
+ auto* array = static_cast<ArrowArray*>(batch_.array);
+ if (array && array->release) {
+ array->release(array);
+ }
+ if (schema && schema->release) {
+ schema->release(schema);
+ }
+ paimon_arrow_batch_free(batch_);
+ }
+
+ ArrowBatch(const ArrowBatch&) = delete;
+ ArrowBatch& operator=(const ArrowBatch&) = delete;
+
+ ArrowSchema* schema() const { return static_cast<ArrowSchema*>(batch_.schema); }
+ ArrowArray* array() const { return static_cast<ArrowArray*>(batch_.array); }
+
+private:
+ paimon_arrow_batch batch_;
+};
+
+// Render a paimon_error into a string. Takes ownership of `err` via RAII so it
+// is freed on every return path. Safe to call with nullptr.
+std::string consume_error(paimon_error* err) {
+ error_ptr owned(err);
+ if (!owned) {
+ return "unknown error";
+ }
+ std::string msg;
+ if (owned->message.data != nullptr && owned->message.len > 0) {
+ msg.assign(reinterpret_cast<const char*>(owned->message.data), owned->message.len);
+ }
+ return "code=" + std::to_string(owned->code) + ", msg=" + msg;
+}
+
+// Render storage option KEYS for diagnostics. Values are never rendered:
+// credential keys arrive under many spellings and cases (AWS_SECRET_KEY,
+// AWS_TOKEN, fs.oss.accessKeySecret, s3.secret-key, ...), and a key-name
+// blocklist that misses one alias leaks the value into the INFO log, so
+// only the key names are printed at all.
+std::string format_options(const std::map<std::string, std::string>& options) {
+ std::string out;
+ for (const auto& kv : options) {
+ if (!out.empty()) {
+ out += ", ";
+ }
+ out += kv.first;
+ }
+ return out;
+}
+
+} // namespace
+
+// Paimon-rust handles. Order of members matters: destruction runs in reverse
+// declaration order, and the read_builder depends on the table while the arrow
+// reader depends on the whole pipeline above it. So the table MUST be declared
+// first (destroyed last) and the record batch reader last.
+struct PaimonRustTableReader::PaimonHandles {
+ table_ptr table;
+ read_builder_ptr read_builder;
+ plan_ptr plan;
+ table_read_ptr table_read;
+ record_batch_reader_ptr reader;
+};
+
+PaimonRustTableReader::PaimonRustTableReader() = default;
+
+PaimonRustTableReader::~PaimonRustTableReader() = default;
+
+Status PaimonRustTableReader::init(format::TableReadOptions&& options) {
+ RETURN_IF_ERROR(format::TableReader::init(std::move(options)));
+ {
+ // Base and derived scopes must not overlap on the same counter: RuntimeProfile timers
+ // add deltas, so nested use would double-count instead of extending lifecycle coverage.
+ SCOPED_TIMER(_profile.total_timer);
+ SCOPED_TIMER(_profile.init_timer);
+ // Materialize TIMESTAMP_LTZ in the session timezone — the same
+ // convention as the JNI reader (PaimonJniScanner reads time_zone from
+ // its scan params) and lance_reader. Timezone-naive (paimon TIMESTAMP)
+ // arrow values are decoded in UTC by the DateTimeV2 serde regardless
+ // of _ctz, so NTZ wall-clock semantics are preserved.
+ DORIS_CHECK(_runtime_state != nullptr);
+ _ctz = _runtime_state->timezone_obj();
+ if (_scanner_profile != nullptr) {
+ file_scan_profile::ensure_hierarchy(_scanner_profile);
+ _rust_total_time = ADD_CHILD_TIMER(_scanner_profile, "PaimonRustReader",
+ file_scan_profile::TABLE_READER);
+ _rust_open_split_time =
+ ADD_CHILD_TIMER(_scanner_profile, "OpenSplitTime", "PaimonRustReader");
+ _rust_read_batch_time =
+ ADD_CHILD_TIMER(_scanner_profile, "ReadBatchTime", "PaimonRustReader");
+ _rust_arrow_to_block_time =
+ ADD_CHILD_TIMER(_scanner_profile, "ArrowToBlockTime", "PaimonRustReader");
+ _rust_predicates_input = ADD_CHILD_COUNTER(_scanner_profile, "RustPredicatesInput",
+ TUnit::UNIT, "PaimonRustReader");
+ _rust_predicates_converted = ADD_CHILD_COUNTER(
+ _scanner_profile, "RustPredicatesConverted", TUnit::UNIT, "PaimonRustReader");
+ _rust_predicates_applied = ADD_CHILD_COUNTER(_scanner_profile, "RustPredicatesApplied",
+ TUnit::UNIT, "PaimonRustReader");
+ _rust_runtime_filters_input = ADD_CHILD_COUNTER(
+ _scanner_profile, "RustRuntimeFiltersInput", TUnit::UNIT, "PaimonRustReader");
+ _rust_runtime_filters_applied = ADD_CHILD_COUNTER(
+ _scanner_profile, "RustRuntimeFiltersApplied", TUnit::UNIT, "PaimonRustReader");
+ }
+ // Projected column name -> fixed output position, registered with both the exact and
+ // the lower-case spelling so mixed-case Rust schema output still resolves (v1
+ // semantics: exact match first, lower-case fallback on lookup).
+ _output_name_to_idx.reserve(_projected_columns.size() * 2);
+ for (size_t idx = 0; idx < _projected_columns.size(); ++idx) {
+ _output_name_to_idx.emplace(_projected_columns[idx].name, idx);
+ _output_name_to_idx.emplace(to_lower(_projected_columns[idx].name), idx);
+ }
+ }
+ return Status::OK();
+}
+
+Status PaimonRustTableReader::prepare_split(const format::SplitReadOptions& options) {
+ // EOF belongs to the previous split. Keep it set after closing that split so repeated reads
+ // are idempotent, and clear it only when a new split is explicitly prepared.
+ _close_split_reader();
+ _split_eof = false;
+ _current_range = options.current_range;
+ RETURN_IF_ERROR(format::TableReader::prepare_split(options));
+ if (current_split_pruned()) {
+ return Status::OK();
+ }
+ if (_is_table_level_count_active()) {
+ // No rust pipeline is opened; get_block emits the synthetic count rows.
+ return Status::OK();
+ }
+ RETURN_IF_ERROR(_validate_rust_split(options.current_range));
+ {
+ SCOPED_TIMER(_profile.total_timer);
+ SCOPED_TIMER(_profile.prepare_split_timer);
+ // Open may dominate the scan or fail before get_block; include it in the parent timer.
+ SCOPED_TIMER(_rust_total_time);
+ SCOPED_TIMER(_rust_open_split_time);
+ RETURN_IF_ERROR(_open_split_reader(options.current_range));
+ }
+ return Status::OK();
+}
+
+Status PaimonRustTableReader::get_block(Block* block, bool* eos) {
+ SCOPED_TIMER(_profile.total_timer);
+ SCOPED_TIMER(_profile.exec_timer);
+ SCOPED_TIMER(_rust_total_time);
+ DORIS_CHECK(block != nullptr);
+ DORIS_CHECK(eos != nullptr);
+ DORIS_CHECK(block->columns() == _projected_columns.size());
+ block->clear_column_data(_projected_columns.size());
+ *eos = false;
+
+ if (_is_table_level_count_active()) {
+ return _read_table_level_count(block, eos);
+ }
+
+ // num_splits == 0 yields an empty (but valid) stream: report EOF.
+ if (_split_eof) {
+ *eos = true;
+ return Status::OK();
+ }
+ if (!_handles || !_handles->reader) {
+ return Status::InternalError("paimon-rust reader is not initialized");
+ }
+
+ while (true) {
+ // Mirror the base TableReader cancellation contract so a cancelled query does not
+ // drain the whole split.
+ if (_io_ctx != nullptr && _io_ctx->should_stop) {
+ _split_eof = true;
+ _close_split_reader();
+ *eos = true;
+ return Status::OK();
+ }
+
+ paimon_result_next_batch next;
+ {
+ SCOPED_TIMER(_rust_read_batch_time);
+ next = paimon_record_batch_reader_next(_handles->reader.get());
+ }
+ if (next.error != nullptr) {
+ return Status::InternalError("paimon-rust read batch failed: {}",
+ consume_error(next.error));
+ }
+ // End of stream: both pointers are null.
+ if (next.batch.array == nullptr && next.batch.schema == nullptr) {
+ _split_eof = true;
+ _close_split_reader();
+ *eos = true;
+ return Status::OK();
+ }
+
+ // RAII: the batch's Arrow release callbacks + container free run when
+ // `batch` leaves this scope, including on any early return.
+ ArrowBatch batch(next.batch);
+
+ auto* c_array = batch.array();
+ auto* c_schema = batch.schema();
+ arrow::Result<std::shared_ptr<arrow::RecordBatch>> import_result =
+ arrow::ImportRecordBatch(c_array, c_schema);
+ if (!import_result.ok()) {
+ return Status::InternalError("failed to import paimon-rust arrow batch: {}",
+ import_result.status().message());
+ }
+
+ auto record_batch = std::move(import_result).ValueUnsafe();
+ const auto rows = static_cast<size_t>(record_batch->num_rows());
+ if (rows == 0) {
+ // Skip empty batches and keep draining the stream.
+ continue;
+ }
+ RETURN_IF_ERROR(_fill_block_from_record_batch(record_batch, block, rows));
+ _record_scan_rows(rows);
+ *eos = false;
+ return Status::OK();
+ }
+}
+
+Status PaimonRustTableReader::abort_split() {
+ {
+ SCOPED_TIMER(_profile.total_timer);
+ SCOPED_TIMER(_profile.close_timer);
+ _close_split_reader();
+ _split_eof = false;
+ }
+ return format::TableReader::abort_split();
+}
+
+#ifdef BE_TEST
+std::string PaimonRustTableReader::TEST_format_options(
+ const std::map<std::string, std::string>& options) {
+ return format_options(options);
+}
+
+std::map<std::string, std::string> PaimonRustTableReader::TEST_build_options(
+ TFileScanRangeParams* scan_params, const TFileRangeDesc& range) {
+ TFileScanRangeParams* previous_params = _scan_params;
+ TFileRangeDesc previous_range = _current_range;
+ _scan_params = scan_params;
+ _current_range = range;
+ std::map<std::string, std::string> options = _build_options();
+ _scan_params = previous_params;
+ _current_range = std::move(previous_range);
+ return options;
+}
+#endif
+
+Status PaimonRustTableReader::close() {
+ {
+ SCOPED_TIMER(_profile.total_timer);
+ SCOPED_TIMER(_profile.close_timer);
+ _close_split_reader();
+ _close_table();
+ }
+ return format::TableReader::close();
+}
+
+Status PaimonRustTableReader::_validate_rust_split(const TFileRangeDesc& range) const {
+ if (!range.__isset.table_format_params || !range.table_format_params.__isset.paimon_params) {
+ return Status::InternalError(
+ "missing paimon_params for paimon rust reader, possibly caused by FE/BE protocol "
+ "mismatch");
+ }
+ const auto& params = range.table_format_params.paimon_params;
+ if (!params.__isset.paimon_split || params.paimon_split.empty()) {
+ return Status::InternalError(
+ "missing paimon_split for paimon rust reader, possibly caused by FE/BE protocol "
+ "mismatch");
+ }
+ if (params.__isset.reader_type && params.reader_type != TPaimonReaderType::PAIMON_RUST) {
+ return Status::InternalError(
+ "invalid reader_type for paimon rust reader, possibly caused by FE/BE protocol "
+ "mismatch");
+ }
+ if (!_resolve_table_path(range).has_value()) {
+ return Status::InternalError(
+ "paimon-rust missing paimon_table; cannot resolve paimon table location");
+ }
+ if (!_resolve_db_name(range).has_value()) {
+ return Status::InternalError(
+ "paimon-rust missing db_name; cannot open paimon table via schema json");
+ }
+ if (!_resolve_table_name(range).has_value()) {
+ return Status::InternalError(
+ "paimon-rust missing table_name; cannot open paimon table via schema json");
+ }
+ if (!_resolve_table_schema_json(range).has_value()) {
+ return Status::InternalError(
+ "paimon-rust missing paimon_table_schema_json; cannot open paimon table via "
+ "schema json");
+ }
+ return Status::OK();
+}
+
+Status PaimonRustTableReader::_open_split_reader(const TFileRangeDesc& range) {
+ // 1. Decode the FE-planned split first so we fail fast (and without any
+ // filesystem IO) when it is missing or malformed.
+ std::string split_bytes;
+ RETURN_IF_ERROR(_decode_split_bytes(&split_bytes));
+
+ // 2. Resolve identifier + table_path + FE-supplied TableSchema JSON.
+ auto table_path = _resolve_table_path(range).value();
+ auto db_name = _resolve_db_name(range).value();
+ auto table_name = _resolve_table_name(range).value();
+ auto schema_json = _resolve_table_schema_json(range).value();
+ auto branch_opt = _resolve_branch(range);
+
+ // 3. Assemble storage options: FE-supplied paimon options + hadoop_conf +
+ // OSS/S3 → AWS_* translations. These feed FileIO only (per
+ // paimon_table_from_schema_json contract); they are NOT merged into the
+ // supplied table schema.
+ auto options = _build_options();
+
+ auto opened_table_key =
+ std::make_tuple(table_path, schema_json, db_name, table_name, branch_opt, options);
+ if (!_handles || !_handles->table || _opened_table_key != opened_table_key) {
+ // A paimon scan reads one table, so the handle is opened at most once per
+ // distinct identity (e.g. re-created after a close); splits of the same
+ // table reuse it and only rebuild the read pipeline below.
+ _close_table();
+ _handles = std::make_unique<PaimonHandles>();
+
+ std::vector<paimon_option> c_options;
+ c_options.reserve(options.size());
+ for (const auto& kv : options) {
+ c_options.push_back(paimon_option {kv.first.c_str(), kv.second.c_str()});
+ }
+
+ LOG(INFO) << "paimon-rust opening table via schema json: db=" << db_name
+ << " table=" << table_name << " path=" << table_path
+ << " branch=" << (branch_opt.has_value() ? branch_opt.value() : "main")
+ << " storage_options=[" << format_options(options) << "]";
+
+ // Build the table directly from the FE-supplied schema JSON. The Rust
+ // side rejects null / empty branch, so we default to paimon's canonical
+ // "main" sentinel when FE did not set paimon_branch (i.e. the table is
+ // on the main branch — matches upstream Identifier.DEFAULT_MAIN_BRANCH).
+ const std::string& branch_str = branch_opt.has_value() ? branch_opt.value() : "main";
+ paimon_result_get_table tbl_res = paimon_table_from_schema_json(
+ table_path.c_str(), schema_json.c_str(), db_name.c_str(), table_name.c_str(),
+ branch_str.c_str(), c_options.empty() ? nullptr : c_options.data(),
+ c_options.size());
+ if (tbl_res.error != nullptr) {
+ return Status::InternalError(
+ "paimon-rust table_from_schema_json failed: db={} table={} err={}", db_name,
+ table_name, consume_error(tbl_res.error));
+ }
+ _handles->table.reset(tbl_res.table);
+ _opened_table_key = std::move(opened_table_key);
+ }
+
+ // 4. Build the read pipeline: read_builder -> case-insensitive -> projection.
+ // Bound the Rust Arrow allocation itself: wide rows must respect the scanner's
+ // adaptive probe size before they are materialized into a Doris block.
+ const auto batch_size = std::to_string(
+ _batch_size > 0 ? _batch_size : std::max(1, _runtime_state->batch_size()));
+ const paimon_option batch_option {"read.batch-size", batch_size.c_str()};
+ paimon_result_read_builder rb_res =
+ paimon_table_new_read_builder_with_options(_handles->table.get(), &batch_option, 1);
+ if (rb_res.error != nullptr) {
+ return Status::InternalError("paimon-rust new read builder failed: {}",
+ consume_error(rb_res.error));
+ }
+ _handles->read_builder.reset(rb_res.read_builder);
+
+ // Fold column casing on the Rust side so FE-normalized lowercase names
+ // resolve against tables with mixed-case column definitions.
+ if (paimon_error* case_err =
+ paimon_read_builder_with_case_sensitive(_handles->read_builder.get(), false)) {
+ return Status::InternalError("paimon-rust set case_sensitive failed: {}",
+ consume_error(case_err));
+ }
+
+ // Partition keys are excluded: they are materialized from split metadata
+ // (see _fill_non_arrow_columns), and paimon-rust does not emit them.
+ auto read_columns = _build_read_columns();
+ std::vector<const char*> projection;
+ projection.reserve(read_columns.size() + 1);
+ for (const auto& col : read_columns) {
+ projection.push_back(col.c_str());
+ }
+ projection.push_back(nullptr);
+ if (paimon_error* proj_err = paimon_read_builder_with_projection(_handles->read_builder.get(),
+ projection.data())) {
+ return Status::InternalError("paimon-rust set projection failed: {}",
+ consume_error(proj_err));
+ }
+
+ // Convert the scanner conjuncts into a paimon-rust filter and apply it.
+ RETURN_IF_ERROR(_apply_predicate());
+
+ // 5. Deserialize the FE-planned split into a one-split plan, so this
+ // scanner reads exactly the split it was assigned rather than replanning
+ // the whole table. The wire form is identical to what paimon-cpp consumes
+ // (`paimon::table::DataSplit::serialize`).
+ paimon_result_plan plan_res = paimon_plan_from_split_bytes(
+ reinterpret_cast<const uint8_t*>(split_bytes.data()), split_bytes.size());
+ if (plan_res.error != nullptr) {
+ return Status::InternalError("paimon-rust build plan failed: {}",
+ consume_error(plan_res.error));
+ }
+ _handles->plan.reset(plan_res.plan);
+
+ size_t num_splits = paimon_plan_num_splits(_handles->plan.get());
+ if (num_splits == 0) {
+ _split_eof = true;
+ return Status::OK();
+ }
+
+ // 6. Open the arrow stream over the plan.
+ paimon_result_new_read read_res = paimon_read_builder_new_read(_handles->read_builder.get());
+ if (read_res.error != nullptr) {
+ return Status::InternalError("paimon-rust new read failed: {}",
+ consume_error(read_res.error));
+ }
+ _handles->table_read.reset(read_res.read);
+
+ paimon_result_record_batch_reader rdr_res = paimon_table_read_to_arrow(
+ _handles->table_read.get(), _handles->plan.get(), /*offset=*/0, /*length=*/num_splits);
+ if (rdr_res.error != nullptr) {
+ return Status::InternalError("paimon-rust open arrow reader failed: {}",
+ consume_error(rdr_res.error));
+ }
+ _handles->reader.reset(rdr_res.reader);
+ return Status::OK();
+}
+
+void PaimonRustTableReader::_close_split_reader() {
+ if (!_handles) {
+ return;
+ }
+ // Reverse of the declaration order in PaimonHandles.
+ _handles->reader.reset();
+ _handles->table_read.reset();
+ _handles->plan.reset();
+ _handles->read_builder.reset();
+}
+
+void PaimonRustTableReader::_close_table() {
+ if (!_handles) {
+ return;
+ }
+ _close_split_reader();
+ _handles->table.reset();
+ _opened_table_key.reset();
+}
+
+Status PaimonRustTableReader::_apply_predicate() {
+ if (_conjuncts.empty() || !_handles || !_handles->table || !_handles->read_builder) {
+ return Status::OK();
+ }
+ if (_scanner_profile != nullptr) {
+ COUNTER_UPDATE(_rust_predicates_input, _conjuncts.size());
+ for (const auto& conjunct : _conjuncts) {
+ if (conjunct && conjunct->root() && conjunct->root()->is_rf_wrapper()) {
+ COUNTER_UPDATE(_rust_runtime_filters_input, 1);
+ }
+ }
+ }
+ LOG(INFO) << "paimon-rust predicate pushdown: " << _conjuncts.size() << " conjunct(s) input";
+ // The conjunct VSlotRefs carry table global indices (positions), so the v2
+ // converter mode resolves fields by the projected column names; partition
+ // keys are excluded because the rust reader does not read them.
+ std::vector<std::string> names;
+ std::vector<DataTypePtr> types;
+ names.reserve(_projected_columns.size());
+ types.reserve(_projected_columns.size());
+ for (const auto& col : _projected_columns) {
+ if (col.is_partition_key) {
+ continue;
+ }
+ names.push_back(col.name);
+ types.push_back(col.type);
+ }
+ PaimonRustPredicateConverter converter(names, types, _handles->table.get());
+ paimon_predicate* predicate = converter.build(_conjuncts);
+ if (_scanner_profile != nullptr) {
+ COUNTER_UPDATE(_rust_predicates_converted, converter.converted_conjuncts());
+ }
+ if (predicate == nullptr) {
+ LOG(INFO) << "paimon-rust predicate pushdown: nothing convertible, no filter applied";
+ return Status::OK();
+ }
+ // paimon_read_builder_with_filter consumes the predicate (ownership moves to
+ // the builder) on every path, so we must not free it here.
+ if (paimon_error* err =
+ paimon_read_builder_with_filter(_handles->read_builder.get(), predicate)) {
+ return Status::InternalError("paimon-rust apply filter failed: {}", consume_error(err));
+ }
+ // Count application only after the C API accepts the filter; reader timers
+ // alone cannot distinguish pushdown from Doris residual-only execution.
+ if (_scanner_profile != nullptr) {
+ COUNTER_UPDATE(_rust_predicates_applied, converter.converted_conjuncts());
+ COUNTER_UPDATE(_rust_runtime_filters_applied, converter.converted_runtime_filters());
+ }
+ LOG(INFO) << "paimon-rust predicate pushdown: applied";
+ return Status::OK();
+}
+
+Status PaimonRustTableReader::_fill_block_from_record_batch(
+ const std::shared_ptr<arrow::RecordBatch>& batch, Block* block, size_t rows) {
+ SCOPED_TIMER(_rust_arrow_to_block_time);
+ DORIS_CHECK(batch != nullptr);
+ DORIS_CHECK(block != nullptr);
+ std::unordered_set<size_t> materialized_indices;
+ materialized_indices.reserve(_projected_columns.size());
+ {
+ auto columns_guard = block->mutate_columns_scoped();
+ auto& columns = columns_guard.mutable_columns();
+ for (int c = 0; c < batch->num_columns(); ++c) {
+ const auto& field = batch->schema()->field(c);
+ if (field->name() == VALUE_KIND_FIELD) {
+ continue;
+ }
+ // Projected column names are FE-normalized to lowercase.
+ // paimon-rust's case_sensitive=false setting also case-folds column
+ // names in the schema output, so exact match works — but tolerate
+ // mixed-case Rust output by folding here as well.
+ auto it = _output_name_to_idx.find(field->name());
+ if (it == _output_name_to_idx.end()) {
+ it = _output_name_to_idx.find(to_lower(field->name()));
+ }
+ if (it == _output_name_to_idx.end()) {
+ // Skip columns that are not in the block (e.g. columns dropped by
+ // slot pruning).
+ continue;
+ }
+ const auto output_idx = it->second;
+ if (!materialized_indices.emplace(output_idx).second) {
+ return Status::InternalError("paimon-rust returned duplicate column '{}'",
+ field->name());
+ }
+ try {
+ RETURN_IF_ERROR(columns_guard.get_datatype_by_position(output_idx)
+ ->get_serde()
+ ->read_column_from_arrow(*columns[output_idx],
+ batch->column(c).get(), 0, rows,
+ _ctz));
+ } catch (Exception& e) {
+ return Status::InternalError("Failed to convert from arrow to block: {}", e.what());
+ }
+ }
+ }
+ // Partition columns and other projected columns absent from the arrow batch
+ // are back-filled from split metadata / defaults.
+ RETURN_IF_ERROR(_fill_non_arrow_columns(block, rows, materialized_indices));
+ // This direct Arrow path bypasses TableReader::finalize_chunk, whose last
+ // step enforces truncate_char_or_varchar_columns — without it, a column
+ // narrowed by schema evolution returns untruncated historical values.
+ RETURN_IF_ERROR(_truncate_char_or_varchar_columns(block));
+ return Status::OK();
+}
+
+Status PaimonRustTableReader::_truncate_char_or_varchar_columns(Block* block) {
+ if (_runtime_state == nullptr ||
+ !_runtime_state->query_options().truncate_char_or_varchar_columns) {
+ return Status::OK();
+ }
+ for (size_t idx = 0; idx < block->columns(); ++idx) {
+ const auto& column_type = block->get_by_position(idx).type;
+ if (column_type == nullptr) {
+ continue;
+ }
+ const auto type = remove_nullable(column_type);
+ const auto primitive = type->get_primitive_type();
+ if (primitive != TYPE_VARCHAR && primitive != TYPE_CHAR) {
+ continue;
+ }
+ const auto target_len = assert_cast<const DataTypeString*>(type.get())->len();
+ if (target_len <= 0) {
+ continue;
+ }
+ // Reuses TableReader's vectorized truncation (substring(column, 1,
+ // len)); the base variant maps through column_mapper metadata, which
+ // the direct rust path does not populate, so iterate the block's own
+ // slot-derived types here — the arrow side is always lengthless Utf8,
+ // so any bounded CHAR/VARCHAR target truncates to its declared length.
+ // Partition/default expressions can produce ColumnConst(ColumnNullable). The shared
+ // truncation helper expects a materialized nullable column, as in finalize_chunk.
+ auto& column = block->get_by_position(idx).column;
+ column = column->convert_to_full_column_if_const();
+ _truncate_char_or_varchar_column(block, idx, target_len);
+ }
+ return Status::OK();
+}
+
+Status PaimonRustTableReader::_fill_non_arrow_columns(
+ Block* block, size_t rows, const std::unordered_set<size_t>& materialized_indices) {
+ for (size_t idx = 0; idx < _projected_columns.size(); ++idx) {
+ if (materialized_indices.count(idx) != 0) {
+ continue;
+ }
+ const auto& column = _projected_columns[idx];
+ VExprContextSPtr constant_expr;
+ if (const Field* value = find_partition_value(column, _partition_values);
+ column.is_partition_key && value != nullptr) {
+ // Partition values are split constants (same materialization the
+ // TableColumnMapper builds for native readers).
+ constant_expr =
+ VExprContext::create_shared(VLiteral::create_shared(column.type, *value));
+ } else if (column.default_expr != nullptr) {
+ constant_expr = column.default_expr;
+ } else {
+ // The column is genuinely absent from the arrow batch. Schema
+ // evolution is handled by paimon-rust itself, so reaching here means
+ // an unexpected schema drift: fill defaults so the scan remains
+ // well-defined instead of failing the query.
+ LOG(WARNING) << "paimon-rust did not return projected column '" << column.name
+ << "'; filling with defaults";
+ auto data = column.type->create_column();
+ data->insert_many_defaults(rows);
+ block->replace_by_position(idx, std::move(data));
+ continue;
+ }
+ ColumnPtr constant_column;
+ RETURN_IF_ERROR(_materialize_constant_column(constant_expr, column.type, column.name, rows,
+ &constant_column));
+ block->replace_by_position(idx, std::move(constant_column));
+ }
+ return Status::OK();
+}
+
+Status PaimonRustTableReader::_materialize_constant_column(const VExprContextSPtr& expr,
+ const DataTypePtr& type,
+ const std::string& name, size_t rows,
+ ColumnPtr* column) {
+ DORIS_CHECK(expr != nullptr);
+ DORIS_CHECK(column != nullptr);
+ RowDescriptor row_desc;
+ RETURN_IF_ERROR(expr->prepare(_runtime_state, row_desc));
+ RETURN_IF_ERROR(expr->open(_runtime_state));
+ // Constants evaluate per input row, so a rows-sized synthetic block yields a
+ // rows-sized result for both plain literals and default expressions.
+ Block eval_block;
+ eval_block.insert({type->create_column_const_with_default_value(rows), type, name});
+ int result_column_id = -1;
+ RETURN_IF_ERROR(expr->execute(&eval_block, &result_column_id));
+ DORIS_CHECK(result_column_id >= 0);
+ ColumnPtr result_column = eval_block.get_by_position(result_column_id).column;
+ if (result_column->size() == 1 && rows > 1) {
+ result_column = ColumnConst::create(std::move(result_column), rows);
+ }
+ *column = std::move(result_column);
+ return Status::OK();
+}
+
+Status PaimonRustTableReader::_decode_split_bytes(std::string* out) const {
+ if (!_current_range.__isset.table_format_params ||
+ !_current_range.table_format_params.__isset.paimon_params ||
+ !_current_range.table_format_params.paimon_params.__isset.paimon_split) {
+ return Status::InternalError("paimon-rust missing paimon_split in scan range");
+ }
+ const auto& encoded_split = _current_range.table_format_params.paimon_params.paimon_split;
+ if (!base64_decode(encoded_split, out)) {
+ return Status::InternalError("paimon-rust base64 decode paimon_split failed");
+ }
+ if (out->empty()) {
+ return Status::InternalError("paimon-rust decoded paimon_split is empty");
+ }
+ return Status::OK();
+}
+
+std::optional<std::string> PaimonRustTableReader::_resolve_table_path(
+ const TFileRangeDesc& range) const {
+ if (range.__isset.table_format_params && range.table_format_params.__isset.paimon_params &&
+ range.table_format_params.paimon_params.__isset.paimon_table &&
+ !range.table_format_params.paimon_params.paimon_table.empty()) {
+ return range.table_format_params.paimon_params.paimon_table;
+ }
+ return std::nullopt;
+}
+
+std::optional<std::string> PaimonRustTableReader::_resolve_db_name(
+ const TFileRangeDesc& range) const {
+ if (range.__isset.table_format_params && range.table_format_params.__isset.paimon_params &&
+ range.table_format_params.paimon_params.__isset.db_name &&
+ !range.table_format_params.paimon_params.db_name.empty()) {
+ return range.table_format_params.paimon_params.db_name;
+ }
+ return std::nullopt;
+}
+
+std::optional<std::string> PaimonRustTableReader::_resolve_table_name(
+ const TFileRangeDesc& range) const {
+ if (range.__isset.table_format_params && range.table_format_params.__isset.paimon_params &&
+ range.table_format_params.paimon_params.__isset.table_name &&
+ !range.table_format_params.paimon_params.table_name.empty()) {
+ return range.table_format_params.paimon_params.table_name;
+ }
+ return std::nullopt;
+}
+
+std::optional<std::string> PaimonRustTableReader::_resolve_table_schema_json(
+ const TFileRangeDesc& range) const {
+ if (range.__isset.table_format_params && range.table_format_params.__isset.paimon_params &&
+ range.table_format_params.paimon_params.__isset.paimon_table_schema_json &&
+ !range.table_format_params.paimon_params.paimon_table_schema_json.empty()) {
+ return range.table_format_params.paimon_params.paimon_table_schema_json;
+ }
+ return std::nullopt;
+}
+
+std::optional<std::string> PaimonRustTableReader::_resolve_branch(
+ const TFileRangeDesc& range) const {
+ // FE only sets paimon_branch when the branch is not `main` (matches
+ // upstream paimon commit 742da63: null-if-DEFAULT_MAIN_BRANCH). Unset here
+ // means main-branch semantics.
+ if (range.__isset.table_format_params && range.table_format_params.__isset.paimon_params &&
+ range.table_format_params.paimon_params.__isset.paimon_branch &&
+ !range.table_format_params.paimon_params.paimon_branch.empty()) {
+ return range.table_format_params.paimon_params.paimon_branch;
+ }
+ return std::nullopt;
+}
+
+std::vector<std::string> PaimonRustTableReader::_build_read_columns() const {
+ std::vector<std::string> columns;
+ columns.reserve(_projected_columns.size());
+ for (const auto& column : _projected_columns) {
+ if (column.is_partition_key) {
+ continue;
+ }
+ columns.emplace_back(column.name);
+ }
+ return columns;
+}
+
+std::map<std::string, std::string> PaimonRustTableReader::_build_options() const {
+ std::map<std::string, std::string> options;
+ if (_scan_params && _scan_params->__isset.paimon_options &&
+ !_scan_params->paimon_options.empty()) {
+ options.insert(_scan_params->paimon_options.begin(), _scan_params->paimon_options.end());
+ } else if (_current_range.__isset.table_format_params &&
+ _current_range.table_format_params.__isset.paimon_params &&
+ _current_range.table_format_params.paimon_params.__isset.paimon_options) {
+ options.insert(_current_range.table_format_params.paimon_params.paimon_options.begin(),
+ _current_range.table_format_params.paimon_params.paimon_options.end());
+ }
+
+ if (_scan_params && _scan_params->__isset.properties && !_scan_params->properties.empty()) {
+ for (const auto& kv : _scan_params->properties) {
+ options[kv.first] = kv.second;
+ }
+ } else if (_current_range.__isset.table_format_params &&
+ _current_range.table_format_params.__isset.paimon_params &&
+ _current_range.table_format_params.paimon_params.__isset.hadoop_conf) {
+ for (const auto& kv : _current_range.table_format_params.paimon_params.hadoop_conf) {
+ options[kv.first] = kv.second;
+ }
+ }
+
+ auto copy_if_missing = [&](const char* from_key, const char* to_key) {
+ if (options.find(to_key) != options.end()) {
+ return;
+ }
+ auto it = options.find(from_key);
+ if (it != options.end() && !it->second.empty()) {
+ options[to_key] = it->second;
+ }
+ };
+
+ // The pinned paimon-rust storage dispatcher (io/storage.rs) selects the
+ // FileIO parser from the table path's URI scheme, and libpaimon_c.a
+ // compiles in separate COS, OBS, GCS and Azdls parsers besides the OSS
+ // and S3 ones. Doris's FE normalizes every object store's credentials
+ // into the AWS_* / use_path_style aliases, which this bridge can only
+ // translate into the two key families those two parsers read:
+ // oss:// -> fs.oss.endpoint / fs.oss.accessKeyId /
+ // fs.oss.accessKeySecret (+ fs.oss.securityToken STS)
+ // s3:// / s3a:// -> paimon-java's s3.* family (access-key, secret-key,
+ // session.token, endpoint, region, path-style-access,
+ // normalized from the fs.s3a. / s3a. / s3. prefixes)
+ // The FE therefore gates the rust reader to exactly these schemes
+ // (PaimonScanNode: cosn:// / obs:// / gs:// / abfs:// warehouses fall
+ // back to JNI before the split is encoded — their parsers read the
+ // fs.cosn.userinfo.* / fs.obs.* / gcs.* / azure.* families the AWS_*
+ // aliases cannot express). The mapping here is scoped the same way so a
+ // version-skewed FE cannot smuggle AWS_* aliases into another parser's
+ // property map: a cosn:// table reaching this bridge with synthesized
+ // s3.* keys would hit the COS parser without
+ // fs.cosn.userinfo.secretId / secretKey and fail the open with a
+ // misleading auth error instead of the JNI fallback.
+ const std::string table_path = _resolve_table_path(_current_range).value_or("");
+ std::string scheme;
+ if (const auto sep = table_path.find("://"); sep != std::string::npos) {
+ scheme = to_lower(table_path.substr(0, sep));
+ }
+ if (scheme == "oss") {
+ // The OSS parser reads only the four fs.oss.* keys (and retry
+ // settings); native fs.oss.* options pass through untouched. It has
+ // no region / anonymous / assume-role handling, so nothing else is
+ // mapped for this scheme.
+ copy_if_missing("AWS_ENDPOINT", "fs.oss.endpoint");
+ copy_if_missing("AWS_ACCESS_KEY", "fs.oss.accessKeyId");
+ copy_if_missing("AWS_SECRET_KEY", "fs.oss.accessKeySecret");
+ copy_if_missing("AWS_TOKEN", "fs.oss.securityToken");
+ return options;
+ }
+ if (scheme == "s3" || scheme == "s3a") {
+ copy_if_missing("AWS_ACCESS_KEY", "s3.access-key");
+ copy_if_missing("AWS_SECRET_KEY", "s3.secret-key");
+ copy_if_missing("AWS_TOKEN", "s3.session.token");
+ copy_if_missing("AWS_ENDPOINT", "s3.endpoint");
+ copy_if_missing("AWS_REGION", "s3.region");
+ copy_if_missing("use_path_style", "s3.path-style-access");
+ // Authentication modes: the FE storage-properties channel marks anonymous
+ // access with AWS_CREDENTIALS_PROVIDER_TYPE=ANONYMOUS (emitted when no
+ // static credentials are configured) and assume-role with
+ // AWS_ROLE_ARN / AWS_EXTERNAL_ID (from the s3.role_arn / s3.external_id
+ // catalog properties). The crate reads s3.anonymous (skip_signature) and
+ // the s3.assumed.role.* family, so map both; without these, anonymous
+ // catalogs would consult the ambient credential chain and role-only
+ // catalogs would never assume the requested role. The remaining provider
+ // modes are ambient JVM credential chains (ENV, SYSTEM_PROPERTIES,
+ // WEB_IDENTITY, CONTAINER, INSTANCE_PROFILE) with no paimon-rust
+ // equivalent — the FE gates those away from the rust reader before the
+ // split is encoded.
+ if (options.contains("AWS_CREDENTIALS_PROVIDER_TYPE") &&
+ options.at("AWS_CREDENTIALS_PROVIDER_TYPE") == "ANONYMOUS") {
+ options["s3.anonymous"] = "true";
+ }
+ copy_if_missing("AWS_ROLE_ARN", "s3.assumed.role.arn");
+ copy_if_missing("AWS_EXTERNAL_ID", "s3.assumed.role.externalId");
+ // OSS-shaped options on an S3 warehouse (cross-protocol alias): map them
+ // to the s3.* family as well.
+ copy_if_missing("fs.oss.accessKeyId", "s3.access-key");
+ copy_if_missing("fs.oss.accessKeySecret", "s3.secret-key");
+ copy_if_missing("fs.oss.sessionToken", "s3.session.token");
+ copy_if_missing("fs.oss.endpoint", "s3.endpoint");
+ copy_if_missing("fs.oss.region", "s3.region");
+ }
+ // Every other scheme passes through untouched: hdfs:// tables read their
+ // hadoop conf as delivered, scheme-less / file:// paths need no
+ // credentials, and the COS / OBS / GCS / Azdls families are gated to JNI
+ // on the FE — synthesizing s3.* keys for any of them would be dead
+ // weight at best and a silent misconfiguration at worst.
+ return options;
+}
+
+} // namespace doris::format::paimon
diff --git a/be/src/format_v2/table/paimon_rust_table_reader.h b/be/src/format_v2/table/paimon_rust_table_reader.h
new file mode 100644
index 00000000000..0c04f2f1f04
--- /dev/null
+++ b/be/src/format_v2/table/paimon_rust_table_reader.h
@@ -0,0 +1,188 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <tuple>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "cctz/time_zone.h"
+#include "common/status.h"
+#include "format_v2/table_reader.h"
+#include "runtime/runtime_profile.h"
+
+namespace arrow {
+class RecordBatch;
+}
+
+namespace doris::format::paimon {
+
+// Reads one FE-planned Paimon DataSplit per split through the paimon-rust C
+// bindings (libpaimon_c), using the schema-json pipeline:
+//
+// paimon_table_from_schema_json(table_path, schema_json, db, table, branch, options)
+// -> read_builder (case-insensitive, projection, filter)
+// -> paimon_plan_from_split_bytes(base64-decoded FE split)
+// -> paimon_read_builder_new_read -> paimon_table_read_to_arrow
+// -> per-batch paimon_record_batch_reader_next (Arrow C Data Interface).
+//
+// Leaf reader inside PaimonHybridReader: a DataSplit is a logical multi-file
+// split, not a file range, so this reader does not go through
+// FileReader/TableColumnMapper and fills the table-schema output block directly
+// (same shape as LanceTableReader). Partition columns that paimon-rust does not
+// emit are materialized from the partition values captured by the TableReader
+// base class; columns with default expressions are materialized from those
+// expressions.
+class PaimonRustTableReader final : public format::TableReader {
+public:
+ // Out-of-line (in the .cpp) so TUs that construct or destroy the reader do not
+ // need the complete PaimonHandles pimpl type.
+ PaimonRustTableReader();
+ ~PaimonRustTableReader() override;
+
+ Status init(format::TableReadOptions&& options) override;
+ // Each split's Rust read builder applies the batch size stored by the base class
+ // before opening the Arrow reader.
+ Status prepare_split(const format::SplitReadOptions& options) override;
+ Status get_block(Block* block, bool* eos) override;
+ Status abort_split() override;
+ Status close() override;
+
+#ifdef BE_TEST
+ Status TEST_validate_rust_split(const TFileRangeDesc& range) const {
+ return _validate_rust_split(range);
+ }
+ Status TEST_fill_non_arrow_columns(
+ Block* block, size_t rows,
+ const std::unordered_set<size_t>& materialized_indices = {}) {
+ return _fill_non_arrow_columns(block, rows, materialized_indices);
+ }
+ // The direct-path bounded-string truncation, exposed for unit tests.
+ Status TEST_truncate_char_or_varchar_columns(Block* block) {
+ return _truncate_char_or_varchar_columns(block);
+ }
+ void TEST_set_projected_columns(std::vector<format::ColumnDefinition> columns) {
+ _projected_columns = std::move(columns);
+ }
+ void TEST_set_partition_values(std::map<std::string, Field> values) {
+ _partition_values = std::move(values);
+ }
+ // The session timezone the reader materializes TIMESTAMP_LTZ with.
+ const cctz::time_zone& TEST_ctz() const { return _ctz; }
+ // Exposed for unit tests so the keys-only option-logging contract can be
+ // checked without a live table open.
+ static std::string TEST_format_options(const std::map<std::string, std::string>& options);
+ // Storage options the reader would build for the given FE-planned scan
+ // params and split (S3 key remapping, provider modes, scheme dispatch);
+ // both must outlive the call. Exposed for unit tests.
+ std::map<std::string, std::string> TEST_build_options(TFileScanRangeParams* scan_params,
+ const TFileRangeDesc& range);
+#endif
+
+private:
+ // Opaque holder for the paimon-rust C handles (defined in the .cpp so that
+ // paimon_rust/paimon.h does not leak into other translation units).
+ struct PaimonHandles;
+
+ // Fail fast (without filesystem IO) when the schema-json pipeline fields are
+ // missing from the split; the error text points at an FE/BE protocol mismatch.
+ Status _validate_rust_split(const TFileRangeDesc& range) const;
+ // Open (or reuse the cached) paimon_table and build the per-split read
+ // pipeline: read_builder -> projection -> filter -> plan -> arrow reader.
+ Status _open_split_reader(const TFileRangeDesc& range);
+ // Drop the per-split pipeline (read_builder .. record batch reader), keeping
+ // the cached table handle for the next split of the same table.
+ void _close_split_reader();
+ // Drop the cached paimon_table handle.
+ void _close_table();
+ // Convert _conjuncts (global-index VSlotRefs) into a paimon-rust filter and
+ // apply it to the read builder. Best effort: non-convertible conjuncts are
+ // dropped silently and re-applied by the scanner row-level filter.
+ Status _apply_predicate();
+ // Fill the output block from one arrow record batch: arrow columns are
+ // matched by name (exact then lower-case, v1 semantics) onto fixed projected
+ // positions, then _fill_non_arrow_columns back-fills the rest.
+ Status _fill_block_from_record_batch(const std::shared_ptr<arrow::RecordBatch>& batch,
+ Block* block, size_t rows);
+ // Materialize projected columns the arrow batch did not contain: partition
+ // constants, default expressions, or a default/NULL fill as a last resort.
+ Status _fill_non_arrow_columns(Block* block, size_t rows,
+ const std::unordered_set<size_t>& materialized_indices);
+ // Enforce truncate_char_or_varchar_columns on the direct Arrow path: the
+ // pinned rust reader maps paimon CHAR(n)/VARCHAR(n) to lengthless Arrow
+ // Utf8, so a column narrowed by schema evolution would return untruncated
+ // historical values here while the normal (finalize_chunk) path truncates.
+ // The block's own column types carry the table-side bounds (the reader
+ // builds them from the FE slot types); the arrow side is always boundless,
+ // so every bounded CHAR/VARCHAR column truncates to its declared length.
+ Status _truncate_char_or_varchar_columns(Block* block);
+ // Evaluate a constant expression (VLiteral or default_expr) on a one-row
+ // block and broadcast the result to `rows` as a ColumnConst.
+ Status _materialize_constant_column(const VExprContextSPtr& expr, const DataTypePtr& type,
+ const std::string& name, size_t rows, ColumnPtr* column);
+
+ std::optional<std::string> _resolve_table_path(const TFileRangeDesc& range) const;
+ std::optional<std::string> _resolve_db_name(const TFileRangeDesc& range) const;
+ std::optional<std::string> _resolve_table_name(const TFileRangeDesc& range) const;
+ std::optional<std::string> _resolve_table_schema_json(const TFileRangeDesc& range) const;
+ // Non-default branch name (unset means main branch, matching upstream
+ // paimon commit 742da63: null-if-DEFAULT_MAIN_BRANCH).
+ std::optional<std::string> _resolve_branch(const TFileRangeDesc& range) const;
+ // Base64-decode the FE-planned split into the raw serialized DataSplit bytes.
+ Status _decode_split_bytes(std::string* out) const;
+ // Projected column names the rust reader actually reads (partition keys are
+ // excluded; they are materialized from split metadata, not from arrow).
+ std::vector<std::string> _build_read_columns() const;
+ // Storage options: FE paimon_options + hadoop conf/properties, with the
+ // OSS/S3 -> AWS_* key remap (v1 semantics).
+ std::map<std::string, std::string> _build_options() const;
+
+ std::unique_ptr<PaimonHandles> _handles; // cached table + per-split pipeline
+ // Identity of the opened table so consecutive splits reuse the handle.
+ std::optional<
+ std::tuple<std::string /*table_path*/, std::string /*schema_json*/, std::string /*db*/,
+ std::string /*table*/, std::optional<std::string> /*branch*/,
+ std::map<std::string, std::string> /*options*/>>
+ _opened_table_key;
+ // Projected column name (exact and lower-case) -> output block position.
+ std::unordered_map<std::string, size_t> _output_name_to_idx;
+ TFileRangeDesc _current_range;
+ cctz::time_zone _ctz;
+ // EOF belongs to the current split; reset when a new split is prepared.
+ bool _split_eof = false;
+
+ // Profile counters under the "PaimonRustReader" profile group
+ // (file_scan_profile::TABLE_READER parent, mirroring JniTableReader).
+ RuntimeProfile::Counter* _rust_predicates_input = nullptr;
+ RuntimeProfile::Counter* _rust_predicates_converted = nullptr;
+ RuntimeProfile::Counter* _rust_predicates_applied = nullptr;
+ RuntimeProfile::Counter* _rust_runtime_filters_input = nullptr;
+ RuntimeProfile::Counter* _rust_runtime_filters_applied = nullptr;
+ RuntimeProfile::Counter* _rust_total_time = nullptr;
+ RuntimeProfile::Counter* _rust_open_split_time = nullptr;
+ RuntimeProfile::Counter* _rust_read_batch_time = nullptr;
+ RuntimeProfile::Counter* _rust_arrow_to_block_time = nullptr;
+};
+
+} // namespace doris::format::paimon
diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp
index 57a17a79e5a..ef7c6ab19e2 100644
--- a/be/test/exec/scan/file_scanner_v2_test.cpp
+++ b/be/test/exec/scan/file_scanner_v2_test.cpp
@@ -911,6 +911,22 @@ TEST(FileScannerV2Test, GeneratedChildrenKeepOneGlobalRowIdSourceMapping) {
EXPECT_EQ(first_id, second_id);
}
+TFileRangeDesc paimon_rust_jni_range(bool with_schema_json = true) {
+ auto range = range_with_format("paimon", TFileFormatType::FORMAT_JNI);
+ TPaimonFileDesc paimon_params;
+ paimon_params.__set_reader_type(TPaimonReaderType::PAIMON_RUST);
+ paimon_params.__set_file_format("parquet");
+ paimon_params.__set_paimon_split("serialized-rust-split");
+ paimon_params.__set_paimon_table("/paimon/warehouse/db.db/t");
+ paimon_params.__set_db_name("db");
+ paimon_params.__set_table_name("t");
+ if (with_schema_json) {
+ paimon_params.__set_paimon_table_schema_json("{}");
+ }
+ range.table_format_params.__set_paimon_params(std::move(paimon_params));
+ return range;
+}
+
TEST(FileScannerV2Test, JniCompatibilityShapesUseV2Scanner) {
TQueryOptions query_options;
query_options.__set_enable_file_scanner_v2(true);
@@ -921,6 +937,32 @@ TEST(FileScannerV2Test, JniCompatibilityShapesUseV2Scanner) {
EXPECT_TRUE(FileScannerV2::is_supported(params, legacy_paimon_jni_range_without_reader_type()));
}
+TEST(FileScannerV2Test, PaimonRustSplitUsesV2Scanner) {
+ TFileScanRangeParams params;
+ params.__set_format_type(TFileFormatType::FORMAT_JNI);
+
+ // A complete rust split (schema json + serialized split) is supported by V2.
+ EXPECT_TRUE(FileScannerV2::is_supported(params, paimon_rust_jni_range()));
+ EXPECT_TRUE(FileScannerV2::TEST_validate_scan_range(params, paimon_rust_jni_range()).ok());
+
+ // The schema-json pipeline field is required: a rust split without it cannot open the
+ // table and is rejected at validation instead of failing deep inside the reader.
+ const auto rust_range_without_schema = paimon_rust_jni_range(/*with_schema_json=*/false);
+ EXPECT_FALSE(FileScannerV2::is_supported(params, rust_range_without_schema));
+ const auto status = FileScannerV2::TEST_validate_scan_range(params, rust_range_without_schema);
+ EXPECT_TRUE(status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>());
+
+ // Regression guard: PAIMON_CPP splits must remain on the V1 fallback.
+ auto cpp_range = range_with_format("paimon", TFileFormatType::FORMAT_JNI);
+ {
+ TPaimonFileDesc cpp_paimon_params;
+ cpp_paimon_params.__set_reader_type(TPaimonReaderType::PAIMON_CPP);
+ cpp_paimon_params.__set_file_format("parquet");
+ cpp_range.table_format_params.__set_paimon_params(std::move(cpp_paimon_params));
+ }
+ EXPECT_FALSE(FileScannerV2::is_supported(params, cpp_range));
+}
+
TEST(FileScannerV2Test, IcebergDeleteSplitKeepsInitialScannerCountCap) {
TQueryOptions query_options;
query_options.__set_enable_file_scanner_v2(true);
diff --git a/be/test/exec/test_data/paimon_rust_batch_size/README.md b/be/test/exec/test_data/paimon_rust_batch_size/README.md
new file mode 100644
index 00000000000..b9e939b66ec
--- /dev/null
+++ b/be/test/exec/test_data/paimon_rust_batch_size/README.md
@@ -0,0 +1,32 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Paimon batch-size fixture
+
+Generated with Apache Paimon 1.3.1 as an append-only Parquet table with
+`bucket=-1` and `write-only=true`. It contains 65 rows: `k` ranges from 0 to 64,
+and every `v` is 128 KiB of the ASCII character `x`.
+
+`schema/schema-0` is the table schema. `split.bin` is a Java `DataSplit`
+serialization whose bucket path is the neutral `__BUCKET__` placeholder.
+The reader test replaces that Java `writeUTF` field with the fixture's bucket
+path at runtime. No catalog, object store, or Java process is needed to read it.
+
+The fixture exercises the actual Rust Arrow reader with small batch sizes and
+checks all values across batch boundaries and repeated split preparation.
diff --git a/be/test/exec/test_data/paimon_rust_batch_size/bucket-0/data-3b6dc957-0664-4ec4-946c-9535d228957a-0.parquet b/be/test/exec/test_data/paimon_rust_batch_size/bucket-0/data-3b6dc957-0664-4ec4-946c-9535d228957a-0.parquet
new file mode 100644
index 00000000000..160939df2fb
Binary files /dev/null and b/be/test/exec/test_data/paimon_rust_batch_size/bucket-0/data-3b6dc957-0664-4ec4-946c-9535d228957a-0.parquet differ
diff --git a/be/test/exec/test_data/paimon_rust_batch_size/schema/schema-0 b/be/test/exec/test_data/paimon_rust_batch_size/schema/schema-0
new file mode 100644
index 00000000000..2c057fbb6d2
--- /dev/null
+++ b/be/test/exec/test_data/paimon_rust_batch_size/schema/schema-0
@@ -0,0 +1,22 @@
+{
+ "version" : 3,
+ "id" : 0,
+ "fields" : [ {
+ "id" : 0,
+ "name" : "k",
+ "type" : "INT"
+ }, {
+ "id" : 1,
+ "name" : "v",
+ "type" : "STRING"
+ } ],
+ "highestFieldId" : 1,
+ "partitionKeys" : [ ],
+ "primaryKeys" : [ ],
+ "options" : {
+ "bucket" : "-1",
+ "write-only" : "true",
+ "file.format" : "parquet"
+ },
+ "timeMillis" : 1790690378190
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/paimon_rust_batch_size/split.bin b/be/test/exec/test_data/paimon_rust_batch_size/split.bin
new file mode 100644
index 00000000000..10135ae00fb
Binary files /dev/null and b/be/test/exec/test_data/paimon_rust_batch_size/split.bin differ
diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp
index 91900dffd99..1eaa841b507 100644
--- a/be/test/format_v2/table/paimon_reader_test.cpp
+++ b/be/test/format_v2/table/paimon_reader_test.cpp
@@ -356,6 +356,25 @@ TFileRangeDesc make_paimon_jni_range() {
return range;
}
+TFileRangeDesc make_paimon_rust_range() {
+ TFileRangeDesc range;
+ range.__set_path("/data-placeholder.parquet");
+ range.__set_format_type(TFileFormatType::FORMAT_JNI);
+ TTableFormatFileDesc table_format_params;
+ table_format_params.__set_table_format_type("paimon");
+ TPaimonFileDesc paimon_params;
+ paimon_params.__set_file_format("parquet");
+ paimon_params.__set_reader_type(TPaimonReaderType::PAIMON_RUST);
+ paimon_params.__set_paimon_split("serialized-paimon-split");
+ paimon_params.__set_paimon_table("/paimon/warehouse/db.db/t");
+ paimon_params.__set_db_name("db");
+ paimon_params.__set_table_name("t");
+ paimon_params.__set_paimon_table_schema_json("{}");
+ table_format_params.__set_paimon_params(paimon_params);
+ range.__set_table_format_params(table_format_params);
+ return range;
+}
+
TFileRangeDesc make_legacy_paimon_native_range(TFileFormatType::type physical_format_type) {
TFileRangeDesc range = make_paimon_native_range(physical_format_type);
range.__set_format_type(TFileFormatType::FORMAT_JNI);
@@ -988,6 +1007,24 @@ TEST(PaimonHybridReaderTest, ClassifiesJniSplitByReaderType) {
EXPECT_TRUE(paimon::PaimonHybridReader::TEST_is_jni_split(make_paimon_jni_range()));
}
+TEST(PaimonHybridReaderTest, ClassifiesRustSplitByReaderType) {
+ // A rust split is neither a JNI split nor a native split; the dispatch order in
+ // _ensure_current_split_reader relies on the rust check running before the JNI one.
+ EXPECT_TRUE(paimon::PaimonHybridReader::TEST_is_rust_split(make_paimon_rust_range()));
+ EXPECT_FALSE(paimon::PaimonHybridReader::TEST_is_rust_split(make_paimon_jni_range()));
+ EXPECT_FALSE(paimon::PaimonHybridReader::TEST_is_rust_split(
+ make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)));
+ EXPECT_FALSE(paimon::PaimonHybridReader::TEST_is_rust_split(
+ make_legacy_paimon_native_range(TFileFormatType::FORMAT_PARQUET)));
+ EXPECT_FALSE(paimon::PaimonHybridReader::TEST_is_jni_split(make_paimon_rust_range()));
+
+ // A serialized split without a reader_type stays on the legacy JNI classification.
+ TFileRangeDesc legacy_split = make_paimon_rust_range();
+ legacy_split.table_format_params.paimon_params.__isset.reader_type = false;
+ EXPECT_FALSE(paimon::PaimonHybridReader::TEST_is_rust_split(legacy_split));
+ EXPECT_TRUE(paimon::PaimonHybridReader::TEST_is_jni_split(legacy_split));
+}
+
TEST(PaimonHybridReaderTest, ConvertsNativeSplitFileFormat) {
FileFormat file_format;
ASSERT_TRUE(paimon::PaimonHybridReader::TEST_to_file_format(
@@ -1028,6 +1065,7 @@ TEST(PaimonHybridReaderTest, NormalizesLegacyNativeSplitFormatBeforeChildPrepare
tracking_reader = child.get();
return child;
},
+ [] { return std::make_unique<TableReader>(); },
[] { return std::make_unique<TableReader>(); });
ASSERT_TRUE(reader.init({
.projected_columns = {},
@@ -1048,20 +1086,21 @@ TEST(PaimonHybridReaderTest, NormalizesLegacyNativeSplitFormatBeforeChildPrepare
EXPECT_EQ(tracking_reader->prepared_format, FileFormat::PARQUET);
}
-TEST(PaimonHybridReaderTest, AdaptiveBatchSizeReachesBothChildReaders) {
+TEST(PaimonHybridReaderTest, AdaptiveBatchSizeReachesAllChildReaders) {
paimon::PaimonHybridReader reader;
reader.TEST_install_batch_size_children();
reader.set_batch_size(321);
const auto child_batch_sizes = reader.TEST_child_batch_sizes();
- EXPECT_EQ(child_batch_sizes.first, 321);
- EXPECT_EQ(child_batch_sizes.second, 321);
+ EXPECT_EQ(std::get<0>(child_batch_sizes), 321);
+ EXPECT_EQ(std::get<1>(child_batch_sizes), 321);
+ EXPECT_EQ(std::get<2>(child_batch_sizes), 321);
}
-TEST(PaimonHybridReaderTest, AggregatesConditionCacheHitsFromBothChildren) {
+TEST(PaimonHybridReaderTest, AggregatesConditionCacheHitsFromAllChildren) {
paimon::PaimonHybridReader reader;
reader.TEST_install_batch_size_children();
- reader.TEST_set_child_condition_cache_hits(3, 5);
- EXPECT_EQ(reader.condition_cache_hit_count(), 8);
+ reader.TEST_set_child_condition_cache_hits(3, 5, 7);
+ EXPECT_EQ(reader.condition_cache_hit_count(), 15);
}
TEST(PaimonHybridReaderTest, ForwardsLatePredicatesToActiveChild) {
@@ -1075,6 +1114,7 @@ TEST(PaimonHybridReaderTest, ForwardsLatePredicatesToActiveChild) {
child = tracking.get();
return tracking;
},
+ [] { return std::make_unique<TableReader>(); },
[] { return std::make_unique<TableReader>(); });
ASSERT_TRUE(reader.init({
.projected_columns = {},
@@ -1184,6 +1224,120 @@ TEST(PaimonHybridReaderTest, DispatchesNativeThenJniSplitToMatchingReader) {
ASSERT_TRUE(reader.close().ok());
}
+TEST(PaimonHybridReaderTest, DispatchesRustSplitToRustChild) {
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ auto scan_params = make_local_parquet_scan_params();
+ paimon::PaimonHybridReader reader;
+ SplitFormatTrackingTableReader* tracking_reader = nullptr;
+ reader.TEST_set_child_reader_factories(
+ [] { return std::make_unique<TableReader>(); },
+ [] { return std::make_unique<TableReader>(); },
+ [&] {
+ auto child = std::make_unique<SplitFormatTrackingTableReader>();
+ tracking_reader = child.get();
+ return child;
+ });
+ ASSERT_TRUE(reader.init({
+ .projected_columns = {},
+ .conjuncts = {},
+ .format = FileFormat::JNI,
+ .scan_params = &scan_params,
+ .io_ctx = nullptr,
+ .runtime_state = &state,
+ .scanner_profile = nullptr,
+ })
+ .ok());
+
+ SplitReadOptions rust_split;
+ rust_split.current_range = make_paimon_rust_range();
+ rust_split.current_split_format = FileFormat::JNI;
+ ASSERT_TRUE(reader.prepare_split(rust_split).ok());
+ ASSERT_NE(tracking_reader, nullptr);
+ // A serialized rust split stays on the JNI wrapper format. FE also sets
+ // paimon_params.file_format="parquet" on serialized splits, so normalizing it
+ // to PARQUET would mis-dispatch the split to the native child.
+ EXPECT_EQ(tracking_reader->prepared_format, FileFormat::JNI);
+
+ ASSERT_TRUE(reader.close().ok());
+}
+
+TEST(PaimonHybridReaderTest, DispatchesNativeThenRustSplitToMatchingReader) {
+ RuntimeProfile profile("test_profile");
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ auto scan_params = make_local_parquet_scan_params();
+ io::FileReaderStats file_reader_stats;
+ io::FileCacheStatistics file_cache_stats;
+ auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats);
+
+ paimon::PaimonHybridReader reader;
+ // The native child is the real PaimonReader over a placeholder path; the rust
+ // child is a tracking stub, proving dispatch reaches it without native
+ // normalization.
+ SplitFormatTrackingTableReader* rust_tracking_reader = nullptr;
+ reader.TEST_set_child_reader_factories(nullptr, nullptr, [&] {
+ auto child = std::make_unique<SplitFormatTrackingTableReader>();
+ rust_tracking_reader = child.get();
+ return child;
+ });
+ ASSERT_TRUE(reader.init({
+ .projected_columns = {},
+ .conjuncts = {},
+ .format = FileFormat::PARQUET,
+ .scan_params = &scan_params,
+ .io_ctx = io_ctx,
+ .runtime_state = &state,
+ .scanner_profile = &profile,
+ })
+ .ok());
+
+ SplitReadOptions native_split;
+ native_split.current_range = make_paimon_native_range(TFileFormatType::FORMAT_PARQUET);
+ native_split.current_split_format = FileFormat::PARQUET;
+ ASSERT_TRUE(reader.prepare_split(native_split).ok());
+
+ SplitReadOptions rust_split;
+ rust_split.current_range = make_paimon_rust_range();
+ rust_split.current_split_format = FileFormat::JNI;
+ ASSERT_TRUE(reader.prepare_split(rust_split).ok());
+ ASSERT_NE(rust_tracking_reader, nullptr);
+ EXPECT_EQ(rust_tracking_reader->prepared_format, FileFormat::JNI);
+
+ ASSERT_TRUE(reader.close().ok());
+}
+
+TEST(PaimonHybridReaderTest, ForwardsLatePredicatesToRustChild) {
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ auto scan_params = make_local_parquet_scan_params();
+ paimon::PaimonHybridReader reader;
+ RefreshTrackingTableReader* rust_child = nullptr;
+ reader.TEST_set_child_reader_factories(
+ [] { return std::make_unique<TableReader>(); },
+ [] { return std::make_unique<TableReader>(); },
+ [&] {
+ auto tracking = std::make_unique<RefreshTrackingTableReader>();
+ rust_child = tracking.get();
+ return tracking;
+ });
+ ASSERT_TRUE(reader.init({
+ .projected_columns = {},
+ .conjuncts = {},
+ .format = FileFormat::JNI,
+ .scan_params = &scan_params,
+ .io_ctx = nullptr,
+ .runtime_state = &state,
+ .scanner_profile = nullptr,
+ })
+ .ok());
+
+ SplitReadOptions rust_split;
+ rust_split.current_range = make_paimon_rust_range();
+ rust_split.current_split_format = FileFormat::JNI;
+ ASSERT_TRUE(reader.prepare_split(rust_split).ok());
+ ASSERT_NE(rust_child, nullptr);
+ ASSERT_TRUE(reader.refresh_conjuncts({}).ok());
+ EXPECT_EQ(rust_child->refresh_count, 1);
+}
+
TEST(PaimonHybridReaderTest, FirstNativeAndJniChildInitAreCountedOnce) {
RuntimeProfile profile("test_profile");
RuntimeState state {TQueryOptions(), TQueryGlobals()};
@@ -1203,6 +1357,7 @@ TEST(PaimonHybridReaderTest, FirstNativeAndJniChildInitAreCountedOnce) {
})
.ok());
reader.TEST_set_child_reader_factories([] { return std::make_unique<SlowInitTableReader>(); },
+ [] { return std::make_unique<SlowInitTableReader>(); },
[] { return std::make_unique<SlowInitTableReader>(); });
auto* total = profile.get_counter("TableReader");
diff --git a/be/test/format_v2/table/paimon_rust_predicate_converter_test.cpp b/be/test/format_v2/table/paimon_rust_predicate_converter_test.cpp
new file mode 100644
index 00000000000..6aee77faa8d
--- /dev/null
+++ b/be/test/format_v2/table/paimon_rust_predicate_converter_test.cpp
@@ -0,0 +1,674 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format_v2/table/paimon_rust_predicate_converter.h"
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "common/status.h"
+#include "core/block/block.h"
+#include "core/column/column_vector.h"
+#include "core/data_type/data_type_date_or_datetime_v2.h"
+#include "core/data_type/data_type_decimal.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/primitive_type.h"
+#include "core/field.h"
+#include "core/types.h"
+#include "core/value/vdatetime_value.h"
+#include "exprs/runtime_filter_expr.h"
+#include "exprs/vectorized_fn_call.h"
+#include "exprs/vexpr.h"
+#include "exprs/vexpr_context.h"
+#include "exprs/vin_predicate.h"
+#include "exprs/vliteral.h"
+#include "exprs/vslot_ref.h"
+
+namespace doris {
+namespace {
+
+struct table_deleter {
+ void operator()(paimon_table* table) const {
+ if (table) {
+ paimon_table_free(table);
+ }
+ }
+};
+using table_ptr = std::unique_ptr<paimon_table, table_deleter>;
+
+struct predicate_deleter {
+ void operator()(paimon_predicate* predicate) const {
+ if (predicate) {
+ paimon_predicate_free(predicate);
+ }
+ }
+};
+using predicate_ptr = std::unique_ptr<paimon_predicate, predicate_deleter>;
+
+// A binary predicate node with a hand-set opcode. The converter only inspects
+// op() and the children, never executes the expr, so execute_column_impl is a
+// stub that is never reached.
+class TestBinaryPredicate final : public VExpr {
+public:
+ TestBinaryPredicate(TExprOpcode::type opcode, VExprSPtr left, VExprSPtr right)
+ : VExpr(std::make_shared<DataTypeUInt8>(), false) {
+ _node_type = opcode == TExprOpcode::EQ_FOR_NULL ? TExprNodeType::NULL_AWARE_BINARY_PRED
+ : TExprNodeType::BINARY_PRED;
+ _opcode = opcode;
+ add_child(std::move(left));
+ add_child(std::move(right));
+ }
+
+ const std::string& expr_name() const override { return _name; }
+
+ Status execute_column_impl(VExprContext* /*context*/, const Block* /*block*/,
+ const Selector* /*selector*/, size_t /*count*/,
+ ColumnPtr& result_column) const override {
+ result_column = ColumnUInt8::create();
+ return Status::OK();
+ }
+
+private:
+ const std::string _name = "test_binary_predicate";
+};
+
+// A cast node wrapping an operand, like the BE tree of
+// `CAST(amount AS DECIMAL(10,1)) = 1.2`. The converter only inspects
+// node_type() and the children, never executes the expr.
+class TestCastExpr final : public VExpr {
+public:
+ TestCastExpr(const DataTypePtr& target_type, VExprSPtr child) : VExpr(target_type, false) {
+ _node_type = TExprNodeType::CAST_EXPR;
+ add_child(std::move(child));
+ }
+
+ const std::string& expr_name() const override { return _name; }
+
+ Status execute_column_impl(VExprContext* /*context*/, const Block* /*block*/,
+ const Selector* /*selector*/, size_t /*count*/,
+ ColumnPtr& result_column) const override {
+ result_column = ColumnUInt8::create();
+ return Status::OK();
+ }
+
+private:
+ const std::string _name = "test_cast";
+};
+
+// An error-preserving conjunct like assert_true(...): executing it only on
+// rows selected by other predicates could discard the error it must raise, so
+// VExpr::is_safe_to_execute_on_selected_rows reports it unsafe (a failing
+// VCastExpr behaves the same way).
+class UnsafeExpr final : public VExpr {
+public:
+ explicit UnsafeExpr(VExprSPtr child)
+ : VExpr(make_nullable(std::make_shared<DataTypeInt32>()), false) {
+ _node_type = TExprNodeType::BINARY_PRED;
+ add_child(std::move(child));
+ }
+
+ bool is_safe_to_execute_on_selected_rows() const override { return false; }
+
+ const std::string& expr_name() const override { return _name; }
+
+ Status execute_column_impl(VExprContext* /*context*/, const Block* /*block*/,
+ const Selector* /*selector*/, size_t /*count*/,
+ ColumnPtr& result_column) const override {
+ result_column = ColumnUInt8::create();
+ return Status::OK();
+ }
+
+private:
+ const std::string _name = "unsafe_expr";
+};
+
+// A slot ref resolved by column name, like FileScannerV2's rewritten conjuncts.
+VExprSPtr slot_ref(const std::string& column_name, const DataTypePtr& type) {
+ return std::make_shared<VSlotRef>(0, 0, -1, type, column_name);
+}
+
+VExprSPtr slot_ref(const std::string& column_name) {
+ return slot_ref(column_name, make_nullable(std::make_shared<DataTypeInt32>()));
+}
+
+VExprSPtr int_literal(int32_t value) {
+ return VLiteral::create_shared(std::make_shared<DataTypeInt32>(),
+ Field::create_field<TYPE_INT>(value));
+}
+
+// A STRING column type.
+const DataTypePtr& string_type() {
+ static const auto type = make_nullable(std::make_shared<DataTypeString>());
+ return type;
+}
+
+VExprSPtr string_literal(std::string value) {
+ return VLiteral::create_shared(std::make_shared<DataTypeString>(),
+ Field::create_field<TYPE_STRING>(std::move(value)));
+}
+
+// A like(col, pattern[, escape]) function call. The converter only inspects
+// function_name() and the children, never executes the expr.
+VExprSPtr like_call(std::vector<VExprSPtr> children) {
+ TExprNode node;
+ node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
+ node.__set_node_type(TExprNodeType::FUNCTION_CALL);
+ node.__set_fn(TFunction());
+ node.fn.__set_name(TFunctionName());
+ node.fn.name.function_name = "like";
+ node.__set_is_nullable(false);
+ auto call = VectorizedFnCall::create_shared(node);
+ for (auto& child : children) {
+ call->add_child(std::move(child));
+ }
+ return call;
+}
+
+// An IN / NOT IN predicate over the given children (built through the
+// TExprNode path like VDirectInPredicate::get_slot_in_expr does, so
+// is_not_in() is initialized). The converter only inspects node_type(),
+// op() and the children.
+VExprSPtr in_predicate(bool is_not, std::vector<VExprSPtr> children) {
+ TExprNode node;
+ node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
+ node.__set_node_type(TExprNodeType::IN_PRED);
+ node.in_predicate.__set_is_not_in(is_not);
+ node.__set_opcode(is_not ? TExprOpcode::FILTER_NOT_IN : TExprOpcode::FILTER_IN);
+ node.__set_is_nullable(false);
+ auto predicate = VInPredicate::create_shared(node);
+ for (auto& child : children) {
+ predicate->add_child(std::move(child));
+ }
+ return predicate;
+}
+
+// 2024-01-01 00:00:00.<microsecond> as a DATETIMEV2(6) literal.
+const DataTypePtr& datetimev2_type() {
+ static const auto type = make_nullable(std::make_shared<DataTypeDateTimeV2>(6));
+ return type;
+}
+
+VExprSPtr datetimev2_literal(uint32_t microsecond) {
+ DateV2Value<DateTimeV2ValueType> value;
+ value.unchecked_set_time(2024, 1, 1, 0, 0, 0, microsecond);
+ return VLiteral::create_shared(std::make_shared<DataTypeDateTimeV2>(6),
+ Field::create_field<TYPE_DATETIMEV2>(value));
+}
+
+// A DECIMAL(10,2) column type.
+const DataTypePtr& decimal_type() {
+ static const auto type = make_nullable(std::make_shared<DataTypeDecimal64>(10, 2));
+ return type;
+}
+
+// A DECIMAL(10,2) literal, e.g. decimal_literal(1, 24) is 1.24.
+VExprSPtr decimal_literal(int64_t integer, int64_t fraction) {
+ return VLiteral::create_shared(
+ std::make_shared<DataTypeDecimal64>(10, 2),
+ Field::create_field<TYPE_DECIMAL64>(Decimal64::from_int_frac(integer, fraction, 2)));
+}
+
+VExprSPtr runtime_in_filter(VExprSPtr predicate) {
+ TExprNode node;
+ node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
+ node.__set_node_type(TExprNodeType::IN_PRED);
+ node.__set_is_nullable(true);
+ return RuntimeFilterExpr::create_shared(node, std::move(predicate), 0.0, false, 1);
+}
+
+// A DOUBLE column type.
+const DataTypePtr& double_type() {
+ static const auto type = make_nullable(std::make_shared<DataTypeFloat64>());
+ return type;
+}
+
+} // namespace
+
+// Pins the EQ_FOR_NULL (<=>) decision matrix of the converter. Semantics are
+// aligned with the FE PaimonPredicateConverter, which only converts a binary
+// conjunct when the RHS is a convertible literal: a column-to-column
+// `a <=> b` must NOT be pushed down — the old code pushed `a IS NULL` and
+// wrongly discarded rows like (1, 1), which the residual conjunct cannot
+// recover.
+class PaimonRustPredicateConverterTest : public testing::Test {
+protected:
+ void SetUp() override {
+ // The table handle is built in memory from the schema JSON (the same
+ // format as an on-disk schema/schema-N file): building predicates only
+ // resolves fields against the schema, so the path never needs to exist.
+ // NOTE: the custom `json` delimiter matters — the schema contains
+ // `TIMESTAMP(6)"`, whose `)"` would terminate a plain R"(...)" early.
+ static constexpr const char* kSchemaJson =
+ R"json({"version":3,"id":0,"fields":[{"id":0,"name":"a","type":"INT"},)json"
+ R"json({"id":1,"name":"b","type":"INT"},{"id":2,"name":"ts",)json"
+ R"json("type":"TIMESTAMP(6)"},{"id":3,"name":"amount",)json"
+ R"json("type":"DECIMAL(10, 2)"},{"id":4,"name":"d",)json"
+ R"json("type":"DOUBLE"},{"id":5,"name":"s","type":"STRING"}],)json"
+ R"json("highestFieldId":5,"partitionKeys":[],"primaryKeys":[],)json"
+ R"json("options":{},"timeMillis":0})json";
+ auto result = paimon_table_from_schema_json("/tmp/paimon_rust_predicate_converter_test",
+ kSchemaJson, "db", "t", "main", nullptr, 0);
+ if (result.error != nullptr) {
+ std::string message;
+ if (result.error->message.data != nullptr && result.error->message.len > 0) {
+ message.assign(reinterpret_cast<const char*>(result.error->message.data),
+ result.error->message.len);
+ }
+ const int32_t code = result.error->code;
+ paimon_error_free(result.error);
+ FAIL() << "paimon_table_from_schema_json failed: code=" << code << ", msg=" << message;
+ }
+ _table.reset(result.table);
+
+ _column_names = {"a", "b", "ts", "amount", "d", "s"};
+ _column_types = {make_nullable(std::make_shared<DataTypeInt32>()),
+ make_nullable(std::make_shared<DataTypeInt32>()),
+ datetimev2_type(),
+ decimal_type(),
+ double_type(),
+ string_type()};
+ }
+
+ // Runs one conjunct through a fresh converter; a null return means the
+ // conjunct was rejected (left to the Doris residual).
+ predicate_ptr push(TExprOpcode::type opcode, VExprSPtr left, VExprSPtr right) {
+ PaimonRustPredicateConverter converter(_column_names, _column_types, _table.get());
+ VExprSPtr root =
+ std::make_shared<TestBinaryPredicate>(opcode, std::move(left), std::move(right));
+ VExprContextSPtrs conjuncts {VExprContext::create_shared(std::move(root))};
+ return predicate_ptr(converter.build(conjuncts));
+ }
+
+ // Runs one pre-built expr (e.g. an IN predicate) through a fresh
+ // converter; a null return means the conjunct was rejected.
+ predicate_ptr push_expr(VExprSPtr expr) {
+ PaimonRustPredicateConverter converter(_column_names, _column_types, _table.get());
+ VExprContextSPtrs conjuncts {VExprContext::create_shared(std::move(expr))};
+ return predicate_ptr(converter.build(conjuncts));
+ }
+
+ // Converts a literal expr against a column type so tests can inspect the
+ // resulting paimon_datum (tag / int_val / int_val2).
+ auto convert_literal(const VExprSPtr& expr, const DataTypePtr& column_type) {
+ PaimonRustPredicateConverter converter(_column_names, _column_types, _table.get());
+ return converter.TEST_convert_literal(expr, column_type);
+ }
+
+ // Base epoch millis of 2024-01-01 00:00:00 through the same conversion;
+ // the absolute value depends on the converter's timezone resolution.
+ int64_t converted_base_millis() {
+ auto holder = convert_literal(datetimev2_literal(0), datetimev2_type());
+ EXPECT_TRUE(holder.has_value());
+ return holder->datum.int_val;
+ }
+
+ table_ptr _table;
+ std::vector<std::string> _column_names;
+ std::vector<DataTypePtr> _column_types;
+};
+
+TEST_F(PaimonRustPredicateConverterTest, EqForNullColumnToColumnIsNotPushed) {
+ // The P1 case: `a <=> b` must stay in the residual. Pushing `a IS NULL`
+ // would keep only the (NULL, NULL) row and drop (1, 1) forever.
+ // (The literal forms `a <=> NULL` / `a <=> 1` are not covered: FE's
+ // NullSafeEqualToEqual rewrite turns them into IS NULL / plain equality
+ // before they ever reach a BE converter, so EQ_FOR_NULL arriving here is
+ // always column-to-column.)
+ EXPECT_EQ(push(TExprOpcode::EQ_FOR_NULL, slot_ref("a"), slot_ref("b")).get(), nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, EqLiteralIsPushed) {
+ // Sanity: plain equality still converts.
+ auto predicate = push(TExprOpcode::EQ, slot_ref("a"), int_literal(1));
+ EXPECT_NE(predicate.get(), nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, UnknownColumnIsNotPushed) {
+ // A slot outside the reader's projected columns cannot be resolved.
+ EXPECT_EQ(push(TExprOpcode::EQ, slot_ref("nope"), int_literal(1)).get(), nullptr);
+}
+
+// ---- pushdown stops at unsafe conjuncts (safe-prefix rule) ----
+
+TEST_F(PaimonRustPredicateConverterTest, UnsafeConjunctStopsLaterPredicatesFromBeingPushed) {
+ // The review's P1 case: an error-preserving conjunct (assert_true(...),
+ // a failing cast) that precedes a convertible predicate — in practice an
+ // arrived IN runtime filter, modeled here by a plain equality — must stop
+ // the pushdown. Pushing the later predicate would let rust prune the row
+ // on which the unsafe conjunct must still raise.
+ PaimonRustPredicateConverter converter(_column_names, _column_types, _table.get());
+ VExprContextSPtrs conjuncts;
+ conjuncts.push_back(VExprContext::create_shared(std::make_shared<UnsafeExpr>(slot_ref("a"))));
+ conjuncts.push_back(VExprContext::create_shared(
+ std::make_shared<TestBinaryPredicate>(TExprOpcode::EQ, slot_ref("b"), int_literal(1))));
+
+ predicate_ptr predicate(converter.build(conjuncts));
+ EXPECT_EQ(predicate.get(), nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, SafePrefixBeforeUnsafeConjunctIsStillPushed) {
+ // The prefix itself stays pushable: the convertible conjunct before the
+ // unsafe one is pushed, and the unsafe one only blocks everything after
+ // it (the later equality must not be pushed).
+ PaimonRustPredicateConverter converter(_column_names, _column_types, _table.get());
+ VExprContextSPtrs conjuncts;
+ conjuncts.push_back(VExprContext::create_shared(
+ std::make_shared<TestBinaryPredicate>(TExprOpcode::EQ, slot_ref("a"), int_literal(1))));
+ conjuncts.push_back(VExprContext::create_shared(std::make_shared<UnsafeExpr>(slot_ref("b"))));
+ conjuncts.push_back(VExprContext::create_shared(
+ std::make_shared<TestBinaryPredicate>(TExprOpcode::EQ, slot_ref("b"), int_literal(2))));
+
+ predicate_ptr predicate(converter.build(conjuncts));
+ EXPECT_NE(predicate.get(), nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, DoubleColumnPredicatesAreNotPushed) {
+ // Doris defines NaN as equal to itself and greater than every finite
+ // value, but the pinned rust evaluator compares doubles with
+ // f64::partial_cmp (IEEE partial ordering: NaN unordered, NaN != NaN), so
+ // a pushed `d > 1.0` would drop a stored NaN row Doris retains and a
+ // pushed IN (NaN) would reject it — rows pruned by the rust filter cannot
+ // be recovered by the residual. DOUBLE pushdown is skipped entirely
+ // (paimon-java's JNI path is unaffected: its CompareUtils goes through
+ // Double.compareTo, which is Doris's total ordering).
+ EXPECT_EQ(push(TExprOpcode::EQ, slot_ref("d", double_type()), int_literal(1)).get(), nullptr);
+ EXPECT_EQ(push(TExprOpcode::GT, slot_ref("d", double_type()), int_literal(1)).get(), nullptr);
+}
+
+// ---- IN list values must be bare literals (FE doInPredicate parity) ----
+
+TEST_F(PaimonRustPredicateConverterTest, InListIsPushed) {
+ // Sanity: an IN over bare literals still converts, so the rejections
+ // below come from the casted children, not from IN handling.
+ auto predicate =
+ push_expr(in_predicate(false, {slot_ref("a"), int_literal(1), int_literal(2)}));
+ EXPECT_NE(predicate.get(), nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, OversizedRuntimeInFilterStaysResidual) {
+ // Arrived runtime filters can exceed the 16-bit expression child count even
+ // though SQL IN lists normally cannot. Never push only a wrapped prefix.
+ for (size_t values : {65534, 65535, 65536, 65537}) {
+ SCOPED_TRACE(values);
+ std::vector<VExprSPtr> children {slot_ref("a")};
+ for (size_t i = 0; i < values; ++i) {
+ children.emplace_back(int_literal(static_cast<int32_t>(i)));
+ }
+ auto predicate = push_expr(runtime_in_filter(in_predicate(false, std::move(children))));
+ EXPECT_EQ(predicate != nullptr, values < 65535);
+ }
+}
+
+TEST_F(PaimonRustPredicateConverterTest, NullAwareRuntimeFilterStaysResidual) {
+ // A null-aware runtime filter (arrived from an EQ_FOR_NULL join) restores
+ // NULL probe rows to true in its residual execution
+ // (RuntimeFilterExpr::change_null_to_true). Unwrapping it to the ordinary
+ // IN set — rebuilt through VDirectInPredicate::get_slot_in_expr — treats
+ // NULL as not-in-set and would prune the NULL probes permanently before
+ // the join sees them. The wrapper is kept instead: it fails every
+ // dispatch, so the conjunct stays in the residual, while a non-null-aware
+ // wrapper still unwraps and pushes its impl (the arrived-IN case).
+ // VExpr(node) derives the wrapper's data type from the node's type
+ // descriptor, so it needs a valid one (a bare node aborts in the thrift
+ // type-vector walk).
+ TExprNode node;
+ node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
+ node.__set_node_type(TExprNodeType::IN_PRED);
+ node.__set_is_nullable(true);
+ VExprSPtr null_aware_rf = RuntimeFilterExpr::create_shared(
+ node, in_predicate(false, {slot_ref("a"), int_literal(1), int_literal(2)}), 0.0,
+ /*null_aware=*/true, 1);
+ EXPECT_EQ(push_expr(std::move(null_aware_rf)).get(), nullptr);
+
+ VExprSPtr plain_rf = RuntimeFilterExpr::create_shared(
+ node, in_predicate(false, {slot_ref("a"), int_literal(1), int_literal(2)}), 0.0,
+ /*null_aware=*/false, 2);
+ EXPECT_NE(push_expr(std::move(plain_rf)).get(), nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, CastedInListValueIsNotPushed) {
+ // The review's P1 case (IN form): with debug_skip_fold_constant=true, a
+ // cast in the IN list reaches the BE un-folded. Unwrapping it (the old
+ // one-level cast rule from _convert_literal) would push the pre-cast
+ // value, but Doris compares against the cast result — `amount IN
+ // (CAST(1.24 AS DECIMAL(10,1)))` must compare against 1.2, and the
+ // unwrapped 1.24 push would permanently remove the 1.2 rows. _convert_literal
+ // rejects the casted value, which rejects the whole predicate; the
+ // residual applies the cast correctly.
+ auto casted = std::make_shared<TestCastExpr>(make_nullable(std::make_shared<DataTypeInt64>()),
+ int_literal(1));
+ EXPECT_EQ(push_expr(in_predicate(false, {slot_ref("a"), int_literal(1), std::move(casted)}))
+ .get(),
+ nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, CastedNotInListValueIsNotPushed) {
+ // Same rule for NOT IN: rejecting the push keeps both readers on the
+ // Doris residual, where the cast is evaluated correctly.
+ auto casted = std::make_shared<TestCastExpr>(make_nullable(std::make_shared<DataTypeInt64>()),
+ int_literal(1));
+ EXPECT_EQ(push_expr(in_predicate(true, {slot_ref("a"), std::move(casted)})).get(), nullptr);
+}
+
+// The sub-millisecond remainder must land in int_val2 (nanos) instead of being
+// truncated — the rust datum holds (millis = int_val, nanos = int_val2) and
+// timestamp comparison includes both. A truncated conversion would turn
+// `ts = '2024-01-01 00:00:00.123456'` into `.123000` and wrongly drop the
+// matching rows — the runtime-filter IN scenario from the review. (The base
+// epoch millis depend on the converter's timezone resolution, so expectations
+// are expressed relative to a whole-second literal converted the same way.)
+TEST_F(PaimonRustPredicateConverterTest, TimestampV2FractionalMicrosArePreserved) {
+ const int64_t base = converted_base_millis();
+ auto holder = convert_literal(datetimev2_literal(123456), datetimev2_type());
+ ASSERT_TRUE(holder.has_value());
+ // paimon_datum tag 10 = Timestamp (see paimon.h).
+ EXPECT_EQ(holder->datum.tag, 10);
+ EXPECT_EQ(holder->datum.int_val, base + 123);
+ EXPECT_EQ(holder->datum.int_val2, 456000);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, TimestampV2WholeMillisLeaveNanosZero) {
+ const int64_t base = converted_base_millis();
+ auto holder = convert_literal(datetimev2_literal(123000), datetimev2_type());
+ ASSERT_TRUE(holder.has_value());
+ EXPECT_EQ(holder->datum.int_val, base + 123);
+ EXPECT_EQ(holder->datum.int_val2, 0);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, TimestampV2MaxFractionIsPreserved) {
+ const int64_t base = converted_base_millis();
+ auto holder = convert_literal(datetimev2_literal(999999), datetimev2_type());
+ ASSERT_TRUE(holder.has_value());
+ EXPECT_EQ(holder->datum.int_val, base + 999);
+ EXPECT_EQ(holder->datum.int_val2, 999000);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, TimestampPredicatesWithoutSourcePrecisionStayResidual) {
+ // A DATETIMEV2 slot does not reveal whether its source was TIMESTAMP(9).
+ auto predicate =
+ push(TExprOpcode::EQ, slot_ref("ts", datetimev2_type()), datetimev2_literal(123456));
+ EXPECT_EQ(predicate.get(), nullptr);
+ EXPECT_EQ(push_expr(runtime_in_filter(in_predicate(false, {slot_ref("ts", datetimev2_type()),
+ datetimev2_literal(123456)})))
+ .get(),
+ nullptr);
+}
+
+// ---- casted operands are rejected, mirroring the FE converter ----
+
+TEST_F(PaimonRustPredicateConverterTest, CastedColumnEqIsNotPushed) {
+ // `CAST(a AS BIGINT) = 1` must stay in the residual: a cast on the column
+ // is not equivalence-preserving in general, so like the FE converter's
+ // convertDorisExprToSlotRef the cast is not unwrapped. (The old code
+ // stripped it via expr_without_cast and pushed against the raw column.)
+ auto casted = std::make_shared<TestCastExpr>(make_nullable(std::make_shared<DataTypeInt64>()),
+ slot_ref("a"));
+ EXPECT_EQ(push(TExprOpcode::EQ, std::move(casted), int_literal(1)).get(), nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, DecimalScaleCastColumnIsNotPushed) {
+ // The review's P1 case: for a DECIMAL(10,2) column holding 1.24,
+ // `CAST(amount AS DECIMAL(10,1)) = 1.2` must retain the row, but pushing
+ // the unwrapped `amount = 1.2` against the original column prunes it. The
+ // rust filter does not save us either: it accepts decimal literals with
+ // different scales and compares their mathematical values (1.24 != 1.2),
+ // so the unsafe predicate is not rejected as a type mismatch. The conjunct
+ // must stay in the Doris residual.
+ auto casted = std::make_shared<TestCastExpr>(
+ make_nullable(std::make_shared<DataTypeDecimal64>(10, 1)),
+ slot_ref("amount", decimal_type()));
+ EXPECT_EQ(push(TExprOpcode::EQ, std::move(casted), decimal_literal(1, 2)).get(), nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, DecimalPredicatesWithoutFileScaleStayResidual) {
+ // Historical file statistics may still have a larger scale than this slot.
+ auto predicate =
+ push(TExprOpcode::EQ, slot_ref("amount", decimal_type()), decimal_literal(1, 24));
+ EXPECT_EQ(predicate.get(), nullptr);
+ EXPECT_EQ(push_expr(runtime_in_filter(in_predicate(false, {slot_ref("amount", decimal_type()),
+ decimal_literal(1, 23)})))
+ .get(),
+ nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, SingleCastLiteralRhsIsNotPushed) {
+ // The review's P1 case (binary form): a single cast wrapping a literal on
+ // the RHS must stay in the residual. Unwrapping it (the old one-level cast
+ // rule from _convert_literal) pushed the pre-cast value, but Doris compares
+ // against the cast result, so with debug_skip_fold_constant=true the rust
+ // filter and the residual disagree on lossy casts. The cast is not
+ // evaluated here, so the predicate cannot be built from the raw literal.
+ auto casted = std::make_shared<TestCastExpr>(make_nullable(std::make_shared<DataTypeInt64>()),
+ int_literal(1));
+ EXPECT_EQ(push(TExprOpcode::EQ, slot_ref("a"), std::move(casted)).get(), nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, DecimalScaleCastLiteralRhsIsNotPushed) {
+ // The differential case: `amount = CAST(1.24 AS DECIMAL(10,1))` on a
+ // DECIMAL(10,2) column. The cast evaluates to 1.2, so Doris's residual
+ // keeps a stored 1.20 row; the unwrapped literal 1.24 pushed against the
+ // raw column would prune it before the residual runs, and pruned rows
+ // cannot be recovered. The rust filter does not reject this as a type
+ // mismatch either — it accepts decimals of different scales and compares
+ // their mathematical values (1.24 != 1.20) — so the only safe conversion
+ // is none: the conjunct stays in the residual.
+ auto casted = std::make_shared<TestCastExpr>(
+ make_nullable(std::make_shared<DataTypeDecimal64>(10, 1)), decimal_literal(1, 24));
+ EXPECT_EQ(push(TExprOpcode::EQ, slot_ref("amount", decimal_type()), std::move(casted)).get(),
+ nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, NestedCastLiteralIsNotPushed) {
+ // `CAST(CAST(1 AS BIGINT) AS DOUBLE)`: cast trees are rejected at the
+ // outer level now — the old code stripped all cast levels, and the
+ // interim code unwrapped exactly one.
+ auto inner = std::make_shared<TestCastExpr>(make_nullable(std::make_shared<DataTypeInt64>()),
+ int_literal(1));
+ auto outer = std::make_shared<TestCastExpr>(make_nullable(std::make_shared<DataTypeFloat64>()),
+ std::move(inner));
+ EXPECT_EQ(push(TExprOpcode::EQ, slot_ref("a"), std::move(outer)).get(), nullptr);
+}
+
+// ---- LIKE patterns must be bare literals ----
+
+TEST_F(PaimonRustPredicateConverterTest, LikeIsPushed) {
+ // Sanity: a plain `s LIKE 'abc%'` converts, so the rejections below come
+ // from the pattern / escape handling, not from LIKE support itself.
+ auto predicate = push_expr(like_call({slot_ref("s", string_type()), string_literal("abc%")}));
+ EXPECT_NE(predicate.get(), nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, CastedLikePatternIsNotPushed) {
+ // A cast wrapping the pattern must stay in the residual: _extract_string_
+ // literal does not execute the cast, so the pushed pattern would be the
+ // pre-cast value while the Doris residual matches the cast result (a
+ // truncating cast like CAST('abc%' AS CHAR(2)) changes the prefix). Same
+ // rule as _convert_literal for the binary RHS and IN-list values.
+ auto casted = std::make_shared<TestCastExpr>(make_nullable(std::make_shared<DataTypeString>()),
+ string_literal("abc%"));
+ EXPECT_EQ(push_expr(like_call({slot_ref("s", string_type()), std::move(casted)})).get(),
+ nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, LikeWithDivergentEscapeIsNotPushed) {
+ // A backslash before an ordinary character diverges between Doris and the
+ // pinned rust like (Doris keeps both characters, rust consumes the
+ // backslash), so such patterns stay in the residual. Escapes before %, _
+ // and \\ (and a trailing backslash) have identical semantics on both
+ // sides and still convert.
+ EXPECT_EQ(push_expr(like_call({slot_ref("s", string_type()), string_literal("a\\qb%")})).get(),
+ nullptr);
+ EXPECT_NE(push_expr(like_call({slot_ref("s", string_type()), string_literal("a\\%b")})).get(),
+ nullptr);
+ EXPECT_NE(push_expr(like_call({slot_ref("s", string_type()), string_literal("a\\\\b_")})).get(),
+ nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, LikeWithNonDefaultEscapeIsNotPushed) {
+ // The 3-arg like(col, pattern, escape) form only converts for the Doris
+ // default escape '\'; the rust predicate builder rejects anything else.
+ EXPECT_NE(push_expr(like_call({slot_ref("s", string_type()), string_literal("abc%"),
+ string_literal("\\")}))
+ .get(),
+ nullptr);
+ EXPECT_EQ(push_expr(like_call({slot_ref("s", string_type()), string_literal("a!b"),
+ string_literal("!")}))
+ .get(),
+ nullptr);
+ // A casted escape char is rejected like a casted pattern.
+ auto casted_escape = std::make_shared<TestCastExpr>(
+ make_nullable(std::make_shared<DataTypeString>()), string_literal("\\"));
+ EXPECT_EQ(push_expr(like_call({slot_ref("s", string_type()), string_literal("abc%"),
+ std::move(casted_escape)}))
+ .get(),
+ nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, BoundedStringPredicatesStayResidual) {
+ // An old physical value can be longer than the current VARCHAR bound.
+ auto bounded = make_nullable(std::make_shared<DataTypeString>(3, TYPE_VARCHAR));
+ _column_types.back() = bounded;
+ EXPECT_EQ(push(TExprOpcode::EQ, slot_ref("s", bounded), string_literal("abc")).get(), nullptr);
+ EXPECT_EQ(push_expr(runtime_in_filter(in_predicate(
+ false, {slot_ref("s", bounded), string_literal("abc")})))
+ .get(),
+ nullptr);
+}
+
+TEST_F(PaimonRustPredicateConverterTest, ConversionCountersExcludeResidualsAndReset) {
+ auto rf = runtime_in_filter(in_predicate(false, {slot_ref("a"), int_literal(1)}));
+ auto timestamp =
+ in_predicate(false, {slot_ref("ts", datetimev2_type()), datetimev2_literal(123456)});
+ PaimonRustPredicateConverter converter(_column_names, _column_types, _table.get());
+ predicate_ptr predicate(converter.build(
+ {VExprContext::create_shared(rf), VExprContext::create_shared(timestamp)}));
+ ASSERT_NE(predicate.get(), nullptr);
+ EXPECT_EQ(converter.converted_conjuncts(), 1);
+ EXPECT_EQ(converter.converted_runtime_filters(), 1);
+ predicate.reset(converter.build({VExprContext::create_shared(timestamp)}));
+ EXPECT_EQ(predicate.get(), nullptr);
+ EXPECT_EQ(converter.converted_conjuncts(), 0);
+ EXPECT_EQ(converter.converted_runtime_filters(), 0);
+}
+
+} // namespace doris
diff --git a/be/test/format_v2/table/paimon_rust_table_reader_test.cpp b/be/test/format_v2/table/paimon_rust_table_reader_test.cpp
new file mode 100644
index 00000000000..72094f3e020
--- /dev/null
+++ b/be/test/format_v2/table/paimon_rust_table_reader_test.cpp
@@ -0,0 +1,712 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format_v2/table/paimon_rust_table_reader.h"
+
+#include <gtest/gtest.h>
+
+#include <chrono>
+#include <filesystem>
+#include <fstream>
+#include <iterator>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "cctz/time_zone.h"
+#include "core/block/block.h"
+#include "core/column/column_const.h"
+#include "core/column/column_nullable.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/field.h"
+#include "exprs/vexpr_context.h"
+#include "exprs/vliteral.h"
+#include "format_v2/column_data.h"
+#include "gen_cpp/PlanNodes_types.h"
+#include "runtime/runtime_profile.h"
+#include "runtime/runtime_state.h"
+#include "util/url_coding.h"
+
+namespace doris::format::paimon {
+namespace {
+
+ColumnDefinition make_column(const std::string& name, const DataTypePtr& type,
+ bool is_partition_key = false) {
+ ColumnDefinition column;
+ column.name = name;
+ column.type = type->is_nullable() ? type : make_nullable(type);
+ column.is_partition_key = is_partition_key;
+ return column;
+}
+
+TFileRangeDesc make_rust_range() {
+ TFileRangeDesc range;
+ TTableFormatFileDesc table_format_params;
+ table_format_params.__set_table_format_type("paimon");
+ TPaimonFileDesc paimon_params;
+ paimon_params.__set_reader_type(TPaimonReaderType::PAIMON_RUST);
+ std::string encoded;
+ base64_encode("dummy-split-bytes", &encoded);
+ paimon_params.__set_paimon_split(encoded);
+ paimon_params.__set_paimon_table("/paimon/warehouse/db.db/t");
+ paimon_params.__set_db_name("db");
+ paimon_params.__set_table_name("t");
+ paimon_params.__set_paimon_table_schema_json("{}");
+ table_format_params.__set_paimon_params(paimon_params);
+ range.__set_table_format_params(table_format_params);
+ return range;
+}
+
+// A range whose table path carries the given URI scheme — the production
+// shape FE ships in paimon_table (PaimonScanNode sets it from the catalog's
+// table location).
+TFileRangeDesc make_rust_range_with_table_path(const std::string& table_path) {
+ auto range = make_rust_range();
+ range.table_format_params.paimon_params.__set_paimon_table(table_path);
+ return range;
+}
+
+} // namespace
+
+class PaimonRustTableReaderTest : public testing::Test {
+protected:
+ void SetUp() override {
+ _query_options.__set_batch_size(3);
+ _runtime_state = RuntimeState::create_unique(_query_options, _query_globals);
+ }
+
+ Status init_reader_with_count(PaimonRustTableReader* reader,
+ std::vector<GlobalIndex> count_columns) {
+ return reader->init({.projected_columns = {_projected_column},
+ .conjuncts = {},
+ .format = FileFormat::JNI,
+ .scan_params = nullptr,
+ .io_ctx = nullptr,
+ .runtime_state = _runtime_state.get(),
+ .scanner_profile = nullptr,
+ .push_down_agg_type = TPushAggOp::type::COUNT,
+ .push_down_count_columns = std::move(count_columns)});
+ }
+
+ TQueryOptions _query_options;
+ TQueryGlobals _query_globals;
+ std::unique_ptr<RuntimeState> _runtime_state;
+ ColumnDefinition _projected_column = make_column("k", std::make_shared<DataTypeInt32>());
+};
+
+TEST_F(PaimonRustTableReaderTest, ValidatesRustSplit) {
+ PaimonRustTableReader reader;
+
+ // Missing paimon_split.
+ auto range = make_rust_range();
+ range.table_format_params.paimon_params.__isset.paimon_split = false;
+ auto status = reader.TEST_validate_rust_split(range);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("missing paimon_split"), std::string::npos) << status;
+
+ // Missing paimon_table (table path).
+ range = make_rust_range();
+ range.table_format_params.paimon_params.__isset.paimon_table = false;
+ status = reader.TEST_validate_rust_split(range);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("missing paimon_table"), std::string::npos) << status;
+
+ // Missing db_name.
+ range = make_rust_range();
+ range.table_format_params.paimon_params.__isset.db_name = false;
+ status = reader.TEST_validate_rust_split(range);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("missing db_name"), std::string::npos) << status;
+
+ // Missing table_name.
+ range = make_rust_range();
+ range.table_format_params.paimon_params.__isset.table_name = false;
+ status = reader.TEST_validate_rust_split(range);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("missing table_name"), std::string::npos) << status;
+
+ // Missing paimon_table_schema_json.
+ range = make_rust_range();
+ range.table_format_params.paimon_params.__isset.paimon_table_schema_json = false;
+ status = reader.TEST_validate_rust_split(range);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("missing paimon_table_schema_json"), std::string::npos)
+ << status;
+
+ // A mismatched reader_type is a protocol error.
+ range = make_rust_range();
+ range.table_format_params.paimon_params.__set_reader_type(TPaimonReaderType::PAIMON_JNI);
+ status = reader.TEST_validate_rust_split(range);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("invalid reader_type"), std::string::npos) << status;
+
+ // A complete range validates cleanly.
+ EXPECT_TRUE(reader.TEST_validate_rust_split(make_rust_range()).ok());
+}
+
+TEST_F(PaimonRustTableReaderTest, AdaptiveBatchSizeBoundsWideStringReads) {
+ const auto fixture = std::filesystem::path(__FILE__).parent_path() /
+ "../../exec/test_data/paimon_rust_batch_size";
+ const auto read_file = [](const std::filesystem::path& path) {
+ std::ifstream input(path, std::ios::binary);
+ return std::string(std::istreambuf_iterator<char>(input), std::istreambuf_iterator<char>());
+ };
+ auto split = read_file(fixture / "split.bin");
+ const auto placeholder = split.find("__BUCKET__");
+ ASSERT_NE(placeholder, std::string::npos);
+ ASSERT_GE(placeholder, 2);
+ const auto bucket = std::filesystem::canonical(fixture / "bucket-0").string();
+ ASSERT_LT(bucket.size(), 65536);
+ // The Java DataSplit fixture uses a neutral bucket path, encoded with writeUTF.
+ const std::string length = {static_cast<char>(bucket.size() >> 8),
+ static_cast<char>(bucket.size() & 0xff)};
+ split.replace(placeholder - 2, 2 + std::string("__BUCKET__").size(), length + bucket);
+ std::string encoded;
+ base64_encode(split, &encoded);
+ SplitReadOptions options;
+ options.current_range = make_rust_range();
+ options.current_split_format = FileFormat::JNI;
+ options.all_runtime_filters_applied = true;
+ auto& params = options.current_range.table_format_params.paimon_params;
+ params.__set_paimon_table(std::filesystem::canonical(fixture).string());
+ params.__set_paimon_table_schema_json(read_file(fixture / "schema/schema-0"));
+ params.__set_paimon_split(encoded);
+
+ const auto value_column = make_column("v", std::make_shared<DataTypeString>());
+ PaimonRustTableReader reader;
+ ASSERT_TRUE(reader.init({.projected_columns = {_projected_column, value_column},
+ .conjuncts = {},
+ .format = FileFormat::JNI,
+ .scan_params = nullptr,
+ .io_ctx = nullptr,
+ .runtime_state = _runtime_state.get(),
+ .scanner_profile = nullptr})
+ .ok());
+ // Wide values must be bounded before Arrow decoding, including when reusing a table handle.
+ for (const size_t requested : {0, 32, 7}) {
+ if (requested != 0) {
+ reader.set_batch_size(requested);
+ }
+ const size_t expected = requested == 0 ? 3 : requested;
+ auto status = reader.prepare_split(options);
+ ASSERT_TRUE(status.ok()) << status;
+ Block block({ColumnWithTypeAndName(_projected_column.type->create_column(),
+ _projected_column.type, "k"),
+ ColumnWithTypeAndName(value_column.type->create_column(), value_column.type,
+ "v")});
+ size_t total = 0;
+ bool eos = false;
+ while (!eos) {
+ status = reader.get_block(&block, &eos);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_LE(block.rows(), expected);
+ const auto& keys = assert_cast<const ColumnNullable&>(*block.get_by_position(0).column);
+ const auto& values =
+ assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ for (size_t row = 0; row < block.rows(); ++row) {
+ ASSERT_FALSE(keys.is_null_at(row));
+ ASSERT_FALSE(values.is_null_at(row));
+ EXPECT_EQ(keys.get_nested_column().get_int(row), total + row);
+ EXPECT_EQ(values.get_nested_column().get_data_at(row).to_string(),
+ std::string(128 * 1024, 'x'));
+ }
+ total += block.rows();
+ }
+ EXPECT_EQ(total, 65);
+ ASSERT_TRUE(reader.abort_split().ok());
+ }
+ ASSERT_TRUE(reader.close().ok());
+}
+
+TEST_F(PaimonRustTableReaderTest, TableLevelCountEmitsSyntheticRows) {
+ // COUNT(*) with a table-level row count takes the base-class metadata path:
+ // prepare_split never opens the rust pipeline and get_block emits synthetic rows.
+ PaimonRustTableReader reader;
+ ASSERT_TRUE(init_reader_with_count(&reader, std::vector<GlobalIndex> {}).ok());
+
+ SplitReadOptions options;
+ options.current_range = make_rust_range();
+ options.current_split_format = FileFormat::JNI;
+ options.all_runtime_filters_applied = true;
+ options.current_range.table_format_params.__set_table_level_row_count(5);
+ ASSERT_TRUE(reader.prepare_split(options).ok());
+ EXPECT_TRUE(reader.current_split_uses_metadata_count());
+
+ Block block = Block({ColumnWithTypeAndName(_projected_column.type->create_column(),
+ _projected_column.type, _projected_column.name)});
+ bool eos = false;
+ // The base-class count contract emits batches until a call finds remaining==0:
+ // batch_size(3) splits 5 rows into 3 + 2, and only the following call reports eos.
+ ASSERT_TRUE(reader.get_block(&block, &eos).ok());
+ EXPECT_EQ(block.rows(), 3);
+ EXPECT_FALSE(eos);
+
+ ASSERT_TRUE(reader.get_block(&block, &eos).ok());
+ EXPECT_EQ(block.rows(), 2);
+ EXPECT_FALSE(eos);
+
+ ASSERT_TRUE(reader.get_block(&block, &eos).ok());
+ EXPECT_EQ(block.rows(), 0);
+ EXPECT_TRUE(eos);
+}
+
+TEST_F(PaimonRustTableReaderTest, TableLevelCountDisabledByConjuncts) {
+ // A row predicate makes the metadata shortcut unsafe: the split must not report a
+ // metadata count. It proceeds to the rust pipeline instead, which fails on the dummy
+ // split bytes of this test range rather than emitting synthetic count rows.
+ PaimonRustTableReader reader;
+ ASSERT_TRUE(init_reader_with_count(&reader, std::vector<GlobalIndex> {}).ok());
+
+ SplitReadOptions options;
+ options.current_range = make_rust_range();
+ options.current_split_format = FileFormat::JNI;
+ options.all_runtime_filters_applied = true;
+ options.current_range.table_format_params.__set_table_level_row_count(5);
+ options.conjuncts = VExprContextSPtrs {};
+ options.conjuncts->push_back(VExprContext::create_shared(VLiteral::create_shared(
+ std::make_shared<DataTypeInt32>(), Field::create_field<TYPE_INT>(1))));
+
+ const auto status = reader.prepare_split(options);
+ EXPECT_FALSE(status.ok());
+ EXPECT_FALSE(reader.current_split_uses_metadata_count());
+
+ Block block = Block({ColumnWithTypeAndName(_projected_column.type->create_column(),
+ _projected_column.type, _projected_column.name)});
+ bool eos = false;
+ const auto get_block_status = reader.get_block(&block, &eos);
+ EXPECT_FALSE(get_block_status.ok());
+ EXPECT_NE(get_block_status.to_string().find("paimon-rust reader is not initialized"),
+ std::string::npos)
+ << get_block_status;
+}
+
+TEST_F(PaimonRustTableReaderTest, FailedSplitOpenIsIncludedInRustTotalTime) {
+ RuntimeProfile profile("scanner");
+ PaimonRustTableReader reader;
+ ASSERT_TRUE(reader.init({.projected_columns = {_projected_column},
+ .conjuncts = {},
+ .format = FileFormat::JNI,
+ .scan_params = nullptr,
+ .io_ctx = nullptr,
+ .runtime_state = _runtime_state.get(),
+ .scanner_profile = &profile})
+ .ok());
+ SplitReadOptions options;
+ options.current_range = make_rust_range();
+ options.current_split_format = FileFormat::JNI;
+ // The invalid split fails during open, before get_block can contribute any time.
+ ASSERT_FALSE(reader.prepare_split(options).ok());
+ auto* total = profile.get_counter("PaimonRustReader");
+ auto* open = profile.get_counter("OpenSplitTime");
+ ASSERT_NE(total, nullptr);
+ ASSERT_NE(open, nullptr);
+ EXPECT_GT(open->value(), 0);
+ EXPECT_GE(total->value(), open->value());
+}
+
+TEST_F(PaimonRustTableReaderTest, FillsPartitionConstantsForMissingArrowColumns) {
+ PaimonRustTableReader reader;
+ const auto data_type = make_nullable(std::make_shared<DataTypeInt32>());
+ const auto partition_type = make_nullable(std::make_shared<DataTypeString>());
+ // The output block matches the projected columns exactly (get_block contract):
+ // position 0 is the data column k (absent from the arrow batch -> default fill),
+ // position 1 is the partition key dt (materialized from split metadata).
+ reader.TEST_set_projected_columns(
+ {make_column("k", std::make_shared<DataTypeInt32>()),
+ make_column("dt", std::make_shared<DataTypeString>(), /*is_partition_key=*/true)});
+ std::map<std::string, Field> partition_values;
+ partition_values.emplace("dt", Field::create_field<TYPE_STRING>("2024-01-01"));
+ reader.TEST_set_partition_values(std::move(partition_values));
+
+ Block block =
+ Block({ColumnWithTypeAndName(data_type->create_column(), data_type, "k"),
+ ColumnWithTypeAndName(partition_type->create_column(), partition_type, "dt")});
+ const size_t rows = 4;
+ ASSERT_TRUE(reader.TEST_fill_non_arrow_columns(&block, rows).ok());
+
+ // The data column is absent from both the arrow batch and split metadata: filled
+ // with defaults.
+ EXPECT_EQ(block.get_by_position(0).column->size(), rows);
+
+ // The partition position is a constant column with the split value broadcast
+ // to every row.
+ const auto& column_with_type = block.get_by_position(1);
+ EXPECT_EQ(column_with_type.column->size(), rows);
+ const auto* const_column = check_and_get_column<ColumnConst>(*column_with_type.column);
+ ASSERT_NE(const_column, nullptr);
+ EXPECT_EQ(const_column->size(), rows);
+ const auto value_field = const_column->get_field();
+ const auto& value = value_field.get<TYPE_STRING>();
+ EXPECT_EQ(std::string(value.data(), value.size()), "2024-01-01");
+}
+
+TEST_F(PaimonRustTableReaderTest, TruncatesNullablePartitionAndDefaultConstants) {
+ _query_options.__set_truncate_char_or_varchar_columns(true);
+ _runtime_state = RuntimeState::create_unique(_query_options, _query_globals);
+ for (const auto primitive : {PrimitiveType::TYPE_CHAR, PrimitiveType::TYPE_VARCHAR}) {
+ for (bool is_null : {false, true}) {
+ PaimonRustTableReader reader;
+ ASSERT_TRUE(init_reader_with_count(&reader, {}).ok());
+ const auto type = make_nullable(std::make_shared<DataTypeString>(3, primitive));
+ auto partition = make_column("partition_key", type, true);
+ auto default_column = make_column("default_value", type);
+ const auto value = is_null ? Field() : Field::create_field<TYPE_STRING>("abcdef");
+ default_column.default_expr =
+ VExprContext::create_shared(VLiteral::create_shared(type, value));
+ reader.TEST_set_projected_columns({partition, default_column});
+ reader.TEST_set_partition_values({{"partition_key", value}});
+ Block block({ColumnWithTypeAndName(type->create_column(), type, "partition_key"),
+ ColumnWithTypeAndName(type->create_column(), type, "default_value")});
+ ASSERT_TRUE(reader.TEST_fill_non_arrow_columns(&block, 3).ok());
+ ASSERT_TRUE(is_column_const(*block.get_by_position(0).column));
+ // Split constants wrap nullable data; truncation must preserve both the row count
+ // and null map when it materializes that outer wrapper.
+ ASSERT_TRUE(reader.TEST_truncate_char_or_varchar_columns(&block).ok());
+ ASSERT_EQ(block.rows(), 3);
+ for (size_t column = 0; column < 2; ++column) {
+ for (size_t row = 0; row < 3; ++row) {
+ Field actual;
+ block.get_by_position(column).column->get(row, actual);
+ if (is_null) {
+ EXPECT_TRUE(actual.is_null());
+ } else {
+ const auto& text = actual.get<TYPE_STRING>();
+ EXPECT_EQ(std::string(text.data(), text.size()), "abc");
+ }
+ }
+ }
+ }
+ }
+}
+
+TEST_F(PaimonRustTableReaderTest, DirectPathTruncatesBoundedStringColumns) {
+ // The pinned rust reader maps paimon CHAR(n)/VARCHAR(n) to lengthless Arrow
+ // Utf8, so the direct Arrow path must enforce truncate_char_or_varchar_
+ // columns itself: a value written under an old wider schema and read into
+ // a column narrowed to VARCHAR(3) truncates to "abc" — the semantics
+ // TableReader::finalize_chunk applies on the normal path, which this
+ // direct path bypasses. Unbounded STRING columns and non-string columns
+ // stay untouched, and with the option off nothing truncates.
+ const auto fill_block = [] {
+ const auto varchar3 =
+ make_nullable(std::make_shared<DataTypeString>(3, PrimitiveType::TYPE_VARCHAR));
+ const auto string_type = make_nullable(std::make_shared<DataTypeString>());
+ const auto int_type = std::make_shared<DataTypeInt32>();
+ Block block = Block({ColumnWithTypeAndName(varchar3->create_column(), varchar3, "v"),
+ ColumnWithTypeAndName(string_type->create_column(), string_type, "s"),
+ ColumnWithTypeAndName(int_type->create_column(), int_type, "k")});
+ for (auto [idx, field] : std::initializer_list<std::pair<size_t, Field>> {
+ {0, Field::create_field<TYPE_STRING>("abcdefghij")},
+ {1, Field::create_field<TYPE_STRING>("0123456789ABCDEF")},
+ {2, Field::create_field<TYPE_INT>(1)},
+ {0, Field::create_field<TYPE_STRING>("ab")},
+ {1, Field::create_field<TYPE_STRING>("xyz")},
+ {2, Field::create_field<TYPE_INT>(2)}}) {
+ auto column = IColumn::mutate(block.get_by_position(idx).column);
+ column->insert(field);
+ block.get_by_position(idx).column = std::move(column);
+ }
+ return block;
+ };
+ const auto value_at = [](const Block& block, size_t idx, size_t row) {
+ Field field;
+ block.get_by_position(idx).column->get(row, field);
+ const auto& value = field.get<TYPE_STRING>();
+ return std::string(value.data(), value.size());
+ };
+
+ // Option off (the SetUp default): historical values return unchanged.
+ {
+ PaimonRustTableReader reader;
+ ASSERT_TRUE(init_reader_with_count(&reader, std::vector<GlobalIndex> {}).ok());
+ Block block = fill_block();
+ ASSERT_TRUE(reader.TEST_truncate_char_or_varchar_columns(&block).ok());
+ EXPECT_EQ(value_at(block, 0, 0), "abcdefghij");
+ }
+
+ // Option on: the VARCHAR(3) column truncates to its declared length; the
+ // unbounded STRING column and the INT column stay untouched.
+ _query_options.__set_truncate_char_or_varchar_columns(true);
+ _runtime_state = RuntimeState::create_unique(_query_options, _query_globals);
+ {
+ PaimonRustTableReader reader;
+ ASSERT_TRUE(init_reader_with_count(&reader, std::vector<GlobalIndex> {}).ok());
+ Block block = fill_block();
+ ASSERT_TRUE(reader.TEST_truncate_char_or_varchar_columns(&block).ok());
+ EXPECT_EQ(value_at(block, 0, 0), "abc");
+ EXPECT_EQ(value_at(block, 0, 1), "ab");
+ EXPECT_EQ(value_at(block, 1, 0), "0123456789ABCDEF");
+ EXPECT_EQ(value_at(block, 1, 1), "xyz");
+ Field int_field;
+ block.get_by_position(2).column->get(0, int_field);
+ EXPECT_EQ(int_field.get<TYPE_INT>(), 1);
+ }
+}
+
+TEST_F(PaimonRustTableReaderTest, MaterializesInSessionTimezone) {
+ // TIMESTAMP_LTZ values materialize as session-local civil times: the
+ // materialization timezone must come from the session, not a fixed default
+ // (epoch 0 reads as 08:00 in a +08:00 session and 00:00 in UTC).
+ const auto hour_of_epoch_zero = [](const cctz::time_zone& tz) {
+ return tz.lookup(cctz::time_point<cctz::seconds>(std::chrono::seconds(0))).cs.hour();
+ };
+
+ PaimonRustTableReader reader;
+ _runtime_state->set_timezone("+08:00");
+ ASSERT_TRUE(init_reader_with_count(&reader, std::vector<GlobalIndex> {}).ok());
+ EXPECT_EQ(hour_of_epoch_zero(reader.TEST_ctz()), 8);
+
+ PaimonRustTableReader utc_reader;
+ _runtime_state->set_timezone("UTC");
+ ASSERT_TRUE(init_reader_with_count(&utc_reader, std::vector<GlobalIndex> {}).ok());
+ EXPECT_EQ(hour_of_epoch_zero(utc_reader.TEST_ctz()), 0);
+}
+
+TEST_F(PaimonRustTableReaderTest, OptionLogRendersKeysOnly) {
+ // Storage-option values must never reach the log: credential keys arrive
+ // under many spellings and cases (AWS_SECRET_KEY, AWS_TOKEN,
+ // fs.oss.accessKeySecret, s3.secret-key, ...), and a key-name blocklist
+ // that misses one alias leaks the value, so the diagnostics rendering
+ // prints key names only.
+ const std::map<std::string, std::string> options {
+ {"AWS_ACCESS_KEY", "admin"},
+ {"AWS_SECRET_KEY", "leak-if-logged-1"},
+ {"AWS_TOKEN", "leak-if-logged-2"},
+ {"fs.oss.accessKeySecret", "leak-if-logged-3"},
+ {"s3.access-key", "leak-if-logged-4"},
+ {"s3.secret-key", "leak-if-logged-5"},
+ {"s3.endpoint", "http://leak-if-logged-6:19001"},
+ };
+ const std::string rendered = PaimonRustTableReader::TEST_format_options(options);
+
+ // Every key is rendered, no value and no '=' separator ever is.
+ EXPECT_NE(rendered.find("AWS_SECRET_KEY"), std::string::npos);
+ EXPECT_NE(rendered.find("fs.oss.accessKeySecret"), std::string::npos);
+ EXPECT_NE(rendered.find("s3.secret-key"), std::string::npos);
+ EXPECT_EQ(rendered.find('='), std::string::npos);
+ for (int i = 1; i <= 6; ++i) {
+ EXPECT_EQ(rendered.find("leak-if-logged-" + std::to_string(i)), std::string::npos)
+ << rendered;
+ }
+ EXPECT_EQ(rendered.find("admin"), std::string::npos) << rendered;
+}
+
+TEST_F(PaimonRustTableReaderTest, MapsAnonymousAndAssumeRoleProviderModes) {
+ // The FE storage-properties channel marks anonymous access with
+ // AWS_CREDENTIALS_PROVIDER_TYPE=ANONYMOUS and assume-role with
+ // AWS_ROLE_ARN / AWS_EXTERNAL_ID; the pinned rust S3 parser reads
+ // s3.anonymous (skip_signature) and the s3.assumed.role.* family, so the
+ // bridge must map both — without the mapping, anonymous catalogs would
+ // consult the ambient credential chain and role-only catalogs would
+ // never assume the requested role. The ambient JVM provider modes (ENV,
+ // SYSTEM_PROPERTIES, WEB_IDENTITY, CONTAINER, INSTANCE_PROFILE) have no
+ // rust equivalent and are gated away from the rust reader on the FE.
+ TFileScanRangeParams params;
+ params.properties["AWS_CREDENTIALS_PROVIDER_TYPE"] = "ANONYMOUS";
+ params.properties["AWS_ROLE_ARN"] = "arn:aws:iam::123:role/reader";
+ params.properties["AWS_EXTERNAL_ID"] = "external-123";
+ params.properties["AWS_ACCESS_KEY"] = "admin";
+ params.properties["AWS_SECRET_KEY"] = "password";
+ params.properties["AWS_ENDPOINT"] = "http://127.0.0.1:19001";
+ params.properties["AWS_REGION"] = "us-east-1";
+ params.properties["use_path_style"] = "true";
+ params.__isset.properties = true;
+
+ PaimonRustTableReader reader;
+ const auto options = reader.TEST_build_options(
+ ¶ms, make_rust_range_with_table_path("s3://bucket/wh/db.db/t"));
+ EXPECT_EQ(options.at("s3.anonymous"), "true");
+ EXPECT_EQ(options.at("s3.assumed.role.arn"), "arn:aws:iam::123:role/reader");
+ EXPECT_EQ(options.at("s3.assumed.role.externalId"), "external-123");
+ // Static credentials and connection settings keep mapping (regression
+ // for the base remap).
+ EXPECT_EQ(options.at("s3.access-key"), "admin");
+ EXPECT_EQ(options.at("s3.secret-key"), "password");
+ EXPECT_EQ(options.at("s3.endpoint"), "http://127.0.0.1:19001");
+ EXPECT_EQ(options.at("s3.region"), "us-east-1");
+ EXPECT_EQ(options.at("s3.path-style-access"), "true");
+}
+
+TEST_F(PaimonRustTableReaderTest, StaticCredentialsMapWithoutProviderMode) {
+ // Default shape: no provider-mode marker, plain static credentials.
+ TFileScanRangeParams params;
+ params.properties["AWS_ACCESS_KEY"] = "admin";
+ params.properties["AWS_SECRET_KEY"] = "password";
+ params.__isset.properties = true;
+
+ PaimonRustTableReader reader;
+ const auto options = reader.TEST_build_options(
+ ¶ms, make_rust_range_with_table_path("s3://bucket/wh/db.db/t"));
+ EXPECT_EQ(options.at("s3.access-key"), "admin");
+ EXPECT_EQ(options.at("s3.secret-key"), "password");
+ EXPECT_EQ(options.count("s3.anonymous"), 0);
+}
+
+TEST_F(PaimonRustTableReaderTest, OssSchemeMapsToOssFileIOKeys) {
+ // The pinned storage dispatcher selects the parser from the table path's
+ // URI scheme: oss:// tables read the OSS parser, which requires
+ // fs.oss.endpoint / fs.oss.accessKeyId / fs.oss.accessKeySecret (plus
+ // optional fs.oss.securityToken for STS). A production-shaped FE map
+ // (OSSProperties emits the AWS_* aliases plus the connection settings)
+ // must therefore map to the fs.oss.* family — mapping everything to
+ // s3.* leaves OSS catalogs failing to open ("Missing required OSS
+ // config: fs.oss.endpoint").
+ TFileRangeDesc range;
+ TTableFormatFileDesc table_format_params;
+ TPaimonFileDesc paimon_params;
+ paimon_params.__set_paimon_table("oss://bucket/wh/db.db/t");
+ table_format_params.__set_paimon_params(paimon_params);
+ range.__set_table_format_params(table_format_params);
+
+ TFileScanRangeParams params;
+ params.properties["AWS_ENDPOINT"] = "http://oss.internal:8080";
+ params.properties["AWS_ACCESS_KEY"] = "oss-admin";
+ params.properties["AWS_SECRET_KEY"] = "oss-password";
+ params.properties["AWS_TOKEN"] = "oss-sts-token";
+ params.properties["AWS_REGION"] = "cn-beijing";
+ params.properties["use_path_style"] = "true";
+ params.__isset.properties = true;
+
+ PaimonRustTableReader reader;
+ const auto options = reader.TEST_build_options(¶ms, range);
+ EXPECT_EQ(options.at("fs.oss.endpoint"), "http://oss.internal:8080");
+ EXPECT_EQ(options.at("fs.oss.accessKeyId"), "oss-admin");
+ EXPECT_EQ(options.at("fs.oss.accessKeySecret"), "oss-password");
+ EXPECT_EQ(options.at("fs.oss.securityToken"), "oss-sts-token");
+ // The OSS parser reads no s3.* keys; nothing but the fs.oss.* family may
+ // be synthesized for this scheme.
+ EXPECT_EQ(options.count("s3.access-key"), 0);
+ EXPECT_EQ(options.count("s3.endpoint"), 0);
+ EXPECT_EQ(options.count("s3.path-style-access"), 0);
+}
+
+TEST_F(PaimonRustTableReaderTest, OssSchemeKeepsNativeFsOssKeysUnmapped) {
+ // Native fs.oss.* options (delivered by configs that already speak the
+ // paimon-java OSS dialect) must pass through untouched instead of being
+ // remapped into the s3.* family.
+ TFileRangeDesc range;
+ TTableFormatFileDesc table_format_params;
+ TPaimonFileDesc paimon_params;
+ paimon_params.__set_paimon_table("oss://bucket/wh/db.db/t");
+ table_format_params.__set_paimon_params(paimon_params);
+ range.__set_table_format_params(table_format_params);
+
+ TFileScanRangeParams params;
+ params.properties["fs.oss.endpoint"] = "http://oss.internal:8080";
+ params.properties["fs.oss.accessKeyId"] = "oss-admin";
+ params.properties["fs.oss.accessKeySecret"] = "oss-password";
+ params.__isset.properties = true;
+
+ PaimonRustTableReader reader;
+ const auto options = reader.TEST_build_options(¶ms, range);
+ EXPECT_EQ(options.at("fs.oss.endpoint"), "http://oss.internal:8080");
+ EXPECT_EQ(options.at("fs.oss.accessKeyId"), "oss-admin");
+ EXPECT_EQ(options.at("fs.oss.accessKeySecret"), "oss-password");
+ EXPECT_EQ(options.count("s3.access-key"), 0);
+}
+
+TEST_F(PaimonRustTableReaderTest, S3aSchemeMapsToS3FileIOKeys) {
+ // The crate dispatches both s3:// and s3a:// to the S3 parser, so the
+ // legacy Hadoop-style spelling must translate to the s3.* family too.
+ TFileScanRangeParams params;
+ params.properties["AWS_ACCESS_KEY"] = "admin";
+ params.properties["AWS_SECRET_KEY"] = "password";
+ params.properties["AWS_ENDPOINT"] = "http://127.0.0.1:19001";
+ params.__isset.properties = true;
+
+ PaimonRustTableReader reader;
+ const auto options = reader.TEST_build_options(
+ ¶ms, make_rust_range_with_table_path("s3a://bucket/wh/db.db/t"));
+ EXPECT_EQ(options.at("s3.access-key"), "admin");
+ EXPECT_EQ(options.at("s3.secret-key"), "password");
+ EXPECT_EQ(options.at("s3.endpoint"), "http://127.0.0.1:19001");
+}
+
+TEST_F(PaimonRustTableReaderTest, UnverifiedSchemesPassThroughWithoutS3Aliases) {
+ // libpaimon_c.a compiles in separate COS, OBS, GCS and Azdls parsers,
+ // each reading its own key family (fs.cosn.userinfo.* / fs.obs.* /
+ // gcs.* / azure.*), while Doris's FE normalizes those object stores'
+ // credentials into the AWS_* aliases (the *Properties storage classes).
+ // The bridge only implements the s3.* / fs.oss.* translations, and the
+ // FE gates every other scheme to JNI before the split is encoded — so
+ // here nothing may be synthesized for them: a production-shaped AWS_*
+ // map must pass through untouched instead of being silently translated
+ // into s3.* keys its scheme's parser ignores (which would fail the open
+ // with a misleading auth error, e.g. "Missing required COS config").
+ const std::vector<std::string> gated_schemes {
+ "cosn://bucket/wh/db.db/t", "cos://bucket/wh/db.db/t", "obs://bucket/wh/db.db/t",
+ "gs://bucket/wh/db.db/t", "gcs://bucket/wh/db.db/t", "abfs://bucket/wh/db.db/t",
+ "abfss://bucket/wh/db.db/t", "az://bucket/wh/db.db/t", "azure://bucket/wh/db.db/t"};
+ for (const auto& table_path : gated_schemes) {
+ TFileScanRangeParams params;
+ params.properties["AWS_CREDENTIALS_PROVIDER_TYPE"] = "DEFAULT";
+ params.properties["AWS_ACCESS_KEY"] = "admin";
+ params.properties["AWS_SECRET_KEY"] = "password";
+ params.properties["AWS_ENDPOINT"] = "http://127.0.0.1:19001";
+ params.properties["AWS_REGION"] = "us-east-1";
+ params.properties["AWS_TOKEN"] = "sts-token";
+ params.properties["use_path_style"] = "true";
+ params.__isset.properties = true;
+
+ PaimonRustTableReader reader;
+ const auto options =
+ reader.TEST_build_options(¶ms, make_rust_range_with_table_path(table_path));
+ // No key of the s3.* / fs.oss.* families is synthesized for a scheme
+ // whose parser reads neither.
+ for (const char* key :
+ {"s3.access-key", "s3.secret-key", "s3.session.token", "s3.endpoint", "s3.region",
+ "s3.path-style-access", "s3.anonymous", "s3.assumed.role.arn",
+ "s3.assumed.role.externalId", "fs.oss.endpoint", "fs.oss.accessKeyId",
+ "fs.oss.accessKeySecret"}) {
+ EXPECT_EQ(options.count(key), 0) << "scheme " << table_path << " synthesized " << key;
+ }
+ // The FE-supplied property map passes through untouched.
+ EXPECT_EQ(options.at("AWS_ACCESS_KEY"), "admin");
+ EXPECT_EQ(options.at("AWS_SECRET_KEY"), "password");
+ EXPECT_EQ(options.at("AWS_ENDPOINT"), "http://127.0.0.1:19001");
+ }
+}
+
+TEST_F(PaimonRustTableReaderTest, HdfsAndLocalSchemesPassThroughHadoopConf) {
+ // hdfs:// tables read their hadoop conf as delivered (the native HDFS
+ // parser has its own config keys) and credential-free local paths need
+ // nothing at all: no AWS_* -> s3.* aliasing may be synthesized for
+ // either, even if an S3-shaped property map is present.
+ const std::vector<std::string> passthrough_schemes {"hdfs://nn/wh/db.db/t",
+ "/paimon/warehouse/db.db/t"};
+ for (const auto& table_path : passthrough_schemes) {
+ TFileScanRangeParams params;
+ params.properties["AWS_ACCESS_KEY"] = "admin";
+ params.properties["AWS_SECRET_KEY"] = "password";
+ params.properties["AWS_ENDPOINT"] = "http://127.0.0.1:19001";
+ params.__isset.properties = true;
+
+ PaimonRustTableReader reader;
+ const auto options =
+ reader.TEST_build_options(¶ms, make_rust_range_with_table_path(table_path));
+ EXPECT_EQ(options.count("s3.access-key"), 0) << table_path;
+ EXPECT_EQ(options.count("s3.secret-key"), 0) << table_path;
+ EXPECT_EQ(options.count("s3.endpoint"), 0) << table_path;
+ EXPECT_EQ(options.count("fs.oss.accessKeyId"), 0) << table_path;
+ EXPECT_EQ(options.at("AWS_ACCESS_KEY"), "admin");
+ }
+}
+
+} // namespace doris::format::paimon
diff --git a/build.sh b/build.sh
index 270c64f11d0..8449eab271a 100755
--- a/build.sh
+++ b/build.sh
@@ -447,7 +447,22 @@ else
LAST_THIRDPARTY_LIB='hadoop_hdfs/native/libhdfs.a'
fi
-if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then
+# Old installs can contain the sentinel but predate Paimon. Rebuild the whole
+# dependency set so Lance and Paimon also use the same selected Rust toolchain.
+# An interrupted header/archive publication must be repaired before either file is reused.
+if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ||
+ ! -f "${DORIS_THIRDPARTY}/installed/lib64/liblance_c.a" ||
+ ! -f "${DORIS_THIRDPARTY}/installed/lib64/libpaimon_c.a" ||
+ ! -f "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ||
+ ! -s "${DORIS_THIRDPARTY}/installed/lib64/libpaimon_c.a" ||
+ ! -s "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ||
+ -e "${DORIS_THIRDPARTY}/installed/lib64/.paimon-installing" ]]; then
+ # Compilation images may contain only installed artifacts; never erase them without a rebuild source.
+ if [[ ! -f "${DORIS_THIRDPARTY}/build-thirdparty.sh" ]]; then
+ echo "Third-party dependencies require a rebuild, but build-thirdparty.sh is missing." >&2
+ echo "Refresh the compilation image or set DORIS_THIRDPARTY to a complete third-party source tree." >&2
+ exit 1
+ fi
echo "Thirdparty libraries need to be build ..."
# need remove all installed pkgs because some lib like lz4 will throw error if its lib alreay exists
rm -rf "${DORIS_THIRDPARTY}/installed"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
index 2f6b0992ef6..185fce52d35 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
@@ -35,9 +35,11 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
@@ -71,6 +73,7 @@ public class ExecutionProfile {
// use to merge profile from multi be
private Map<Integer, Map<TNetworkAddress, List<RuntimeProfile>>> multiBeProfile = null;
+ private Map<Integer, Set<TNetworkAddress>> fragmentIdDoneBackends;
private ReentrantReadWriteLock multiBeProfileLock = new ReentrantReadWriteLock();
// Not serialize this property, it is only used to get profile id.
@@ -88,6 +91,7 @@ public class ExecutionProfile {
root.addChild(fragmentsProfile, true);
fragmentProfiles = Maps.newHashMap();
multiBeProfile = Maps.newHashMap();
+ fragmentIdDoneBackends = Maps.newHashMap();
fragmentIdBeNum = Maps.newHashMap();
seqNoToFragmentId = Maps.newHashMap();
int i = 0;
@@ -96,6 +100,7 @@ public class ExecutionProfile {
fragmentProfiles.put(fragmentId, runtimeProfile);
fragmentsProfile.addChild(runtimeProfile, true);
multiBeProfile.put(fragmentId, Maps.newHashMap());
+ fragmentIdDoneBackends.put(fragmentId, new HashSet<>());
fragmentIdBeNum.put(fragmentId, 0);
seqNoToFragmentId.put(i, fragmentId);
++i;
@@ -147,7 +152,7 @@ public class ExecutionProfile {
}
protected void setMultiBeProfile(int fragmentId, TNetworkAddress backendHBAddress,
- List<RuntimeProfile> taskProfile) {
+ List<RuntimeProfile> taskProfile) {
multiBeProfileLock.writeLock().lock();
try {
multiBeProfile.get(fragmentId).put(backendHBAddress, taskProfile);
@@ -156,6 +161,40 @@ public class ExecutionProfile {
}
}
+ private boolean updateMultiBeProfile(int fragmentId, TNetworkAddress backendHBAddress,
+ List<RuntimeProfile> taskProfile, List<RuntimeProfile> profileNodes, boolean isDone) {
+ multiBeProfileLock.writeLock().lock();
+ try {
+ if (!isDone && fragmentIdDoneBackends.get(fragmentId).contains(backendHBAddress)) {
+ return false;
+ }
+ multiBeProfile.get(fragmentId).put(backendHBAddress, taskProfile);
+ // Publish both rendered forms before completion becomes visible to profile readers.
+ for (RuntimeProfile profileNode : profileNodes) {
+ fragmentProfiles.get(fragmentId).addChild(profileNode, true);
+ }
+ if (isDone) {
+ fragmentIdDoneBackends.get(fragmentId).add(backendHBAddress);
+ }
+ return true;
+ } finally {
+ multiBeProfileLock.writeLock().unlock();
+ }
+ }
+
+ private boolean areAllBackendProfilesDone(int fragmentId, int expectedBackendNum) {
+ if (expectedBackendNum == 0) {
+ return false;
+ }
+
+ multiBeProfileLock.readLock().lock();
+ try {
+ return fragmentIdDoneBackends.get(fragmentId).size() >= expectedBackendNum;
+ } finally {
+ multiBeProfileLock.readLock().unlock();
+ }
+ }
+
protected RuntimeProfile getPipelineAggregatedProfile(Map<Integer, String> planNodeMap) {
RuntimeProfile fragmentsProfile = new RuntimeProfile("Fragments");
for (int i = 0; i < fragmentProfiles.size(); ++i) {
@@ -233,6 +272,7 @@ public class ExecutionProfile {
int fragmentId = entry.getKey();
List<TDetailedReportParams> fragmentProfile = entry.getValue();
int pipelineIdx = 0;
+ List<RuntimeProfile> profileNodes = Lists.newArrayList();
List<RuntimeProfile> taskProfile = Lists.newArrayList();
String suffix = "(host=" + backendHBAddress + ")";
for (TDetailedReportParams pipelineProfile : fragmentProfile) {
@@ -259,9 +299,11 @@ public class ExecutionProfile {
profileNode.update(pipelineProfile.profile);
profileNode.setIsDone(isDone);
- fragmentProfiles.get(fragmentId).addChild(profileNode, true);
+ profileNodes.add(profileNode);
+ }
+ if (!updateMultiBeProfile(fragmentId, backendHBAddress, taskProfile, profileNodes, isDone)) {
+ continue;
}
- setMultiBeProfile(fragmentId, backendHBAddress, taskProfile);
}
LOG.info("Profile update finished query: {} fragments: {} isDone: {}",
@@ -277,29 +319,24 @@ public class ExecutionProfile {
}
public synchronized void addFragmentBackend(PlanFragmentId fragmentId, Long backendId) {
- fragmentIdBeNum.put(fragmentId.asInt(), fragmentIdBeNum.get(fragmentId.asInt()) + 1);
+ addFragmentBackend(fragmentId.asInt(), backendId);
+ }
+
+ public synchronized void addFragmentBackend(int fragmentId, Long backendId) {
+ fragmentIdBeNum.put(fragmentId, fragmentIdBeNum.get(fragmentId) + 1);
}
public TUniqueId getQueryId() {
return queryId;
}
- // Check all fragments's child, if all finished, then this execution profile is finished
+ // Check all fragments' backend reports. A backend can contribute multiple profile nodes.
public boolean isCompleted() {
for (Entry<Integer, RuntimeProfile> element : fragmentProfiles.entrySet()) {
- RuntimeProfile fragmentProfile = element.getValue();
- // If any fragment is empty, it means BE does not report the profile, then the total
- // execution profile is not completed.
- if (fragmentProfile.isEmpty()
- || fragmentProfile.getChildList().size() < fragmentIdBeNum.get(element.getKey())) {
+ int fragmentId = element.getKey();
+ if (!areAllBackendProfilesDone(fragmentId, fragmentIdBeNum.get(fragmentId))) {
return false;
}
- for (Pair<RuntimeProfile, Boolean> runtimeProfile : fragmentProfile.getChildList()) {
- // If any child instance profile is not ready, then return false.
- if (!(runtimeProfile.first.getIsDone() || runtimeProfile.first.getIsCancel())) {
- return false;
- }
- }
}
return true;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
index f8e5b7884a3..a7e84f78487 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
@@ -117,6 +117,8 @@ public class Profile {
// 1. profile is stored to storage
// 2. or profile is loaded from storage
private String profileStoragePath = "";
+ private volatile String storageProfileCompletionState;
+ private volatile boolean executionProfilesReleased;
// isQueryFinished means the coordinator or stmt executor is finished.
// does not mean the profile report has finished, since the report is async.
// finish of collection of profile is marked by isCompleted of ExecutionProfiles.
@@ -346,6 +348,7 @@ public class Profile {
LOG.info("DebugPoint:Profile.profileSizeLimit, MAX_PROFILE_SIZE = {}", maxProfileSize);
}
// add summary to builder
+ updateProfileCompletionStateForDisplay();
summaryProfile.prettyPrint(builder);
if (!builder.isTruncated()) {
getChangedSessionVars(builder);
@@ -557,6 +560,9 @@ public class Profile {
}
public void releaseMemory() {
+ // The manager also releases reports after a failed spill. An empty list then means
+ // lost reports, not successful collection; retain that distinction across retries.
+ executionProfilesReleased = true;
this.executionProfiles.clear();
this.changedSessionVarCache = "";
this.physicalPlan = null;
@@ -619,6 +625,66 @@ public class Profile {
return !Strings.isNullOrEmpty(profileStoragePath);
}
+ // Query completion precedes asynchronous BE reports; persisted profiles must retain
+ // their terminal state even after the in-memory reports have been released.
+ public String getProfileCompletionState() {
+ String storageState = storageProfileCompletionState;
+ if (storageState != null) {
+ return storageState;
+ }
+ if (profileHasBeenStored()) {
+ String storedState = summaryProfile.getSummary().getInfoString(SummaryProfile.PROFILE_COMPLETION_STATE);
+ if (!Strings.isNullOrEmpty(storedState)) {
+ return storedState;
+ }
+ return SummaryProfile.PROFILE_COMPLETION_STATE_UNKNOWN;
+ }
+
+ if (executionProfilesReleased) {
+ return SummaryProfile.PROFILE_COMPLETION_STATE_INCOMPLETE;
+ }
+
+ if (!isQueryFinished) {
+ return SummaryProfile.PROFILE_COMPLETION_STATE_RUNNING;
+ }
+
+ for (int i = 0; i < executionProfiles.size(); i++) {
+ ExecutionProfile executionProfile = executionProfiles.get(i);
+ if (!executionProfile.isCompleted()) {
+ return SummaryProfile.PROFILE_COMPLETION_STATE_COLLECTING;
+ }
+ }
+ return SummaryProfile.PROFILE_COMPLETION_STATE_COMPLETE;
+ }
+
+ private String getProfileCompletionStateForStorage() {
+ if (executionProfilesReleased) {
+ return SummaryProfile.PROFILE_COMPLETION_STATE_INCOMPLETE;
+ }
+ if (!isQueryFinished) {
+ return SummaryProfile.PROFILE_COMPLETION_STATE_RUNNING;
+ }
+
+ for (int i = 0; i < executionProfiles.size(); i++) {
+ ExecutionProfile executionProfile = executionProfiles.get(i);
+ if (!executionProfile.isCompleted()) {
+ return SummaryProfile.PROFILE_COMPLETION_STATE_INCOMPLETE;
+ }
+ }
+ return SummaryProfile.PROFILE_COMPLETION_STATE_COMPLETE;
+ }
+
+ private synchronized void updateProfileCompletionStateForStorage() {
+ // Freeze the terminal state before serialization. Concurrent rendering must not
+ // replace INCOMPLETE with COLLECTING while the storage path is still unpublished.
+ storageProfileCompletionState = getProfileCompletionStateForStorage();
+ summaryProfile.setProfileCompletionState(storageProfileCompletionState);
+ }
+
+ private synchronized void updateProfileCompletionStateForDisplay() {
+ summaryProfile.setProfileCompletionState(getProfileCompletionState());
+ }
+
// Profile IO threads races with Coordinator threads.
public void markQueryFinished() {
try {
@@ -668,6 +734,7 @@ public class Profile {
DataOutputStream memoryDataStream = new DataOutputStream(memoryStream);
// Write summary profile and execution profile content to memory
+ updateProfileCompletionStateForStorage();
this.summaryProfile.write(memoryDataStream);
SafeStringBuilder builder = new SafeStringBuilder();
@@ -688,6 +755,7 @@ public class Profile {
this.profileStoragePath = profileFilePath;
} catch (Exception e) {
+ storageProfileCompletionState = null;
LOG.error("write {} summary profile failed", getId(), e);
return;
} finally {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
index 15e5503bced..fe703ddde0f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
@@ -62,6 +62,12 @@ public class SummaryProfile {
public static final String END_TIME = "End Time";
public static final String TOTAL_TIME = "Total";
public static final String TASK_STATE = "Task State";
+ public static final String PROFILE_COMPLETION_STATE = "Profile Completion State";
+ public static final String PROFILE_COMPLETION_STATE_RUNNING = "RUNNING";
+ public static final String PROFILE_COMPLETION_STATE_COLLECTING = "COLLECTING";
+ public static final String PROFILE_COMPLETION_STATE_COMPLETE = "COMPLETE";
+ public static final String PROFILE_COMPLETION_STATE_INCOMPLETE = "INCOMPLETE";
+ public static final String PROFILE_COMPLETION_STATE_UNKNOWN = "UNKNOWN";
public static final String USER = "User";
public static final String DEFAULT_CATALOG = "Default Catalog";
public static final String DEFAULT_DB = "Default Db";
@@ -470,6 +476,10 @@ public class SummaryProfile {
return summaryProfile;
}
+ public void setProfileCompletionState(String profileCompletionState) {
+ summaryProfile.addInfoString(PROFILE_COMPLETION_STATE, profileCompletionState);
+ }
+
public RuntimeProfile getExecutionSummary() {
return executionSummaryProfile;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/DorisToPaimonTypeVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/DorisToPaimonTypeVisitor.java
index 9eca19a1584..464a1c02ea0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/DorisToPaimonTypeVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/DorisToPaimonTypeVisitor.java
@@ -28,6 +28,7 @@ import org.apache.doris.datasource.DorisTypeVisitor;
import org.apache.paimon.types.BigIntType;
import org.apache.paimon.types.BooleanType;
+import org.apache.paimon.types.CharType;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DateType;
@@ -98,17 +99,21 @@ public class DorisToPaimonTypeVisitor extends DorisTypeVisitor<DataType> {
return new FloatType();
} else if (primitiveType.equals(PrimitiveType.DOUBLE)) {
return new DoubleType();
- } else if (primitiveType.isCharFamily()) {
+ } else if (primitiveType.equals(PrimitiveType.CHAR)) {
+ // DDL must retain bounds: widening to STRING changes historical narrowing casts.
+ return new CharType(((ScalarType) atomic).getLength());
+ } else if (primitiveType.equals(PrimitiveType.VARCHAR)) {
+ return new VarCharType(((ScalarType) atomic).getLength());
+ } else if (primitiveType.equals(PrimitiveType.STRING)) {
return new VarCharType(VarCharType.MAX_LENGTH);
} else if (primitiveType.equals(PrimitiveType.DATE) || primitiveType.equals(PrimitiveType.DATEV2)) {
return new DateType();
} else if (primitiveType.equals(PrimitiveType.DECIMALV2) || primitiveType.isDecimalV3Type()) {
return new DecimalType(((ScalarType) atomic).getScalarPrecision(), ((ScalarType) atomic).getScalarScale());
- } else if (primitiveType.equals(PrimitiveType.DATETIME)) {
- return new TimestampType();
- } else if (primitiveType.equals(PrimitiveType.DATETIMEV2)) {
- // Preserve wall-clock semantics and fractional precision in the Paimon schema.
- return new TimestampType(((ScalarType) atomic).getScalarScale());
+ } else if (primitiveType.equals(PrimitiveType.DATETIME) || primitiveType.equals(PrimitiveType.DATETIMEV2)) {
+ // Paimon interprets numeric timestamp strings in units derived from this precision.
+ return new TimestampType(primitiveType.equals(PrimitiveType.DATETIMEV2)
+ ? ((ScalarType) atomic).getScalarScale() : 0);
} else if (primitiveType.equals(PrimitiveType.TIMESTAMPTZ)) {
// TIMESTAMPTZ represents an instant, so map it to Paimon's local-zoned timestamp.
return new LocalZonedTimestampType(((ScalarType) atomic).getScalarScale());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonScanParams.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonScanParams.java
index 7a1b9718c00..2905952aa79 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonScanParams.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonScanParams.java
@@ -24,6 +24,7 @@ import org.apache.paimon.CoreOptions;
import org.apache.paimon.Snapshot;
import org.apache.paimon.options.ConfigOption;
import org.apache.paimon.options.FallbackKey;
+import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.Table;
import org.apache.paimon.table.source.snapshot.FullCompactedStartingScanner;
@@ -74,6 +75,28 @@ public final class PaimonScanParams {
private static final Set<String> INHERITED_READ_STATE_KEYS = inheritedReadStateKeys();
+ // The selectors the pinned paimon-rust reader re-resolves in copy_with_time_travel; the
+ // inverse of isolateSnapshotRead for schema transport (see withoutTimeTravelSelectors).
+ private static final Set<String> TIME_TRAVEL_SELECTOR_KEYS = ImmutableSet.of(
+ CoreOptions.SCAN_TIMESTAMP_MILLIS.key(),
+ CoreOptions.SCAN_WATERMARK.key(),
+ CoreOptions.SCAN_VERSION.key(),
+ CoreOptions.SCAN_SNAPSHOT_ID.key(),
+ CoreOptions.SCAN_TAG_NAME.key());
+
+ // Paimon's copyInternal materializes the derived scan mode into the copied schema's
+ // options when the merged options carry a selector — a statement fence pinning
+ // scan.snapshot-id leaves a bare "scan.mode=from-snapshot" in schema.options(). Each
+ // of these modes requires a selector (the rust ReadBuilder rejects them otherwise),
+ // so withoutTimeTravelSelectors drops them together with the selector they describe.
+ private static final Set<String> SELECTOR_DEPENDENT_SCAN_MODES = ImmutableSet.of(
+ "from-snapshot",
+ "from-snapshot-full",
+ "from-timestamp",
+ "from-timestamp-full",
+ "from-file-creation-time",
+ "from-creation-timestamp");
+
// FilesScan enumerates the latest partitions before applying its range-aware per-partition scan,
// so it cannot safely read a range when a partition in that range has since been dropped.
private static final Set<String> INCREMENTAL_SYSTEM_TABLES = ImmutableSet.of(
@@ -403,6 +426,42 @@ public final class PaimonScanParams {
return isolatedOptions;
}
+ /**
+ * Strips the time-travel planning selectors — and any derived scan mode Paimon
+ * materialized for them — from a resolved schema's options, keeping every
+ * field and every other option intact. This is the transport-side inverse of
+ * {@link #isolateSnapshotRead(long)}: a statement fence pins the data snapshot by merging
+ * {@code scan.snapshot-id} into the schema options, but the pinned paimon-rust reader
+ * re-resolves a transported selector inside {@code copy_with_time_travel} and swaps the
+ * shipped fields for the pinned snapshot's older schema — a column added after the last
+ * data commit then fails projection before per-file schema evolution can fill it. The rust
+ * reader instead pins data through the serialized DataSplit, so these selectors are
+ * planning-only state and must not be transported.
+ */
+ public static TableSchema withoutTimeTravelSelectors(TableSchema schema) {
+ if (TIME_TRAVEL_SELECTOR_KEYS.stream().noneMatch(schema.options()::containsKey)) {
+ return schema;
+ }
+ Map<String, String> options = new HashMap<>(schema.options());
+ TIME_TRAVEL_SELECTOR_KEYS.forEach(options.keySet()::remove);
+ // Paimon's copyInternal may have materialized the derived scan mode for the
+ // stripped selector; without the selector the rust ReadBuilder validation
+ // ("from-snapshot requires one of scan.snapshot-id, ... to be set") rejects
+ // the open. The reader instead pins data through the serialized DataSplit.
+ // Paimon accepts enum option values case-insensitively but preserves the
+ // original spelling in the options map, so a table can persist
+ // scan.mode=FROM-SNAPSHOT; the mode is normalized before the membership
+ // check because the rust reader recognizes the value case-insensitively and
+ // would reject the now-bare uppercase mode for lacking its selector, while
+ // JNI accepts the original option.
+ String scanMode = options.get(CoreOptions.SCAN_MODE.key());
+ if (scanMode != null && SELECTOR_DEPENDENT_SCAN_MODES.contains(
+ scanMode.trim().toLowerCase(Locale.ROOT))) {
+ options.remove(CoreOptions.SCAN_MODE.key());
+ }
+ return schema.copy(options);
+ }
+
private static boolean isCompatibleStartupMode(String position, String mode) {
if ("default".equals(mode)) {
return true;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
index 649efdd3e31..a3506783707 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
@@ -54,6 +54,7 @@ import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.Timestamp;
import org.apache.paimon.data.serializer.InternalRowSerializer;
+import org.apache.paimon.io.DataOutputViewStreamWrapper;
import org.apache.paimon.manifest.PartitionEntry;
import org.apache.paimon.options.ConfigOption;
import org.apache.paimon.partition.Partition;
@@ -64,6 +65,7 @@ import org.apache.paimon.table.DataTable;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.SpecialFields;
import org.apache.paimon.table.Table;
+import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.table.source.ReadBuilder;
import org.apache.paimon.tag.Tag;
import org.apache.paimon.types.ArrayType;
@@ -81,6 +83,7 @@ import org.apache.paimon.types.VarCharType;
import org.apache.paimon.utils.DateTimeUtils;
import org.apache.paimon.utils.InstantiationUtil;
import org.apache.paimon.utils.InternalRowPartitionComputer;
+import org.apache.paimon.utils.JsonSerdeUtil;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.PartitionPathUtils;
import org.apache.paimon.utils.Projection;
@@ -728,6 +731,36 @@ public class PaimonUtil {
private static final class SerializationSizeLimitException extends IOException {
}
+ /**
+ * Serialize DataSplit using Paimon's native binary format.
+ * This format is compatible with paimon-cpp reader.
+ * Uses standard Base64 encoding (not URL-safe) for BE compatibility.
+ */
+ public static String encodeDataSplitToString(DataSplit split) {
+ try {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos);
+ split.serialize(out);
+ byte[] bytes = baos.toByteArray();
+ return Base64.getEncoder().encodeToString(bytes);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to serialize DataSplit using Paimon native format", e);
+ }
+ }
+
+ /**
+ * Serialize a paimon {@link TableSchema} to JSON via Paimon's registered
+ * serde. Output is identical to an on-disk schema/schema-N file and is
+ * consumed by the paimon-rust BE reader through paimon_table_from_schema_json.
+ */
+ public static String encodeTableSchemaToJson(TableSchema tableSchema) {
+ try {
+ return JsonSerdeUtil.toJson(tableSchema);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to serialize paimon TableSchema to JSON", e);
+ }
+ }
+
public static Map<String, String> getPartitionInfoMap(Table table, BinaryRow partitionValues, String timeZone) {
return getPartitionInfoMap(table, partitionValues, timeZone, false);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java
new file mode 100644
index 00000000000..d45ff4ac70c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java
@@ -0,0 +1,282 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon.source;
+
+import org.apache.doris.analysis.SlotDescriptor;
+import org.apache.doris.analysis.TupleDescriptor;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypeChecks;
+import org.apache.paimon.types.DataTypeRoot;
+import org.apache.paimon.types.DecimalType;
+import org.apache.paimon.types.MapType;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.types.VarCharType;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/** Compatibility checks for the pinned paimon-rust reader, beyond storage capabilities. */
+final class PaimonRustReaderCapabilities {
+ private static final Set<String> SUPPORTED_AGGREGATE_NAMES = ImmutableSet.of(
+ "sum", "product", "min", "max", "last_value", "first_value", "last_non_null_value",
+ "first_non_null_value", "first_not_null_value", "bool_and", "bool_or", "listagg");
+ private final FileStoreTable table;
+ private final TableSchema schema;
+ private final boolean tableCompatible;
+ private final Map<Long, Boolean> compatibleFileSchemas = new ConcurrentHashMap<>();
+
+ PaimonRustReaderCapabilities(FileStoreTable table, TupleDescriptor tuple) {
+ this.table = table;
+ this.schema = table.schema();
+ this.tableCompatible = schema != null && hasCompatibleSchemaVersion(schema)
+ && hasFullNestedProjection(tuple) && hasCompatibleMergeEngine(schema)
+ && hasCompatibleAggregates(schema);
+ }
+
+ boolean canRead(DataSplit split) {
+ if (!tableCompatible) {
+ return false;
+ }
+ // The pinned merge reader retains losing input batches until an output batch fills.
+ // Neither zero deletes nor a small read.batch-size bounds this across multiple files.
+ if (!schema.primaryKeys().isEmpty() && split.dataFiles().size() > 1) {
+ return false;
+ }
+ for (DataFileMeta file : split.dataFiles()) {
+ // Sort-merge retains consumed batches until it emits enough rows. Retracts can
+ // produce an unbounded zero-output prefix; aggregation also rejects retracts.
+ // Unknown counts must stay on JNI, including old files without this statistic.
+ if (!schema.primaryKeys().isEmpty() && file.deleteRowCount().orElse(-1L) != 0L) {
+ return false;
+ }
+ if (file.schemaId() != schema.id() && !compatibleFileSchemas.computeIfAbsent(
+ file.schemaId(), this::hasCompatibleFileSchema)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean hasFullNestedProjection(TupleDescriptor tuple) {
+ for (SlotDescriptor slot : tuple.getSlots()) {
+ // The ABI projects only root names, while Arrow struct SerDes bind by ordinal.
+ // A pruned slot must use JNI's recursive read type, even inside arrays or maps.
+ if (slot.getType().isComplexType() && slot.getColumn() != null
+ && !slot.getType().equals(slot.getColumn().getType())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean hasCompatibleFileSchema(long id) {
+ try {
+ TableSchema fileSchema = table.schemaManager().schema(id);
+ // Match historical types by ID: renames and added fields do not require value casts.
+ return fileSchema != null && hasCompatibleSchemaVersion(fileSchema)
+ && !hasIncompatibleEvolution(fileSchema.fields(), schema.fields());
+ } catch (RuntimeException e) {
+ // Failure to establish compatibility must not opt a historical file into Rust.
+ return false;
+ }
+ }
+
+ private static boolean hasCompatibleSchemaVersion(TableSchema schema) {
+ // Java defaults a missing JSON version to 1; Rust requires that field when opening
+ // both current and historical schemas. Metadata cannot distinguish an explicit 1.
+ return schema.version() > TableSchema.PAIMON_07_VERSION;
+ }
+
+ private static boolean hasCompatibleMergeEngine(TableSchema schema) {
+ CoreOptions.MergeEngine engine = new CoreOptions(schema.options()).mergeEngine();
+ // Older partial-update/aggregate writers retained repeated INSERT keys within one
+ // file. Zero deletes cannot bound Rust's retained losing batches for those producers.
+ return schema.primaryKeys().isEmpty()
+ || (engine != CoreOptions.MergeEngine.PARTIAL_UPDATE && engine != CoreOptions.MergeEngine.AGGREGATE);
+ }
+
+ private static boolean hasIncompatibleEvolution(List<DataField> oldFields, List<DataField> newFields) {
+ Map<Integer, DataType> oldTypes = new HashMap<>();
+ for (DataField field : oldFields) {
+ oldTypes.put(field.id(), field.type());
+ }
+ for (DataField field : newFields) {
+ DataType oldType = oldTypes.get(field.id());
+ if (oldType != null && hasIncompatibleEvolution(oldType, field.type())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean hasIncompatibleEvolution(DataType oldType, DataType newType) {
+ if (oldType.equalsIgnoreNullable(newType)) {
+ return false;
+ }
+ if (oldType instanceof RowType && newType instanceof RowType) {
+ return hasIncompatibleEvolution(((RowType) oldType).getFields(), ((RowType) newType).getFields());
+ }
+ if (oldType instanceof ArrayType && newType instanceof ArrayType) {
+ return hasIncompatibleEvolution(((ArrayType) oldType).getElementType(),
+ ((ArrayType) newType).getElementType());
+ }
+ if (oldType instanceof MapType && newType instanceof MapType) {
+ MapType oldMap = (MapType) oldType;
+ MapType newMap = (MapType) newType;
+ return hasIncompatibleEvolution(oldMap.getKeyType(), newMap.getKeyType())
+ || hasIncompatibleEvolution(oldMap.getValueType(), newMap.getValueType());
+ }
+ DataTypeRoot oldRoot = oldType.getTypeRoot();
+ DataTypeRoot newRoot = newType.getTypeRoot();
+ int oldWidth = integerWidth(oldRoot);
+ int newWidth = integerWidth(newRoot);
+ if (oldWidth > 0) {
+ return !((newWidth >= oldWidth) || newRoot == DataTypeRoot.FLOAT
+ || newRoot == DataTypeRoot.DOUBLE || newRoot == DataTypeRoot.DECIMAL);
+ }
+ if (oldRoot == DataTypeRoot.FLOAT && newRoot == DataTypeRoot.DOUBLE) {
+ return false;
+ }
+ if (oldRoot == DataTypeRoot.DECIMAL && newRoot == DataTypeRoot.DECIMAL) {
+ return false;
+ }
+ if ((oldRoot == DataTypeRoot.CHAR || oldRoot == DataTypeRoot.VARCHAR)
+ && newRoot == DataTypeRoot.VARCHAR) {
+ return DataTypeChecks.getLength(oldType) > DataTypeChecks.getLength(newType);
+ }
+ if (oldRoot == newRoot && (newRoot == DataTypeRoot.TIMESTAMP_WITHOUT_TIME_ZONE
+ || newRoot == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE)) {
+ int oldPrecision = DataTypeChecks.getPrecision(oldType);
+ int newPrecision = DataTypeChecks.getPrecision(newType);
+ // Arrow's nanosecond range is narrower than historical millis/micros; Java
+ // preserves out-of-range dates while Rust's checked cast turns them into NULL.
+ return oldPrecision > newPrecision || (oldPrecision <= 6 && newPrecision > 6);
+ }
+ // Only established cast equivalence may use Rust. Java's temporal casts use the
+ // default zone and truncate negative epochs; its string casts format/pad values.
+ // Arrow differs there, in numeric narrowing/decimal rounding, and in supported
+ // constructed casts. This allowlist applies at every nested leaf and map key.
+ return true;
+ }
+
+ private static int integerWidth(DataTypeRoot type) {
+ switch (type) {
+ case TINYINT:
+ return 8;
+ case SMALLINT:
+ return 16;
+ case INTEGER:
+ return 32;
+ case BIGINT:
+ return 64;
+ default:
+ return 0;
+ }
+ }
+
+ private static boolean hasCompatibleAggregates(TableSchema schema) {
+ Map<String, String> options = schema.options();
+ CoreOptions.MergeEngine engine = new CoreOptions(options).mergeEngine();
+ if (engine != CoreOptions.MergeEngine.AGGREGATE && engine != CoreOptions.MergeEngine.PARTIAL_UPDATE) {
+ return true;
+ }
+ // Rust validates every configured name before resolving overrides or excluding keys
+ // and sequence fields. Java may accept an unused SPI function that Rust cannot open.
+ for (Map.Entry<String, String> option : options.entrySet()) {
+ String key = option.getKey();
+ if (("fields.default-aggregate-function".equals(key)
+ || (key.startsWith("fields.") && key.endsWith(".aggregate-function")))
+ && !SUPPORTED_AGGREGATE_NAMES.contains(option.getValue())) {
+ return false;
+ }
+ }
+ Set<String> sequenceFields = new HashSet<>();
+ if (engine == CoreOptions.MergeEngine.PARTIAL_UPDATE) {
+ for (String key : options.keySet()) {
+ if (key.startsWith("fields.") && key.endsWith(".sequence-group")) {
+ sequenceFields.addAll(Arrays.asList(key.substring("fields.".length(),
+ key.length() - ".sequence-group".length()).split(",")));
+ }
+ }
+ }
+ // Validate values and types, not just option keys. Java's SPI includes functions
+ // such as collect which the pinned Rust aggregator factory does not implement.
+ for (DataField field : schema.fields()) {
+ if (schema.primaryKeys().contains(field.name()) || sequenceFields.contains(field.name())) {
+ continue;
+ }
+ String function = options.get("fields." + field.name() + ".aggregate-function");
+ // Partial-update applies the default too, except to keys and sequence fields.
+ // A field override must win before checking the effective function's type support.
+ if (function == null) {
+ function = options.get("fields.default-aggregate-function");
+ }
+ if (function != null && !supportsAggregate(function, field.type())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean supportsAggregate(String function, DataType type) {
+ DataTypeRoot root = type.getTypeRoot();
+ switch (function) {
+ case "sum":
+ // Java wraps integer arithmetic and retains compact-decimal intermediate
+ // sums beyond declared precision. Rust errors or resets the accumulator.
+ return root == DataTypeRoot.FLOAT || root == DataTypeRoot.DOUBLE
+ || (type instanceof DecimalType && ((DecimalType) type).getPrecision() > 18);
+ case "product":
+ return root == DataTypeRoot.FLOAT || root == DataTypeRoot.DOUBLE;
+ case "min":
+ case "max":
+ return integerWidth(root) != 0 || root == DataTypeRoot.FLOAT || root == DataTypeRoot.DOUBLE
+ || root == DataTypeRoot.DECIMAL || root == DataTypeRoot.DATE
+ || root == DataTypeRoot.TIME_WITHOUT_TIME_ZONE
+ || root == DataTypeRoot.TIMESTAMP_WITHOUT_TIME_ZONE
+ || root == DataTypeRoot.CHAR || root == DataTypeRoot.VARCHAR;
+ case "bool_and":
+ case "bool_or":
+ return root == DataTypeRoot.BOOLEAN;
+ case "listagg":
+ return type instanceof VarCharType && ((VarCharType) type).getLength() == Integer.MAX_VALUE;
+ case "last_value":
+ case "first_value":
+ case "last_non_null_value":
+ case "first_non_null_value":
+ case "first_not_null_value":
+ return true;
+ default:
+ return false;
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
index 7b5e0e56a63..111bbc905c0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
@@ -53,6 +53,7 @@ import org.apache.doris.planner.ScanContext;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.spi.Split;
import org.apache.doris.statistics.StatisticalType;
+import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.thrift.TFileRangeDesc;
@@ -63,16 +64,21 @@ import org.apache.doris.thrift.TTableFormatFileDesc;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.format.OrcOptions;
+import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.rest.RESTTokenFileIO;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.BucketMode;
import org.apache.paimon.table.DataTable;
+import org.apache.paimon.table.FallbackReadFileStoreTable;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.Table;
import org.apache.paimon.table.source.DataSplit;
@@ -85,7 +91,7 @@ import org.apache.paimon.table.source.TableScan;
import org.apache.paimon.table.source.snapshot.SnapshotReader;
import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.DataType;
-import org.apache.paimon.types.LocalZonedTimestampType;
+import org.apache.paimon.types.DataTypeRoot;
import org.apache.paimon.types.MapType;
import org.apache.paimon.types.MultisetType;
import org.apache.paimon.types.RowType;
@@ -95,6 +101,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -102,6 +109,7 @@ import java.util.Objects;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.OptionalLong;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@@ -111,6 +119,7 @@ public class PaimonScanNode extends FileQueryScanNode {
private static final long COUNT_WITH_PARALLEL_SPLITS = 10000;
private static final long MAX_RETAINED_SERIALIZED_TASK_BYTES = 16L * 1024 * 1024;
+ private volatile PaimonRustReaderCapabilities rustReaderCapabilities;
private long maxRetainedSerializedTaskBytes = MAX_RETAINED_SERIALIZED_TASK_BYTES;
// The keys of incremental read params for Paimon SDK
private static final String PAIMON_INCREMENTAL_BETWEEN = "incremental-between";
@@ -131,11 +140,39 @@ public class PaimonScanNode extends FileQueryScanNode {
"doris.scan.manifest.parallelism-cap";
private static final String DORIS_SERIALIZED_SYSTEM_SOURCE = "doris.serialized-system-source";
private static final String DORIS_SYSTEM_TABLE_TYPE = "doris.system-table-type";
+ // Same key as the rust CoreOptions DELETION_VECTORS_MERGE_ON_READ_OPTION: paimon
+ // 1.4 exposes no Java accessor for it, so the raw TableSchema option is read.
+ private static final String DELETION_VECTORS_MERGE_ON_READ = "deletion-vectors.merge-on-read";
private static final List<String> BACKEND_PAIMON_OPTIONS = Arrays.asList(
DORIS_ENABLE_JNI_IO_MANAGER,
DORIS_JNI_IO_MANAGER_TMP_DIR,
DORIS_JNI_IO_MANAGER_IMPL_CLASS,
DORIS_ENABLE_FILE_READER_ASYNC);
+ // The table-location URI schemes whose property translation into the
+ // paimon-rust FileIO key families is implemented (BE bridge) and open
+ // tested: s3 / s3a -> the s3.* family, oss -> the fs.oss.* family, plus
+ // the credential-free hdfs and local-filesystem parsers (hadoop conf and
+ // local paths pass through untouched). See isRustVerifiedLocationScheme.
+ // An hdfs:// location additionally requires the credential-free backend
+ // shape of isRustVerifiedHdfsBackend: the same storage properties carry a
+ // Kerberized catalog's principal / keytab, its proxy user and its HA
+ // nameservice config, none of which the rust HDFS parser can honor.
+ private static final Set<String> RUST_VERIFIED_LOCATION_SCHEMES =
+ new HashSet<>(Arrays.asList("s3", "s3a", "oss", "hdfs", "file"));
+
+ // The dfs./hadoop./fs. keys a credential-free HDFS catalog may transport
+ // without changing what the pinned rust reader does: the warehouse path
+ // identity (fs.defaultFS, always shipped from the location), the
+ // authentication-type markers HdfsProperties always writes (validated to
+ // simple above), and the always-written fallback flag whose semantics only
+ // matter under Kerberos (already rejected above). Every other client
+ // setting is dropped by the rust parser and would diverge from JNI.
+ private static final Set<String> RUST_VERIFIED_HDFS_OPTION_KEYS =
+ new HashSet<>(Arrays.asList(
+ "fs.defaultFS",
+ "hadoop.security.authentication",
+ "hdfs.security.authentication",
+ "ipc.client.fallback-to-simple-auth-allowed"));
private enum SplitReadType {
JNI,
@@ -348,6 +385,283 @@ public class PaimonScanNode extends FileQueryScanNode {
}
}
+ /**
+ * Whether the table location's URI scheme is served by a paimon-rust FileIO parser whose
+ * property translation the FE/BE bridge implements (the s3.* / fs.oss.* key families for
+ * s3 / s3a / oss) or that needs no credentials at all (hdfs hadoop conf and local
+ * filesystem paths). Every other scheme the pinned crate dispatches to its own parser
+ * (cosn / obs / gs / abfs and friends) must fall back to the JNI reader because Doris
+ * delivers those credentials only as AWS_* aliases that those parsers do not read.
+ * A null location cannot be verified (and cannot ship paimon_table either), so it is
+ * not rust-eligible.
+ *
+ * <p>The scheme must also appear in the exact lowercase form the pinned crate consumes.
+ * URI schemes are case-insensitive, but the crate lowercases only its storage
+ * dispatch: its object-store path extraction strips a lowercase {@code s3://} prefix
+ * from the original string, and the hdfs / file helpers likewise match only lowercase
+ * prefixes. A {@code S3://} or {@code Hdfs://} warehouse also produces DataSplit file
+ * paths in that original casing (serialized by the paimon SDK before the FE sees
+ * them), so the mixed-case shape fails the rust open beyond the transported location.
+ * Those valid URI variants must therefore route to JNI, whose Java stack is
+ * case-insensitive everywhere.
+ */
+ @VisibleForTesting
+ static boolean isRustVerifiedLocationScheme(String location) {
+ if (location == null) {
+ return false;
+ }
+ int sep = location.indexOf("://");
+ if (sep <= 0) {
+ // No URI scheme: a plain local path reads through the crate's local-filesystem
+ // parser, which needs no credentials.
+ return true;
+ }
+ return RUST_VERIFIED_LOCATION_SCHEMES.contains(location.substring(0, sep));
+ }
+
+ // Whether the location is an hdfs:// table (the only HDFS-family scheme in
+ // RUST_VERIFIED_LOCATION_SCHEMES; viewfs / jfs never pass it).
+ private static boolean isHdfsLocationScheme(String location) {
+ if (location == null) {
+ return false;
+ }
+ int sep = location.indexOf("://");
+ return sep > 0 && "hdfs".equalsIgnoreCase(location.substring(0, sep));
+ }
+
+ // Whether any member file of the split is ORC. Paimon allows per-level
+ // file.format, so one DataSplit can mix Parquet and ORC files; the split
+ // path's suffix (the first file) cannot speak for the whole split, and the
+ // shifted ORC LTZ decode applies to whichever ORC members rust reads.
+ @VisibleForTesting
+ static boolean splitHasOrcFile(DataSplit dataSplit) {
+ if (dataSplit == null) {
+ return false;
+ }
+ for (DataFileMeta fileMeta : dataSplit.dataFiles()) {
+ String format = fileMeta.fileFormat();
+ if (format != null && "orc".equalsIgnoreCase(format)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // Mirrors the pinned paimon-rust read-mode option matrix
+ // (PartialUpdateConfig::read_unsupported_option_keys and
+ // AggregationConfig's runtime-unsupported keys). Both validate option-key
+ // PRESENCE, not values, so a table carrying the key with an off value is
+ // still rejected by the rust merge construction while Java reads it — the
+ // gate must mirror presence exactly.
+ //
+ // Partial-update reads support basic mode, sequence groups and field
+ // aggregation; unsupported keys are the remove-record-on-delete family,
+ // per-field ignore-delete / ignore-retract / distinct / nested-key /
+ // count-limit options.
+ private static boolean isRustUnsupportedPartialUpdateReadOption(String key) {
+ return (key.endsWith(".ignore-delete")
+ && !"ignore-delete".equals(key)
+ && !"partial-update.ignore-delete".equals(key))
+ || "partial-update.remove-record-on-delete".equals(key)
+ || "partial-update.remove-record-on-sequence-group".equals(key)
+ || hasFieldOptionSuffix(key, ".ignore-retract")
+ || hasFieldOptionSuffix(key, ".distinct")
+ || hasFieldOptionSuffix(key, ".nested-key")
+ || hasFieldOptionSuffix(key, ".count-limit");
+ }
+
+ // Aggregation reads support the per-field aggregate-function /
+ // list-agg-delimiter / default-aggregate-function matrix; unsupported keys
+ // are the remove-record-on-delete family, every ignore-delete spelling
+ // (including the bare one), and per-field sequence-group / ignore-retract /
+ // distinct / nested-key / count-limit options.
+ private static boolean isRustUnsupportedAggregationRuntimeOption(String key) {
+ return "ignore-delete".equals(key)
+ || key.endsWith(".ignore-delete")
+ || "aggregation.remove-record-on-delete".equals(key)
+ || hasFieldOptionSuffix(key, ".sequence-group")
+ || hasFieldOptionSuffix(key, ".ignore-retract")
+ || hasFieldOptionSuffix(key, ".distinct")
+ || hasFieldOptionSuffix(key, ".nested-key")
+ || hasFieldOptionSuffix(key, ".count-limit");
+ }
+
+ private static boolean hasFieldOptionSuffix(String key, String suffix) {
+ return key.startsWith("fields.") && key.endsWith(suffix);
+ }
+
+ // Whether the schema options carry any option key the pinned paimon-rust
+ // read rejects for this merge engine.
+ @VisibleForTesting
+ static boolean hasRustUnsupportedMergeOption(Map<String, String> options,
+ CoreOptions.MergeEngine mergeEngine) {
+ boolean partialUpdate = mergeEngine == CoreOptions.MergeEngine.PARTIAL_UPDATE;
+ for (String key : options.keySet()) {
+ if (key == null) {
+ continue;
+ }
+ if (partialUpdate ? isRustUnsupportedPartialUpdateReadOption(key)
+ : isRustUnsupportedAggregationRuntimeOption(key)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // Whether any member file of the split carries a data-file.external-paths
+ // location. Paimon can store an absolute location in each DataFileMeta, and
+ // both Java and the serialized rust split prefer it over the bucket path —
+ // but the pinned rust table builds ONE FileIO from paimon_table, whose
+ // storage enum parses every file with that warehouse-selected backend: an
+ // admitted hdfs table with an s3:// external file (or an s3 table with an
+ // oss:// file) reaches the wrong parser and fails the open, while JNI
+ // reads it. The shipped options describe only the warehouse, so any
+ // external file keeps the split on JNI.
+ @VisibleForTesting
+ static boolean splitHasExternalFiles(DataSplit dataSplit) {
+ if (dataSplit == null) {
+ return false;
+ }
+ for (DataFileMeta fileMeta : dataSplit.dataFiles()) {
+ if (fileMeta.externalPath().isPresent()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // Native and Rust ORC routing must share this recursive check: nested LTZ values
+ // require the same JVM-zone conversion, including elements of a MULTISET.
+ @VisibleForTesting
+ static boolean containsTimestampLtz(DataType type) {
+ if (type == null) {
+ return false;
+ }
+ if (type.getTypeRoot() == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) {
+ return true;
+ }
+ if (type instanceof ArrayType) {
+ return containsTimestampLtz(((ArrayType) type).getElementType());
+ }
+ if (type instanceof MapType) {
+ MapType mapType = (MapType) type;
+ return containsTimestampLtz(mapType.getKeyType())
+ || containsTimestampLtz(mapType.getValueType());
+ }
+ if (type instanceof MultisetType) {
+ return containsTimestampLtz(((MultisetType) type).getElementType());
+ }
+ if (type instanceof RowType) {
+ return ((RowType) type).getFields().stream()
+ .anyMatch(field -> containsTimestampLtz(field.type()));
+ }
+ return false;
+ }
+
+ /**
+ * Whether an HDFS catalog's shipped backend properties describe a shape the
+ * pinned paimon-rust HDFS reader can serve identically to JNI. The rust
+ * storage_hdfs parser reads only the hdfs.name-node / hdfs.enable-append
+ * keys — no kerberos, no proxy user, no HA resolution, and no Hadoop client
+ * option map (HdfsNativeConfig.options stays empty) — so a catalog whose
+ * settings need any of those would open as the BE process's ambient
+ * identity, resolve the wrong DataNode, or miss the catalog's configured
+ * access while the same query works through JNI. Only the open-tested
+ * credential-free shape stays rust-eligible: simple (or unset)
+ * authentication, no principal / keytab / proxy user, no HA nameservice
+ * resolution, and no client option beyond the always-shipped inert keys of
+ * {@link #RUST_VERIFIED_HDFS_OPTION_KEYS} — everything else in the
+ * dfs./hadoop./fs. namespaces is dropped by the rust parser and keeps the
+ * catalog on JNI (dfs.client.use.datanode.hostname=true is the canonical
+ * simple-auth example: hdfs-native defaults to false and connects to the
+ * DataNode's advertised IP instead of its hostname, which commonly fails
+ * behind containers/NAT).
+ */
+ @VisibleForTesting
+ static boolean isRustVerifiedHdfsBackend(Map<String, String> backendStorageProperties) {
+ if (backendStorageProperties == null) {
+ return true;
+ }
+ // Authentication type: "simple" (or unset) authenticates as the same
+ // ambient OS user on both readers; kerberos (or anything else) needs the
+ // channel the rust parser does not read.
+ for (String authKey : new String[] {"hadoop.security.authentication",
+ "hdfs.security.authentication"}) {
+ String value = backendStorageProperties.get(authKey);
+ if (value != null && !"simple".equalsIgnoreCase(value.trim())) {
+ return false;
+ }
+ }
+ // Kerberos identity and the proxy user: any of these configured means
+ // the open must carry a specific identity, which only JNI can honor.
+ for (String identityKey : new String[] {"hadoop.kerberos.principal",
+ "hadoop.kerberos.keytab", "hadoop.username"}) {
+ String value = backendStorageProperties.get(identityKey);
+ if (value != null && !value.trim().isEmpty()) {
+ return false;
+ }
+ }
+ // HA nameservice resolution: the rust parser receives no dfs.* config,
+ // so a nameservice-authority location (dfs.nameservices / dfs.ha.*)
+ // cannot be resolved; the proven shape is a single name-node URI.
+ String nameServices = backendStorageProperties.get("dfs.nameservices");
+ if (nameServices != null && !nameServices.trim().isEmpty()) {
+ return false;
+ }
+ for (Map.Entry<String, String> entry : backendStorageProperties.entrySet()) {
+ if (entry.getKey() != null && entry.getKey().startsWith("dfs.ha.")
+ && entry.getValue() != null && !entry.getValue().trim().isEmpty()) {
+ return false;
+ }
+ }
+ // Client options: the pinned rust storage_hdfs parser reads only the
+ // hdfs.name-node / hdfs.enable-append keys and leaves
+ // HdfsNativeConfig.options empty, so any other transported dfs./hadoop./fs.
+ // setting is silently dropped and hdfs-native runs with its own
+ // defaults. dfs.client.use.datanode.hostname=true is the canonical
+ // simple-auth example: valid without Kerberos, but hdfs-native
+ // defaults to false and connects to the DataNode's advertised IP
+ // instead of its hostname, which commonly fails behind containers/NAT
+ // while JNI succeeds. Only the keys HdfsProperties always writes for a
+ // credential-free catalog (or already validated above) are proven
+ // inert; anything else in these namespaces keeps the catalog on JNI.
+ for (Map.Entry<String, String> entry : backendStorageProperties.entrySet()) {
+ String key = entry.getKey();
+ // A blank value is the unset case (the producer null-filters; a
+ // blank site-config entry is inert), matching the identity and HA
+ // checks above.
+ if (key == null || entry.getValue() == null || entry.getValue().trim().isEmpty()) {
+ continue;
+ }
+ if ((key.startsWith("dfs.") || key.startsWith("hadoop.") || key.startsWith("fs."))
+ && !RUST_VERIFIED_HDFS_OPTION_KEYS.contains(key)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Mirrors the pinned paimon-rust {@code DataSplit::is_fully_materialized_pk_dv}: a
+ * primary-key split is safe to read raw under deletion vectors only when scan planning
+ * marked it raw convertible AND every data file is compacted (level != 0) and known to
+ * carry no retract rows (delete_row_count == Some(0)). The rust read_pk path requires
+ * this for partial-update / aggregation DV tables; anything weaker must stay on JNI.
+ */
+ @VisibleForTesting
+ static boolean isFullyMaterializedPkDvSplit(DataSplit dataSplit) {
+ if (!dataSplit.rawConvertible()) {
+ return false;
+ }
+ for (DataFileMeta fileMeta : dataSplit.dataFiles()) {
+ Optional<Long> deleteRowCount = fileMeta.deleteRowCount();
+ if (fileMeta.level() == 0 || !deleteRowCount.isPresent() || deleteRowCount.get() != 0L) {
+ return false;
+ }
+ }
+ return true;
+ }
+
private List<String> getOrderedPathPartitionKeys() {
if (source == null) {
return Collections.emptyList();
@@ -432,10 +746,348 @@ public class PaimonScanNode extends FileQueryScanNode {
String fileFormat = getFileFormat(paimonSplit.getPathString());
if (split != null) {
+ // use jni reader / paimon-cpp reader / paimon-rust reader
rangeDesc.setFormatType(TFileFormatType.FORMAT_JNI);
- // A logical DataSplit may span multiple files, so keep it intact for the JNI reader.
- fileDesc.setReaderType(TPaimonReaderType.PAIMON_JNI);
- fileDesc.setPaimonSplit(PaimonUtil.encodeObjectToString(split));
+ // paimon-cpp and paimon-rust both consume Paimon native binary serialization,
+ // which only supports DataSplit. Any other split type falls back to JNI.
+ boolean nativeSplit = split instanceof DataSplit;
+ // Fallback-read splits stay on JNI: FallbackDataSplit extends
+ // DataSplit, so the instanceof above passes, but its serializer
+ // appends an isFallback byte after the ordinary split that the
+ // pinned rust decoder rejects outright ("trailing bytes after
+ // DataSplit" — it requires full-buffer consumption), and even a
+ // permissive decode would still lack the second table identity
+ // needed to honor the fallback-side discriminator. Both sides of a
+ // FallbackReadFileStoreTable wrap their splits, so the table
+ // wrapper is gated as a whole (any split from it routes to JNI)
+ // until the rust ABI represents both sides; the FallbackSplit
+ // interface also catches a wrapper split regardless of how the
+ // table was resolved here.
+ boolean fallbackRead = split instanceof FallbackReadFileStoreTable.FallbackSplit
+ || processedTable instanceof FallbackReadFileStoreTable;
+ // Serialize the same effective table that planning and the JNI reader use.
+ // Relation options such as t@options('read.batch-size'='1') are applied by
+ // getProcessedTable() (doInitialize caches it in processedTable), and the
+ // rust reader derives its read batch size from the schema options — the raw
+ // cached table would silently drop the override. Copies, delegates and
+ // fallback wrappers of getProcessedTable() are still FileStoreTable, so the
+ // instanceof gate keeps its semantics.
+ Table paimonTable = processedTable;
+ FileStoreTable paimonFileStoreTable =
+ paimonTable instanceof FileStoreTable ? (FileStoreTable) paimonTable : null;
+ // query-auth.enabled tables stay on JNI: when catalog authorization
+ // succeeds with no row filter or column mask, Paimon still leaves an
+ // ordinary DataSplit (restricted results use QueryAuthSplit and are
+ // already handled by the nativeSplit gate above), so this table shape
+ // passes the compound gate — but the shipped schema keeps
+ // query-auth.enabled=true and the pinned rust ReadBuilder rejects
+ // every such table (its CoreOptions::ensure_read_authorized fails
+ // closed because the client cannot enforce the row filter / column
+ // masking), turning a valid authorized scan into a BE-open failure.
+ // Until the authorization result can be transported and enforced by
+ // the rust ABI, these tables route to JNI.
+ boolean queryAuthTable = false;
+ // REST-token tables stay on JNI: doInitialize snapshots
+ // RESTTokenFileIO.validToken().token() into the backend storage
+ // properties, discarding expireAtMillis and the REST refresh
+ // context, so the shipped credentials look static — but the
+ // pinned rust table reuses one option map with no refresh
+ // callback, while paimon 1.4.2's JNI RESTTokenFileIO checks
+ // expiry before each file operation and obtains a replacement
+ // token. A queued or long scan that crosses the token TTL would
+ // start on rust and later fail authentication. Gate until the
+ // rust ABI can refresh and atomically update credentials.
+ boolean restTokenTable = false;
+ // Partial-update / aggregation tables with deletion vectors only pass
+ // the rust reader in the fully materialized shape: the pinned rust
+ // read_pk rejects merge-engine=partial-update/aggregation with
+ // deletion-vectors.merge-on-read=true outright, and otherwise requires
+ // every split to be compacted and known free of retract rows
+ // (DataSplit::is_fully_materialized_pk_dv). Their ordinary DataSplits
+ // sail through the compound gate above, so without this check a valid
+ // Java/JNI scan reaches BE and the rust open fails. Deduplicate stays
+ // rust-eligible: its read_pk routes uncompacted splits to the KV
+ // reader, which applies the attached per-file DVs. merge-on-read=true
+ // is a table option, so the whole table routes to JNI;
+ // non-materialized splits are gated per split below.
+ boolean puAggDeletionVectors = false;
+ boolean dvMergeOnRead = false;
+ boolean deduplicateIgnoreDelete = false;
+ boolean rustUnsupportedMergeOption = false;
+ if (paimonFileStoreTable != null) {
+ // A renewable REST token reached the shipped properties as a
+ // plain value; only the table's FileIO type reveals it expires.
+ // Null-safe: a table handle whose FileIO is not resolved stays
+ // rust-eligible, mirroring the CoreOptions null-safety below.
+ restTokenTable = paimonFileStoreTable.fileIO() instanceof RESTTokenFileIO;
+ CoreOptions resolvedCoreOptions = paimonFileStoreTable.coreOptions();
+ // Null-safe: a table handle whose CoreOptions is not resolved
+ // (e.g. some wrapper shapes) stays rust-eligible rather than
+ // failing the scan here — the rust open itself rejects such a
+ // table if the option is really set.
+ if (resolvedCoreOptions != null) {
+ queryAuthTable = resolvedCoreOptions.queryAuthEnabled();
+ CoreOptions.MergeEngine mergeEngine = resolvedCoreOptions.mergeEngine();
+ if (resolvedCoreOptions.deletionVectorsEnabled()
+ && (mergeEngine == CoreOptions.MergeEngine.PARTIAL_UPDATE
+ || mergeEngine == CoreOptions.MergeEngine.AGGREGATE)) {
+ puAggDeletionVectors = true;
+ // The merge-engine and deletion-vectors.enabled checks
+ // above resolve through the Java CoreOptions accessors,
+ // which the table builds from this same schema options
+ // map — the one the BE rust reader deserializes from
+ // the shipped schema JSON — so they cannot diverge from
+ // what BE sees. merge-on-read has no Java accessor in
+ // paimon 1.4, so it is read raw from the map, with the
+ // rust parsing semantics (any case-insensitive "true"
+ // is on, default false).
+ TableSchema dvSchema = paimonFileStoreTable.schema();
+ Map<String, String> dvOptions = dvSchema == null ? null : dvSchema.options();
+ String mergeOnRead = dvOptions == null
+ ? null : dvOptions.get(DELETION_VECTORS_MERGE_ON_READ);
+ dvMergeOnRead = "true".equalsIgnoreCase(mergeOnRead);
+ }
+ // deduplicate.ignore-delete=true tables stay on JNI:
+ // Java's DeduplicateMergeFunction skips retract records
+ // when the option is set — including old, uncompacted
+ // files that still contain them — but the pinned rust
+ // read_pk does not pass table options into its
+ // deduplicate merge: it picks the latest row and omits
+ // the key when that row is DELETE/UPDATE_BEFORE. An
+ // uncompacted insert followed by a delete therefore
+ // returns the insert through JNI but silently disappears
+ // through rust. Gate the option until the rust merge
+ // implements it.
+ if (mergeEngine == CoreOptions.MergeEngine.DEDUPLICATE
+ && resolvedCoreOptions.ignoreDelete()) {
+ deduplicateIgnoreDelete = true;
+ }
+ // Non-DV merge options the pinned rust read rejects: Java
+ // supports partial-update.remove-record-on-delete /
+ // aggregation.remove-record-on-delete and the wider
+ // per-field retract matrix, but the rust
+ // PartialUpdateConfig / AggregationConfig validations
+ // return Unsupported for them — and the DV-derived gates
+ // above only cover deletion-vector tables, so an ordinary
+ // non-DV DataSplit with one of these options would pass the
+ // compound gate and fail during the rust merge
+ // construction. Mirror the exact rust key matrix (presence,
+ // not values) against the same schema options map BE
+ // deserializes.
+ if (mergeEngine == CoreOptions.MergeEngine.PARTIAL_UPDATE
+ || mergeEngine == CoreOptions.MergeEngine.AGGREGATE) {
+ TableSchema mergeSchema = paimonFileStoreTable.schema();
+ Map<String, String> mergeOptions =
+ mergeSchema == null ? null : mergeSchema.options();
+ rustUnsupportedMergeOption = mergeOptions != null
+ && hasRustUnsupportedMergeOption(mergeOptions, mergeEngine);
+ }
+ }
+ }
+ // paimon-rust additionally requires (a) FileScannerV2: the V1 FileScanner
+ // explicitly rejects PAIMON_RUST, so with enable_file_scanner_v2 disabled
+ // the split falls back to JNI instead of encoding a rust request that the
+ // selected scanner cannot consume, and (b) a FileStoreTable: BE opens the
+ // table via paimon_table_from_schema_json, which needs the resolved
+ // TableSchema that only FileStoreTable exposes via schema(). If the table
+ // is not a FileStoreTable (e.g. a sys table backed by DataSplit), we cannot
+ // ship a schema JSON, so fall back to CPP / JNI rather than sending an
+ // incomplete PAIMON_RUST request that BE would reject.
+ //
+ // The paimon-rust S3 bridge maps static credentials, anonymous
+ // access (AWS_CREDENTIALS_PROVIDER_TYPE=ANONYMOUS -> s3.anonymous)
+ // and assume-role (AWS_ROLE_ARN / AWS_EXTERNAL_ID ->
+ // s3.assumed.role.*), but the remaining credential-provider modes
+ // are ambient JVM provider chains (ENV, SYSTEM_PROPERTIES,
+ // WEB_IDENTITY, CONTAINER, INSTANCE_PROFILE) with no paimon-rust
+ // equivalent — rust would silently sign with whatever the ambient
+ // chain resolves to. Gate those modes away from the rust reader
+ // here so the configured provider is honored via the JNI path.
+ boolean providerModeTranslatable = true;
+ String providerType = backendStorageProperties == null
+ ? null : backendStorageProperties.get("AWS_CREDENTIALS_PROVIDER_TYPE");
+ String mode = providerType == null ? "DEFAULT" : providerType.trim().toUpperCase(Locale.ROOT);
+ if (mode.isEmpty()) {
+ mode = "DEFAULT";
+ }
+ String location = source.getTableLocation();
+ if (location != null && (location.startsWith("s3://") || location.startsWith("s3a://"))) {
+ // Java DEFAULT may resolve JVM properties or anonymous credentials;
+ // Rust's ambient chain is different. Only explicit keys or explicit
+ // anonymous access can cross this boundary without changing identity.
+ String accessKey = backendStorageProperties == null
+ ? null : backendStorageProperties.get("AWS_ACCESS_KEY");
+ String secretKey = backendStorageProperties == null
+ ? null : backendStorageProperties.get("AWS_SECRET_KEY");
+ boolean staticKeys = accessKey != null && !accessKey.trim().isEmpty()
+ && secretKey != null && !secretKey.trim().isEmpty();
+ // Hadoop's Simple provider gives static keys precedence over anonymous,
+ // role and token settings. Rust interprets those settings differently.
+ boolean conflictingStaticSettings = staticKeys && (mode.equals("ANONYMOUS")
+ || StringUtils.isNotEmpty(backendStorageProperties.get("AWS_ROLE_ARN"))
+ || StringUtils.isNotEmpty(backendStorageProperties.get("AWS_TOKEN")));
+ providerModeTranslatable = !conflictingStaticSettings
+ && (mode.equals("ANONYMOUS") || (mode.equals("DEFAULT") && staticKeys));
+ } else if (providerType != null) {
+ providerModeTranslatable = mode.equals("DEFAULT") || mode.equals("ANONYMOUS");
+ // OSS has no anonymous FileIO mode in the pinned Rust dependency.
+ if (mode.equals("ANONYMOUS") && location != null && location.startsWith("oss://")) {
+ providerModeTranslatable = false;
+ }
+ }
+ // Incremental scans (binlog / changelog / delta / diff) must stay
+ // on the JNI path: this wire format carries only an ordinary
+ // DataSplit and the rust reader invokes TableRead::to_arrow, but
+ // paimon 1.4 marks incremental splits as streaming (which the
+ // pinned rust deserializer rejects), diff requires a separate
+ // IncrementalPlan instead of an ordinary plan, and ordinary
+ // primary-key reads can merge versions rather than return the
+ // changes — until the C ABI transports the mode and plan, the
+ // rust reader cannot express any of these.
+ TableScanParams incrementalParams = getScanParams();
+ boolean isIncremental = incrementalParams != null && incrementalParams.incrementalRead();
+ // ORC TIMESTAMP_WITH_LOCAL_TIME_ZONE schemas stay on JNI: the pinned
+ // paimon-rust ORC decoder materializes LTZ instants shifted by the
+ // writer timezone (an upstream crate limitation), so a logical ORC
+ // DataSplit that selects rust (e.g. with force_jni_scanner=true or
+ // when raw conversion is unavailable) returns a different instant
+ // than JNI — applying the session timezone in BE cannot repair an
+ // epoch already shifted during decode. Two bypasses are covered:
+ // (a) the format must come from EVERY member file — paimon allows
+ // per-level file.format, so one DataSplit can mix Parquet and ORC
+ // files and the split path's suffix (the first file) would hide
+ // the ORC members; (b) the LTZ search must recurse into nested
+ // types — an LTZ under MAP/ARRAY/ROW reaches the same shifted ORC
+ // decode through the container's field materialization. Parquet
+ // files with any LTZ, and ORC without any recursive LTZ, stay
+ // rust-eligible. nativeSplit only guards the cast — non-DataSplit
+ // splits already route to JNI.
+ boolean orcLtzSchema = paimonFileStoreTable != null
+ && nativeSplit
+ && splitHasOrcFile((DataSplit) split)
+ && paimonFileStoreTable.schema().fields().stream()
+ .anyMatch(field -> containsTimestampLtz(field.type()));
+ // data-file.external-paths splits stay on JNI (see
+ // splitHasExternalFiles): the rust table's single FileIO cannot
+ // serve an external file's backend. nativeSplit only guards the
+ // cast — non-DataSplit splits already route to JNI.
+ boolean externalFileSplit = nativeSplit && splitHasExternalFiles((DataSplit) split);
+ // Projected VARIANT columns stay on JNI: the rust leaf feeds its
+ // Arrow arrays to the slot serdes, and DataTypeVariantV2SerDe::
+ // read_column_from_arrow unconditionally returns
+ // NOT_IMPLEMENTED_ERROR — a nested Variant (ARRAY / MAP / STRUCT
+ // containing one) reaches the same decoder through the container
+ // serdes. desc carries only the slots this query projects, so a
+ // table whose VARIANT column is not projected still scans on
+ // rust. Gate until the rust leaf has a Variant Arrow decoder.
+ boolean projectedVariant = desc.getSlots().stream()
+ .anyMatch(slot -> PaimonUtil.containsVariant(slot.getType()));
+ // Scheme capability gate: the pinned paimon-rust storage
+ // dispatcher (io/storage.rs) selects the FileIO parser from the
+ // table location's URI scheme, and libpaimon_c.a compiles in
+ // separate COS, OBS, GCS and Azdls parsers besides the OSS and S3
+ // ones. Doris normalizes every object store's credentials into
+ // the AWS_* / use_path_style aliases (see the *Properties storage
+ // classes), which the BE rust bridge translates only into the
+ // fs.oss.* and s3.* key families — a cosn:// / obs:// / gs:// /
+ // abfs:// warehouse would reach its scheme's parser without the
+ // key family it reads (fs.cosn.userinfo.*, fs.obs.*, gcs.*,
+ // azure.*) and fail the open instead of using JNI. Only the
+ // schemes whose property translation is implemented and
+ // open-tested (s3 / s3a / oss, via RUST_VERIFIED_LOCATION_SCHEMES)
+ // plus the credential-free hdfs and local-filesystem parsers stay
+ // rust-eligible; every other scheme falls back to JNI. A null
+ // location also routes to JNI: the rust path needs the
+ // paimon_table that only a real location can provide (BE rejects
+ // a split without it).
+ boolean schemeCapabilityVerified = isRustVerifiedLocationScheme(source.getTableLocation());
+ // An hdfs:// location is scheme-verified only together with the credential-free
+ // backend shape: the backend storage properties that ship to BE also carry an
+ // HDFS catalog's authentication (kerberos principal / keytab, proxy user, HA
+ // nameservice config), none of which the pinned rust HDFS parser reads — the
+ // scan would open as the BE process's ambient identity instead of the
+ // catalog's configured one and fail the access JNI honors. See
+ // isRustVerifiedHdfsBackend.
+ boolean hdfsBackendVerified = !isHdfsLocationScheme(source.getTableLocation())
+ || isRustVerifiedHdfsBackend(backendStorageProperties);
+ // With merge-on-read=true the whole table already routes to JNI (dvMergeOnRead);
+ // for the remaining partial-update/aggregation DV tables, a split that is
+ // not fully materialized (uncompacted level-0 data, or retractions not
+ // known to be applied — even a split with no deletion file attached yet)
+ // fails the rust is_fully_materialized_pk_dv guard, so it falls back per
+ // split instead of turning into a BE-open failure. nativeSplit and
+ // !fallbackRead only guard the cast — those splits already route to JNI.
+ boolean splitDvNotMaterialized = puAggDeletionVectors && !dvMergeOnRead
+ && nativeSplit && !fallbackRead
+ && !isFullyMaterializedPkDvSplit((DataSplit) split);
+ boolean canUseRust = sessionVariable.isEnablePaimonRustReader()
+ && sessionVariable.enableFileScannerV2 && nativeSplit && !fallbackRead
+ && !isIncremental && providerModeTranslatable && !queryAuthTable
+ && !restTokenTable
+ && !dvMergeOnRead && !splitDvNotMaterialized
+ && !orcLtzSchema && !projectedVariant && !externalFileSplit
+ && !deduplicateIgnoreDelete && !rustUnsupportedMergeOption
+ && schemeCapabilityVerified && hdfsBackendVerified
+ && paimonFileStoreTable != null;
+ if (canUseRust) {
+ // Branch tables can retain persisted modes without a selector. Validate the
+ // transported options too: the pinned Rust builder only accepts default here.
+ String scanMode = PaimonScanParams.withoutTimeTravelSelectors(paimonFileStoreTable.schema())
+ .options().get(CoreOptions.SCAN_MODE.key());
+ canUseRust = scanMode == null || "default".equalsIgnoreCase(scanMode);
+ }
+ if (canUseRust) {
+ // Every candidate can receive this range; older BEs cannot decode reader type 3.
+ canUseRust = !backendPolicy.getBackends().isEmpty()
+ && backendPolicy.getBackends().stream().allMatch(Backend::isPaimonRustReaderSupported);
+ }
+ if (canUseRust) {
+ if (rustReaderCapabilities == null) {
+ rustReaderCapabilities = new PaimonRustReaderCapabilities(paimonFileStoreTable, desc);
+ }
+ canUseRust = rustReaderCapabilities.canRead((DataSplit) split);
+ }
+ if (canUseRust) {
+ fileDesc.setReaderType(TPaimonReaderType.PAIMON_RUST);
+ fileDesc.setPaimonSplit(PaimonUtil.encodeDataSplitToString((DataSplit) split));
+ } else {
+ // A logical DataSplit may span multiple files, so keep it intact for the JNI reader.
+ fileDesc.setReaderType(TPaimonReaderType.PAIMON_JNI);
+ fileDesc.setPaimonSplit(PaimonUtil.encodeObjectToString(split));
+ }
+ // Set table location for paimon-cpp / paimon-rust reader
+ String tableLocation = source.getTableLocation();
+ if (tableLocation != null) {
+ fileDesc.setPaimonTable(tableLocation);
+ }
+ // paimon-rust reader opens tables via paimon_table_from_schema_json:
+ // ship db/table + resolved TableSchema JSON + non-default branch.
+ if (canUseRust) {
+ ExternalTable extTable = source.getExternalTable();
+ fileDesc.setDbName(extTable.getDbName());
+ fileDesc.setTableName(extTable.getName());
+
+ // No catalog / warehouse needed. FE ships the resolved TableSchema JSON
+ // and the branch (null-if-main; matches upstream paimon commit 742da63)
+ // so BE can skip a schema-file round trip. The FileStoreTable cast is
+ // safe here because canUseRust gates on it above. The fence's time-travel
+ // selector is stripped before transport: the rust reader pins data via
+ // the serialized DataSplit, and a shipped selector (scan.snapshot-id) is
+ // re-resolved by paimon-rust's copy_with_time_travel, which would swap
+ // these resolved fields for the pinned snapshot's older schema — a
+ // column added after the last data commit would then fail projection
+ // before per-file schema evolution could fill it (JNI keeps the resolved
+ // schema, so stripping restores rust/JNI parity).
+ TableSchema tableSchema =
+ PaimonScanParams.withoutTimeTravelSelectors(
+ ((FileStoreTable) paimonTable).schema());
+ fileDesc.setPaimonTableSchemaJson(PaimonUtil.encodeTableSchemaToJson(tableSchema));
+
+ String branch = CoreOptions.branch(tableSchema.options());
+ if (!Identifier.DEFAULT_MAIN_BRANCH.equals(branch)) {
+ fileDesc.setPaimonBranch(branch);
+ }
+ }
rangeDesc.setSelfSplitWeight(paimonSplit.getSelfSplitWeight());
} else {
// use native reader
@@ -584,6 +1236,9 @@ public class PaimonScanNode extends FileQueryScanNode {
pushDownCountSplits.add(split);
pushDownCountSum += count;
} else if (!forceJniScanner && !forceJniForSystemTable && supportNativeReader(optRawFiles)) {
+ // Keep raw-file reads ahead of logical JNI/Rust dispatch: only the native file
+ // reader can use footer aggregates for COUNT(col) and MIN/MAX. Logical DataSplits
+ // may require merge/delete semantics and cannot reuse independent file statistics.
if (ignoreSplitType == SessionVariable.IgnoreSplitType.IGNORE_NATIVE) {
continue;
}
@@ -1099,26 +1754,6 @@ public class PaimonScanNode extends FileQueryScanNode {
return true;
}
- private static boolean containsTimestampLtz(DataType type) {
- if (type instanceof LocalZonedTimestampType) {
- return true;
- }
- if (type instanceof ArrayType) {
- return containsTimestampLtz(((ArrayType) type).getElementType());
- }
- if (type instanceof MapType) {
- MapType map = (MapType) type;
- return containsTimestampLtz(map.getKeyType()) || containsTimestampLtz(map.getValueType());
- }
- if (type instanceof MultisetType) {
- return containsTimestampLtz(((MultisetType) type).getElementType());
- }
- if (type instanceof RowType) {
- return ((RowType) type).getFieldTypes().stream().anyMatch(PaimonScanNode::containsTimestampLtz);
- }
- return false;
- }
-
@Override
public TFileFormatType getFileFormatType() throws DdlException, MetaNotFoundException {
return TFileFormatType.FORMAT_JNI;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index bfb5b358b99..495aa82d96b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -821,6 +821,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String FORCE_JNI_SCANNER = "force_jni_scanner";
+ public static final String ENABLE_PAIMON_RUST_READER = "enable_paimon_rust_reader";
+
public static final String ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE = "enable_count_push_down_for_external_table";
public static final String FETCH_ALL_FE_FOR_SYSTEM_TABLE = "fetch_all_fe_for_system_table";
@@ -3147,6 +3149,12 @@ public class SessionVariable implements Serializable, Writable {
description = {"强制使用 jni 方式读取外表", "Force the use of jni mode to read external table"})
private boolean forceJniScanner = false;
+ @VariableMgr.VarAttr(name = ENABLE_PAIMON_RUST_READER,
+ fuzzy = true,
+ description = {"Paimon 非原生文件读取使用 paimon-rust,优先级高于 paimon-cpp",
+ "Use paimon-rust for non-native Paimon reads, higher priority than paimon-cpp"})
+ private boolean enablePaimonRustReader = false;
+
@VariableMgr.VarAttr(name = ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE,
fuzzy = true,
description = {"对外表启用 count(*) 下推优化", "enable count(*) pushdown optimization for external table"})
@@ -4076,6 +4084,8 @@ public class SessionVariable implements Serializable, Writable {
// jni
this.forceJniScanner = random.nextBoolean();
+ this.enablePaimonRustReader = random.nextBoolean();
+
// statistics
this.fetchHiveRowCountSync = random.nextBoolean();
@@ -5875,6 +5885,7 @@ public class SessionVariable implements Serializable, Writable {
tResult.setEnableOrcFilterByMinMax(enableOrcFilterByMinMax);
tResult.setEnableExprZonemapFilter(enableExprZonemapFilter);
tResult.setFilePresignedUrlTtlSeconds(filePresignedUrlTtlSeconds);
+ tResult.setEnablePaimonRustReader(enablePaimonRustReader);
tResult.setCheckOrcInitSargsSuccess(checkOrcInitSargsSuccess);
tResult.setTruncateCharOrVarcharColumns(truncateCharOrVarcharColumns);
@@ -6704,6 +6715,10 @@ public class SessionVariable implements Serializable, Writable {
return forceJniScanner;
}
+ public boolean isEnablePaimonRustReader() {
+ return enablePaimonRustReader;
+ }
+
public String getIgnoreSplitType() {
return ignoreSplitType;
}
@@ -6727,6 +6742,10 @@ public class SessionVariable implements Serializable, Writable {
forceJniScanner = force;
}
+ public void setEnablePaimonRustReader(boolean enable) {
+ enablePaimonRustReader = enable;
+ }
+
public boolean isEnableCountPushDownForExternalTable() {
return enableCountPushDownForExternalTable;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilder.java
index d9503e3145a..03f6f923dc2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilder.java
@@ -18,6 +18,7 @@
package org.apache.doris.qe.runtime;
import org.apache.doris.common.Pair;
+import org.apache.doris.common.profile.ExecutionProfile;
import org.apache.doris.common.profile.SummaryProfile;
import org.apache.doris.nereids.trees.plans.distribute.worker.BackendWorker;
import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
@@ -86,15 +87,16 @@ public class PipelineExecutionTaskBuilder {
backend,
backendServiceProxy,
serializeFragments,
- buildSingleFragmentPipelineTask(backend, fragmentParamsList)
+ buildSingleFragmentPipelineTask(
+ coordinatorContext.executionProfile, backend, fragmentParamsList)
)
);
}
return fragmentTasks;
}
- private Map<Integer, SingleFragmentPipelineTask> buildSingleFragmentPipelineTask(
- Backend backend, TPipelineFragmentParamsList fragmentParamsList) {
+ static Map<Integer, SingleFragmentPipelineTask> buildSingleFragmentPipelineTask(
+ ExecutionProfile executionProfile, Backend backend, TPipelineFragmentParamsList fragmentParamsList) {
Map<Integer, SingleFragmentPipelineTask> tasks = Maps.newLinkedHashMap();
for (TPipelineFragmentParams fragmentParams : fragmentParamsList.getParamsList()) {
int fragmentId = fragmentParams.getFragmentId();
@@ -103,6 +105,7 @@ public class PipelineExecutionTaskBuilder {
.map(TPipelineInstanceParams::getFragmentInstanceId)
.collect(Collectors.toSet());
tasks.put(fragmentId, new SingleFragmentPipelineTask(backend, fragmentId, instanceIds));
+ executionProfile.addFragmentBackend(fragmentId, backend.getId());
}
return tasks;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
index 8e475bc07dc..adb4a57d448 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
@@ -145,6 +145,8 @@ public class Backend implements Writable {
// The physical memory available for use by BE.
@SerializedName("beMemory")
private long beMemory = 0;
+ @SerializedName("supportsPaimonRustReader")
+ private volatile boolean supportsPaimonRustReader;
// from config::pipeline_executor_size , default equal cpuCores
@SerializedName("pipelineExecutorSize")
private int pipelineExecutorSize = 1;
@@ -257,6 +259,10 @@ public class Backend implements Writable {
return version;
}
+ public boolean isPaimonRustReaderSupported() {
+ return supportsPaimonRustReader;
+ }
+
public int getBePort() {
return bePort;
}
@@ -871,6 +877,11 @@ public class Backend implements Writable {
public boolean handleHbResponse(BackendHbResponse hbResponse, boolean isReplay) {
boolean isChanged = false;
if (hbResponse.getStatus() == HbStatus.OK) {
+ // An absent capability also clears support after a downgrade or legacy heartbeat replay.
+ if (supportsPaimonRustReader != hbResponse.isPaimonRustReaderSupported()) {
+ isChanged = true;
+ supportsPaimonRustReader = hbResponse.isPaimonRustReaderSupported();
+ }
if (!this.version.equals(hbResponse.getVersion())) {
isChanged = true;
this.version = hbResponse.getVersion();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java b/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java
index 41fad9961a4..55fb0c9f456 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java
@@ -51,6 +51,8 @@ public class BackendHbResponse extends HeartbeatResponse implements Writable {
private boolean isShutDown = false;
// The physical memory available for use by BE.
private long beMemory = 0;
+ @SerializedName("supportsPaimonRustReader")
+ private boolean supportsPaimonRustReader;
public BackendHbResponse() {
super(HeartbeatResponse.Type.BACKEND);
@@ -152,6 +154,14 @@ public class BackendHbResponse extends HeartbeatResponse implements Writable {
return beMemory;
}
+ public boolean isPaimonRustReaderSupported() {
+ return supportsPaimonRustReader;
+ }
+
+ public void setPaimonRustReaderSupported(boolean supported) {
+ this.supportsPaimonRustReader = supported;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java
index a8906e43df1..5e13e9191af 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java
@@ -369,9 +369,12 @@ public class HeartbeatMgr extends MasterDaemon {
isShutDown = tBackendInfo.isIsShutdown();
}
long beMemory = tBackendInfo.isSetBeMem() ? tBackendInfo.getBeMem() : 0;
- return new BackendHbResponse(backendId, bePort, httpPort, brpcPort,
+ BackendHbResponse response = new BackendHbResponse(backendId, bePort, httpPort, brpcPort,
System.currentTimeMillis(), beStartTime, version, nodeRole,
fragmentNum, lastFragmentUpdateTime, isShutDown, arrowFlightSqlPort, beMemory);
+ response.setPaimonRustReaderSupported(tBackendInfo.isSetSupportsPaimonRustReader()
+ && tBackendInfo.isSupportsPaimonRustReader());
+ return response;
} else {
return new BackendHbResponse(backendId, backend.getHost(), backend.getLastUpdateMs(),
result.getStatus().getErrorMsgs().isEmpty()
diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfileCompletionStateTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfileCompletionStateTest.java
new file mode 100644
index 00000000000..f263ff0b99f
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfileCompletionStateTest.java
@@ -0,0 +1,217 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.common.profile;
+
+import org.apache.doris.common.util.DebugUtil;
+import org.apache.doris.planner.PlanFragmentId;
+import org.apache.doris.thrift.TDetailedReportParams;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TQueryProfile;
+import org.apache.doris.thrift.TRuntimeProfileNode;
+import org.apache.doris.thrift.TRuntimeProfileTree;
+import org.apache.doris.thrift.TUniqueId;
+
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+public class ProfileCompletionStateTest {
+ @Rule
+ public TemporaryFolder temporary = new TemporaryFolder();
+
+ private Profile profile() {
+ UUID id = UUID.randomUUID();
+ TUniqueId queryId = new TUniqueId(id.getMostSignificantBits(), id.getLeastSignificantBits());
+ Profile profile = new Profile();
+ profile.getSummaryProfile().getSummary().addInfoString(SummaryProfile.PROFILE_ID, DebugUtil.printId(queryId));
+ profile.addExecutionProfile(new ExecutionProfile(queryId, Collections.singletonList(0)));
+ return profile;
+ }
+
+ private void report(ExecutionProfile execution, String host, boolean done) {
+ TRuntimeProfileNode node = new TRuntimeProfileNode();
+ node.setName("PipelineProfile");
+ node.setNumChildren(0);
+ node.setCounters(new ArrayList<>());
+ node.setMetadata(0);
+ node.setIndent(false);
+ node.setInfoStrings(new HashMap<>());
+ node.setInfoStringsDisplayOrder(new ArrayList<>());
+ node.setChildCountersMap(new HashMap<>());
+ node.setTimestamp(0);
+ TDetailedReportParams params = new TDetailedReportParams();
+ params.setProfile(new TRuntimeProfileTree(Collections.singletonList(node)));
+ params.setIsFragmentLevel(false);
+ TQueryProfile report = new TQueryProfile();
+ report.setQueryId(execution.getQueryId());
+ // Multiple pipeline nodes from one BE must not count as multiple BE reports.
+ report.putToFragmentIdToProfile(0, Arrays.asList(params, params));
+ execution.updateProfile(report, new TNetworkAddress(host, 9050), done);
+ }
+
+ @Test
+ public void waitsForEveryBackendAndRejectsStaleReports() {
+ Profile profile = profile();
+ ExecutionProfile execution = profile.getExecutionProfiles().get(0);
+ execution.addFragmentBackend(new PlanFragmentId(0), 1L);
+ execution.addFragmentBackend(new PlanFragmentId(0), 2L);
+ Assert.assertEquals("RUNNING", profile.getProfileCompletionState());
+ profile.markQueryFinished();
+ Assert.assertEquals("COLLECTING", profile.getProfileCompletionState());
+ report(execution, "127.0.0.1", true);
+ Assert.assertEquals("COLLECTING", profile.getProfileCompletionState());
+ report(execution, "127.0.0.2", true);
+ Assert.assertEquals("COMPLETE", profile.getProfileCompletionState());
+ report(execution, "127.0.0.1", false);
+ Assert.assertEquals("COMPLETE", profile.getProfileCompletionState());
+ Assert.assertTrue(profile.getProfileByLevel().contains("Profile Completion State: COMPLETE"));
+ }
+
+ private static class PausingSummaryProfile extends SummaryProfile {
+ private final transient boolean beforeSerialization;
+ private final transient CountDownLatch storagePaused;
+ private final transient CountDownLatch resumeStorage;
+
+ private PausingSummaryProfile(boolean beforeSerialization, CountDownLatch storagePaused,
+ CountDownLatch resumeStorage) {
+ this.beforeSerialization = beforeSerialization;
+ this.storagePaused = storagePaused;
+ this.resumeStorage = resumeStorage;
+ }
+
+ @Override
+ public void write(DataOutput output) throws IOException {
+ if (!beforeSerialization) {
+ super.write(output);
+ }
+ storagePaused.countDown();
+ try {
+ if (!resumeStorage.await(10, TimeUnit.SECONDS)) {
+ throw new IOException("Timed out waiting for concurrent profile rendering");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException(e);
+ }
+ if (beforeSerialization) {
+ super.write(output);
+ }
+ }
+ }
+
+ private static class FailingSummaryProfile extends SummaryProfile {
+ private transient boolean fail = true;
+
+ @Override
+ public void write(DataOutput output) throws IOException {
+ if (fail) {
+ fail = false;
+ throw new IOException("Injected storage failure");
+ }
+ super.write(output);
+ }
+ }
+
+ @Test
+ public void failedStorageDoesNotFreezeCompletionState() throws Exception {
+ Profile profile = profile();
+ SummaryProfile summary = new FailingSummaryProfile();
+ summary.getSummary().addInfoString(SummaryProfile.PROFILE_ID, profile.getId());
+ profile.setSummaryProfile(summary);
+ ExecutionProfile execution = profile.getExecutionProfiles().get(0);
+ execution.addFragmentBackend(new PlanFragmentId(0), 1L);
+ profile.markQueryFinished();
+ String directory = temporary.newFolder().getAbsolutePath();
+ profile.writeToStorage(directory);
+ Assert.assertFalse(profile.profileHasBeenStored());
+ Assert.assertEquals("COLLECTING", profile.getProfileCompletionState());
+ report(execution, "127.0.0.1", true);
+ profile.writeToStorage(directory);
+ Assert.assertTrue(profile.profileHasBeenStored());
+ Assert.assertEquals("COMPLETE", Profile.read(profile.getProfileStoragePath()).getProfileCompletionState());
+ }
+
+ @Test
+ public void renderingDuringStoragePreservesTerminalState() throws Exception {
+ for (boolean beforeSerialization : new boolean[] {true, false}) {
+ Profile profile = profile();
+ CountDownLatch storagePaused = new CountDownLatch(1);
+ CountDownLatch resumeStorage = new CountDownLatch(1);
+ SummaryProfile summary = new PausingSummaryProfile(beforeSerialization, storagePaused, resumeStorage);
+ summary.getSummary().addInfoString(SummaryProfile.PROFILE_ID, profile.getId());
+ profile.setSummaryProfile(summary);
+ profile.getExecutionProfiles().get(0).addFragmentBackend(new PlanFragmentId(0), 1L);
+ profile.markQueryFinished();
+ String directory = temporary.newFolder().getAbsolutePath();
+ ExecutorService writer = Executors.newSingleThreadExecutor();
+ try {
+ Future<?> stored = writer.submit(() -> profile.writeToStorage(directory));
+ Assert.assertTrue(storagePaused.await(10, TimeUnit.SECONDS));
+ // Render both before serialization and before path publication, while storage is paused.
+ profile.getProfileByLevel();
+ resumeStorage.countDown();
+ stored.get(10, TimeUnit.SECONDS);
+ Assert.assertTrue(profile.profileHasBeenStored());
+ Assert.assertEquals("INCOMPLETE", profile.getProfileCompletionState());
+ Profile restored = Profile.read(profile.getProfileStoragePath());
+ Assert.assertNotNull(restored);
+ Assert.assertEquals("INCOMPLETE", restored.getProfileCompletionState());
+ } finally {
+ resumeStorage.countDown();
+ writer.shutdownNow();
+ Assert.assertTrue(writer.awaitTermination(10, TimeUnit.SECONDS));
+ }
+ }
+ }
+
+ @Test
+ public void persistsTerminalStateBeforeReleasingReports() throws Exception {
+ for (boolean complete : new boolean[] {false, true}) {
+ Profile profile = profile();
+ ExecutionProfile execution = profile.getExecutionProfiles().get(0);
+ execution.addFragmentBackend(new PlanFragmentId(0), 1L);
+ profile.markQueryFinished();
+ if (complete) {
+ report(execution, "127.0.0.1", true);
+ }
+ profile.writeToStorage(temporary.newFolder().getAbsolutePath());
+ Assert.assertTrue(profile.profileHasBeenStored());
+ profile.releaseMemory();
+ String expected = complete ? "COMPLETE" : "INCOMPLETE";
+ Assert.assertEquals(expected, profile.getProfileCompletionState());
+ Profile restored = Profile.read(profile.getProfileStoragePath());
+ Assert.assertNotNull(restored);
+ Assert.assertEquals(expected, restored.getProfileCompletionState());
+ Assert.assertTrue(restored.getProfileByLevel().contains("Profile Completion State: " + expected));
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfileManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfileManagerTest.java
index 65778b8c512..f6d37c6d3b4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfileManagerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfileManagerTest.java
@@ -430,6 +430,35 @@ class ProfileManagerTest {
Assertions.assertEquals(30 / 2, profiles.size());
}
+ @Test
+ void testFailedSpillRemainsIncompleteAfterMemoryRelease() throws Exception {
+ UUID id = UUID.randomUUID();
+ TUniqueId queryId = new TUniqueId(id.getMostSignificantBits(), id.getLeastSignificantBits());
+ Profile profile = constructProfile(DebugUtil.printId(queryId));
+ ExecutionProfile execution = new ExecutionProfile(queryId, Lists.newArrayList(0));
+ execution.addFragmentBackend(0, 1L);
+ profile.addExecutionProfile(execution);
+ profile.markQueryFinished();
+ profile.setQueryFinishTimestamp(0);
+ profileManager.pushProfile(profile);
+ profileManager.addExecutionProfile(execution);
+ File invalidDirectory = new File(tempDir, "not_a_directory");
+ Assertions.assertTrue(invalidDirectory.createNewFile());
+ ProfileManager.PROFILE_STORAGE_PATH = invalidDirectory.getAbsolutePath();
+
+ profileManager.writeProfileToStorage();
+
+ Assertions.assertFalse(profile.profileHasBeenStored());
+ Assertions.assertTrue(profile.getExecutionProfiles().isEmpty());
+ Assertions.assertEquals("INCOMPLETE", profile.getProfileCompletionState());
+ Assertions.assertTrue(profile.getProfileByLevel().contains("Profile Completion State: INCOMPLETE"));
+ // A later successful spill cannot recreate the reports discarded after the failed write.
+ ProfileManager.PROFILE_STORAGE_PATH = tempDir.getAbsolutePath();
+ profileManager.writeProfileToStorage();
+ Assertions.assertTrue(profile.profileHasBeenStored());
+ Assertions.assertEquals("INCOMPLETE", Profile.read(profile.getProfileStoragePath()).getProfileCompletionState());
+ }
+
@Test
void testWriteProfileToStorage() throws InterruptedException {
// Create some test profile files
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/DorisToPaimonTypeVisitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/DorisToPaimonTypeVisitorTest.java
new file mode 100644
index 00000000000..e54a1a2ff47
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/DorisToPaimonTypeVisitorTest.java
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon;
+
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.datasource.DorisTypeVisitor;
+
+import org.apache.paimon.casting.CastExecutor;
+import org.apache.paimon.casting.CastExecutors;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.types.CharType;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.TimestampType;
+import org.apache.paimon.types.VarCharType;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class DorisToPaimonTypeVisitorTest {
+ private DataType convert(Type type) {
+ return DorisTypeVisitor.visit(type, new DorisToPaimonTypeVisitor());
+ }
+
+ @Test
+ public void testCharacterBounds() {
+ Assert.assertEquals(new CharType(3), convert(ScalarType.createCharType(3)));
+ Assert.assertEquals(new VarCharType(3), convert(ScalarType.createVarcharType(3)));
+ Assert.assertEquals(new VarCharType(VarCharType.MAX_LENGTH), convert(Type.STRING));
+ }
+
+ @Test
+ public void testTimestampPrecision() {
+ for (int precision : new int[] {0, 3, 6}) {
+ Assert.assertEquals(new TimestampType(precision),
+ convert(ScalarType.createDatetimeV2Type(precision)));
+ }
+ Assert.assertEquals(new TimestampType(0), convert(Type.DATETIME));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testMappedHistoricalCasts() {
+ CastExecutor<BinaryString, ?> varcharCast = (CastExecutor<BinaryString, ?>) CastExecutors.resolve(
+ new VarCharType(10), convert(ScalarType.createVarcharType(3)));
+ Assert.assertNotNull(varcharCast);
+ Assert.assertEquals("abc", varcharCast.cast(BinaryString.fromString("abcdef")).toString());
+ CastExecutor<BinaryString, ?> timestampCast = (CastExecutor<BinaryString, ?>) CastExecutors.resolve(
+ new VarCharType(VarCharType.MAX_LENGTH), convert(ScalarType.createDatetimeV2Type(3)));
+ Assert.assertNotNull(timestampCast);
+ Assert.assertEquals("1970-01-01T00:00:00.042", timestampCast.cast(BinaryString.fromString("42")).toString());
+ }
+
+ @Test
+ public void testNestedBoundsAndPrecision() {
+ ArrayType array = new ArrayType(ScalarType.createVarcharType(3));
+ Assert.assertEquals(new org.apache.paimon.types.ArrayType(new VarCharType(3)), convert(array));
+ MapType map = new MapType(ScalarType.createCharType(2), ScalarType.createDatetimeV2Type(3));
+ Assert.assertEquals(new org.apache.paimon.types.MapType(new CharType(false, 2), new TimestampType(3)),
+ convert(map));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustBackendCapabilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustBackendCapabilityTest.java
new file mode 100644
index 00000000000..24f0383cd87
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustBackendCapabilityTest.java
@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon.source;
+
+import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.analysis.TupleId;
+import org.apache.doris.datasource.ExternalScanNode;
+import org.apache.doris.datasource.FederationBackendPolicy;
+import org.apache.doris.datasource.paimon.PaimonExternalTable;
+import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.planner.PlanNodeId;
+import org.apache.doris.planner.ScanContext;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.system.Backend;
+import org.apache.doris.thrift.TFileRangeDesc;
+import org.apache.doris.thrift.TPaimonFileDesc;
+import org.apache.doris.thrift.TPaimonReaderType;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.manifest.FileSource;
+import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.stats.SimpleStats;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.IntType;
+import org.apache.paimon.utils.InstantiationUtil;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.List;
+
+public class PaimonRustBackendCapabilityTest {
+ @Test
+ public void testUnknownAndMixedBackendsUseJni() throws Exception {
+ Backend oldBackend = new Backend(1, "127.0.0.1", 9050);
+ Backend newBackend = capableBackend(2);
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, reader(Collections.emptyList(), true));
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, reader(Collections.singletonList(oldBackend), true));
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, reader(Arrays.asList(newBackend, oldBackend), true));
+ Assert.assertEquals(TPaimonReaderType.PAIMON_RUST, reader(Collections.singletonList(newBackend), true));
+ Assert.assertEquals(TPaimonReaderType.PAIMON_RUST, reader(Arrays.asList(newBackend, capableBackend(3)), true));
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, reader(Collections.singletonList(newBackend), false));
+ }
+
+ private Backend capableBackend(long id) {
+ // Exercise the same persisted capability used when a follower plans the scan.
+ return GsonUtils.GSON.fromJson("{\"id\":" + id + ",\"supportsPaimonRustReader\":true}", Backend.class);
+ }
+
+ private TPaimonReaderType reader(List<Backend> backends, boolean enabled) throws Exception {
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(enabled);
+ vars.enableFileScannerV2 = true;
+ PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0), new TupleDescriptor(new TupleId(0)),
+ false, vars, ScanContext.EMPTY);
+ Field policyField = ExternalScanNode.class.getDeclaredField("backendPolicy");
+ policyField.setAccessible(true);
+ ((FederationBackendPolicy) policyField.get(node)).replaceBackendOrder(backends);
+
+ PaimonSource source = Mockito.mock(PaimonSource.class);
+ FileStoreTable table = Mockito.mock(FileStoreTable.class);
+ Mockito.when(table.schema()).thenReturn(new TableSchema(1,
+ Collections.singletonList(new DataField(0, "id", new IntType())), 0,
+ Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), null));
+ PaimonExternalTable external = Mockito.mock(PaimonExternalTable.class);
+ Mockito.when(source.getExternalTable()).thenReturn(external);
+ Mockito.when(external.getDbName()).thenReturn("db");
+ Mockito.when(external.getName()).thenReturn("t");
+ Mockito.when(source.getTableLocation()).thenReturn("file:///warehouse/db/t");
+ node.setSource(source);
+ Field tableField = PaimonScanNode.class.getDeclaredField("processedTable");
+ tableField.setAccessible(true);
+ tableField.set(node, table);
+
+ DataFileMeta file = DataFileMeta.forAppend("data.parquet", 1024, 1, SimpleStats.EMPTY_STATS,
+ 1, 1, 1, Collections.emptyList(), null, FileSource.APPEND,
+ Collections.emptyList(), null, null, Collections.emptyList());
+ DataSplit split = DataSplit.builder().rawConvertible(true).withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0).withBucketPath("file:///warehouse/db/t/bucket-0")
+ .withDataFiles(Collections.singletonList(file)).build();
+ TFileRangeDesc range = new TFileRangeDesc();
+ Method setParams = PaimonScanNode.class.getDeclaredMethod("setPaimonParams",
+ TFileRangeDesc.class, PaimonSplit.class);
+ setParams.setAccessible(true);
+ setParams.invoke(node, range, new PaimonSplit(split));
+ TPaimonFileDesc fileDesc = range.getTableFormatParams().getPaimonParams();
+ if (fileDesc.getReaderType() == TPaimonReaderType.PAIMON_JNI) {
+ // Legacy BEs require the Java object encoding as well as the JNI reader enum.
+ DataSplit decoded = InstantiationUtil.deserializeObject(
+ Base64.getUrlDecoder().decode(fileDesc.getPaimonSplit()), getClass().getClassLoader());
+ Assert.assertEquals("data.parquet", decoded.dataFiles().get(0).fileName());
+ }
+ return fileDesc.getReaderType();
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java
new file mode 100644
index 00000000000..4e44aa67809
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java
@@ -0,0 +1,496 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon.source;
+
+import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.analysis.TupleId;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.Decimal;
+import org.apache.paimon.data.GenericArray;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.Timestamp;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.reader.RecordReader;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.SchemaChange;
+import org.apache.paimon.schema.SchemaManager;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.FileStoreTableFactory;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.table.source.ReadBuilder;
+import org.apache.paimon.table.source.Split;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowKind;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.function.Function;
+
+/** Real persisted files verify the Java result contract and the split metadata used by the gate. */
+public class PaimonRustReaderCompatibilityTest {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Test
+ public void testPersistedIntegerNarrowing() throws Exception {
+ FileStoreTable table = table(DataTypes.INT(), Collections.emptyMap());
+ commit(table, GenericRow.of(1, 383), GenericRow.of(2, -129), GenericRow.of(3, null));
+ assertRead(table, true, row -> row.isNullAt(1) ? null : row.getInt(1), Arrays.asList(383, -129, null));
+ table.schemaManager().commitChanges(Collections.singletonList(
+ SchemaChange.updateColumnType("v", DataTypes.TINYINT(), true)));
+ table = FileStoreTableFactory.create(table.fileIO(), table.location());
+ assertRead(table, false, row -> row.isNullAt(1) ? null : row.getByte(1),
+ Arrays.asList((byte) 127, (byte) 127, null));
+ }
+
+ @Test
+ public void testPersistedStringToTemporal() throws Exception {
+ for (boolean date : new boolean[] {true, false}) {
+ for (boolean nested : new boolean[] {false, true}) {
+ DataType sourceType = nested ? DataTypes.ROW(DataTypes.FIELD(2, "value", DataTypes.STRING()))
+ : DataTypes.STRING();
+ DataType target = date ? DataTypes.DATE() : DataTypes.TIMESTAMP(3);
+ FileStoreTable table = table(Schema.newBuilder().column("id", DataTypes.INT())
+ .column("v", sourceType), ImmutableMap.of("bucket", "-1"));
+ Object value = BinaryString.fromString("42");
+ commit(table, GenericRow.of(1, nested ? GenericRow.of(value) : value),
+ GenericRow.of(2, nested ? GenericRow.of((Object) null) : null));
+ assertRead(table, true, row -> row.getInt(0), Arrays.asList(1, 2));
+ table.schemaManager().commitChanges(Collections.singletonList(SchemaChange.updateColumnType(
+ nested ? new String[] {"v", "value"} : new String[] {"v"}, target, true)));
+ table = FileStoreTableFactory.create(table.fileIO(), table.location());
+ // Java accepts epoch-day/epoch-millisecond strings that Arrow's date parser rejects.
+ assertRead(table, false, row -> {
+ InternalRow valueRow = nested ? row.getRow(1, 1) : row;
+ int pos = nested ? 0 : 1;
+ if (valueRow.isNullAt(pos)) {
+ return null;
+ }
+ return date ? java.time.LocalDate.ofEpochDay(valueRow.getInt(pos)).toString()
+ : valueRow.getTimestamp(pos, 3).toString();
+ }, Arrays.asList(date ? "1970-02-12" : "1970-01-01T00:00:00.042", null));
+ }
+ }
+ }
+
+ @Test
+ public void testPersistedIntegerToTimestamp() throws Exception {
+ for (boolean bigint : new boolean[] {false, true}) {
+ for (int precision : new int[] {0, 3, 6}) {
+ for (boolean nested : new boolean[] {false, true}) {
+ DataType sourceType = bigint ? DataTypes.BIGINT() : DataTypes.INT();
+ FileStoreTable table = table(Schema.newBuilder().column("id", DataTypes.INT())
+ .column("v", nested ? DataTypes.ROW(DataTypes.FIELD(2, "value", sourceType)) : sourceType),
+ ImmutableMap.of("bucket", "-1"));
+ Object value = bigint ? (Object) 1700000000L : 1700000000;
+ commit(table, GenericRow.of(1, nested ? GenericRow.of(value) : value),
+ GenericRow.of(2, nested ? GenericRow.of((Object) null) : null));
+ assertRead(table, true, row -> row.getInt(0), Arrays.asList(1, 2));
+ table.schemaManager().commitChanges(Collections.singletonList(SchemaChange.updateColumnType(
+ nested ? new String[] {"v", "value"} : new String[] {"v"},
+ DataTypes.TIMESTAMP(precision), true)));
+ table = FileStoreTableFactory.create(table.fileIO(), table.location());
+ // Paimon numeric timestamps use seconds independently of the target precision.
+ assertRead(table, false, row -> {
+ InternalRow valueRow = nested ? row.getRow(1, 1) : row;
+ int pos = nested ? 0 : 1;
+ return valueRow.isNullAt(pos) ? null : valueRow.getTimestamp(pos, precision).toString();
+ }, Arrays.asList("2023-11-14T22:13:20", null));
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testPersistedTimestampToDateBeforeEpoch() throws Exception {
+ // Java divides toward zero; Arrow takes the preceding calendar date before the epoch.
+ assertPersistedEvolution(DataTypes.TIMESTAMP(3), DataTypes.DATE(), Timestamp.fromEpochMillis(-1),
+ false, row -> row.getInt(1), 0);
+ }
+
+ @Test
+ public void testPersistedTimestampZoneChanges() throws Exception {
+ TimeZone previous = TimeZone.getDefault();
+ try {
+ TimeZone.setDefault(TimeZone.getTimeZone("Asia/Shanghai"));
+ // Paimon interprets NTZ/LTZ changes in the JVM zone, unlike an Arrow timezone cast.
+ assertPersistedEvolution(DataTypes.TIMESTAMP(3), DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3),
+ Timestamp.fromEpochMillis(0), false, row -> row.getTimestamp(1, 3).getMillisecond(),
+ -8L * 60 * 60 * 1000);
+ assertPersistedEvolution(DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3), DataTypes.TIMESTAMP(3),
+ Timestamp.fromEpochMillis(0), false, row -> row.getTimestamp(1, 3).getMillisecond(),
+ 8L * 60 * 60 * 1000);
+ } finally {
+ TimeZone.setDefault(previous);
+ }
+ }
+
+ @Test
+ public void testPersistedTimeToTimestamp() throws Exception {
+ assertPersistedEvolution(DataTypes.TIME(3), DataTypes.TIMESTAMP(3), 1234, false,
+ row -> row.getTimestamp(1, 3).toString(), "1970-01-01T00:00:01.234");
+ }
+
+ @Test
+ public void testPersistedFormattedStrings() throws Exception {
+ assertPersistedEvolution(DataTypes.FLOAT(), DataTypes.STRING(), 0.0001f, false,
+ row -> row.getString(1).toString(), "1.0E-4");
+ assertPersistedEvolution(DataTypes.DOUBLE(), DataTypes.STRING(), 1.0e20, false,
+ row -> row.getString(1).toString(), "1.0E20");
+ assertPersistedEvolution(DataTypes.TIMESTAMP(3), DataTypes.STRING(), Timestamp.fromEpochMillis(1230),
+ false, row -> row.getString(1).toString(), "1970-01-01 00:00:01.230");
+ assertPersistedEvolution(DataTypes.TIME(3), DataTypes.STRING(), 1230, false,
+ row -> row.getString(1).toString(), "00:00:01.23");
+ }
+
+ @Test
+ public void testPersistedStringBounds() throws Exception {
+ assertPersistedEvolution(DataTypes.VARCHAR(10), DataTypes.VARCHAR(3), BinaryString.fromString("abcdef"),
+ false, row -> row.getString(1).toString(), "abc");
+ assertPersistedEvolution(DataTypes.CHAR(3), DataTypes.CHAR(10), BinaryString.fromString("x"),
+ false, row -> row.getString(1).toString(), "x ");
+ assertPersistedEvolution(DataTypes.VARCHAR(3), DataTypes.VARCHAR(10), BinaryString.fromString("abc"),
+ true, row -> row.getString(1).toString(), "abc");
+ }
+
+ @Test
+ public void testPersistedRowToString() throws Exception {
+ DataType rowType = DataTypes.ROW(DataTypes.FIELD(2, "a", DataTypes.INT()),
+ DataTypes.FIELD(3, "b", DataTypes.STRING()));
+ assertPersistedEvolution(rowType, DataTypes.STRING(), GenericRow.of(11, BinaryString.fromString("x")),
+ false, row -> row.getString(1).toString(), "{11, x}");
+ }
+
+ @Test
+ public void testPersistedTimestampEvolutionIntoNanosecondRange() throws Exception {
+ List<String> expected = Arrays.asList("2300-01-01T00:00:00.123", "1600-01-01T00:00:00.123",
+ "2024-01-01T00:00:00.123", null);
+ for (int oldPrecision : new int[] {3, 6}) {
+ for (boolean localZone : new boolean[] {false, true}) {
+ DataType source = localZone ? DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(oldPrecision)
+ : DataTypes.TIMESTAMP(oldPrecision);
+ FileStoreTable table = table(Schema.newBuilder().column("id", DataTypes.INT()).column("v", source),
+ ImmutableMap.of("bucket", "-1"));
+ commit(table, GenericRow.of(1, Timestamp.fromLocalDateTime(LocalDateTime.parse(expected.get(0)))),
+ GenericRow.of(2, Timestamp.fromLocalDateTime(LocalDateTime.parse(expected.get(1)))),
+ GenericRow.of(3, Timestamp.fromLocalDateTime(LocalDateTime.parse(expected.get(2)))),
+ GenericRow.of(4, null));
+ assertRead(table, true, row -> row.isNullAt(1) ? null
+ : row.getTimestamp(1, oldPrecision).toString(), expected);
+ DataType target = localZone ? DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(9) : DataTypes.TIMESTAMP(9);
+ table.schemaManager().commitChanges(Collections.singletonList(
+ SchemaChange.updateColumnType("v", target, true)));
+ table = FileStoreTableFactory.create(table.fileIO(), table.location());
+ // Java retains the full historical range; Arrow's nanosecond cast overflows these dates to NULL.
+ assertRead(table, false, row -> row.isNullAt(1) ? null : row.getTimestamp(1, 9).toString(), expected);
+ }
+ }
+ }
+
+ @Test
+ public void testPersistedSafeEvolutionControls() throws Exception {
+ assertPersistedEvolution(DataTypes.INT(), DataTypes.BIGINT(), 123, true, row -> row.getLong(1), 123L);
+ assertPersistedEvolution(DataTypes.INT(), DataTypes.DECIMAL(12, 2), 123, true,
+ row -> row.getDecimal(1, 12, 2).toBigDecimal().toPlainString(), "123.00");
+ assertPersistedEvolution(DataTypes.DECIMAL(10, 3), DataTypes.DECIMAL(10, 2),
+ Decimal.fromUnscaledLong(1235, 10, 3), true,
+ row -> row.getDecimal(1, 10, 2).toBigDecimal().toPlainString(), "1.24");
+ assertPersistedEvolution(DataTypes.TIMESTAMP(3), DataTypes.TIMESTAMP(6), Timestamp.fromEpochMillis(1234),
+ true, row -> row.getTimestamp(1, 6).toString(), "1970-01-01T00:00:01.234");
+ }
+
+ private <T> void assertPersistedEvolution(DataType sourceType, DataType targetType, Object value,
+ boolean rustExpected, Function<InternalRow, T> extract, T expected) throws Exception {
+ FileStoreTable table = table(Schema.newBuilder().column("id", DataTypes.INT()).column("v", sourceType),
+ ImmutableMap.of("bucket", "-1"));
+ commit(table, GenericRow.of(1, value), GenericRow.of(2, null));
+ assertRead(table, true, row -> row.getInt(0), Arrays.asList(1, 2));
+ table.schemaManager().commitChanges(Collections.singletonList(
+ SchemaChange.updateColumnType("v", targetType, true)));
+ table = FileStoreTableFactory.create(table.fileIO(), table.location());
+ assertRead(table, rustExpected, row -> row.isNullAt(1) ? null : extract.apply(row),
+ Arrays.asList(expected, null));
+ }
+
+ @Test
+ public void testPersistedFloatingToDecimalRounding() throws Exception {
+ FileStoreTable table = table(Schema.newBuilder().column("id", DataTypes.INT())
+ .column("v", DataTypes.DOUBLE()), ImmutableMap.of("bucket", "-1"));
+ commit(table, GenericRow.of(1, 1.005), GenericRow.of(2, -1.005), GenericRow.of(3, 1.25),
+ GenericRow.of(4, null));
+ assertRead(table, true, row -> row.getInt(0), Arrays.asList(1, 2, 3, 4));
+ table.schemaManager().commitChanges(Collections.singletonList(
+ SchemaChange.updateColumnType("v", DataTypes.DECIMAL(3, 2), true)));
+ table = FileStoreTableFactory.create(table.fileIO(), table.location());
+ assertRead(table, false, row -> row.isNullAt(1) ? null
+ : row.getDecimal(1, 3, 2).toBigDecimal().toPlainString(), Arrays.asList("1.01", "-1.01", "1.25", null));
+ }
+
+ @Test
+ public void testPersistedUnusedAggregateDefault() throws Exception {
+ for (String engine : Arrays.asList("aggregation", "partial-update")) {
+ for (String function : Arrays.asList("collect", "sum")) {
+ Map<String, String> options = new HashMap<>();
+ options.put("merge-engine", engine);
+ options.put("fields.default-aggregate-function", function);
+ options.put("fields.v.aggregate-function", "max");
+ Schema.Builder schema = Schema.newBuilder().column("id", DataTypes.INT().notNull())
+ .column("v", DataTypes.INT()).primaryKey("id");
+ boolean partial = "partial-update".equals(engine);
+ if (partial) {
+ schema.column("seq", DataTypes.INT());
+ options.put("fields.seq.sequence-group", "v");
+ }
+ FileStoreTable table = table(schema, options);
+ commit(table, partial ? GenericRow.of(1, 7, 1) : GenericRow.of(1, 7));
+ // An unused function override cannot establish that historical files contain unique keys.
+ assertRead(table, false, row -> row.getInt(1), Collections.singletonList(7));
+ }
+ }
+ }
+
+ @Test
+ public void testPersistedNonIntegerNarrowing() throws Exception {
+ for (DataType type : Arrays.asList(DataTypes.DOUBLE(), DataTypes.DECIMAL(10, 2))) {
+ boolean floating = type.equals(DataTypes.DOUBLE());
+ FileStoreTable table = table(type, Collections.emptyMap());
+ commit(table, GenericRow.of(1, floating ? 383.0 : Decimal.fromUnscaledLong(38300, 10, 2)),
+ GenericRow.of(2, floating ? -129.0 : Decimal.fromUnscaledLong(-12900, 10, 2)),
+ GenericRow.of(3, floating ? 12.0 : Decimal.fromUnscaledLong(1200, 10, 2)),
+ GenericRow.of(4, null));
+ assertRead(table, true, row -> row.getInt(0), Arrays.asList(1, 2, 3, 4));
+ table.schemaManager().commitChanges(Collections.singletonList(
+ SchemaChange.updateColumnType("v", DataTypes.TINYINT(), true)));
+ table = FileStoreTableFactory.create(table.fileIO(), table.location());
+ assertRead(table, false, row -> row.isNullAt(1) ? null : row.getByte(1),
+ Arrays.asList((byte) 127, (byte) 127, (byte) 12, null));
+ }
+ }
+
+ @Test
+ public void testPersistedPartialUpdateDefaultAggregate() throws Exception {
+ for (boolean decimal : new boolean[] {false, true}) {
+ DataType type = decimal ? DataTypes.DECIMAL(2, 0) : DataTypes.TINYINT();
+ for (boolean override : new boolean[] {false, true}) {
+ Map<String, String> options = new HashMap<>();
+ options.put("merge-engine", "partial-update");
+ options.put("fields.seq.sequence-group", "v");
+ options.put("fields.default-aggregate-function", "sum");
+ if (override) {
+ options.put("fields.v.aggregate-function", "max");
+ }
+ FileStoreTable table = table(Schema.newBuilder().column("id", DataTypes.INT().notNull())
+ .column("v", type).column("seq", DataTypes.INT()).primaryKey("id"), options);
+ commit(table, GenericRow.of(1, decimal ? Decimal.fromUnscaledLong(99, 2, 0) : (byte) 127, 1));
+ // Single-file partial updates must also fall back: old writers retained repeated keys.
+ assertRead(table, false, row -> decimal
+ ? row.getDecimal(1, 2, 0).toBigDecimal().toPlainString()
+ : Byte.toString(row.getByte(1)), Collections.singletonList(decimal ? "99" : "127"));
+ commit(table, GenericRow.of(1, decimal ? Decimal.fromUnscaledLong(1, 2, 0) : (byte) 1, 2));
+ if (decimal) {
+ commit(table, GenericRow.of(1, Decimal.fromUnscaledLong(-1, 2, 0), 3));
+ }
+ assertRead(table, false, row -> decimal
+ ? row.getDecimal(1, 2, 0).toBigDecimal().toPlainString()
+ : Byte.toString(row.getByte(1)), Collections.singletonList(
+ decimal ? "99" : override ? "127" : "-128"));
+ }
+ }
+ }
+
+ @Test
+ public void testPersistedInsertOnlyOverlappingRunsFallback() throws Exception {
+ FileStoreTable table = table(DataTypes.STRING(), ImmutableMap.of("read.batch-size", "1"));
+ for (int run = 0; run < 3; run++) {
+ List<GenericRow> rows = new ArrayList<>();
+ String payload = run + String.join("", Collections.nCopies(4096, "x"));
+ for (int key = 0; key < 1025; key++) {
+ rows.add(GenericRow.of(key, BinaryString.fromString(payload)));
+ }
+ commit(table, rows.toArray(new GenericRow[0]));
+ }
+ List<Split> splits = table.newReadBuilder().newScan().plan().splits();
+ Assert.assertEquals("Overlapping runs must be read in one logical split", 1, splits.size());
+ DataSplit split = (DataSplit) splits.get(0);
+ Assert.assertEquals(3, split.dataFiles().size());
+ split.dataFiles().forEach(file -> {
+ Assert.assertEquals(1025L, file.rowCount());
+ Assert.assertEquals(Long.valueOf(0), file.deleteRowCount().get());
+ });
+ assertRead(table, false, row -> {
+ String payload = row.getString(1).toString();
+ Assert.assertEquals(4097, payload.length());
+ return payload.substring(0, 1);
+ }, Collections.nCopies(1025, "2"));
+ }
+
+ @Test
+ public void testPersistedAppendOnlyMultiFileControl() throws Exception {
+ FileStoreTable table = table(Schema.newBuilder().column("id", DataTypes.INT()).column("v", DataTypes.INT()),
+ ImmutableMap.of("bucket", "-1"));
+ commit(table, GenericRow.of(1, 11));
+ commit(table, GenericRow.of(2, 22));
+ Assert.assertTrue(table.newReadBuilder().newScan().plan().splits().stream()
+ .anyMatch(split -> ((DataSplit) split).dataFiles().size() > 1));
+ assertRead(table, true, row -> row.getInt(1), Arrays.asList(11, 22));
+ }
+
+ @Test
+ public void testPersistedIntegerSumAndProductOverflow() throws Exception {
+ DataType[] types = {DataTypes.TINYINT(), DataTypes.SMALLINT(), DataTypes.INT(), DataTypes.BIGINT()};
+ Object[] maximums = {(byte) 127, (short) 32767, Integer.MAX_VALUE, Long.MAX_VALUE};
+ Object[] ones = {(byte) 1, (short) 1, 1, 1L};
+ Object[] halves = {(byte) 64, (short) 16384, 1073741824, 4611686018427387904L};
+ Object[] twos = {(byte) 2, (short) 2, 2, 2L};
+ for (String function : Arrays.asList("sum", "product")) {
+ for (int i = 0; i < types.length; i++) {
+ FileStoreTable table = table(types[i], aggregate(function));
+ commit(table, GenericRow.of(1, "sum".equals(function) ? maximums[i] : halves[i]));
+ commit(table, GenericRow.of(1, "sum".equals(function) ? ones[i] : twos[i]));
+ InternalRow.FieldGetter getter = InternalRow.createFieldGetter(types[i], 1);
+ Object minimum = i == 0 ? (byte) -128 : i == 1 ? (short) -32768
+ : i == 2 ? Integer.MIN_VALUE : Long.MIN_VALUE;
+ // Compare decimal strings to avoid Java's conditional numeric promotion.
+ assertRead(table, false, row -> getter.getFieldOrNull(row).toString(),
+ Collections.singletonList(minimum.toString()));
+ }
+ }
+ }
+
+ @Test
+ public void testPersistedCompactDecimalIntermediateOverflow() throws Exception {
+ FileStoreTable table = table(DataTypes.DECIMAL(2, 0), aggregate("sum"));
+ for (long value : new long[] {99, 1, -1}) {
+ commit(table, GenericRow.of(1, Decimal.fromUnscaledLong(value, 2, 0)));
+ }
+ assertRead(table, false, row -> row.getDecimal(1, 2, 0).toBigDecimal().toPlainString(),
+ Collections.singletonList("99"));
+ }
+
+ @Test
+ public void testPersistedCollect() throws Exception {
+ FileStoreTable table = table(DataTypes.ARRAY(DataTypes.INT()), aggregate("collect"));
+ commit(table, GenericRow.of(1, new GenericArray(new Integer[] {11})));
+ commit(table, GenericRow.of(1, new GenericArray(new Integer[] {22})));
+ assertRead(table, false, row -> {
+ List<Integer> values = Arrays.asList(row.getArray(1).getInt(0), row.getArray(1).getInt(1));
+ Collections.sort(values);
+ return values;
+ }, Collections.singletonList(Arrays.asList(11, 22)));
+ }
+
+ @Test
+ public void testPersistedAggregateInsertsAndRetractsFallback() throws Exception {
+ for (RowKind kind : Arrays.asList(RowKind.DELETE, RowKind.UPDATE_BEFORE)) {
+ FileStoreTable table = table(DataTypes.DOUBLE(), aggregate("sum"));
+ commit(table, GenericRow.of(1, 10.0));
+ assertRead(table, false, row -> row.getDouble(1), Collections.singletonList(10.0));
+ commit(table, GenericRow.of(1, 3.0));
+ assertRead(table, false, row -> row.getDouble(1), Collections.singletonList(13.0));
+ commit(table, GenericRow.ofKind(kind, 1, 3.0));
+ assertRead(table, false, row -> row.getDouble(1), Collections.singletonList(10.0));
+ }
+ }
+
+ @Test
+ public void testPersistedZeroAndSparseOutputMerge() throws Exception {
+ for (boolean survivor : new boolean[] {false, true}) {
+ FileStoreTable table = table(DataTypes.INT(), Collections.emptyMap());
+ List<GenericRow> inserts = new ArrayList<>();
+ List<GenericRow> deletes = new ArrayList<>();
+ for (int i = 0; i < 2048; i++) {
+ inserts.add(GenericRow.of(i, i));
+ deletes.add(GenericRow.ofKind(RowKind.DELETE, i, i));
+ }
+ if (survivor) {
+ inserts.add(GenericRow.of(2048, 2048));
+ }
+ commit(table, inserts.toArray(new GenericRow[0]));
+ commit(table, deletes.toArray(new GenericRow[0]));
+ assertRead(table, false, row -> row.getInt(1), survivor
+ ? Collections.singletonList(2048) : Collections.emptyList());
+ }
+ }
+
+ private static Map<String, String> aggregate(String function) {
+ return ImmutableMap.of("merge-engine", "aggregation", "fields.v.aggregate-function", function);
+ }
+
+ private FileStoreTable table(DataType type, Map<String, String> options) throws Exception {
+ return table(Schema.newBuilder().column("id", DataTypes.INT().notNull()).column("v", type)
+ .primaryKey("id"), options);
+ }
+
+ private FileStoreTable table(Schema.Builder schema, Map<String, String> options) throws Exception {
+ Path path = new Path(temporaryFolder.newFolder().toURI());
+ LocalFileIO fileIO = LocalFileIO.create();
+ schema.option("bucket", "1").option("write-only", "true")
+ .option("file.format", "parquet").option("deletion-vectors.enabled", "false")
+ .option("read.batch-size", "16").option("scan.manifest.parallelism", "1");
+ options.forEach(schema::option);
+ new SchemaManager(fileIO, path).createTable(schema.build());
+ return FileStoreTableFactory.create(fileIO, path);
+ }
+
+ private static void commit(FileStoreTable table, GenericRow... rows) throws Exception {
+ // Separate writers force persisted, uncompacted inputs instead of a write-time reduction.
+ BatchWriteBuilder builder = table.newBatchWriteBuilder();
+ try (BatchTableWrite write = builder.newWrite(); BatchTableCommit commit = builder.newCommit()) {
+ for (GenericRow row : rows) {
+ write.write(row);
+ }
+ commit.commit(write.prepareCommit());
+ }
+ }
+
+ private static <T> void assertRead(FileStoreTable table, boolean rustExpected,
+ Function<InternalRow, T> extract, List<T> expected) throws Exception {
+ ReadBuilder builder = table.newReadBuilder();
+ List<Split> splits = builder.newScan().plan().splits();
+ Assert.assertFalse("Persisted files must reach scan planning", splits.isEmpty());
+ PaimonRustReaderCapabilities capabilities = new PaimonRustReaderCapabilities(table,
+ new TupleDescriptor(new TupleId(0)));
+ List<T> actual = new ArrayList<>();
+ for (Split split : splits) {
+ Assert.assertEquals("reader eligibility for persisted input", rustExpected,
+ capabilities.canRead((DataSplit) split));
+ try (RecordReader<InternalRow> reader = builder.newRead().createReader(split)) {
+ reader.forEachRemaining(row -> actual.add(extract.apply(row)));
+ }
+ }
+ Assert.assertEquals(expected, actual);
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderEvolutionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderEvolutionTest.java
new file mode 100644
index 00000000000..e3ee311e38c
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderEvolutionTest.java
@@ -0,0 +1,184 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon.source;
+
+import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.analysis.TupleId;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.manifest.FileSource;
+import org.apache.paimon.schema.SchemaManager;
+import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.stats.SimpleStats;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Optional;
+
+public class PaimonRustReaderEvolutionTest {
+ @Test
+ public void testPrimaryKeyMergeEnginesWithOneInsertOnlyFile() {
+ for (String engine : Arrays.asList("deduplicate", "first-row", "partial-update", "aggregation")) {
+ FileStoreTable table = Mockito.mock(FileStoreTable.class);
+ TableSchema schema = new TableSchema(2, Arrays.asList(new DataField(0, "id", DataTypes.INT()),
+ new DataField(1, "v", DataTypes.DOUBLE())), 1, Collections.emptyList(),
+ Collections.singletonList("id"), Collections.singletonMap("merge-engine", engine), null);
+ Mockito.when(table.schema()).thenReturn(schema);
+ DataFileMeta file = Mockito.mock(DataFileMeta.class);
+ Mockito.when(file.schemaId()).thenReturn(2L);
+ Mockito.when(file.deleteRowCount()).thenReturn(Optional.of(0L));
+ DataSplit split = DataSplit.builder().withPartition(BinaryRow.EMPTY_ROW).withBucket(0)
+ .withBucketPath("file:///warehouse/table/bucket-0")
+ .withDataFiles(Collections.singletonList(file)).build();
+ // Older writers can keep repeated inserts inside one aggregate/partial-update file.
+ Assert.assertEquals(engine, "deduplicate".equals(engine) || "first-row".equals(engine),
+ new PaimonRustReaderCapabilities(table, new TupleDescriptor(new TupleId(0))).canRead(split));
+ }
+ }
+
+ @Test
+ public void testTimestampEvolutionIntoNanosecondRangeFallback() {
+ for (int oldPrecision : new int[] {0, 3, 6}) {
+ for (int newPrecision : new int[] {7, 8, 9}) {
+ assertNestedEvolution(DataTypes.TIMESTAMP(oldPrecision), DataTypes.TIMESTAMP(newPrecision), false);
+ assertNestedEvolution(DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(oldPrecision),
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(newPrecision), false);
+ }
+ }
+ assertNestedEvolution(DataTypes.TIMESTAMP(7), DataTypes.TIMESTAMP(9), true);
+ }
+
+ @Test
+ public void testTimestampToDateFallback() {
+ assertNestedEvolution(DataTypes.TIMESTAMP(3), DataTypes.DATE(), false);
+ assertNestedEvolution(DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3), DataTypes.DATE(), false);
+ }
+
+ @Test
+ public void testTimestampZoneChangeFallback() {
+ assertNestedEvolution(DataTypes.TIMESTAMP(3), DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3), false);
+ assertNestedEvolution(DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3), DataTypes.TIMESTAMP(3), false);
+ }
+
+ @Test
+ public void testTimeToTimestampFallback() {
+ assertNestedEvolution(DataTypes.TIME(3), DataTypes.TIMESTAMP(3), false);
+ assertNestedEvolution(DataTypes.TIME(3), DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3), false);
+ }
+
+ @Test
+ public void testFormattedStringFallback() {
+ for (DataType source : Arrays.asList(DataTypes.FLOAT(), DataTypes.DOUBLE(), DataTypes.TIME(3),
+ DataTypes.TIMESTAMP(3), DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3))) {
+ for (DataType target : Arrays.asList(DataTypes.STRING(), DataTypes.VARCHAR(30), DataTypes.CHAR(30))) {
+ assertNestedEvolution(source, target, false);
+ }
+ }
+ }
+
+ @Test
+ public void testConstructedStringFallback() {
+ for (DataType source : Arrays.asList(DataTypes.ROW(DataTypes.FIELD(2, "a", DataTypes.INT())),
+ DataTypes.ARRAY(DataTypes.INT()), DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()))) {
+ assertNestedEvolution(source, DataTypes.STRING(), false);
+ }
+ }
+
+ @Test
+ public void testUnprovenScalarCastsFallback() {
+ assertNestedEvolution(DataTypes.STRING(), DataTypes.TIME(3), false);
+ assertNestedEvolution(DataTypes.DATE(), DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3), false);
+ assertNestedEvolution(DataTypes.TIMESTAMP(6), DataTypes.TIMESTAMP(3), false);
+ assertNestedEvolution(DataTypes.VARCHAR(10), DataTypes.VARCHAR(3), false);
+ assertNestedEvolution(DataTypes.CHAR(3), DataTypes.CHAR(10), false);
+ }
+
+ @Test
+ public void testSafeEvolutionControls() {
+ DataType[][] pairs = {
+ {DataTypes.TINYINT(), DataTypes.SMALLINT()},
+ {DataTypes.INT(), DataTypes.BIGINT()},
+ {DataTypes.INT(), DataTypes.DOUBLE()},
+ {DataTypes.INT(), DataTypes.DECIMAL(12, 2)},
+ {DataTypes.FLOAT(), DataTypes.DOUBLE()},
+ {DataTypes.DECIMAL(10, 3), DataTypes.DECIMAL(10, 2)},
+ {DataTypes.VARCHAR(10), DataTypes.VARCHAR(30)},
+ {DataTypes.CHAR(3), DataTypes.VARCHAR(10)},
+ {DataTypes.TIMESTAMP(3), DataTypes.TIMESTAMP(6)},
+ {DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3), DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(6)},
+ {DataTypes.DATE(), DataTypes.DATE()}
+ };
+ for (DataType[] pair : pairs) {
+ assertNestedEvolution(pair[0], pair[1], true);
+ }
+ }
+
+ @Test
+ public void testNestedFieldsMatchById() {
+ DataType oldType = DataTypes.ROW(DataTypes.FIELD(2, "removed", DataTypes.TIMESTAMP(3)),
+ DataTypes.FIELD(3, "old_name", DataTypes.INT()));
+ DataType newType = DataTypes.ROW(DataTypes.FIELD(3, "renamed", DataTypes.BIGINT()),
+ DataTypes.FIELD(4, "added", DataTypes.STRING()));
+ assertNestedEvolution(oldType, newType, true);
+ }
+
+ private void assertNestedEvolution(DataType source, DataType target, boolean expected) {
+ DataType[][] shapes = {
+ {source, target},
+ {DataTypes.ROW(new DataField(10, "old_name", source)),
+ DataTypes.ROW(new DataField(10, "renamed", target))},
+ {DataTypes.ARRAY(source), DataTypes.ARRAY(target)},
+ {DataTypes.MAP(source, DataTypes.INT()), DataTypes.MAP(target, DataTypes.INT())},
+ {DataTypes.MAP(DataTypes.INT(), source), DataTypes.MAP(DataTypes.INT(), target)}
+ };
+ for (DataType[] shape : shapes) {
+ Assert.assertTrue("Unchanged history must remain eligible: " + shape[0], canRead(shape[0], shape[0]));
+ Assert.assertEquals(Arrays.toString(shape), expected, canRead(shape[0], shape[1]));
+ }
+ }
+
+ private boolean canRead(DataType source, DataType target) {
+ FileStoreTable table = Mockito.mock(FileStoreTable.class);
+ SchemaManager schemas = Mockito.mock(SchemaManager.class);
+ Mockito.when(table.schema()).thenReturn(schema(2, target));
+ Mockito.when(table.schemaManager()).thenReturn(schemas);
+ Mockito.when(schemas.schema(1)).thenReturn(schema(1, source));
+ DataFileMeta file = DataFileMeta.forAppend("historical.parquet", 100L, 1L, SimpleStats.EMPTY_STATS,
+ 0L, 0L, 1L, Collections.emptyList(), null, FileSource.APPEND,
+ Collections.emptyList(), null, null, Collections.emptyList());
+ DataSplit split = DataSplit.builder().withPartition(BinaryRow.EMPTY_ROW).withBucket(0)
+ .withBucketPath("file:///warehouse/table/bucket-0")
+ .withDataFiles(Collections.singletonList(file)).build();
+ return new PaimonRustReaderCapabilities(table, new TupleDescriptor(new TupleId(0))).canRead(split);
+ }
+
+ private TableSchema schema(long id, DataType valueType) {
+ return new TableSchema(id, Arrays.asList(new DataField(0, "id", DataTypes.INT()),
+ new DataField(1, "v", valueType)), 10, Collections.emptyList(), Collections.emptyList(),
+ Collections.emptyMap(), null);
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderLegacySchemaTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderLegacySchemaTest.java
new file mode 100644
index 00000000000..1ed35e94007
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderLegacySchemaTest.java
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon.source;
+
+import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.analysis.TupleId;
+
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.reader.RecordReader;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.SchemaChange;
+import org.apache.paimon.schema.SchemaManager;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.FileStoreTableFactory;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.table.source.Split;
+import org.apache.paimon.types.DataTypes;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class PaimonRustReaderLegacySchemaTest {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Test
+ public void testPersistedLegacySchemaFallsBackBeforeAndAfterEvolution() throws Exception {
+ java.nio.file.Path directory = temporaryFolder.newFolder().toPath();
+ Path location = new Path(directory.toUri());
+ LocalFileIO fileIO = LocalFileIO.create();
+ new SchemaManager(fileIO, location).createTable(Schema.newBuilder()
+ .column("id", DataTypes.INT()).option("bucket", "-1")
+ .option("file.format", "parquet").build());
+ FileStoreTable table = FileStoreTableFactory.create(fileIO, location);
+ BatchWriteBuilder builder = table.newBatchWriteBuilder();
+ try (BatchTableWrite write = builder.newWrite(); BatchTableCommit commit = builder.newCommit()) {
+ write.write(GenericRow.of(7));
+ commit.commit(write.prepareCommit());
+ }
+ assertReadable(table, true);
+ // Older producers omitted version. Java accepts that schema, but Rust requires it
+ // even when reopening a historical schema for otherwise compatible field evolution.
+ java.nio.file.Path schemaFile = directory.resolve("schema/schema-0");
+ // Fixture I/O must also work with the FE test compilation target of Java 8.
+ String json = new String(Files.readAllBytes(schemaFile), StandardCharsets.UTF_8);
+ String legacy = json.replaceFirst("\"version\"\\s*:\\s*\\d+\\s*,", "");
+ Assert.assertNotEquals(json, legacy);
+ Files.write(schemaFile, legacy.getBytes(StandardCharsets.UTF_8));
+ table = FileStoreTableFactory.create(fileIO, location);
+ Assert.assertEquals(1, table.schema().version());
+ assertReadable(table, false);
+ table.schemaManager().commitChanges(Collections.singletonList(
+ SchemaChange.addColumn("extra", DataTypes.STRING())));
+ table = FileStoreTableFactory.create(fileIO, location);
+ Assert.assertTrue(table.schema().version() > 0);
+ assertReadable(table, false);
+ }
+
+ private static void assertReadable(FileStoreTable table, boolean rustExpected) throws Exception {
+ List<Split> splits = table.newScan().plan().splits();
+ Assert.assertFalse(splits.isEmpty());
+ PaimonRustReaderCapabilities capabilities = new PaimonRustReaderCapabilities(table,
+ new TupleDescriptor(new TupleId(0)));
+ List<Integer> actual = new ArrayList<>();
+ for (Split split : splits) {
+ Assert.assertEquals(rustExpected, capabilities.canRead((DataSplit) split));
+ try (RecordReader<InternalRow> reader = table.newRead().createReader(split)) {
+ reader.forEachRemaining(row -> actual.add(row.getInt(0)));
+ }
+ }
+ Assert.assertEquals(Collections.singletonList(7), actual);
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java
index d744973a566..2508f3c7bbe 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java
@@ -35,6 +35,7 @@ import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.CatalogProperty;
+import org.apache.doris.datasource.FederationBackendPolicy;
import org.apache.doris.datasource.FileQueryScanNode;
import org.apache.doris.datasource.FileSplitter;
import org.apache.doris.datasource.hive.HMSExternalTable;
@@ -54,10 +55,12 @@ import org.apache.doris.datasource.paimon.PaimonUtils;
import org.apache.doris.datasource.property.metastore.MetastoreProperties;
import org.apache.doris.datasource.property.metastore.PaimonJdbcMetaStoreProperties;
import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.ScanContext;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.thrift.TFileRangeDesc;
import org.apache.doris.thrift.TFileScanRangeParams;
@@ -65,21 +68,26 @@ import org.apache.doris.thrift.TPaimonReaderType;
import org.apache.doris.thrift.TPushAggOp;
import com.google.common.collect.ImmutableMap;
+import mockit.MockUp;
import org.apache.paimon.CoreOptions;
import org.apache.paimon.Snapshot;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.io.DataInputDeserializer;
+import org.apache.paimon.io.DataOutputSerializer;
import org.apache.paimon.manifest.FileSource;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
+import org.apache.paimon.rest.RESTTokenFileIO;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.stats.SimpleStats;
import org.apache.paimon.table.AppendOnlyFileStoreTable;
import org.apache.paimon.table.BucketMode;
import org.apache.paimon.table.CatalogEnvironment;
import org.apache.paimon.table.DataTable;
+import org.apache.paimon.table.FallbackReadFileStoreTable;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.Table;
import org.apache.paimon.table.source.DataSplit;
@@ -91,7 +99,9 @@ import org.apache.paimon.table.source.snapshot.SnapshotReader;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.IntType;
+import org.apache.paimon.types.LocalZonedTimestampType;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.types.TimestampType;
import org.apache.paimon.utils.InstantiationUtil;
import org.junit.After;
import org.junit.Assert;
@@ -110,6 +120,7 @@ import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -117,6 +128,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.OptionalLong;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
@RunWith(MockitoJUnitRunner.class)
public class PaimonScanNodeTest {
@@ -134,6 +146,14 @@ public class PaimonScanNodeTest {
// Statement-scoped scan task reuse is on by default; the mock's field would otherwise
// read false and bypass the cache under test.
sv.enableExternalScanTaskReuse = true;
+ // These routing fixtures bypass scan initialization and use an upgraded eligible BE.
+ Backend backend = GsonUtils.GSON.fromJson("{\"supportsPaimonRustReader\":true}", Backend.class);
+ new MockUp<FederationBackendPolicy>() {
+ @mockit.Mock
+ public Collection<Backend> getBackends() {
+ return Collections.singletonList(backend);
+ }
+ };
}
@After
@@ -474,6 +494,43 @@ public class PaimonScanNodeTest {
Mockito.verify(dataSplit, Mockito.never()).mergedRowCount();
}
+ @Test
+ public void testRustEnabledKeepsNativeAggregateSplits() throws Exception {
+ SessionVariable session = new SessionVariable();
+ session.setEnablePaimonRustReader(true);
+ session.enableFileScannerV2 = true;
+ PaimonScanNode node = Mockito.spy(newTestNode(new PlanNodeId(1), new TupleId(3), session));
+ PaimonSource source = Mockito.mock(PaimonSource.class);
+ DataTable table = Mockito.mock(DataTable.class, Mockito.RETURNS_DEEP_STUBS);
+ RowType rowType = DataTypes.ROW(DataTypes.INT());
+ Mockito.when(table.partitionKeys()).thenReturn(Collections.emptyList());
+ Mockito.when(table.rowType()).thenReturn(rowType);
+ // Native ORC eligibility now inspects historical types even for file aggregates.
+ Mockito.when(table.schemaManager().schema(1L).logicalRowType()).thenReturn(rowType);
+ Mockito.when(source.getPaimonTable()).thenReturn(table);
+ Mockito.when(source.getExternalTable()).thenReturn(Mockito.mock(PaimonExternalTable.class));
+ node.setSource(source);
+ setField(FileQueryScanNode.class, node, "fileSplitter",
+ new FileSplitter(64L * 1024 * 1024, 64L * 1024 * 1024, 0));
+ setField(PaimonScanNode.class, node, "storagePropertiesMap", Collections.emptyMap());
+ DataSplit dataSplit = Mockito.spy(createDataSplit(Arrays.asList("first.parquet", "second.orc")));
+ Mockito.doReturn(Collections.singletonList(dataSplit)).when(node).getPaimonSplitFromAPI();
+
+ // File aggregates need native file readers; enabling Rust must not replace raw-file splits
+ // with logical DataSplits, which require row reads for COUNT(col) and MIN/MAX.
+ for (TPushAggOp aggregate : Arrays.asList(TPushAggOp.COUNT, TPushAggOp.MINMAX)) {
+ node.setPushDownAggNoGrouping(aggregate);
+ node.setPushDownCountSlotIds(Collections.singletonList(new SlotId(7)));
+ List<org.apache.doris.spi.Split> splits = node.getSplits(1);
+ Assert.assertEquals(2, splits.size());
+ for (org.apache.doris.spi.Split split : splits) {
+ Assert.assertNull(((PaimonSplit) split).getSplit());
+ Assert.assertFalse(((PaimonSplit) split).getRowCount().isPresent());
+ }
+ }
+ Mockito.verify(dataSplit, Mockito.never()).mergedRowCount();
+ }
+
@Test
public void testCountColumnKeepsAllSplitsWhileCountStarUsesMergedRowCount() throws UserException {
PaimonScanNode node = Mockito.spy(newTestNode(new PlanNodeId(1), new TupleId(3), sv));
@@ -1846,6 +1903,173 @@ public class PaimonScanNodeTest {
Assert.assertTrue(rangeDesc.getTableFormatParams().getPaimonParams().isSetPaimonSplit());
}
+ @Test
+ public void testRustReaderSelectionRequiresFileScannerV2() throws Exception {
+ // The V1 FileScanner explicitly rejects PAIMON_RUST, so a rust request may
+ // only be encoded when enable_file_scanner_v2 is on; with V2 disabled the
+ // split falls back to JNI so the selected scanner can consume it.
+ for (boolean scannerV2 : Arrays.asList(true, false)) {
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = scannerV2;
+
+ PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0),
+ new TupleDescriptor(new TupleId(0)), false, vars, ScanContext.EMPTY);
+ PaimonSource source = Mockito.mock(PaimonSource.class);
+ FileStoreTable paimonTable = Mockito.mock(FileStoreTable.class);
+ Mockito.when(paimonTable.schema()).thenReturn(new TableSchema(
+ 1, Collections.singletonList(new DataField(0, "id", new IntType())),
+ 0, Collections.emptyList(), Collections.emptyList(),
+ Collections.emptyMap(), null));
+ PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class);
+ Mockito.when(source.getExternalTable()).thenReturn(externalTable);
+ Mockito.when(source.getTableLocation()).thenReturn("s3://warehouse/wh/db.db/t");
+ setField(PaimonScanNode.class, node, "backendStorageProperties",
+ ImmutableMap.of("AWS_ACCESS_KEY", "example-key", "AWS_SECRET_KEY", "example-secret"));
+ Mockito.when(externalTable.getDbName()).thenReturn("db");
+ Mockito.when(externalTable.getName()).thenReturn("t");
+ node.setSource(source);
+ // The rust gate and schema serialization use doInitialize's cached
+ // getProcessedTable() result, not the raw source table.
+ setField(PaimonScanNode.class, node, "processedTable", paimonTable);
+ setField(PaimonScanNode.class, node, "storagePropertiesMap", Collections.emptyMap());
+
+ TFileRangeDesc rangeDesc = new TFileRangeDesc();
+ invokePrivateMethod(node, "setPaimonParams",
+ new Class<?>[] {TFileRangeDesc.class, PaimonSplit.class},
+ rangeDesc, new PaimonSplit(createDataSplit("rust_gate.parquet")));
+
+ Assert.assertEquals(scannerV2 ? TPaimonReaderType.PAIMON_RUST : TPaimonReaderType.PAIMON_JNI,
+ rangeDesc.getTableFormatParams().getPaimonParams().getReaderType());
+ }
+ }
+
+ @Test
+ public void testRustSchemaJsonShipsProcessedTableRelationOptions() throws Exception {
+ // Relation options such as t@options('read.batch-size'='1') are applied by
+ // getProcessedTable(); the JNI reader serializes that effective table, and the
+ // rust reader rebuilds its table from the shipped schema JSON, deriving its
+ // read batch size from the schema options. Only the processed table's schema
+ // carries the override here, and source.getPaimonTable() is deliberately left
+ // unstubbed (it returns null): if the serialization ever regresses to the raw
+ // cached table, the rust gate falls back to JNI and this test fails.
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = true;
+
+ PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0),
+ new TupleDescriptor(new TupleId(0)), false, vars, ScanContext.EMPTY);
+
+ // The processed table: the relation override is merged into its schema.
+ FileStoreTable processed = Mockito.mock(FileStoreTable.class);
+ Mockito.when(processed.schema()).thenReturn(new TableSchema(
+ 1, Collections.singletonList(new DataField(0, "id", new IntType())),
+ 0, Collections.emptyList(), Collections.emptyList(),
+ ImmutableMap.of("read.batch-size", "1"), null));
+
+ PaimonSource source = Mockito.mock(PaimonSource.class);
+ PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class);
+ Mockito.when(source.getExternalTable()).thenReturn(externalTable);
+ Mockito.when(source.getTableLocation()).thenReturn("s3://warehouse/wh/db.db/t");
+ setField(PaimonScanNode.class, node, "backendStorageProperties",
+ ImmutableMap.of("AWS_ACCESS_KEY", "example-key", "AWS_SECRET_KEY", "example-secret"));
+ Mockito.when(externalTable.getDbName()).thenReturn("db");
+ Mockito.when(externalTable.getName()).thenReturn("t");
+ node.setSource(source);
+ setField(PaimonScanNode.class, node, "processedTable", processed);
+ setField(PaimonScanNode.class, node, "storagePropertiesMap", Collections.emptyMap());
+
+ TFileRangeDesc rangeDesc = new TFileRangeDesc();
+ invokePrivateMethod(node, "setPaimonParams",
+ new Class<?>[] {TFileRangeDesc.class, PaimonSplit.class},
+ rangeDesc, new PaimonSplit(createDataSplit("relation_options.parquet")));
+
+ org.apache.doris.thrift.TPaimonFileDesc fileDesc =
+ rangeDesc.getTableFormatParams().getPaimonParams();
+ Assert.assertEquals(TPaimonReaderType.PAIMON_RUST, fileDesc.getReaderType());
+ TableSchema shipped = org.apache.paimon.utils.JsonSerdeUtil.fromJson(
+ fileDesc.getPaimonTableSchemaJson(), TableSchema.class);
+ Assert.assertEquals("1", shipped.options().get("read.batch-size"));
+ }
+
+ @Test
+ public void testRustSchemaJsonStripsTimeTravelSelectors() throws Exception {
+ // A statement fence pins the data snapshot by merging scan.snapshot-id into the
+ // schema options (isolateSnapshotRead) while the fields keep the latest schema —
+ // a schema-only ALTER after the last data commit must stay visible. The rust
+ // reader pins data via the serialized DataSplit and re-resolves a shipped
+ // selector in copy_with_time_travel, swapping the fields back to the pinned
+ // snapshot's older schema; the shipped JSON must therefore carry the resolved
+ // fields without the planning selectors, keeping every other option.
+ // The derived scan.mode is stripped with it in every spelling: paimon accepts
+ // enum option values case-insensitively but preserves the original string in
+ // the options map, so a table can persist scan.mode=FROM-SNAPSHOT — the rust
+ // reader recognizes the value case-insensitively and would reject the now-bare
+ // uppercase mode for lacking its selector, while JNI accepts the original
+ // option.
+ for (String scanMode : new String[] {"from-snapshot", "FROM-SNAPSHOT", " From-Snapshot "}) {
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = true;
+
+ PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0),
+ new TupleDescriptor(new TupleId(0)), false, vars, ScanContext.EMPTY);
+
+ // Resolved (latest) schema: the column added after the last data commit is a
+ // field, and the fence snapshot id is only planning state in the options.
+ FileStoreTable processed = Mockito.mock(FileStoreTable.class);
+ Mockito.when(processed.schema()).thenReturn(new TableSchema(
+ 1, Arrays.asList(
+ new DataField(0, "id", new IntType()),
+ new DataField(1, "added_after", DataTypes.STRING())),
+ 1, Collections.emptyList(), Collections.emptyList(),
+ ImmutableMap.of(
+ "scan.snapshot-id", "3",
+ "scan.tag-name", "stale-tag",
+ // Paimon's copyInternal materializes the derived scan mode for
+ // the merged selector; it dangles once the selector is stripped.
+ "scan.mode", scanMode,
+ "read.batch-size", "1"),
+ null));
+
+ PaimonSource source = Mockito.mock(PaimonSource.class);
+ PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class);
+ Mockito.when(source.getExternalTable()).thenReturn(externalTable);
+ Mockito.when(source.getTableLocation()).thenReturn("s3://warehouse/wh/db.db/t");
+ setField(PaimonScanNode.class, node, "backendStorageProperties",
+ ImmutableMap.of("AWS_ACCESS_KEY", "example-key", "AWS_SECRET_KEY", "example-secret"));
+ Mockito.when(externalTable.getDbName()).thenReturn("db");
+ Mockito.when(externalTable.getName()).thenReturn("t");
+ node.setSource(source);
+ setField(PaimonScanNode.class, node, "processedTable", processed);
+ setField(PaimonScanNode.class, node, "storagePropertiesMap", Collections.emptyMap());
+
+ TFileRangeDesc rangeDesc = new TFileRangeDesc();
+ invokePrivateMethod(node, "setPaimonParams",
+ new Class<?>[] {TFileRangeDesc.class, PaimonSplit.class},
+ rangeDesc, new PaimonSplit(createDataSplit("strip_selectors.parquet")));
+
+ org.apache.doris.thrift.TPaimonFileDesc fileDesc =
+ rangeDesc.getTableFormatParams().getPaimonParams();
+ Assert.assertEquals("scan.mode " + scanMode, TPaimonReaderType.PAIMON_RUST,
+ fileDesc.getReaderType());
+ TableSchema shipped = org.apache.paimon.utils.JsonSerdeUtil.fromJson(
+ fileDesc.getPaimonTableSchemaJson(), TableSchema.class);
+ Assert.assertNull(shipped.options().get(CoreOptions.SCAN_SNAPSHOT_ID.key()));
+ Assert.assertNull(shipped.options().get(CoreOptions.SCAN_TAG_NAME.key()));
+ // The fence-materialized "from-snapshot" would otherwise dangle: the rust
+ // ReadBuilder requires a selector for it and rejects the open.
+ Assert.assertNull("scan.mode " + scanMode + " must be stripped",
+ shipped.options().get(CoreOptions.SCAN_MODE.key()));
+ Assert.assertEquals("1", shipped.options().get("read.batch-size"));
+ // The resolved fields are transported untouched, including the column added
+ // after the last data commit.
+ Assert.assertEquals(
+ Arrays.asList("id", "added_after"),
+ shipped.fields().stream().map(DataField::name).collect(Collectors.toList()));
+ }
+ }
+
@Test
public void testGetFieldIndexMatchesMixedCaseColumns() {
List<String> fieldNames = Arrays.asList("data", "mIxEd_COL", "PART");
@@ -2085,15 +2309,21 @@ public class PaimonScanNodeTest {
}
private DataSplit createDataSplit(String fileName) {
- DataFileMeta dataFileMeta = DataFileMeta.forAppend(fileName, 64L * 1024 * 1024, 1L, SimpleStats.EMPTY_STATS,
- 1L, 1L, 1L, Collections.<String>emptyList(), null, FileSource.APPEND,
- Collections.<String>emptyList(), null, null, Collections.<String>emptyList());
+ return createDataSplit(Collections.singletonList(fileName));
+ }
+
+ private DataSplit createDataSplit(List<String> fileNames) {
+ List<DataFileMeta> dataFileMetas = fileNames.stream().map(fileName ->
+ DataFileMeta.forAppend(fileName, 64L * 1024 * 1024, 1L, SimpleStats.EMPTY_STATS,
+ 1L, 1L, 1L, Collections.<String>emptyList(), null, FileSource.APPEND,
+ Collections.<String>emptyList(), null, null, Collections.<String>emptyList()))
+ .collect(Collectors.toList());
return DataSplit.builder()
.rawConvertible(true)
.withPartition(BinaryRow.singleColumn(1))
.withBucket(1)
.withBucketPath("file://b1")
- .withDataFiles(Collections.singletonList(dataFileMeta))
+ .withDataFiles(dataFileMetas)
.build();
}
@@ -2111,4 +2341,1311 @@ public class PaimonScanNodeTest {
Mockito.when(dataSplit.deletionFiles()).thenReturn(Optional.empty());
return dataSplit;
}
+
+ @Test
+ public void testRustReaderSelectionRejectsFallbackSplits() throws Exception {
+ RustRoutingFixture f = new RustRoutingFixture();
+ DataOutputSerializer out = new DataOutputSerializer(1024);
+ f.split.serialize(out);
+ out.writeBoolean(true);
+ DataInputDeserializer in = new DataInputDeserializer();
+ in.setBuffer(out.getSharedBuffer(), 0, out.length());
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI,
+ f.reader(FallbackReadFileStoreTable.FallbackDataSplit.deserialize(in)));
+ setField(PaimonScanNode.class, f.node, "processedTable", Mockito.mock(FallbackReadFileStoreTable.class));
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, f.reader(f.split));
+ }
+
+ @Test
+ public void testRustReaderSelectionRejectsQueryAuthTables() throws Exception {
+ RustRoutingFixture f = new RustRoutingFixture();
+ f.schema(1, new IntType(), ImmutableMap.of("query-auth.enabled", "true"), false);
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, f.reader(f.split));
+ }
+
+ @Test
+ public void testRustReaderSelectionRejectsUnsupportedPkDvModes() throws Exception {
+ // The pinned rust read_pk supports partial-update / aggregation tables
+ // with deletion vectors only in the fully materialized shape: it rejects
+ // deletion-vectors.merge-on-read=true outright, and otherwise requires
+ // every split to be compacted (level != 0) and known free of retract rows
+ // (DataSplit::is_fully_materialized_pk_dv). All other shapes are valid
+ // Java/JNI scans, so they must fall back here instead of failing the
+ // BE open. Deduplicate stays rust-eligible: its read_pk routes
+ // uncompacted DV splits to the KV reader, which applies the DVs.
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = true;
+
+ // createDataSplit builds a level-0 file — the ordinary uncompacted shape.
+ DataSplit levelZeroSplit = createDataSplit("dv_level0.parquet");
+ DataSplit unknownDeleteCountSplit = dvSplitOverFiles(
+ dvCompactedFile("dv_unknown.parquet", null));
+ DataSplit retractsSplit = dvSplitOverFiles(
+ dvCompactedFile("dv_retracts.parquet", 1L));
+ DataSplit materializedSplit = dvSplitOverFiles(
+ dvCompactedFile("dv_materialized.parquet", 0L));
+
+ for (CoreOptions.MergeEngine engine : Arrays.asList(
+ CoreOptions.MergeEngine.PARTIAL_UPDATE, CoreOptions.MergeEngine.AGGREGATE)) {
+ // deletion-vectors.merge-on-read=true: rejected table-wide.
+ Assert.assertEquals("merge-on-read " + engine, TPaimonReaderType.PAIMON_JNI,
+ dvReaderTypeOf(vars, engine, true,
+ ImmutableMap.of("deletion-vectors.merge-on-read", "true"),
+ materializedSplit));
+ // merge-on-read=false but the split still needs per-key merge work:
+ // level-0 data, unknown delete count, or known retracts — all fall
+ // back per split.
+ Assert.assertEquals("level-0 " + engine, TPaimonReaderType.PAIMON_JNI,
+ dvReaderTypeOf(vars, engine, true, Collections.emptyMap(), levelZeroSplit));
+ Assert.assertEquals("unknown delete count " + engine, TPaimonReaderType.PAIMON_JNI,
+ dvReaderTypeOf(vars, engine, true, Collections.emptyMap(), unknownDeleteCountSplit));
+ Assert.assertEquals("retracts " + engine, TPaimonReaderType.PAIMON_JNI,
+ dvReaderTypeOf(vars, engine, true, Collections.emptyMap(), retractsSplit));
+ // A fully materialized compacted split reads raw on rust, and without
+ // deletion vectors the KV reader handles any split.
+ Assert.assertEquals("materialized " + engine, TPaimonReaderType.PAIMON_RUST,
+ dvReaderTypeOf(vars, engine, true, Collections.emptyMap(), materializedSplit));
+ Assert.assertEquals("no dv " + engine, TPaimonReaderType.PAIMON_RUST,
+ dvReaderTypeOf(vars, engine, false, Collections.emptyMap(), levelZeroSplit));
+ }
+
+ // Deduplicate keeps rust eligibility for uncompacted DV splits and for
+ // merge-on-read=true: the KV reader applies the attached per-file DVs.
+ Assert.assertEquals(TPaimonReaderType.PAIMON_RUST,
+ dvReaderTypeOf(vars, CoreOptions.MergeEngine.DEDUPLICATE, true,
+ ImmutableMap.of("deletion-vectors.merge-on-read", "true"), levelZeroSplit));
+ }
+
+ // Builds a node whose processed table is a primary-key table with the given
+ // merge engine and deletion-vector options (all other gates open: no
+ // query-auth, v2 on, ordinary DataSplit) and returns the reader type chosen
+ // for the given split.
+ private TPaimonReaderType dvReaderTypeOf(SessionVariable vars, CoreOptions.MergeEngine mergeEngine,
+ boolean deletionVectorsEnabled, Map<String, String> schemaOptions, DataSplit split)
+ throws Exception {
+ PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0),
+ new TupleDescriptor(new TupleId(0)), false, vars, ScanContext.EMPTY);
+ PaimonSource source = Mockito.mock(PaimonSource.class);
+ FileStoreTable paimonTable = Mockito.mock(FileStoreTable.class);
+ // lenient: the schema / external-table stubs are only consumed when the
+ // split stays rust-eligible.
+ Mockito.lenient().when(paimonTable.schema()).thenReturn(new TableSchema(
+ 1, Collections.singletonList(new DataField(0, "id", new IntType())),
+ 0, Collections.emptyList(), Collections.emptyList(), schemaOptions, null));
+ CoreOptions options = Mockito.mock(CoreOptions.class);
+ Mockito.when(paimonTable.coreOptions()).thenReturn(options);
+ Mockito.when(options.queryAuthEnabled()).thenReturn(false);
+ Mockito.when(options.mergeEngine()).thenReturn(mergeEngine);
+ Mockito.when(options.deletionVectorsEnabled()).thenReturn(deletionVectorsEnabled);
+ PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class);
+ Mockito.lenient().when(source.getExternalTable()).thenReturn(externalTable);
+ Mockito.when(source.getTableLocation()).thenReturn("s3://warehouse/wh/db.db/t");
+ setField(PaimonScanNode.class, node, "backendStorageProperties",
+ ImmutableMap.of("AWS_ACCESS_KEY", "example-key", "AWS_SECRET_KEY", "example-secret"));
+ Mockito.lenient().when(externalTable.getDbName()).thenReturn("db");
+ Mockito.lenient().when(externalTable.getName()).thenReturn("t");
+ node.setSource(source);
+ setField(PaimonScanNode.class, node, "processedTable", paimonTable);
+
+ TFileRangeDesc rangeDesc = new TFileRangeDesc();
+ invokePrivateMethod(node, "setPaimonParams",
+ new Class<?>[] {TFileRangeDesc.class, PaimonSplit.class},
+ rangeDesc, new PaimonSplit(split));
+ return rangeDesc.getTableFormatParams().getPaimonParams().getReaderType();
+ }
+
+ // A compacted (level != 0) data file with an explicit retract-row count:
+ // null = unknown, 0 = fully materialized, >0 = rows still to retract.
+ private static DataFileMeta dvCompactedFile(String fileName, Long deleteRowCount) {
+ return DataFileMeta.create(fileName, 64L * 1024 * 1024, 1L,
+ DataFileMeta.EMPTY_MIN_KEY, DataFileMeta.EMPTY_MAX_KEY,
+ SimpleStats.EMPTY_STATS, SimpleStats.EMPTY_STATS,
+ 1L, 1L, 1L, 5, deleteRowCount, null,
+ FileSource.APPEND, Collections.emptyList(), null, Collections.emptyList());
+ }
+
+ private static DataSplit dvSplitOverFiles(DataFileMeta... files) {
+ return DataSplit.builder()
+ .rawConvertible(true)
+ .withPartition(BinaryRow.singleColumn(1))
+ .withBucket(1)
+ .withBucketPath("file://b1")
+ .withDataFiles(Arrays.asList(files))
+ .build();
+ }
+
+ @Test
+ public void testRustReaderSelectionRejectsOrcTimestampLtzSchemas() throws Exception {
+ RustRoutingFixture f = new RustRoutingFixture();
+ Assert.assertEquals(TPaimonReaderType.PAIMON_RUST, f.reader(createDataSplit("plain.orc")));
+ f.schema(1, new org.apache.paimon.types.LocalZonedTimestampType(6), Collections.emptyMap(), false);
+ Assert.assertEquals(TPaimonReaderType.PAIMON_RUST, f.reader(f.split));
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, f.reader(createDataSplit("ltz.orc")));
+ }
+
+ @Test
+ public void testRustReaderSelectionRejectsMixedFormatOrcLtzSplits() throws Exception {
+ // Paimon allows per-level file.format, so one DataSplit can mix
+ // Parquet and ORC members. The old gate read only the split path's
+ // suffix (the first file), so a first-Parquet/later-ORC split with an
+ // LTZ schema passed while rust still applied the shifted ORC decode
+ // to the ORC members. The format now comes from every member file.
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = true;
+ List<DataField> ltzFields = Arrays.asList(
+ new DataField(0, "id", new IntType()),
+ new DataField(1, "ts_ltz", new LocalZonedTimestampType(6)));
+
+ // First file Parquet, later member ORC: the first file's suffix no
+ // longer speaks for the split -> JNI.
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI,
+ readerTypeOf(vars, ltzFields, "first.parquet", "second.orc"));
+ // ORC first, later Parquet member (the shape the suffix check did
+ // catch) stays JNI.
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI,
+ readerTypeOf(vars, ltzFields, "first.orc", "second.parquet"));
+ // All-Parquet members with the same LTZ schema stay rust-eligible.
+ Assert.assertEquals(TPaimonReaderType.PAIMON_RUST,
+ readerTypeOf(vars, ltzFields, "a.parquet", "b.parquet"));
+ }
+
+ @Test
+ public void testRustReaderSelectionRejectsNestedTimestampLtzSchemas() throws Exception {
+ // An LTZ nested under MAP/ARRAY/ROW reaches the same shifted ORC
+ // decode through the container's field materialization; the old
+ // top-level-only type-root check missed it. The search recurses; the
+ // same containers without LTZ stay rust-eligible, and Parquet splits
+ // with nested LTZ are unaffected (only the ORC decoder diverges).
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = true;
+ LocalZonedTimestampType ltz = new LocalZonedTimestampType(6);
+
+ // LTZ under each container kind, uniform-ORC split -> JNI.
+ for (DataField nested : Arrays.asList(
+ new DataField(1, "m", new org.apache.paimon.types.MapType(new IntType(), ltz)),
+ new DataField(1, "arr", new org.apache.paimon.types.ArrayType(ltz)),
+ new DataField(1, "r",
+ new RowType(Collections.singletonList(new DataField(0, "ts_ltz", ltz)))))) {
+ Assert.assertEquals("nested " + nested.name() + " under ORC",
+ TPaimonReaderType.PAIMON_JNI,
+ readerTypeOf(vars, Arrays.asList(
+ new DataField(0, "id", new IntType()), nested),
+ "nested.orc"));
+ // The same nested-LTZ schema over Parquet stays rust-eligible.
+ Assert.assertEquals("nested " + nested.name() + " over Parquet",
+ TPaimonReaderType.PAIMON_RUST,
+ readerTypeOf(vars, Arrays.asList(
+ new DataField(0, "id", new IntType()), nested),
+ "nested.parquet"));
+ }
+
+ // Containers without any LTZ over ORC stay rust-eligible.
+ Assert.assertEquals(TPaimonReaderType.PAIMON_RUST,
+ readerTypeOf(vars, Arrays.asList(
+ new DataField(0, "id", new IntType()),
+ new DataField(1, "m",
+ new org.apache.paimon.types.MapType(new IntType(), new IntType()))),
+ "map_int.orc"));
+ }
+
+ @Test
+ public void testRustReaderSelectionRejectsExternalDataFileSplits() throws Exception {
+ // data-file.external-paths: paimon can store an absolute location in
+ // each DataFileMeta, and both Java and the serialized rust split prefer
+ // it over the bucket path — but the pinned rust table builds one FileIO
+ // from paimon_table, whose storage enum parses every file with the
+ // warehouse-selected backend: an hdfs table with an s3:// external file
+ // (or an s3 table with an oss:// file) reaches the wrong parser and
+ // fails the open, while JNI reads it. Any split carrying an external
+ // file therefore routes to JNI, regardless of the external path's own
+ // scheme (the shipped options describe only the warehouse).
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = true;
+ List<DataField> fields = Arrays.asList(
+ new DataField(0, "id", new IntType()),
+ new DataField(1, "ts", new TimestampType(6)));
+
+ // A split whose file carries an external path -> JNI.
+ DataFileMeta external = DataFileMeta.forAppend(
+ "external.parquet", 64L * 1024 * 1024, 1L, SimpleStats.EMPTY_STATS,
+ 1L, 1L, 1L, Collections.<String>emptyList(), null, FileSource.APPEND,
+ Collections.<String>emptyList(), null, null, Collections.<String>emptyList())
+ .newExternalPath("hdfs://other-nn/external/wh/external.parquet");
+ DataSplit externalSplit = DataSplit.builder()
+ .rawConvertible(true)
+ .withPartition(BinaryRow.singleColumn(1))
+ .withBucket(1)
+ .withBucketPath("file://b1")
+ .withDataFiles(Collections.singletonList(external))
+ .build();
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI,
+ readerTypeOfSplit(vars, fields, externalSplit));
+
+ // The same split without an external path stays rust-eligible.
+ DataSplit plainSplit = DataSplit.builder()
+ .rawConvertible(true)
+ .withPartition(BinaryRow.singleColumn(1))
+ .withBucket(1)
+ .withBucketPath("file://b1")
+ .withDataFiles(Collections.singletonList(DataFileMeta.forAppend(
+ "plain.parquet", 64L * 1024 * 1024, 1L, SimpleStats.EMPTY_STATS,
+ 1L, 1L, 1L, Collections.<String>emptyList(), null, FileSource.APPEND,
+ Collections.<String>emptyList(), null, null, Collections.<String>emptyList())))
+ .build();
+ Assert.assertEquals(TPaimonReaderType.PAIMON_RUST,
+ readerTypeOfSplit(vars, fields, plainSplit));
+ }
+
+ @Test
+ public void testRustReaderSelectionRejectsDeduplicateIgnoreDeleteTables() throws Exception {
+ // deduplicate.ignore-delete=true tables stay on JNI: Java's
+ // DeduplicateMergeFunction skips retract records when the option is set
+ // (including old, uncompacted files that still contain them), but the
+ // pinned rust read_pk does not pass table options into its deduplicate
+ // merge — it picks the latest row and omits the key when that row is
+ // DELETE/UPDATE_BEFORE. An uncompacted insert followed by a delete
+ // therefore returns the insert through JNI but silently disappears
+ // through rust.
+ for (Object[] shape : new Object[][] {
+ // DEDUPLICATE engine with the option on -> JNI.
+ {CoreOptions.MergeEngine.DEDUPLICATE, true, TPaimonReaderType.PAIMON_JNI},
+ // Option off, same engine -> rust.
+ {CoreOptions.MergeEngine.DEDUPLICATE, false, TPaimonReaderType.PAIMON_RUST},
+ // Option on under a non-deduplicate engine: meaningless there,
+ // the gate must not reject -> rust.
+ {CoreOptions.MergeEngine.PARTIAL_UPDATE, true, TPaimonReaderType.PAIMON_RUST}}) {
+ CoreOptions.MergeEngine mergeEngine = (CoreOptions.MergeEngine) shape[0];
+ boolean ignoreDelete = (Boolean) shape[1];
+ TPaimonReaderType expected = (TPaimonReaderType) shape[2];
+
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = true;
+
+ PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0),
+ new TupleDescriptor(new TupleId(0)), false, vars, ScanContext.EMPTY);
+ PaimonSource source = Mockito.mock(PaimonSource.class);
+ FileStoreTable paimonTable = Mockito.mock(FileStoreTable.class);
+ CoreOptions options = Mockito.mock(CoreOptions.class);
+ Mockito.when(paimonTable.coreOptions()).thenReturn(options);
+ Mockito.when(options.queryAuthEnabled()).thenReturn(false);
+ Mockito.when(options.mergeEngine()).thenReturn(mergeEngine);
+ Mockito.when(options.ignoreDelete()).thenReturn(ignoreDelete);
+ PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class);
+ Mockito.when(source.getExternalTable()).thenReturn(externalTable);
+ Mockito.when(source.getTableLocation()).thenReturn("s3://warehouse/wh/db.db/t");
+ setField(PaimonScanNode.class, node, "backendStorageProperties",
+ ImmutableMap.of("AWS_ACCESS_KEY", "example-key", "AWS_SECRET_KEY", "example-secret"));
+ Mockito.when(externalTable.getDbName()).thenReturn("db");
+ Mockito.when(externalTable.getName()).thenReturn("t");
+ node.setSource(source);
+ setField(PaimonScanNode.class, node, "processedTable", paimonTable);
+ if (expected == TPaimonReaderType.PAIMON_RUST) {
+ // Only the rust branch serializes the schema (a JNI-expected
+ // stub would trip Mockito's strict unused-stubbing check).
+ Mockito.when(paimonTable.schema()).thenReturn(new TableSchema(
+ 1, Collections.singletonList(new DataField(0, "id", new IntType())),
+ 0, Collections.emptyList(), Collections.emptyList(),
+ Collections.emptyMap(), null));
+ }
+
+ TFileRangeDesc rangeDesc = new TFileRangeDesc();
+ invokePrivateMethod(node, "setPaimonParams",
+ new Class<?>[] {TFileRangeDesc.class, PaimonSplit.class},
+ rangeDesc, new PaimonSplit(createDataSplit("dedup_ignore_del.parquet")));
+ Assert.assertEquals("engine " + mergeEngine + ", ignore-delete " + ignoreDelete,
+ expected, rangeDesc.getTableFormatParams().getPaimonParams().getReaderType());
+ }
+ }
+
+ @Test
+ public void testRustReaderSelectionRejectsRestTokenTables() throws Exception {
+ // doInitialize snapshots RESTTokenFileIO.validToken().token() into
+ // the backend storage properties, discarding expireAtMillis and the
+ // REST refresh context, so the shipped credentials look static — but
+ // the pinned rust table reuses one option map with no refresh
+ // callback, while paimon 1.4.2's JNI RESTTokenFileIO checks expiry
+ // before each file operation and obtains a replacement token. A scan
+ // crossing the token TTL would start on rust and later fail
+ // authentication; only the table's FileIO type reveals the token is
+ // renewable.
+ for (boolean restToken : new boolean[] {true, false}) {
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = true;
+
+ PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0),
+ new TupleDescriptor(new TupleId(0)), false, vars, ScanContext.EMPTY);
+ PaimonSource source = Mockito.mock(PaimonSource.class);
+ FileStoreTable paimonTable = Mockito.mock(FileStoreTable.class);
+ CoreOptions options = Mockito.mock(CoreOptions.class);
+ Mockito.when(paimonTable.coreOptions()).thenReturn(options);
+ Mockito.when(options.queryAuthEnabled()).thenReturn(false);
+ // S3-location static credentials, an ordinary shape the
+ // REST-token snapshot also produces.
+ setField(PaimonScanNode.class, node, "backendStorageProperties", ImmutableMap.of(
+ "AWS_CREDENTIALS_PROVIDER_TYPE", "DEFAULT",
+ "AWS_ACCESS_KEY", "ak",
+ "AWS_SECRET_KEY", "sk",
+ "AWS_ENDPOINT", "http://127.0.0.1:19001",
+ "AWS_REGION", "us-east-1"));
+ if (restToken) {
+ Mockito.when(paimonTable.fileIO())
+ .thenReturn(Mockito.mock(RESTTokenFileIO.class));
+ // The JNI branch never serializes the schema.
+ } else {
+ Mockito.when(paimonTable.schema()).thenReturn(new TableSchema(
+ 1, Collections.singletonList(new DataField(0, "id", new IntType())),
+ 0, Collections.emptyList(), Collections.emptyList(),
+ Collections.emptyMap(), null));
+ // A plain (or unresolved) FileIO shape stays rust-eligible.
+ Mockito.when(paimonTable.fileIO()).thenReturn(null);
+ }
+ PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class);
+ Mockito.when(source.getExternalTable()).thenReturn(externalTable);
+ Mockito.when(source.getTableLocation()).thenReturn("s3://warehouse/wh/db.db/t");
+ Mockito.when(externalTable.getDbName()).thenReturn("db");
+ Mockito.when(externalTable.getName()).thenReturn("t");
+ node.setSource(source);
+ setField(PaimonScanNode.class, node, "processedTable", paimonTable);
+
+ TFileRangeDesc rangeDesc = new TFileRangeDesc();
+ invokePrivateMethod(node, "setPaimonParams",
+ new Class<?>[] {TFileRangeDesc.class, PaimonSplit.class},
+ rangeDesc, new PaimonSplit(createDataSplit("rest_token.parquet")));
+ Assert.assertEquals("RESTTokenFileIO=" + restToken,
+ restToken ? TPaimonReaderType.PAIMON_JNI : TPaimonReaderType.PAIMON_RUST,
+ rangeDesc.getTableFormatParams().getPaimonParams().getReaderType());
+ }
+ }
+
+ @Test
+ public void testRustReaderSelectionRejectsRustUnsupportedMergeOptions() throws Exception {
+ // Java supports partial-update.remove-record-on-delete /
+ // aggregation.remove-record-on-delete and the wider per-field option
+ // matrix, but the pinned paimon-rust PartialUpdateConfig /
+ // AggregationConfig validations return Unsupported for them — the
+ // rust predicates check option-key PRESENCE, so the FE gate mirrors
+ // presence exactly (an off value still routes to JNI, because the
+ // rust merge construction rejects the bare key). Non-DV tables are
+ // the review's case: without this gate an ordinary DataSplit passes
+ // the compound gate and fails during the rust merge construction
+ // instead of scanning through JNI.
+ for (Object[] shape : new Object[][] {
+ // Partial-update unsupported keys -> JNI.
+ {CoreOptions.MergeEngine.PARTIAL_UPDATE, "partial-update.remove-record-on-delete",
+ TPaimonReaderType.PAIMON_JNI},
+ {CoreOptions.MergeEngine.PARTIAL_UPDATE,
+ "partial-update.remove-record-on-sequence-group",
+ TPaimonReaderType.PAIMON_JNI},
+ {CoreOptions.MergeEngine.PARTIAL_UPDATE, "fields.v.ignore-retract",
+ TPaimonReaderType.PAIMON_JNI},
+ {CoreOptions.MergeEngine.PARTIAL_UPDATE, "fields.v.distinct",
+ TPaimonReaderType.PAIMON_JNI},
+ {CoreOptions.MergeEngine.PARTIAL_UPDATE, "fields.v.ignore-delete",
+ TPaimonReaderType.PAIMON_JNI},
+ // Aggregation unsupported keys -> JNI.
+ {CoreOptions.MergeEngine.AGGREGATE, "aggregation.remove-record-on-delete",
+ TPaimonReaderType.PAIMON_JNI},
+ {CoreOptions.MergeEngine.AGGREGATE, "fields.v.sequence-group",
+ TPaimonReaderType.PAIMON_JNI},
+ {CoreOptions.MergeEngine.AGGREGATE, "ignore-delete",
+ TPaimonReaderType.PAIMON_JNI},
+ // Keys the rust read path DOES support: bare /
+ // partial-update-prefixed ignore-delete, sequence groups and
+ // field aggregation stay rust-eligible.
+ {CoreOptions.MergeEngine.PARTIAL_UPDATE, "partial-update.ignore-delete",
+ TPaimonReaderType.PAIMON_RUST},
+ {CoreOptions.MergeEngine.PARTIAL_UPDATE, "fields.v.sequence-group",
+ TPaimonReaderType.PAIMON_RUST},
+ {CoreOptions.MergeEngine.PARTIAL_UPDATE, "fields.v.aggregate-function",
+ TPaimonReaderType.PAIMON_RUST},
+ {CoreOptions.MergeEngine.AGGREGATE, "fields.v.aggregate-function",
+ TPaimonReaderType.PAIMON_RUST},
+ {CoreOptions.MergeEngine.AGGREGATE, "fields.default-aggregate-function",
+ TPaimonReaderType.PAIMON_RUST}}) {
+ CoreOptions.MergeEngine mergeEngine = (CoreOptions.MergeEngine) shape[0];
+ String optionKey = (String) shape[1];
+ TPaimonReaderType expected = (TPaimonReaderType) shape[2];
+
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = true;
+
+ PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0),
+ new TupleDescriptor(new TupleId(0)), false, vars, ScanContext.EMPTY);
+ PaimonSource source = Mockito.mock(PaimonSource.class);
+ FileStoreTable paimonTable = Mockito.mock(FileStoreTable.class);
+ CoreOptions options = Mockito.mock(CoreOptions.class);
+ Mockito.when(paimonTable.coreOptions()).thenReturn(options);
+ Mockito.when(options.queryAuthEnabled()).thenReturn(false);
+ Mockito.when(options.mergeEngine()).thenReturn(mergeEngine);
+ Mockito.when(paimonTable.schema()).thenReturn(new TableSchema(
+ 1, Collections.singletonList(new DataField(0, "id", new IntType())),
+ 0, Collections.emptyList(), Collections.emptyList(),
+ ImmutableMap.of(optionKey, "true"), null));
+ PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class);
+ Mockito.when(source.getExternalTable()).thenReturn(externalTable);
+ Mockito.when(source.getTableLocation()).thenReturn("s3://warehouse/wh/db.db/t");
+ setField(PaimonScanNode.class, node, "backendStorageProperties",
+ ImmutableMap.of("AWS_ACCESS_KEY", "example-key", "AWS_SECRET_KEY", "example-secret"));
+ Mockito.when(externalTable.getDbName()).thenReturn("db");
+ Mockito.when(externalTable.getName()).thenReturn("t");
+ node.setSource(source);
+ setField(PaimonScanNode.class, node, "processedTable", paimonTable);
+
+ TFileRangeDesc rangeDesc = new TFileRangeDesc();
+ invokePrivateMethod(node, "setPaimonParams",
+ new Class<?>[] {TFileRangeDesc.class, PaimonSplit.class},
+ rangeDesc, new PaimonSplit(createDataSplit("merge_option_gate.parquet")));
+ Assert.assertEquals("engine " + mergeEngine + ", option " + optionKey, expected,
+ rangeDesc.getTableFormatParams().getPaimonParams().getReaderType());
+ }
+ }
+
+ // Builds a node whose processed table carries the given row-type fields
+ // (all other gates open: no query-auth, v2 on, ordinary DataSplit) and
+ // returns the reader type chosen for a split over the given member files.
+ // Multiple file names build a multi-file split (paimon allows per-level
+ // file.format, so one DataSplit can mix Parquet and ORC members).
+ private TPaimonReaderType readerTypeOf(SessionVariable vars, List<DataField> fields,
+ String... fileNames) throws Exception {
+ return readerTypeOfSplit(vars, fields, createDataSplit(Arrays.asList(fileNames)));
+ }
+
+ // Same harness as readerTypeOf over an externally built DataSplit (e.g. one
+ // whose member files carry data-file.external-paths locations).
+ private TPaimonReaderType readerTypeOfSplit(SessionVariable vars, List<DataField> fields,
+ DataSplit dataSplit) throws Exception {
+ PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0),
+ new TupleDescriptor(new TupleId(0)), false, vars, ScanContext.EMPTY);
+ PaimonSource source = Mockito.mock(PaimonSource.class);
+ FileStoreTable paimonTable = Mockito.mock(FileStoreTable.class);
+ Mockito.when(paimonTable.schema()).thenReturn(new TableSchema(
+ 1, fields, 0, Collections.emptyList(), Collections.emptyList(),
+ Collections.emptyMap(), null));
+ CoreOptions options = Mockito.mock(CoreOptions.class);
+ Mockito.when(paimonTable.coreOptions()).thenReturn(options);
+ Mockito.when(options.queryAuthEnabled()).thenReturn(false);
+ PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class);
+ Mockito.when(source.getExternalTable()).thenReturn(externalTable);
+ Mockito.when(source.getTableLocation()).thenReturn("s3://warehouse/wh/db.db/t");
+ setField(PaimonScanNode.class, node, "backendStorageProperties",
+ ImmutableMap.of("AWS_ACCESS_KEY", "example-key", "AWS_SECRET_KEY", "example-secret"));
+ Mockito.when(externalTable.getDbName()).thenReturn("db");
+ Mockito.when(externalTable.getName()).thenReturn("t");
+ node.setSource(source);
+ setField(PaimonScanNode.class, node, "processedTable", paimonTable);
+
+ TFileRangeDesc rangeDesc = new TFileRangeDesc();
+ invokePrivateMethod(node, "setPaimonParams",
+ new Class<?>[] {TFileRangeDesc.class, PaimonSplit.class},
+ rangeDesc, new PaimonSplit(dataSplit));
+ return rangeDesc.getTableFormatParams().getPaimonParams().getReaderType();
+ }
+
+ @Test
+ public void testRustReaderSelectionRejectsIncrementalScans() throws Exception {
+ for (String mode : Arrays.asList("delta", "changelog", "diff")) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ f.node.setScanParams(new TableScanParams("incr", ImmutableMap.of(
+ "incremental-between", "5,10", "incremental-between-scan-mode", mode), null));
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, f.reader(f.split));
+ }
+ }
+
+ @Test
+ public void testRustReaderSelectionRejectsAmbientCredentialProviderModes() throws Exception {
+ for (String provider : Arrays.asList("ENV", "SYSTEM_PROPERTIES", "WEB_IDENTITY",
+ "CONTAINER", "INSTANCE_PROFILE")) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ f.credentials(provider);
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, f.reader(f.split));
+ }
+ }
+
+ @Test
+ public void testRustReaderSelectionRequiresExplicitS3Credentials() throws Exception {
+ // DEFAULT without static keys may use JVM-only credentials or anonymous fallback.
+ for (String[] shape : new String[][] {
+ {"ANONYMOUS", "s3://warehouse/wh", "PAIMON_RUST", "", ""},
+ {"DEFAULT", "s3://warehouse/wh", "PAIMON_JNI", "", ""},
+ {"DEFAULT", "s3a://warehouse/wh", "PAIMON_JNI", "example-key", ""},
+ {"DEFAULT", "s3://warehouse/wh", "PAIMON_JNI", " ", " "},
+ {"DEFAULT", "s3://warehouse/wh", "PAIMON_RUST", "example-key", "example-secret"},
+ {"", "s3://warehouse/wh", "PAIMON_JNI", "", ""},
+ {"ANONYMOUS", "oss://warehouse/wh", "PAIMON_JNI", "", ""}}) {
+ String mode = shape[0];
+ String location = shape[1];
+ TPaimonReaderType expected =
+ TPaimonReaderType.valueOf(shape[2]);
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = true;
+
+ PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0),
+ new TupleDescriptor(new TupleId(0)), false, vars, ScanContext.EMPTY);
+ PaimonSource source = Mockito.mock(PaimonSource.class);
+ Mockito.when(source.getTableLocation()).thenReturn(location);
+ FileStoreTable paimonTable = Mockito.mock(FileStoreTable.class);
+ PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class);
+ // Supply both paths so an incorrect Rust selection fails the
+ // reader-type assertion rather than unrelated schema setup.
+ Mockito.lenient().when(source.getExternalTable()).thenReturn(externalTable);
+ Mockito.lenient().when(paimonTable.schema()).thenReturn(new TableSchema(
+ 1, Collections.singletonList(new DataField(0, "id", new IntType())),
+ 0, Collections.emptyList(), Collections.emptyList(),
+ Collections.emptyMap(), null));
+ Mockito.lenient().when(externalTable.getDbName()).thenReturn("db");
+ Mockito.lenient().when(externalTable.getName()).thenReturn("t");
+ node.setSource(source);
+ setField(PaimonScanNode.class, node, "processedTable", paimonTable);
+ setField(PaimonScanNode.class, node, "backendStorageProperties",
+ ImmutableMap.of("AWS_CREDENTIALS_PROVIDER_TYPE", mode,
+ "AWS_ACCESS_KEY", shape[3], "AWS_SECRET_KEY", shape[4]));
+
+ TFileRangeDesc rangeDesc = new TFileRangeDesc();
+ invokePrivateMethod(node, "setPaimonParams",
+ new Class<?>[] {TFileRangeDesc.class, PaimonSplit.class},
+ rangeDesc, new PaimonSplit(createDataSplit("provider_ok.parquet")));
+ Assert.assertEquals("mode " + mode + " on " + location, expected,
+ rangeDesc.getTableFormatParams().getPaimonParams().getReaderType());
+ }
+ }
+
+ @Test
+ public void testRustReaderRejectsConflictingStaticCredentials() throws Exception {
+ for (Map<String, String> extra : Arrays.asList(
+ ImmutableMap.of("AWS_CREDENTIALS_PROVIDER_TYPE", "ANONYMOUS"),
+ ImmutableMap.of("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/example"),
+ ImmutableMap.of("AWS_TOKEN", "expired-example-token"),
+ ImmutableMap.of("AWS_TOKEN", " "), ImmutableMap.of("AWS_ROLE_ARN", " "))) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ Map<String, String> properties = new HashMap<>();
+ properties.put("AWS_ACCESS_KEY", "example-key");
+ properties.put("AWS_SECRET_KEY", "example-secret");
+ properties.putAll(extra);
+ setField(PaimonScanNode.class, f.node, "backendStorageProperties", properties);
+ // Hadoop's Simple provider ignores these settings; Rust must not change identity.
+ Assert.assertEquals(extra.toString(), TPaimonReaderType.PAIMON_JNI, f.reader(f.split));
+ }
+ }
+
+ @Test
+ public void testRustReaderRejectsUnsupportedPersistedBranchScanModes() throws Exception {
+ for (String mode : Arrays.asList("latest", "latest-full", "full", "compacted-full", "LATEST-FULL", " default ")) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ f.schema(1, new IntType(), ImmutableMap.of("branch", "audit", "scan.mode", mode), false);
+ // A branch can keep its persisted mode without a time-travel selector to strip it.
+ Assert.assertEquals(mode, TPaimonReaderType.PAIMON_JNI, f.reader(f.split));
+ }
+ }
+
+ @Test
+ public void testRustReaderSelectionGatesUnverifiedLocationSchemes() throws Exception {
+ // The pinned paimon-rust storage dispatcher (io/storage.rs) selects the
+ // FileIO parser from the table location's URI scheme, and libpaimon_c.a
+ // compiles in separate COS, OBS, GCS and Azdls parsers besides OSS and
+ // S3. Doris normalizes those object stores' credentials into the AWS_*
+ // aliases, which the BE rust bridge translates only into the s3.* /
+ // fs.oss.* key families — a cosn:// / obs:// / gs:// / abfs:// table
+ // would reach its scheme's parser without the key family it reads
+ // (fs.cosn.userinfo.*, fs.obs.*, gcs.*, azure.*) and fail the open.
+ // Those schemes must fall back to JNI before the split is encoded;
+ // the verified schemes (s3 / s3a / oss, plus credential-free hdfs and
+ // local paths) stay rust-eligible.
+ for (String[] shape : new String[][] {
+ // Unverified object-store schemes -> JNI (property translation
+ // not implemented; Doris sends these as AWS_* aliases).
+ {"cosn://bucket/wh/db.db/t", "PAIMON_JNI"},
+ {"obs://bucket/wh/db.db/t", "PAIMON_JNI"},
+ {"gs://bucket/wh/db.db/t", "PAIMON_JNI"},
+ {"abfs://bucket/wh/db.db/t", "PAIMON_JNI"},
+ {"abfss://bucket@account/wh/db.db/t", "PAIMON_JNI"},
+ {"az://bucket/wh/db.db/t", "PAIMON_JNI"},
+ {"azure://bucket/wh/db.db/t", "PAIMON_JNI"},
+ // Uppercase URI-scheme variants of otherwise verified schemes:
+ // the crate strips only a lowercase prefix, and the DataSplit's
+ // file paths carry the original casing too -> JNI.
+ {"S3://bucket/wh/db.db/t", "PAIMON_JNI"},
+ {"Hdfs://nn/wh/db.db/t", "PAIMON_JNI"},
+ // A null location cannot be verified (and cannot ship
+ // paimon_table for the rust reader) -> JNI.
+ {null, "PAIMON_JNI"},
+ // Verified schemes with implemented property translation or
+ // credential-free parsers -> rust.
+ {"s3://bucket/wh/db.db/t", "PAIMON_RUST"},
+ {"s3a://bucket/wh/db.db/t", "PAIMON_RUST"},
+ {"oss://bucket/wh/db.db/t", "PAIMON_RUST"},
+ {"hdfs://nn/wh/db.db/t", "PAIMON_RUST"},
+ {"file:///paimon/wh/db.db/t", "PAIMON_RUST"},
+ {"/paimon/wh/db.db/t", "PAIMON_RUST"}}) {
+ String location = shape[0];
+ TPaimonReaderType expected = TPaimonReaderType.valueOf(shape[1]);
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = true;
+
+ PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0),
+ new TupleDescriptor(new TupleId(0)), false, vars, ScanContext.EMPTY);
+ PaimonSource source = Mockito.mock(PaimonSource.class);
+ if (location != null) {
+ Mockito.when(source.getTableLocation()).thenReturn(location);
+ }
+ FileStoreTable paimonTable = Mockito.mock(FileStoreTable.class);
+ PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class);
+ if (expected == TPaimonReaderType.PAIMON_RUST) {
+ // Only the rust path serializes the schema and the table
+ // identity; the JNI iterations must not leave unused stubs.
+ Mockito.when(source.getExternalTable()).thenReturn(externalTable);
+ Mockito.when(paimonTable.schema()).thenReturn(new TableSchema(
+ 1, Collections.singletonList(new DataField(0, "id", new IntType())),
+ 0, Collections.emptyList(), Collections.emptyList(),
+ Collections.emptyMap(), null));
+ Mockito.when(externalTable.getDbName()).thenReturn("db");
+ Mockito.when(externalTable.getName()).thenReturn("t");
+ }
+ node.setSource(source);
+ setField(PaimonScanNode.class, node, "processedTable", paimonTable);
+ // Production-shaped COS/OBS/GCS/Azure map: static credentials
+ // normalized to the AWS_* aliases (plus the S3-compatible
+ // connection settings).
+ setField(PaimonScanNode.class, node, "backendStorageProperties", ImmutableMap.of(
+ "AWS_CREDENTIALS_PROVIDER_TYPE", "DEFAULT",
+ "AWS_ACCESS_KEY", "ak",
+ "AWS_SECRET_KEY", "sk",
+ "AWS_ENDPOINT", "http://127.0.0.1:19001",
+ "AWS_REGION", "us-east-1"));
+
+ TFileRangeDesc rangeDesc = new TFileRangeDesc();
+ invokePrivateMethod(node, "setPaimonParams",
+ new Class<?>[] {TFileRangeDesc.class, PaimonSplit.class},
+ rangeDesc, new PaimonSplit(createDataSplit("scheme_gate.parquet")));
+ Assert.assertEquals("location " + location, expected,
+ rangeDesc.getTableFormatParams().getPaimonParams().getReaderType());
+ }
+ }
+
+ @Test
+ public void testIsRustVerifiedLocationSchemeClassification() {
+ // A plain path without a URI scheme is a local-filesystem location, which
+ // reads through the crate's LocalFs parser and needs no credentials. The
+ // scheme must appear in the exact lowercase form the pinned crate
+ // consumes: it lowercases only its storage dispatch, while its path
+ // extraction strips a lowercase prefix from the original string — a
+ // S3:// or Hdfs:// warehouse must route to JNI instead (the Java stack
+ // is case-insensitive everywhere).
+ Assert.assertTrue(PaimonScanNode.isRustVerifiedLocationScheme("s3://bucket/wh"));
+ Assert.assertTrue(PaimonScanNode.isRustVerifiedLocationScheme("/local/wh"));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedLocationScheme("S3://bucket/wh"));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedLocationScheme("OSS://bucket/wh"));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedLocationScheme("Hdfs://nn/wh"));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedLocationScheme("FILE:///paimon/wh"));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedLocationScheme("cosn://bucket/wh"));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedLocationScheme("COSN://bucket/wh"));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedLocationScheme("viewfs://cluster/wh"));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedLocationScheme(null));
+ }
+
+ @Test
+ public void testRustReaderSelectionRejectsAuthenticatedHdfsBackends() throws Exception {
+ // The pinned paimon-rust HDFS parser reads only the hdfs.name-node /
+ // hdfs.enable-append keys: no kerberos, no proxy user, no hadoop HA
+ // resolution. A Kerberized HDFS catalog (or one with hadoop.username /
+ // HA nameservice config) passes the location scheme gate, so the scan
+ // would open as the BE process's ambient identity and fail the access
+ // JNI honors — those shapes must fall back to JNI; the credential-free
+ // shape stays rust-eligible.
+ for (String[][] shape : new String[][][] {
+ // Production-shaped credential-free HdfsProperties output: an
+ // fs.defaultFS, the always-written simple-auth markers and
+ // pass-through tunables, none of which carry an identity.
+ {new String[] {"fs.defaultFS", "hdfs://nn:8020"},
+ new String[] {"hdfs.security.authentication", "simple"},
+ new String[] {"ipc.client.fallback-to-simple-auth-allowed", "true"},
+ new String[] {"PAIMON_RUST"}},
+ // A null / empty map is the anonymous default the parser
+ // supports (the regression suite's plain hdfs:// table).
+ {new String[] {"PAIMON_RUST"}},
+ // Kerberos: authentication type plus principal / keytab, as
+ // HdfsProperties emits them for a Kerberized catalog.
+ {new String[] {"hadoop.security.authentication", "kerberos"},
+ new String[] {"hadoop.kerberos.principal", "hdfs/_HOST@REALM"},
+ new String[] {"hadoop.kerberos.keytab", "/etc/security/hdfs.keytab"},
+ new String[] {"PAIMON_JNI"}},
+ {new String[] {"hdfs.security.authentication", "kerberos"}, new String[] {"PAIMON_JNI"}},
+ // Proxy user: hadoop.username authenticates as a specific user
+ // the rust parser would drop.
+ {new String[] {"hadoop.username", "hive"}, new String[] {"PAIMON_JNI"}},
+ // HA nameservice resolution: dfs.nameservices / dfs.ha.* config
+ // cannot be resolved without the dfs.* channel.
+ {new String[] {"dfs.nameservices", "ns1"},
+ new String[] {"dfs.ha.namenodes.ns1", "nn1,nn2"},
+ new String[] {"PAIMON_JNI"}},
+ {new String[] {"dfs.ha.namenodes.ns1", "nn1,nn2"}, new String[] {"PAIMON_JNI"}},
+ // Simple-auth client options: valid without Kerberos, but the
+ // rust storage_hdfs parser drops every dfs./hadoop./fs. key
+ // beyond its two, and hdfs-native defaults
+ // dfs.client.use.datanode.hostname to false — behind
+ // containers/NAT it connects to the advertised IP instead of
+ // the hostname and fails while JNI succeeds.
+ {new String[] {"dfs.client.use.datanode.hostname", "true"},
+ new String[] {"PAIMON_JNI"}},
+ {new String[] {"hadoop.proxyuser.kerberos", "x"},
+ new String[] {"PAIMON_JNI"}}}) {
+ Map<String, String> backendProperties = new HashMap<>();
+ for (int i = 0; i < shape.length - 1; i++) {
+ backendProperties.put(shape[i][0], shape[i][1]);
+ }
+ TPaimonReaderType expected = TPaimonReaderType.valueOf(
+ shape[shape.length - 1][0]);
+
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = true;
+
+ PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0),
+ new TupleDescriptor(new TupleId(0)), false, vars, ScanContext.EMPTY);
+ PaimonSource source = Mockito.mock(PaimonSource.class);
+ Mockito.when(source.getTableLocation()).thenReturn("hdfs://nn/wh/db.db/t");
+ FileStoreTable paimonTable = Mockito.mock(FileStoreTable.class);
+ PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class);
+ if (expected == TPaimonReaderType.PAIMON_RUST) {
+ Mockito.when(source.getExternalTable()).thenReturn(externalTable);
+ Mockito.when(paimonTable.schema()).thenReturn(new TableSchema(
+ 1, Collections.singletonList(new DataField(0, "id", new IntType())),
+ 0, Collections.emptyList(), Collections.emptyList(),
+ Collections.emptyMap(), null));
+ Mockito.when(externalTable.getDbName()).thenReturn("db");
+ Mockito.when(externalTable.getName()).thenReturn("t");
+ }
+ node.setSource(source);
+ setField(PaimonScanNode.class, node, "processedTable", paimonTable);
+ setField(PaimonScanNode.class, node, "backendStorageProperties", backendProperties);
+
+ TFileRangeDesc rangeDesc = new TFileRangeDesc();
+ invokePrivateMethod(node, "setPaimonParams",
+ new Class<?>[] {TFileRangeDesc.class, PaimonSplit.class},
+ rangeDesc, new PaimonSplit(createDataSplit("hdfs_auth_gate.parquet")));
+ Assert.assertEquals("backend " + backendProperties, expected,
+ rangeDesc.getTableFormatParams().getPaimonParams().getReaderType());
+ }
+ }
+
+ @Test
+ public void testIsRustVerifiedHdfsBackendClassification() {
+ // The anonymous / null map is the open-tested shape; the always-written
+ // simple-auth markers and fs.defaultFS carry no identity; every
+ // auth-bearing key (type, principal, keytab, proxy user) and the HA
+ // nameservice config route to JNI.
+ Assert.assertTrue(PaimonScanNode.isRustVerifiedHdfsBackend(null));
+ Assert.assertTrue(PaimonScanNode.isRustVerifiedHdfsBackend(Collections.emptyMap()));
+ Assert.assertTrue(PaimonScanNode.isRustVerifiedHdfsBackend(ImmutableMap.of(
+ "fs.defaultFS", "hdfs://nn:8020",
+ "hadoop.security.authentication", "simple")));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedHdfsBackend(ImmutableMap.of(
+ "hadoop.security.authentication", "kerberos")));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedHdfsBackend(ImmutableMap.of(
+ "hadoop.security.authentication", "KERBEROS")));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedHdfsBackend(ImmutableMap.of(
+ "hadoop.kerberos.principal", "hdfs/_HOST@REALM")));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedHdfsBackend(ImmutableMap.of(
+ "hadoop.kerberos.keytab", "/etc/security/hdfs.keytab")));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedHdfsBackend(ImmutableMap.of(
+ "hadoop.username", "hive")));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedHdfsBackend(ImmutableMap.of(
+ "dfs.nameservices", "ns1")));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedHdfsBackend(ImmutableMap.of(
+ "dfs.ha.namenodes.ns1", "nn1,nn2")));
+ // A blank value is the unset case (values are null-filtered upstream,
+ // but a blank site-config entry must not gate an otherwise
+ // credential-free catalog).
+ Assert.assertTrue(PaimonScanNode.isRustVerifiedHdfsBackend(ImmutableMap.of(
+ "hadoop.username", " ")));
+ // The always-shipped inert keys stay rust-eligible together.
+ Assert.assertTrue(PaimonScanNode.isRustVerifiedHdfsBackend(ImmutableMap.of(
+ "fs.defaultFS", "hdfs://nn:8020",
+ "hadoop.security.authentication", "simple",
+ "hdfs.security.authentication", "simple",
+ "ipc.client.fallback-to-simple-auth-allowed", "true")));
+ // Any other dfs./hadoop./fs. client option is dropped by the rust
+ // parser and would diverge from JNI -> JNI (the canonical simple-auth
+ // case is dfs.client.use.datanode.hostname).
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedHdfsBackend(ImmutableMap.of(
+ "dfs.client.use.datanode.hostname", "true")));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedHdfsBackend(ImmutableMap.of(
+ "dfs.client.socket-timeout", "60000")));
+ Assert.assertFalse(PaimonScanNode.isRustVerifiedHdfsBackend(ImmutableMap.of(
+ "fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem")));
+ // Keys outside the hadoop/dfs/fs namespaces cannot change what the
+ // rust HDFS parser does (the BE bridge passes them through untouched
+ // and the parser ignores them).
+ Assert.assertTrue(PaimonScanNode.isRustVerifiedHdfsBackend(ImmutableMap.of(
+ "warehouse", "hdfs://nn/wh", "unrelated.key", "value")));
+ }
+
+ @Test
+ public void testRustReaderSelectionRejectsProjectedVariantColumns() throws Exception {
+ // The rust leaf feeds its Arrow arrays to the slot serdes, and
+ // DataTypeVariantV2SerDe::read_column_from_arrow unconditionally returns
+ // NOT_IMPLEMENTED_ERROR — a nested Variant (ARRAY / MAP / STRUCT
+ // containing one) reaches the same decoder through the container
+ // serdes. Projections containing Variant must route to JNI; a table
+ // whose VARIANT column is not projected by this query stays on rust.
+ Type[][] shapes = new Type[][] {
+ // Plain projected columns -> rust.
+ new Type[] {Type.INT, Type.STRING},
+ // A projected VARIANT -> JNI.
+ new Type[] {Type.INT, Type.VARIANT},
+ // Nested Variant inside every container kind -> JNI.
+ new Type[] {new ArrayType(Type.VARIANT)},
+ new Type[] {new MapType(Type.STRING, Type.VARIANT)},
+ new Type[] {new StructType(new StructField("v", Type.VARIANT))},
+ // The same containers without Variant stay rust-eligible.
+ new Type[] {new ArrayType(Type.INT)},
+ new Type[] {new MapType(Type.STRING, Type.INT)}};
+ boolean[] expectRust = new boolean[] {true, false, false, false, false, true, true};
+
+ for (int i = 0; i < shapes.length; i++) {
+ TupleDescriptor desc = new TupleDescriptor(new TupleId(0));
+ for (int j = 0; j < shapes[i].length; j++) {
+ SlotDescriptor slot = new SlotDescriptor(new SlotId(j), desc);
+ slot.setType(shapes[i][j]);
+ desc.addSlot(slot);
+ }
+
+ SessionVariable vars = new SessionVariable();
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = true;
+
+ PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0), desc, false, vars,
+ ScanContext.EMPTY);
+ PaimonSource source = Mockito.mock(PaimonSource.class);
+ Mockito.when(source.getTableLocation()).thenReturn("s3://bucket/wh/db.db/t");
+ FileStoreTable paimonTable = Mockito.mock(FileStoreTable.class);
+ PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class);
+ if (expectRust[i]) {
+ Mockito.when(source.getExternalTable()).thenReturn(externalTable);
+ Mockito.when(paimonTable.schema()).thenReturn(new TableSchema(
+ 1, Collections.singletonList(new DataField(0, "id", new IntType())),
+ 0, Collections.emptyList(), Collections.emptyList(),
+ Collections.emptyMap(), null));
+ Mockito.when(externalTable.getDbName()).thenReturn("db");
+ Mockito.when(externalTable.getName()).thenReturn("t");
+ }
+ node.setSource(source);
+ setField(PaimonScanNode.class, node, "processedTable", paimonTable);
+ setField(PaimonScanNode.class, node, "backendStorageProperties", ImmutableMap.of(
+ "AWS_CREDENTIALS_PROVIDER_TYPE", "DEFAULT",
+ "AWS_ACCESS_KEY", "ak",
+ "AWS_SECRET_KEY", "sk",
+ "AWS_ENDPOINT", "http://127.0.0.1:19001",
+ "AWS_REGION", "us-east-1"));
+
+ TFileRangeDesc rangeDesc = new TFileRangeDesc();
+ invokePrivateMethod(node, "setPaimonParams",
+ new Class<?>[] {TFileRangeDesc.class, PaimonSplit.class},
+ rangeDesc, new PaimonSplit(createDataSplit("variant_gate.parquet")));
+ Assert.assertEquals("projection " + Arrays.toString(shapes[i]),
+ expectRust[i] ? TPaimonReaderType.PAIMON_RUST : TPaimonReaderType.PAIMON_JNI,
+ rangeDesc.getTableFormatParams().getPaimonParams().getReaderType());
+ }
+ }
+
+ /**
+ * Every negative assertion starts from a complete Rust-eligible fixture, so missing
+ * schema or credentials cannot accidentally satisfy a fallback test.
+ */
+ private class RustRoutingFixture {
+ final SessionVariable vars = new SessionVariable();
+ final TupleDescriptor tuple = new TupleDescriptor(new TupleId(0));
+ final PaimonSource source = Mockito.mock(PaimonSource.class);
+ final FileStoreTable table = Mockito.mock(FileStoreTable.class);
+ final org.apache.paimon.schema.SchemaManager schemas =
+ Mockito.mock(org.apache.paimon.schema.SchemaManager.class);
+ final PaimonScanNode node;
+ final DataSplit split = createDataSplit("capabilities.parquet");
+
+ RustRoutingFixture() throws Exception {
+ vars.setEnablePaimonRustReader(true);
+ vars.enableFileScannerV2 = true;
+ node = new PaimonScanNode(new PlanNodeId(0), tuple, false, vars, ScanContext.EMPTY);
+ PaimonExternalTable external = Mockito.mock(PaimonExternalTable.class);
+ Mockito.when(source.getExternalTable()).thenReturn(external);
+ Mockito.when(external.getDbName()).thenReturn("db");
+ Mockito.when(external.getName()).thenReturn("t");
+ Mockito.when(source.getTableLocation()).thenReturn("s3://warehouse/wh/db.db/t");
+ Mockito.when(table.schemaManager()).thenReturn(schemas);
+ schema(1, new IntType(), Collections.emptyMap(), false);
+ node.setSource(source);
+ setField(PaimonScanNode.class, node, "processedTable", table);
+ credentials("DEFAULT");
+ Assert.assertEquals(TPaimonReaderType.PAIMON_RUST, reader(split));
+ }
+
+ void credentials(String provider) throws Exception {
+ setField(PaimonScanNode.class, node, "backendStorageProperties", ImmutableMap.of(
+ "AWS_CREDENTIALS_PROVIDER_TYPE", provider,
+ "AWS_ACCESS_KEY", "example-key", "AWS_SECRET_KEY", "example-secret"));
+ }
+
+ void schema(long id, org.apache.paimon.types.DataType valueType,
+ Map<String, String> options, boolean pk) throws Exception {
+ TableSchema schema = new TableSchema(id, Arrays.asList(
+ new DataField(0, "id", new IntType(false)), new DataField(1, "v", valueType)),
+ 10, Collections.emptyList(), pk ? Collections.singletonList("id") : Collections.emptyList(),
+ options, null);
+ setField(PaimonScanNode.class, node, "rustReaderCapabilities", null);
+ Mockito.when(table.schema()).thenReturn(schema);
+ Mockito.when(table.coreOptions()).thenReturn(new CoreOptions(options));
+ }
+
+ TPaimonReaderType reader(DataSplit input) throws Exception {
+ TFileRangeDesc range = new TFileRangeDesc();
+ invokePrivateMethod(node, "setPaimonParams",
+ new Class<?>[] {TFileRangeDesc.class, PaimonSplit.class}, range, new PaimonSplit(input));
+ return range.getTableFormatParams().getPaimonParams().getReaderType();
+ }
+
+ void project(Type full, Type pruned) throws Exception {
+ setField(PaimonScanNode.class, node, "rustReaderCapabilities", null);
+ SlotDescriptor slot = new SlotDescriptor(new SlotId(0), tuple);
+ slot.setColumn(new Column("v", full));
+ slot.setType(pruned);
+ tuple.addSlot(slot);
+ }
+ }
+
+ @Test
+ public void testRustPrunedNestedProjectionFallback() throws Exception {
+ StructType full = new StructType(new StructField("a", Type.INT), new StructField("b", Type.INT));
+ StructType pruned = new StructType(new StructField("b", Type.INT));
+ Type[][] shapes = {{full, pruned}, {new ArrayType(full), new ArrayType(pruned)},
+ {new MapType(Type.INT, full), new MapType(Type.INT, pruned)}};
+ for (Type[] shape : shapes) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ f.project(shape[0], shape[0]);
+ Assert.assertEquals(TPaimonReaderType.PAIMON_RUST, f.reader(f.split));
+ f.tuple.getSlots().get(0).setType(shape[1]);
+ setField(PaimonScanNode.class, f.node, "rustReaderCapabilities", null);
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, f.reader(f.split));
+ }
+ }
+
+ @Test
+ public void testRustHistoricalIntegerNarrowingFallback() throws Exception {
+ for (org.apache.paimon.types.DataType target : Arrays.asList(
+ new org.apache.paimon.types.TinyIntType(), new org.apache.paimon.types.SmallIntType(),
+ new IntType())) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ TableSchema old = new TableSchema(1, Arrays.asList(new DataField(0, "id", new IntType(false)),
+ new DataField(1, "v", new org.apache.paimon.types.BigIntType())), 1,
+ Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), null);
+ Mockito.when(f.schemas.schema(1)).thenReturn(old);
+ f.schema(2, target, Collections.emptyMap(), false);
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, f.reader(f.split));
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, f.reader(f.split));
+ Mockito.verify(f.schemas, Mockito.times(1)).schema(1);
+ }
+ }
+
+ @Test
+ public void testRustStringHistoryToTemporalFallback() throws Exception {
+ for (org.apache.paimon.types.DataType sourceType : Arrays.asList(
+ DataTypes.STRING(), DataTypes.VARCHAR(20), DataTypes.CHAR(20))) {
+ for (org.apache.paimon.types.DataType target : Arrays.asList(DataTypes.DATE(),
+ DataTypes.TIMESTAMP(0), DataTypes.TIMESTAMP(3), DataTypes.TIMESTAMP(6),
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3))) {
+ assertHistoricalTemporalRouting(sourceType, target, TPaimonReaderType.PAIMON_JNI);
+ }
+ }
+ assertHistoricalTemporalRouting(DataTypes.DATE(), DataTypes.DATE(), TPaimonReaderType.PAIMON_RUST);
+ }
+
+ @Test
+ public void testRustIntegerHistoryToTimestampFallback() throws Exception {
+ for (int precision : new int[] {0, 3, 6}) {
+ for (org.apache.paimon.types.DataType target : Arrays.asList(DataTypes.TIMESTAMP(precision),
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(precision))) {
+ for (org.apache.paimon.types.DataType sourceType : Arrays.asList(DataTypes.INT(), DataTypes.BIGINT())) {
+ assertHistoricalTemporalRouting(sourceType, target, TPaimonReaderType.PAIMON_JNI);
+ }
+ assertHistoricalTemporalRouting(target, target, TPaimonReaderType.PAIMON_RUST);
+ }
+ }
+ }
+
+ private void assertHistoricalTemporalRouting(org.apache.paimon.types.DataType sourceType,
+ org.apache.paimon.types.DataType target, TPaimonReaderType expected) throws Exception {
+ org.apache.paimon.types.DataType[][] shapes = {
+ {sourceType, target},
+ {DataTypes.ROW(new DataField(2, "old_name", sourceType)),
+ DataTypes.ROW(new DataField(2, "renamed", target))},
+ {DataTypes.ARRAY(sourceType), DataTypes.ARRAY(target)},
+ {DataTypes.MAP(sourceType, DataTypes.INT()), DataTypes.MAP(target, DataTypes.INT())},
+ {DataTypes.MAP(DataTypes.INT(), sourceType), DataTypes.MAP(DataTypes.INT(), target)}};
+ for (org.apache.paimon.types.DataType[] shape : shapes) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ Mockito.when(f.schemas.schema(1)).thenReturn(new TableSchema(1, Arrays.asList(
+ new DataField(0, "id", new IntType(false)), new DataField(1, "v", shape[0])),
+ 10, Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), null));
+ f.schema(2, shape[1], Collections.emptyMap(), false);
+ Assert.assertEquals(Arrays.toString(shape), expected, f.reader(f.split));
+ }
+ }
+
+ @Test
+ public void testRustFloatingHistoryToDecimalFallback() throws Exception {
+ for (org.apache.paimon.types.DataType oldType : Arrays.asList(
+ DataTypes.FLOAT(), DataTypes.DOUBLE(), DataTypes.INT(), DataTypes.DECIMAL(3, 2))) {
+ org.apache.paimon.types.DataType decimal = DataTypes.DECIMAL(10, 2);
+ org.apache.paimon.types.DataType[][] shapes = {
+ {oldType, decimal},
+ {DataTypes.ROW(new DataField(2, "old_name", oldType)),
+ DataTypes.ROW(new DataField(2, "renamed", decimal))},
+ {DataTypes.ARRAY(oldType), DataTypes.ARRAY(decimal)},
+ {DataTypes.MAP(oldType, DataTypes.INT()), DataTypes.MAP(decimal, DataTypes.INT())},
+ {DataTypes.MAP(DataTypes.INT(), oldType), DataTypes.MAP(DataTypes.INT(), decimal)}};
+ for (org.apache.paimon.types.DataType[] shape : shapes) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ Mockito.when(f.schemas.schema(1)).thenReturn(new TableSchema(1, Arrays.asList(
+ new DataField(0, "id", new IntType(false)), new DataField(1, "v", shape[0])),
+ 10, Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), null));
+ f.schema(2, shape[1], Collections.emptyMap(), false);
+ boolean floating = oldType.equals(DataTypes.FLOAT()) || oldType.equals(DataTypes.DOUBLE());
+ Assert.assertEquals(Arrays.toString(shape), floating
+ ? TPaimonReaderType.PAIMON_JNI : TPaimonReaderType.PAIMON_RUST, f.reader(f.split));
+ }
+ }
+ }
+
+ @Test
+ public void testRustUnusedAggregateNamesFallback() throws Exception {
+ for (String engine : Arrays.asList("aggregation", "partial-update")) {
+ List<String> optionKeys = new ArrayList<>(Arrays.asList(
+ "fields.default-aggregate-function", "fields.id.aggregate-function"));
+ if ("partial-update".equals(engine)) {
+ optionKeys.add("fields.seq.aggregate-function");
+ }
+ for (String key : optionKeys) {
+ for (String function : Arrays.asList("collect", "sum")) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ Map<String, String> options = new HashMap<>();
+ options.put("merge-engine", engine);
+ options.put("fields.v.aggregate-function", "max");
+ options.put(key, function);
+ f.schema(1, new IntType(), options, true);
+ if ("partial-update".equals(engine)) {
+ options.put("fields.seq.sequence-group", "v");
+ Mockito.when(f.table.schema()).thenReturn(new TableSchema(1, Arrays.asList(
+ new DataField(0, "id", new IntType(false)), new DataField(1, "v", new IntType()),
+ new DataField(2, "seq", new IntType())), 2, Collections.emptyList(),
+ Collections.singletonList("id"), options, null));
+ }
+ DataFileMeta file = Mockito.spy(f.split.dataFiles().get(0));
+ Mockito.doReturn(Optional.of(0L)).when(file).deleteRowCount();
+ DataSplit split = DataSplit.builder().withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0).withBucketPath("file:///warehouse/db/t/bucket-0")
+ .withDataFiles(Collections.singletonList(file)).build();
+ Assert.assertEquals(engine + " " + key + "=" + function,
+ TPaimonReaderType.PAIMON_JNI, f.reader(split));
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testRustNonIntegerHistoryToIntegerFallback() throws Exception {
+ for (org.apache.paimon.types.DataType sourceType : Arrays.asList(
+ DataTypes.FLOAT(), DataTypes.DOUBLE(), DataTypes.DECIMAL(10, 2), DataTypes.STRING())) {
+ for (org.apache.paimon.types.DataType targetType : Arrays.asList(
+ DataTypes.TINYINT(), DataTypes.SMALLINT(), DataTypes.INT(), DataTypes.BIGINT())) {
+ org.apache.paimon.types.DataType[][] shapes = {
+ {sourceType, targetType},
+ {DataTypes.ROW(new DataField(2, "old_name", sourceType)),
+ DataTypes.ROW(new DataField(2, "renamed", targetType))},
+ {DataTypes.ARRAY(sourceType), DataTypes.ARRAY(targetType)},
+ {DataTypes.MAP(sourceType, DataTypes.INT()), DataTypes.MAP(targetType, DataTypes.INT())},
+ {DataTypes.MAP(DataTypes.INT(), sourceType), DataTypes.MAP(DataTypes.INT(), targetType)}};
+ for (org.apache.paimon.types.DataType[] shape : shapes) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ Mockito.when(f.schemas.schema(1)).thenReturn(new TableSchema(1, Arrays.asList(
+ new DataField(0, "id", new IntType(false)), new DataField(1, "v", shape[0])),
+ 10, Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), null));
+ f.schema(2, shape[1], Collections.emptyMap(), false);
+ Assert.assertEquals(Arrays.toString(shape), TPaimonReaderType.PAIMON_JNI, f.reader(f.split));
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testRustPartialUpdateDefaultAggregateAndSequenceFields() throws Exception {
+ for (org.apache.paimon.types.DataType valueType : Arrays.asList(
+ DataTypes.TINYINT(), DataTypes.DECIMAL(2, 0), DataTypes.DOUBLE())) {
+ for (boolean override : Arrays.asList(false, true)) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ Map<String, String> options = new HashMap<>();
+ options.put("merge-engine", "partial-update");
+ options.put("fields.seq,seq2.sequence-group", "v");
+ options.put("fields.default-aggregate-function", "sum");
+ if (override) {
+ options.put("fields.v.aggregate-function", "max");
+ }
+ f.schema(1, valueType, options, true);
+ Mockito.when(f.table.schema()).thenReturn(new TableSchema(1, Arrays.asList(
+ new DataField(0, "id", new IntType(false)), new DataField(1, "v", valueType),
+ new DataField(2, "seq", new IntType()), new DataField(3, "seq2", new IntType())),
+ 3, Collections.emptyList(), Collections.singletonList("id"), options, null));
+ DataFileMeta file = Mockito.spy(f.split.dataFiles().get(0));
+ Mockito.doReturn(Optional.of(0L)).when(file).deleteRowCount();
+ // Legacy writers can retain an unbounded same-key run even in one file.
+ DataSplit split = DataSplit.builder().withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0).withBucketPath("file:///warehouse/db/t/bucket-0")
+ .withDataFiles(Collections.singletonList(file)).build();
+ Assert.assertEquals(valueType + " override=" + override,
+ TPaimonReaderType.PAIMON_JNI,
+ f.reader(split));
+ }
+ }
+ }
+
+ @Test
+ public void testRustMultiFilePrimaryKeyMergeFallback() throws Exception {
+ for (String engine : Arrays.asList("deduplicate", "aggregation", "partial-update")) {
+ for (boolean primaryKey : Arrays.asList(false, true)) {
+ for (int fileCount : new int[] {1, 2, 8}) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ f.schema(1, new IntType(), ImmutableMap.of("merge-engine", engine,
+ "read.batch-size", "1"), primaryKey);
+ List<DataFileMeta> files = new ArrayList<>();
+ for (int i = 0; i < fileCount; i++) {
+ DataFileMeta file = Mockito.spy(createDataSplit("run-" + i + ".parquet")
+ .dataFiles().get(0));
+ Mockito.doReturn(Optional.of(0L)).when(file).deleteRowCount();
+ files.add(file);
+ }
+ DataSplit split = DataSplit.builder().withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0).withBucketPath("file:///warehouse/db/t/bucket-0")
+ .withDataFiles(files).build();
+ Assert.assertEquals(engine + " pk=" + primaryKey + " files=" + fileCount,
+ primaryKey && (fileCount > 1 || !"deduplicate".equals(engine)) ? TPaimonReaderType.PAIMON_JNI
+ : TPaimonReaderType.PAIMON_RUST, f.reader(split));
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testRustNestedEvolutionAndCompatibleHistory() throws Exception {
+ org.apache.paimon.types.DataType small = new org.apache.paimon.types.TinyIntType();
+ org.apache.paimon.types.DataType large = new org.apache.paimon.types.BigIntType();
+ org.apache.paimon.types.RowType oldRow = new org.apache.paimon.types.RowType(Arrays.asList(
+ new DataField(2, "a", large), new DataField(3, "b", new IntType())));
+ org.apache.paimon.types.RowType newRow = new org.apache.paimon.types.RowType(Arrays.asList(
+ new DataField(3, "b", new IntType()), new DataField(2, "renamed", small)));
+ org.apache.paimon.types.DataType[][] shapes = {
+ {large, small}, {oldRow, newRow},
+ {new org.apache.paimon.types.ArrayType(oldRow), new org.apache.paimon.types.ArrayType(newRow)},
+ {new org.apache.paimon.types.MapType(large, new IntType()),
+ new org.apache.paimon.types.MapType(small, new IntType())},
+ {new org.apache.paimon.types.MapType(new IntType(), oldRow),
+ new org.apache.paimon.types.MapType(new IntType(), newRow)}};
+ for (org.apache.paimon.types.DataType[] shape : shapes) {
+ for (boolean narrow : Arrays.asList(true, false)) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ TableSchema old = new TableSchema(1, Arrays.asList(new DataField(0, "id", new IntType(false)),
+ new DataField(1, "v", shape[narrow ? 0 : 1])), 10,
+ Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), null);
+ Mockito.when(f.schemas.schema(1)).thenReturn(old);
+ f.schema(2, shape[narrow ? 1 : 0], Collections.emptyMap(), false);
+ Assert.assertEquals(narrow ? TPaimonReaderType.PAIMON_JNI : TPaimonReaderType.PAIMON_RUST,
+ f.reader(f.split));
+ }
+ }
+ RustRoutingFixture missing = new RustRoutingFixture();
+ missing.schema(2, new IntType(), Collections.emptyMap(), false);
+ Mockito.when(missing.schemas.schema(1)).thenThrow(new IllegalStateException("missing schema"));
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, missing.reader(missing.split));
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, missing.reader(missing.split));
+ Mockito.verify(missing.schemas, Mockito.times(1)).schema(1);
+ }
+
+ @Test
+ public void testRustDefaultAggregateAndFieldOverride() throws Exception {
+ for (Map<String, String> options : Arrays.asList(
+ ImmutableMap.of("merge-engine", "aggregation", "fields.default-aggregate-function", "sum"),
+ ImmutableMap.of("merge-engine", "aggregation", "fields.default-aggregate-function", "max"),
+ ImmutableMap.of("merge-engine", "aggregation", "fields.default-aggregate-function", "sum",
+ "fields.v.aggregate-function", "max"))) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ f.schema(1, new IntType(), options, true);
+ DataFileMeta file = Mockito.spy(f.split.dataFiles().get(0));
+ Mockito.doReturn(Optional.of(0L)).when(file).deleteRowCount();
+ DataSplit split = DataSplit.builder().withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0).withBucketPath("file:///warehouse/db/t/bucket-0")
+ .withDataFiles(Collections.singletonList(file)).build();
+ Assert.assertEquals(TPaimonReaderType.PAIMON_JNI,
+ f.reader(split));
+ }
+ }
+
+ @Test
+ public void testRustPrimaryKeyRetractsFallback() throws Exception {
+ for (String engine : Arrays.asList("deduplicate", "aggregation", "partial-update")) {
+ for (Long deletes : Arrays.asList(0L, 1L, null)) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ f.schema(1, new IntType(), ImmutableMap.of("merge-engine", engine), true);
+ DataFileMeta file = Mockito.spy(f.split.dataFiles().get(0));
+ Mockito.doReturn(Optional.ofNullable(deletes)).when(file).deleteRowCount();
+ DataSplit split = DataSplit.builder().withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0).withBucketPath("file:///warehouse/db/t/bucket-0")
+ .withDataFiles(Collections.singletonList(file)).build();
+ Assert.assertEquals(engine + " deletes=" + deletes,
+ "deduplicate".equals(engine) && deletes != null && deletes == 0L ? TPaimonReaderType.PAIMON_RUST
+ : TPaimonReaderType.PAIMON_JNI, f.reader(split));
+ }
+ }
+ }
+
+ @Test
+ public void testRustAggregateFunctionAndArithmeticFallback() throws Exception {
+ Object[][] cases = {
+ {"collect", new org.apache.paimon.types.ArrayType(new IntType()), false},
+ {"sum", new org.apache.paimon.types.TinyIntType(), false},
+ {"sum", new org.apache.paimon.types.SmallIntType(), false},
+ {"sum", new IntType(), false},
+ {"sum", new org.apache.paimon.types.BigIntType(), false},
+ {"product", new IntType(), false},
+ {"sum", new org.apache.paimon.types.DecimalType(2, 0), false},
+ {"sum", new org.apache.paimon.types.DecimalType(18, 2), false},
+ {"sum", new org.apache.paimon.types.DecimalType(19, 2), true},
+ {"product", new org.apache.paimon.types.DecimalType(20, 2), false},
+ {"sum", new org.apache.paimon.types.DoubleType(), true},
+ {"max", new IntType(), true},
+ {"max", new org.apache.paimon.types.ArrayType(new IntType()), false},
+ {"bool_and", new org.apache.paimon.types.BooleanType(), true},
+ {"last_non_null_value", new org.apache.paimon.types.ArrayType(new IntType()), true}
+ };
+ for (String engine : Arrays.asList("aggregation", "partial-update")) {
+ for (Object[] shape : cases) {
+ RustRoutingFixture f = new RustRoutingFixture();
+ f.schema(1, (org.apache.paimon.types.DataType) shape[1], ImmutableMap.of(
+ "merge-engine", engine, "fields.v.aggregate-function", (String) shape[0]), true);
+ DataFileMeta file = Mockito.spy(f.split.dataFiles().get(0));
+ Mockito.doReturn(Optional.of(0L)).when(file).deleteRowCount();
+ DataSplit split = DataSplit.builder().withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0).withBucketPath("file:///warehouse/db/t/bucket-0")
+ .withDataFiles(Collections.singletonList(file)).build();
+ Assert.assertEquals(engine + " " + Arrays.toString(shape),
+ TPaimonReaderType.PAIMON_JNI, f.reader(split));
+ }
+ }
+ }
+
+
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilderTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilderTest.java
new file mode 100644
index 00000000000..994853a04cd
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilderTest.java
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.qe.runtime;
+
+import org.apache.doris.common.profile.ExecutionProfile;
+import org.apache.doris.system.Backend;
+import org.apache.doris.thrift.TDetailedReportParams;
+import org.apache.doris.thrift.TPipelineFragmentParams;
+import org.apache.doris.thrift.TPipelineFragmentParamsList;
+import org.apache.doris.thrift.TPipelineInstanceParams;
+import org.apache.doris.thrift.TQueryProfile;
+import org.apache.doris.thrift.TRuntimeProfileNode;
+import org.apache.doris.thrift.TRuntimeProfileTree;
+import org.apache.doris.thrift.TUniqueId;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Map;
+
+public class PipelineExecutionTaskBuilderTest {
+ @Test
+ public void buildSingleFragmentPipelineTaskRegistersBackendWithExecutionProfile() {
+ TUniqueId queryId = new TUniqueId(1, 2);
+ ExecutionProfile executionProfile = new ExecutionProfile(queryId, Lists.newArrayList(0));
+ Backend backend = new Backend(1L, "127.0.0.1", 9050);
+
+ Map<Integer, SingleFragmentPipelineTask> tasks =
+ PipelineExecutionTaskBuilder.buildSingleFragmentPipelineTask(
+ executionProfile, backend, createFragmentParamsList(0));
+
+ Assert.assertEquals(1, tasks.size());
+ Assert.assertTrue(tasks.containsKey(0));
+
+ executionProfile.updateProfile(createQueryProfile(queryId, 0), backend.getHeartbeatAddress(), true);
+ Assert.assertTrue(executionProfile.isCompleted());
+ }
+
+ private static TPipelineFragmentParamsList createFragmentParamsList(int fragmentId) {
+ TPipelineInstanceParams instanceParams = new TPipelineInstanceParams();
+ instanceParams.setFragmentInstanceId(new TUniqueId(3, 4));
+
+ TPipelineFragmentParams fragmentParams = new TPipelineFragmentParams();
+ fragmentParams.setFragmentId(fragmentId);
+ fragmentParams.setLocalParams(Lists.newArrayList(instanceParams));
+
+ TPipelineFragmentParamsList paramsList = new TPipelineFragmentParamsList();
+ paramsList.setParamsList(Lists.newArrayList(fragmentParams));
+ return paramsList;
+ }
+
+ private static TQueryProfile createQueryProfile(TUniqueId queryId, int fragmentId) {
+ TQueryProfile queryProfile = new TQueryProfile();
+ queryProfile.setQueryId(queryId);
+ queryProfile.putToFragmentIdToProfile(fragmentId, Lists.newArrayList(createReportParam()));
+ return queryProfile;
+ }
+
+ private static TDetailedReportParams createReportParam() {
+ TRuntimeProfileNode node = new TRuntimeProfileNode();
+ node.setName("PipelineProfile");
+ node.setNumChildren(0);
+ node.setCounters(Lists.newArrayList());
+ node.setMetadata(0);
+ node.setIndent(false);
+ node.setInfoStrings(Maps.newHashMap());
+ node.setInfoStringsDisplayOrder(Lists.newArrayList());
+ node.setChildCountersMap(Maps.newHashMap());
+ node.setTimestamp(0);
+
+ TRuntimeProfileTree tree = new TRuntimeProfileTree();
+ tree.setNodes(Lists.newArrayList(node));
+
+ TDetailedReportParams params = new TDetailedReportParams();
+ params.setProfile(tree);
+ params.setIsFragmentLevel(false);
+ return params;
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java
new file mode 100644
index 00000000000..9b085b3c7e4
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.system;
+
+import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.resource.Tag;
+import org.apache.doris.thrift.TBackendInfo;
+
+import com.google.gson.JsonObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class BackendPaimonRustCapabilityTest {
+ @Test
+ public void testCapabilitySurvivesHeartbeatReplayAndBackendImage() {
+ Backend backend = new Backend(1, "127.0.0.1", 9050);
+ Assert.assertFalse(backend.isPaimonRustReaderSupported());
+ BackendHbResponse response = heartbeat();
+ response.setPaimonRustReaderSupported(true);
+ BackendHbResponse replay = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(response), BackendHbResponse.class);
+ Assert.assertTrue(replay.isPaimonRustReaderSupported());
+ Assert.assertTrue(backend.handleHbResponse(replay, true));
+ Assert.assertTrue(backend.isPaimonRustReaderSupported());
+ Backend restored = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(backend), Backend.class);
+ Assert.assertTrue(restored.isPaimonRustReaderSupported());
+ }
+
+ @Test
+ public void testLegacyHeartbeatClearsPreviouslyAdvertisedSupport() {
+ Backend backend = new Backend(1, "127.0.0.1", 9050);
+ BackendHbResponse response = heartbeat();
+ response.setPaimonRustReaderSupported(true);
+ backend.handleHbResponse(response, false);
+ Assert.assertTrue(backend.isPaimonRustReaderSupported());
+
+ TBackendInfo legacy = new TBackendInfo();
+ Assert.assertFalse(legacy.isSetSupportsPaimonRustReader());
+ Assert.assertFalse(legacy.isSupportsPaimonRustReader());
+ BackendHbResponse legacyResponse = heartbeat();
+ Assert.assertFalse(legacyResponse.isPaimonRustReaderSupported());
+ Assert.assertTrue(backend.handleHbResponse(legacyResponse, false));
+ Assert.assertFalse(backend.isPaimonRustReaderSupported());
+
+ JsonObject legacyJson = GsonUtils.GSON.toJsonTree(heartbeat()).getAsJsonObject();
+ legacyJson.remove("supportsPaimonRustReader");
+ BackendHbResponse replay = GsonUtils.GSON.fromJson(legacyJson, BackendHbResponse.class);
+ Assert.assertFalse(replay.isPaimonRustReaderSupported());
+ Assert.assertFalse(GsonUtils.GSON.fromJson("{}", Backend.class).isPaimonRustReaderSupported());
+ }
+
+ private BackendHbResponse heartbeat() {
+ return new BackendHbResponse(1, 9060, 8040, 8060, 1000, 1000, "test-version", Tag.VALUE_MIX,
+ 0, 0, false, 8070);
+ }
+}
diff --git a/gensrc/thrift/HeartbeatService.thrift b/gensrc/thrift/HeartbeatService.thrift
index 3fde7fdf2df..8608853b29f 100644
--- a/gensrc/thrift/HeartbeatService.thrift
+++ b/gensrc/thrift/HeartbeatService.thrift
@@ -63,6 +63,7 @@ struct TBackendInfo {
8: optional bool is_shutdown
9: optional Types.TPort arrow_flight_sql_port
10: optional i64 be_mem // The physical memory available for use by BE.
+ 11: optional bool supports_paimon_rust_reader
// For cloud
1000: optional i64 fragment_executing_count
1001: optional i64 fragment_last_active_time
diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
index 5aa105c3608..2044f058052 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -510,6 +510,9 @@ struct TQueryOptions {
// Fall back to RE2 when Hyperscan cannot compile a regular expression.
228: optional bool enable_hyperscan_fallback = true;
+ // Use paimon-rust to read Paimon splits on BE. Higher priority than
+ // enable_paimon_cpp_reader.
+ 229: optional bool enable_paimon_rust_reader = false;
// Master uses 226 for this option; branch-4.2 already uses 226-228, so the id was moved.
229: optional bool enable_prune_nested_column = false;
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index aaf057b6327..3e626e6f0ad 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -357,6 +357,7 @@ enum TPaimonReaderType {
PAIMON_JNI = 1,
// Deprecated wire value kept during rolling upgrades. New plans never emit it.
PAIMON_CPP = 2,
+ PAIMON_RUST = 3,
}
struct TPaimonFileDesc {
@@ -378,6 +379,12 @@ struct TPaimonFileDesc {
16: optional i64 schema_id; // for schema change.
// Reader implementation for logical paimon split. Native file split uses range format type.
17: optional TPaimonReaderType reader_type;
+ // paimon-rust: TableSchema serialized via JsonSerdeUtil.toJson(table.schema()).
+ // Consumed by paimon_table_from_schema_json on BE; skips catalog/warehouse resolution.
+ 18: optional string paimon_table_schema_json
+ // paimon-rust: non-default branch name. Unset means main branch (matches
+ // upstream paimon commit 742da63: null-if-DEFAULT_MAIN_BRANCH).
+ 19: optional string paimon_branch
}
struct TTrinoConnectorFileDesc {
diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_cast_predicates.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_cast_predicates.groovy
new file mode 100644
index 00000000000..5daac9772fa
--- /dev/null
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_cast_predicates.groovy
@@ -0,0 +1,192 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// CASTed operands through the paimon rust reader. The rust predicate converter
+// must NOT strip a cast from a column before pushing: for a DECIMAL(10,2)
+// column holding 1.24, `CAST(amount AS DECIMAL(10,1)) = 1.2` must retain the
+// row, while the unwrapped `amount = 1.2` prunes it — and rows pruned by the
+// pushed filter cannot be recovered by the Doris residual. The FE converter
+// already rejects casted operands (PaimonPredicateConverterTest's
+// rejectDecimalScaleCast / rejectStringToIntegerCast), so JNI never pushes
+// them; the rust reader must match. Cast-induced NULL semantics
+// (`CAST(s AS INT) IS NULL`) are covered the same way.
+//
+// Both differential legs run with force_jni_scanner=true: these parquet
+// append tables convert to raw native splits, which getSplits() would
+// otherwise prefer — both legs would silently use the native reader and
+// never reach the JNI / rust converters. The actual reader path is verified
+// per leg through the query profile (the rust reader's PaimonRustReader
+// timer group).
+import org.apache.doris.regression.action.ProfileAction
+
+suite("test_paimon_rust_reader_cast_predicates", "p0,external,paimon") {
+ String enabled = context.config.otherConfigs.get("enablePaimonTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disabled paimon test")
+ return
+ }
+
+ String catalogName = "test_paimon_rust_cast"
+ String dbName = "test_paimon_rust_cast_db"
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+
+ // Table is created via Spark because Doris does not support Paimon DDL.
+ // t_decimal_cast mirrors the FE rejectDecimalScaleCast shape: a
+ // DECIMAL(10,2) column whose scale-reducing cast must not be pushed.
+ // t_cast_null covers cast-related NULL semantics on a string column.
+ spark_paimon_multi """
+ CREATE DATABASE IF NOT EXISTS paimon.${dbName};
+ DROP TABLE IF EXISTS paimon.${dbName}.t_decimal_cast;
+ CREATE TABLE paimon.${dbName}.t_decimal_cast (
+ id INT, amount DECIMAL(10, 2)
+ ) USING paimon;
+ INSERT INTO paimon.${dbName}.t_decimal_cast VALUES
+ (1, 1.24), (2, 1.20), (3, 1.30), (4, NULL);
+
+ DROP TABLE IF EXISTS paimon.${dbName}.t_cast_null;
+ CREATE TABLE paimon.${dbName}.t_cast_null (
+ id INT, s VARCHAR(10)
+ ) USING paimon;
+ INSERT INTO paimon.${dbName}.t_cast_null VALUES (1, '5'), (2, NULL);
+
+ DROP TABLE IF EXISTS paimon.${dbName}.t_partition_constants;
+ CREATE TABLE paimon.${dbName}.t_partition_constants (id INT, p VARCHAR(3))
+ USING paimon PARTITIONED BY (p) TBLPROPERTIES ('bucket'='-1');
+ INSERT INTO paimon.${dbName}.t_partition_constants VALUES (1,'abc'), (2,'abc'), (3,NULL);
+ """
+
+ // The s3.region property is required: paimon-rust's S3 client rejects a
+ // missing region, while the JNI reader falls back to the SDK default.
+ sql """drop catalog if exists ${catalogName}"""
+ sql """
+ CREATE CATALOG ${catalogName} PROPERTIES (
+ 'type' = 'paimon',
+ 'paimon.catalog.type' = 'filesystem',
+ 'warehouse' = 's3://warehouse/wh',
+ 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}',
+ 's3.access_key' = 'admin',
+ 's3.secret_key' = 'password',
+ 's3.region' = 'us-east-1',
+ 'use_path_style' = 'true'
+ );
+ """
+
+ // Capture the settings this suite overrides so finally can restore them.
+ def originalForceJni = sql("select @@force_jni_scanner")[0][0]
+ def originalEnableProfile = sql("select @@enable_profile")[0][0]
+ def originalTruncate = sql("select @@truncate_char_or_varchar_columns")[0][0]
+
+ try {
+ sql """switch ${catalogName}"""
+ sql """use ${dbName}"""
+ sql """set enable_file_scanner_v2=true"""
+ // These are parquet append tables whose DataSplits convert to raw
+ // native splits; force the logical (JNI / rust) reader path so the
+ // differential actually exercises both converters.
+ sql """set force_jni_scanner=true"""
+ sql """set enable_profile=true"""
+ sql "set truncate_char_or_varchar_columns=true"
+
+ // Use configured HTTP credentials and wait for the Rust group as well as profile
+ // completion: the scanner group can appear before its child reader timers arrive.
+ def profiles = new ProfileAction(context)
+ def profileTextOf = { String query, boolean rustExpected ->
+ sql(query)
+ def queryId = sql("select last_query_id()")[0][0].toString()
+ def required = rustExpected ? ["FileScannerV2", "PaimonRustReader"] : ["FileScannerV2"]
+ profiles.getProfile(queryId, required)
+ }
+
+ def testQueries = [
+ // Scale-reducing cast: 1.24 rounds to 1.2 under the cast, so
+ // the 1.24 row (id 1) must be retained — exactly the row a
+ // wrongly pushed `amount = 1.2` would drop forever.
+ """select id from t_decimal_cast where cast(amount as decimal(10,1)) = 1.2 order by id""",
+ """select id from t_decimal_cast where cast(amount as decimal(10,1)) = 1.3 order by id""",
+ // Plain (uncast) decimal equality still pushes and matches.
+ """select id from t_decimal_cast where amount = 1.24 order by id""",
+ // CAST IS NULL / IS NOT NULL on the casted column keep NULL
+ // semantics: only the NULL row (id 4) is null under the cast.
+ """select id from t_decimal_cast where cast(amount as decimal(10,1)) is null order by id""",
+ """select id from t_decimal_cast where cast(amount as decimal(10,1)) is not null order by id""",
+ // Cast-induced NULL: CAST('5' AS INT) is 5, not NULL.
+ """select id from t_cast_null where cast(s as int) is null order by id""",
+ """select id from t_cast_null where cast(s as int) = 5 order by id""",
+ // IN / NOT IN with a casted list value. FE folds the
+ // list-value cast at plan time (even with
+ // debug_skip_fold_constant) and aligns the column with a
+ // column-side cast instead, so both readers correctly keep
+ // everything in the residual here. The un-folded shape — the
+ // cast still inside the IN list — is covered by the
+ // PaimonRustPredicateConverterTest unit tests, where the
+ // converter rejects any casted IN-list child (FE's
+ // doInPredicate parity).
+ """select id from t_decimal_cast where amount in (cast(1.24 as decimal(10,1)), 1.30) order by id""",
+ """select id from t_decimal_cast where amount not in (cast(1.24 as decimal(10,1))) order by id"""
+ ]
+ def expectedResults = [
+ [[1], [2]],
+ [[3]],
+ [[1]],
+ [[4]],
+ [[1], [2], [3]],
+ [[2]],
+ [[1]],
+ [[2], [3]],
+ [[1], [3]]
+ ]
+ // Partition keys are supplied as nullable constants outside the Arrow batch.
+ testQueries.add("select id,p from t_partition_constants order by id")
+ expectedResults.add([[1,"abc"],[2,"abc"],[3,null]])
+ // Representative scale-cast query reused for the reader-path checks.
+ String scaleCastQuery = testQueries[0]
+
+ sql """set enable_paimon_rust_reader=false"""
+ def jniResults = testQueries.collect { query -> sql(query) }
+ // The JNI leg must ride the logical-split JNI reader: the profile of
+ // the representative query must not contain the rust reader's timer.
+ def jniProfile = profileTextOf(scaleCastQuery, false)
+ assertFalse(jniProfile.contains("PaimonRustReader"), "JNI leg must not use the rust reader")
+
+ sql """set enable_paimon_rust_reader=true"""
+ def rustResults = testQueries.collect { query -> sql(query) }
+ // The rust leg must actually run the rust reader: its profile carries
+ // the PaimonRustReader timer group, which only the rust reader creates.
+ def rustProfile = profileTextOf(scaleCastQuery, true)
+ assertTrue(rustProfile.contains("PaimonRustReader"),
+ "rust leg must use the rust reader (profile timer missing)")
+
+ assertTrue(profileTextOf(testQueries.last(), true).contains("PaimonRustReader"),
+ "partition constants must exercise the Rust materialization path")
+ for (int i = 0; i < testQueries.size(); i++) {
+ // The rust reader must agree with the JNI reader (which, like FE,
+ // never pushes a casted operand) on every form.
+ assertEquals(jniResults[i].toString(), rustResults[i].toString())
+ // And both must be right, not just mutually consistent: id 1 in
+ // the first query is exactly the row a wrongly pushed
+ // `amount = 1.2` would drop.
+ assertEquals(expectedResults[i].toString(), rustResults[i].toString())
+ }
+ } finally {
+ sql """set enable_paimon_rust_reader=false"""
+ sql """set force_jni_scanner=${originalForceJni}"""
+ sql """set enable_profile=${originalEnableProfile}"""
+ sql "set truncate_char_or_varchar_columns=${originalTruncate}"
+ sql """drop catalog if exists ${catalogName}"""
+ }
+}
diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_compatibility.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_compatibility.groovy
new file mode 100644
index 00000000000..cbfaa54525b
--- /dev/null
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_compatibility.groovy
@@ -0,0 +1,308 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.action.ProfileAction
+
+suite("test_paimon_rust_reader_compatibility", "p0,external,paimon") {
+ if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enablePaimonTest"))) {
+ return
+ }
+ def endpoint = context.config.otherConfigs.get("externalEnvIp")
+ def port = context.config.otherConfigs.get("iceberg_minio_port")
+ def catalog = "test_paimon_rust_compatibility"
+ def database = "test_paimon_rust_compatibility_db"
+ def settings = ["enable_paimon_rust_reader", "force_jni_scanner",
+ "enable_file_scanner_v2", "enable_profile", "enable_prune_nested_column",
+ "enable_push_down_no_group_agg", "time_zone"]
+ def saved = settings.collectEntries { [(it): sql("select @@${it}")[0][0]] }
+ sql "DROP CATALOG IF EXISTS ${catalog}"
+ sql """CREATE CATALOG ${catalog} PROPERTIES (
+ 'type'='paimon', 'paimon.catalog.type'='filesystem', 'fs.s3.support'='true',
+ 'warehouse'='s3://warehouse/wh', 's3.endpoint'='http://${endpoint}:${port}',
+ 's3.access_key'='admin', 's3.secret_key'='password',
+ 's3.region'='us-east-1', 'use_path_style'='true')"""
+ sql "SWITCH ${catalog}"
+ sql "DROP DATABASE IF EXISTS ${database} FORCE"
+ sql "CREATE DATABASE ${database}"
+ sql "USE ${database}"
+ try {
+ sql "set enable_profile=true"
+ sql "set time_zone='+00:00'"
+ sql "set enable_prune_nested_column=true"
+ sql "set enable_file_scanner_v2=true"
+ sql "set force_jni_scanner=true"
+ def profiles = new ProfileAction(context)
+ def check = { String query, List expected, boolean rustExpected ->
+ sql "set enable_paimon_rust_reader=false"
+ def baseline = sql(query)
+ assertEquals(expected.toString(), baseline.toString())
+ sql "set enable_paimon_rust_reader=true"
+ def actual = sql(query)
+ def queryId = sql("select last_query_id()")[0][0].toString()
+ // A scanner can be published before its children; absence is meaningful only at completion.
+ def required = ["FileScannerV2", "Profile Completion State: COMPLETE"]
+ if (rustExpected) {
+ required.add("PaimonRustReader")
+ }
+ def profile = profiles.getProfile(queryId, required)
+ assertEquals(expected.toString(), actual.toString())
+ assertEquals(rustExpected, profile.contains("PaimonRustReader"), query)
+ }
+ def createPk = { String name, String valueColumns, String extra ->
+ // Separate commits must remain separate files so the reader performs the merge.
+ sql """CREATE TABLE ${name} (id INT NOT NULL, ${valueColumns}) ENGINE=paimon
+ PROPERTIES ('primary-key'='id', 'bucket'='1', 'write-only'='true',
+ 'deletion-vectors.enabled'='false' ${extra})"""
+ }
+
+ createPk("credential_values", "v INT", "")
+ sql "INSERT INTO credential_values VALUES (1,11)"
+ check("select id,v from credential_values", [[1,11]], true)
+ ["'s3.credentials_provider_type'='ANONYMOUS'",
+ "'s3.role_arn'='arn:aws:iam::123456789012:role/example'",
+ "'s3.session_token'='expired-example-token'"].eachWithIndex { extra, index ->
+ def crossedCatalog = "${catalog}_credentials_${index}"
+ sql "DROP CATALOG IF EXISTS ${crossedCatalog}"
+ try {
+ // Select S3 explicitly: endpoint guessing would choose MinIO and ignore S3 auth modes.
+ // Hadoop chooses static keys first; conflicting settings must retain that identity.
+ sql """CREATE CATALOG ${crossedCatalog} PROPERTIES (
+ 'type'='paimon', 'paimon.catalog.type'='filesystem', 'fs.s3.support'='true',
+ 'warehouse'='s3://warehouse/wh', 's3.endpoint'='http://${endpoint}:${port}',
+ 's3.access_key'='admin', 's3.secret_key'='password',
+ 's3.region'='us-east-1', 'use_path_style'='true', ${extra})"""
+ check("select id,v from ${crossedCatalog}.${database}.credential_values", [[1,11]], false)
+ } finally {
+ sql "DROP CATALOG IF EXISTS ${crossedCatalog}"
+ }
+ }
+
+ createPk("branch_scan_mode", "v INT", ", 'scan.mode'='latest-full'")
+ sql "INSERT INTO branch_scan_mode VALUES (1,11)"
+ spark_paimon """CALL paimon.sys.create_tag(table => '${database}.branch_scan_mode', tag => 'audit_tag')"""
+ spark_paimon """CALL paimon.sys.create_branch(table => '${database}.branch_scan_mode',
+ branch => 'audit', tag => 'audit_tag')"""
+ // A branch preserves the persisted scan mode instead of deriving one from a snapshot selector.
+ check("select id,v from branch_scan_mode@branch(audit)", [[1,11]], false)
+
+ sql """CREATE TABLE footer_aggregates (v INT NULL) ENGINE=paimon
+ PROPERTIES ('bucket'='-1', 'file.format'='parquet')"""
+ sql "INSERT INTO footer_aggregates VALUES (NULL), (-3), (7), (NULL), (2), (5)"
+ def aggregateQueries = [
+ ["select count(v) from footer_aggregates", [[4]], "COUNT"],
+ ["select min(v), max(v) from footer_aggregates", [[-3,7]], "MINMAX"]]
+ def rawRows = { String profile ->
+ def values = (profile =~ /RawRowsRead: ([0-9]+)/).collect { it[1] as long }
+ assertFalse(values.isEmpty(), "missing Parquet RawRowsRead counter")
+ values
+ }
+ // Rust only replaces logical JNI splits. Native raw-file splits must retain footer
+ // aggregation with Rust enabled; result equality alone would hide a full-scan regression.
+ sql "set force_jni_scanner=false"
+ aggregateQueries.each { entry ->
+ [false, true].each { rustEnabled ->
+ sql "set enable_paimon_rust_reader=${rustEnabled}"
+ [false, true].each { pushdown ->
+ sql "set enable_push_down_no_group_agg=${pushdown}"
+ explain {
+ sql(entry[0])
+ contains "pushdown agg=${pushdown ? entry[2] : 'NONE'}"
+ contains "paimonNativeReadSplits="
+ }
+ assertEquals(entry[1].toString(), sql(entry[0]).toString())
+ def queryId = sql("select last_query_id()")[0][0].toString()
+ def profile = profiles.getProfile(queryId, ["FileScannerV2", "ParquetReader", "Profile Completion State: COMPLETE"])
+ assertFalse(profile.contains("PaimonRustReader"))
+ def rows = rawRows(profile)
+ if (pushdown) {
+ assertTrue(rows.every { it == 0 }, "footer aggregate must not read data rows")
+ } else {
+ assertTrue(rows.any { it > 0 }, "full-scan control must read data rows")
+ }
+ }
+ }
+ }
+ sql "set enable_push_down_no_group_agg=true"
+ sql "set force_jni_scanner=true"
+ // Forced logical splits still return real rows to the upper aggregate in both readers.
+ aggregateQueries.each { entry -> check(entry[0], entry[1], true) }
+
+ ["DATE", "DATETIME(3)"].eachWithIndex { target, index ->
+ def name = "string_to_temporal_${index}"
+ sql """CREATE TABLE ${name} (id INT, v STRING) ENGINE=paimon
+ PROPERTIES ('bucket'='-1', 'file.format'='parquet')"""
+ sql "INSERT INTO ${name} VALUES (1,'42'), (2,NULL)"
+ check("select id,v from ${name} order by id", [[1,"42"],[2,null]], true)
+ sql "ALTER TABLE ${name} MODIFY COLUMN v ${target} NULL"
+ // The historical string is an epoch count in Java, not an Arrow calendar literal.
+ check("select id,cast(v as string) from ${name} order by id",
+ [[1,index == 0 ? "1970-02-12" : "1970-01-01 00:00:00.042"],[2,null]], false)
+ }
+ ["INT", "BIGINT"].each { sourceType ->
+ [0, 3, 6].each { precision ->
+ def name = "integer_to_timestamp_${sourceType.toLowerCase()}_${precision}"
+ sql """CREATE TABLE ${name} (id INT, v ${sourceType}) ENGINE=paimon
+ PROPERTIES ('bucket'='-1', 'file.format'='parquet')"""
+ sql "INSERT INTO ${name} VALUES (1,1700000000), (2,NULL)"
+ sql "ALTER TABLE ${name} MODIFY COLUMN v DATETIME(${precision}) NULL"
+ // All target precisions must interpret the source number as epoch seconds.
+ check("select id,date_format(v,'%Y-%m-%d %H:%i:%s') from ${name} order by id",
+ [[1,"2023-11-14 22:13:20"],[2,null]], false)
+ }
+ }
+
+ createPk("nested_values", "v STRUCT<a:INT,b:INT>, arr ARRAY<STRUCT<a:INT,b:INT>>, "
+ + "m MAP<INT,STRUCT<a:INT,b:INT>>", "")
+ // Bracket array literals accept constants only; nested expressions need the array constructor.
+ sql """INSERT INTO nested_values VALUES (1, named_struct('a',11,'b',22),
+ array(named_struct('a',33,'b',44)), map(1,named_struct('a',55,'b',66)))"""
+ check("select id from nested_values", [[1]], true)
+ // Distinct siblings expose ordinal decoding of a pruned second child.
+ check("select struct_element(v,'b') from nested_values", [[22]], false)
+ check("select struct_element(arr[1],'b') from nested_values", [[44]], false)
+ check("select struct_element(m[1],'b') from nested_values", [[66]], false)
+
+ createPk("narrow_values", "v BIGINT NULL", "")
+ sql "INSERT INTO narrow_values VALUES (1,383), (2,-129), (3,NULL)"
+ check("select id,v from narrow_values order by id", [[1,383],[2,-129],[3,null]], true)
+ sql "ALTER TABLE narrow_values MODIFY COLUMN v TINYINT NULL"
+ check("select id,v from narrow_values order by id", [[1,127],[2,127],[3,null]], false)
+ // A narrowing predicate must also be evaluated after the Java-compatible cast.
+ check("select id,v from narrow_values where v=127 order by id", [[1,127],[2,127]], false)
+
+ def integerCases = [
+ ["TINYINT", "127", "1", "-128", "64", "2", "-128"],
+ ["SMALLINT", "32767", "1", "-32768", "16384", "2", "-32768"],
+ ["INT", "2147483647", "1", "-2147483648", "1073741824", "2", "-2147483648"],
+ ["BIGINT", "9223372036854775807", "1", "-9223372036854775808",
+ "4611686018427387904", "2", "-9223372036854775808"]]
+ integerCases.eachWithIndex { c, index ->
+ ["sum", "product"].each { function ->
+ def name = "integer_${function}_${index}"
+ int offset = function == "sum" ? 1 : 4
+ createPk(name, "v ${c[0]}", ", 'merge-engine'='aggregation', "
+ + "'fields.v.aggregate-function'='${function}'")
+ sql "INSERT INTO ${name} VALUES (1,${c[offset]})"
+ sql "INSERT INTO ${name} VALUES (1,${c[offset+1]})"
+ check("select cast(v as string) from ${name}", [[c[offset+2]]], false)
+ }
+ }
+ createPk("decimal_sum", "v DECIMAL(2,0)", ", 'merge-engine'='aggregation', "
+ + "'fields.v.aggregate-function'='sum'")
+ [99, 1, -1].each { sql "INSERT INTO decimal_sum VALUES (1,${it})" }
+ check("select cast(v as string) from decimal_sum", [["99"]], false)
+
+ createPk("collected_values", "v ARRAY<INT>", ", 'merge-engine'='aggregation', "
+ + "'fields.v.aggregate-function'='collect'")
+ sql "INSERT INTO collected_values VALUES (1,[11])"
+ sql "INSERT INTO collected_values VALUES (1,[22])"
+ check("select element_at(array_sort(v),1), element_at(array_sort(v),2) from collected_values",
+ [[11,22]], false)
+
+ // Safe single-file merge engines remain positive controls for the Rust route.
+ ["deduplicate", "first-row"].eachWithIndex { engine, index ->
+ def name = "single_file_merge_${index}"
+ createPk(name, "v INT", ", 'merge-engine'='${engine}'")
+ sql "INSERT INTO ${name} VALUES (1,11)"
+ if (engine == "first-row") {
+ // Paimon hides first-row level-0 files until compaction has deduplicated them.
+ spark_paimon """CALL paimon.sys.compact(
+ table => '${database}.${name}', compact_strategy => 'full')"""
+ }
+ check("select id,v from ${name} order by id", [[1,11]], true)
+ }
+
+ createPk("supported_sum", "v DOUBLE", ", 'merge-engine'='aggregation', "
+ + "'fields.v.aggregate-function'='sum'")
+ sql "INSERT INTO supported_sum VALUES (1,10.0)"
+ // A single physical file can contain multiple versions from older Rust writers.
+ check("select v from supported_sum", [[10.0]], false)
+ sql "INSERT INTO supported_sum VALUES (1,3.0)"
+ check("select v from supported_sum", [[13.0]], false)
+
+ [false, true].each { decimal ->
+ [false, true].each { override ->
+ def name = "partial_default_${decimal}_${override}"
+ def type = decimal ? "DECIMAL(2,0)" : "TINYINT"
+ def first = decimal ? 99 : 127
+ // Keep the trailing operator so Groovy continues the assignment across lines.
+ def extra = ", 'merge-engine'='partial-update', 'fields.seq.sequence-group'='v', " +
+ "'fields.default-aggregate-function'='sum'"
+ if (override) {
+ extra += ", 'fields.v.aggregate-function'='max'"
+ }
+ createPk(name, "v ${type}, seq INT", extra)
+ sql "INSERT INTO ${name} VALUES (1,${first},1)"
+ // The reader must reject these engines even before a second file is written.
+ check("select cast(v as string) from ${name}", [[first.toString()]], false)
+ sql "INSERT INTO ${name} VALUES (1,1,2)"
+ if (decimal) {
+ sql "INSERT INTO ${name} VALUES (1,-1,3)"
+ }
+ def expected = decimal ? "99" : override ? "127" : "-128"
+ check("select cast(v as string) from ${name}", [[expected]], false)
+ }
+ }
+
+ ["aggregation", "partial-update"].eachWithIndex { engine, index ->
+ def name = "unused_default_${index}"
+ def extra = ", 'merge-engine'='${engine}', 'fields.default-aggregate-function'='collect', " +
+ "'fields.v.aggregate-function'='max'"
+ boolean partial = engine == "partial-update"
+ if (partial) {
+ extra += ", 'fields.seq.sequence-group'='v'"
+ }
+ createPk(name, partial ? "v INT, seq INT" : "v INT", extra)
+ sql "INSERT INTO ${name} VALUES (1,7${partial ? ',1' : ''})"
+ // An unused Java SPI default still fails Rust's global function-name validation.
+ check("select v from ${name}", [[7]], false)
+ }
+
+ createPk("insert_only_runs", "v STRING", ", 'read.batch-size'='1'")
+ 3.times { run ->
+ sql """INSERT INTO insert_only_runs
+ SELECT number, concat('${run}', repeat('x',4096)) FROM numbers('number'='1025')"""
+ }
+ // Overlapping INSERT-only files have no tombstones, but Rust retains losing batches.
+ check("select count(*), min(length(v)), max(length(v)) from insert_only_runs "
+ + "where substring(v,1,1)='2'", [[1025,4097,4097]], false)
+
+ createPk("retract_sum", "v DOUBLE, kind STRING", ", 'merge-engine'='aggregation', "
+ + "'fields.v.aggregate-function'='sum', 'rowkind.field'='kind'")
+ sql "INSERT INTO retract_sum VALUES (1,10.0,'+I')"
+ sql "INSERT INTO retract_sum VALUES (1,3.0,'-D')"
+ check("select id,v from retract_sum", [[1,7.0]], false)
+
+ createPk("deleted_prefix", "v STRING, kind STRING", ", 'rowkind.field'='kind'")
+ sql """INSERT INTO deleted_prefix SELECT number, repeat('x',128), '+I'
+ FROM numbers('number'='4096')"""
+ sql """INSERT INTO deleted_prefix SELECT number, repeat('x',128), '-D'
+ FROM numbers('number'='4096')"""
+ // A scan-producing predicate avoids metadata COUNT shortcuts for an empty result.
+ check("select id from deleted_prefix where v is not null", [], false)
+ createPk("sparse_prefix", "v STRING, kind STRING", ", 'rowkind.field'='kind'")
+ sql """INSERT INTO sparse_prefix SELECT number, repeat('x',128), '+I'
+ FROM numbers('number'='4097')"""
+ sql """INSERT INTO sparse_prefix SELECT number, repeat('x',128), '-D'
+ FROM numbers('number'='4096')"""
+ check("select id from sparse_prefix order by id", [[4096]], false)
+ } finally {
+ saved.each { name, value -> sql "set ${name}='${value}'" }
+ sql "DROP DATABASE IF EXISTS ${database} FORCE"
+ sql "DROP CATALOG IF EXISTS ${catalog}"
+ }
+}
diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_eq_for_null.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_eq_for_null.groovy
new file mode 100644
index 00000000000..a5d4b3412ef
--- /dev/null
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_rust_reader_eq_for_null.groovy
@@ -0,0 +1,555 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// NULL-safe equality (`<=>`, EQ_FOR_NULL) through the paimon rust reader.
+// The rust predicate converter must NOT push `a <=> b` down as `a IS NULL`:
+// with rows (NULL, NULL), (1, 1), (1, 2) that would wrongly drop (1, 1), and
+// rows dropped by the pushed filter cannot be recovered by the residual
+// conjunct. FE rewrites the literal forms (`a <=> 1` -> `a = 1`,
+// `a <=> NULL` -> `a IS NULL`) before they reach the BE, so only the
+// column-to-column form exercises EQ_FOR_NULL here; the literal forms still
+// guard the rewrite + pushdown chain end to end.
+//
+// Both differential legs run with force_jni_scanner=true: these parquet
+// append tables convert to raw native splits, which getSplits() would
+// otherwise prefer — both legs would silently use the native reader and
+// never reach the JNI / rust converters. The actual reader path is verified
+// per leg through the query profile (the rust reader's PaimonRustReader
+// timer group), and the join leg forces an IN runtime filter
+// (runtime_filter_type=1 + runtime_filter_wait_infinitely) that must be
+// planned onto the probe scan and arrive before the split opens.
+import org.apache.doris.regression.action.ProfileAction
+
+suite("test_paimon_rust_reader_eq_for_null", "p0,external,paimon") {
+ String enabled = context.config.otherConfigs.get("enablePaimonTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disabled paimon test")
+ return
+ }
+
+ String catalogName = "test_paimon_rust_eq_null"
+ String dbName = "test_paimon_rust_eq_null_db"
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+
+ // Use Spark to prepare the timestamp and nested-schema fixtures.
+ // Both columns stay nullable: `a <=> b` survives FE's NullSafeEqualToEqual
+ // rewrite (which only fires when one side is non-nullable / a NULL literal)
+ // precisely when both sides are nullable.
+ //
+ // t_frac_ts uses Spark TIMESTAMP_NTZ, which maps to Paimon TIMESTAMP
+ // (wall-clock, microsecond precision). Note Spark's plain TIMESTAMP maps
+ // to Paimon TIMESTAMP_LTZ, so the NTZ semantics must be spelled out in the
+ // DDL. spark.sql.timestampType=TIMESTAMP_NTZ makes the TIMESTAMP '...'
+ // literals parse as NTZ civil times too.
+ //
+ // The predicate literals are millisecond-aligned on purpose: FE truncates
+ // plan-time pushed-down timestamp predicates to 3 fractional digits, so a
+ // 6-digit literal would reach the readers truncated to milliseconds and
+ // the exact residual conjunct would then drop every row it kept — for the
+ // JNI, Rust and native readers alike. Literal conversion in the BE retains
+ // microseconds, but timestamp predicates remain
+ // residual because a DATETIMEV2 slot cannot reveal source nanosecond precision.
+ // ---- NaN differential on DOUBLE (total-ordering semantics) ----
+ // t_nan is created below: Doris defines NaN as equal to itself and
+ // greater than every finite value, but the pinned rust evaluator compares
+ // doubles with f64::partial_cmp (IEEE: NaN unordered, NaN != NaN), so the
+ // rust converter must NOT push DOUBLE predicates — a pushed `d > 1.0`
+ // would drop the stored NaN row Doris retains, and rows pruned by the
+ // rust filter cannot be recovered by the residual. The JNI path is
+ // unaffected: paimon-java's CompareUtils compares through Double.compareTo,
+ // which matches Doris's total ordering.
+ spark_paimon_multi """
+ SET spark.sql.timestampType=TIMESTAMP_NTZ;
+ CREATE DATABASE IF NOT EXISTS paimon.${dbName};
+ DROP TABLE IF EXISTS paimon.${dbName}.t_eq_null;
+ CREATE TABLE paimon.${dbName}.t_eq_null (
+ a INT, b INT
+ ) USING paimon;
+ INSERT INTO paimon.${dbName}.t_eq_null VALUES (NULL, NULL), (1, 1), (1, 2);
+
+ DROP TABLE IF EXISTS paimon.${dbName}.t_frac_ts;
+ CREATE TABLE paimon.${dbName}.t_frac_ts (
+ id INT, ts TIMESTAMP_NTZ
+ ) USING paimon;
+ INSERT INTO paimon.${dbName}.t_frac_ts VALUES
+ (1, TIMESTAMP '2024-01-01 00:00:00.123456'),
+ (2, TIMESTAMP '2024-01-01 00:00:00.123000'),
+ (3, TIMESTAMP '2024-01-02 00:00:00.999999');
+
+ DROP TABLE IF EXISTS paimon.${dbName}.t_frac_ts_dim;
+ CREATE TABLE paimon.${dbName}.t_frac_ts_dim (
+ id INT, ts TIMESTAMP_NTZ
+ ) USING paimon;
+ INSERT INTO paimon.${dbName}.t_frac_ts_dim VALUES
+ (1, TIMESTAMP '2024-01-01 00:00:00.123456'),
+ (2, TIMESTAMP '2024-01-02 00:00:00.999999');
+
+ DROP TABLE IF EXISTS paimon.${dbName}.t_narrowed;
+ CREATE TABLE paimon.${dbName}.t_narrowed (
+ id INT, v VARCHAR(10), amount DECIMAL(10,3)
+ ) USING paimon TBLPROPERTIES ('file.format' = 'parquet');
+ INSERT INTO paimon.${dbName}.t_narrowed VALUES (1, 'abcdef', 1.234), (2, 'xyzuvw', 2.345);
+ DROP TABLE IF EXISTS paimon.${dbName}.t_narrowed_dim;
+ CREATE TABLE paimon.${dbName}.t_narrowed_dim (
+ v VARCHAR(3), amount DECIMAL(10,2)
+ ) USING paimon;
+ INSERT INTO paimon.${dbName}.t_narrowed_dim VALUES ('abc', 1.23);
+
+ DROP TABLE IF EXISTS paimon.${dbName}.t_nan;
+ CREATE TABLE paimon.${dbName}.t_nan (
+ id INT, d DOUBLE
+ ) USING paimon;
+ INSERT INTO paimon.${dbName}.t_nan VALUES
+ (1, 1.5), (2, CAST('NaN' AS DOUBLE)), (3, NULL);
+
+ DROP TABLE IF EXISTS paimon.${dbName}.t_dedup_ignore_del;
+ CREATE TABLE paimon.${dbName}.t_dedup_ignore_del (
+ id INT, v INT
+ ) USING paimon TBLPROPERTIES (
+ 'primary-key' = 'id',
+ 'merge-engine' = 'deduplicate',
+ 'deduplicate.ignore-delete' = 'true',
+ 'file.format' = 'parquet'
+ );
+ INSERT INTO paimon.${dbName}.t_dedup_ignore_del VALUES (1, 11), (2, 22);
+ DELETE FROM paimon.${dbName}.t_dedup_ignore_del WHERE id = 1;
+
+ DROP TABLE IF EXISTS paimon.${dbName}.t_pu_remove_record_del;
+ CREATE TABLE paimon.${dbName}.t_pu_remove_record_del (
+ id INT, v INT
+ ) USING paimon TBLPROPERTIES (
+ 'primary-key' = 'id',
+ 'merge-engine' = 'partial-update',
+ 'partial-update.remove-record-on-delete' = 'true',
+ 'file.format' = 'parquet'
+ );
+ INSERT INTO paimon.${dbName}.t_pu_remove_record_del VALUES (1, 11), (2, 22);
+ DELETE FROM paimon.${dbName}.t_pu_remove_record_del WHERE id = 1;
+
+ DROP TABLE IF EXISTS paimon.${dbName}.t_nested_evo;
+ CREATE TABLE paimon.${dbName}.t_nested_evo (
+ id INT, s STRUCT<a: INT, b: STRING>
+ ) USING paimon TBLPROPERTIES (
+ 'bucket' = '-1',
+ 'file.format' = 'parquet'
+ );
+ INSERT INTO paimon.${dbName}.t_nested_evo VALUES (1, struct(10, 'x')), (2, struct(20, 'y'));
+ ALTER TABLE paimon.${dbName}.t_nested_evo ADD COLUMN s.c INT;
+ INSERT INTO paimon.${dbName}.t_nested_evo VALUES (3, struct(30, 'z', 33));
+ """
+
+ def originalSparkTimeZone = spark_paimon("SET spark.sql.session.timeZone")[0][1]
+ try {
+ spark_paimon "SET spark.sql.session.timeZone=Asia/Shanghai"
+ ["parquet", "orc"].each { format ->
+ // Legacy ORC LTZ encodes the writer JVM zone and requires the reader JVM
+ // to match it. Modern encoding keeps this routing test independent of host zones.
+ spark_paimon_multi """
+ DROP TABLE IF EXISTS paimon.${dbName}.t_ltz_${format};
+ CREATE TABLE paimon.${dbName}.t_ltz_${format} (id INT, ts_tz TIMESTAMP_LTZ)
+ USING paimon TBLPROPERTIES (
+ 'bucket'='-1', 'file.format'='${format}',
+ 'orc.timestamp-ltz.legacy.type'='false');
+ INSERT INTO paimon.${dbName}.t_ltz_${format} VALUES
+ (1, TIMESTAMP_LTZ '2025-01-01 00:00:00'),
+ (2, TIMESTAMP_LTZ '2025-06-01 12:34:56.789'),
+ (3, TIMESTAMP_LTZ '2025-12-31 23:59:59.999999'), (4, NULL);
+ """
+ }
+ } finally {
+ spark_paimon "SET spark.sql.session.timeZone=${originalSparkTimeZone}"
+ }
+
+ // The s3.region property is required: paimon-rust's S3 client rejects a
+ // missing region, while the JNI reader falls back to the SDK default.
+ sql """drop catalog if exists ${catalogName}"""
+ sql """
+ CREATE CATALOG ${catalogName} PROPERTIES (
+ 'type' = 'paimon',
+ 'paimon.catalog.type' = 'filesystem',
+ 'warehouse' = 's3://warehouse/wh',
+ 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}',
+ 's3.access_key' = 'admin',
+ 's3.secret_key' = 'password',
+ 's3.region' = 'us-east-1',
+ 'use_path_style' = 'true'
+ );
+ """
+
+ // Capture the settings this suite overrides so finally can restore them.
+ def originalForceJni = sql("select @@force_jni_scanner")[0][0]
+ def originalEnableProfile = sql("select @@enable_profile")[0][0]
+ def originalRfWait = sql("select @@runtime_filter_wait_infinitely")[0][0]
+ def originalRfType = sql("select @@runtime_filter_type")[0][0]
+ def originalRfPrune = sql("select @@enable_runtime_filter_prune")[0][0]
+ def originalRfMaxIn = sql("select @@runtime_filter_max_in_num")[0][0]
+
+ try {
+ sql """switch ${catalogName}"""
+ sql """use ${dbName}"""
+ // Spark rejects narrowing in its analyzer. Doris forwards these explicit
+ // casts to Paimon and publishes a new schema without rewriting old files.
+ sql "ALTER TABLE t_narrowed MODIFY COLUMN v VARCHAR(3) NULL"
+ sql "ALTER TABLE t_narrowed MODIFY COLUMN amount DECIMAL(10,2) NULL"
+ sql """set enable_file_scanner_v2=true"""
+ // These tables are parquet append tables, whose DataSplits convert to
+ // raw native splits; without forcing, getSplits() would hand both legs
+ // to the native reader and bypass the JNI / rust converters entirely.
+ sql """set force_jni_scanner=true"""
+ sql "set enable_paimon_rust_reader=false"
+ // Verify the DDL really narrows historical values before testing filter pushdown.
+ assertEquals([[1, "abc", "1.23"], [2, "xyz", "2.35"]].toString(),
+ sql("select id,v,cast(amount as string) from t_narrowed order by id").toString())
+ // Profile capture for the reader-path verification below.
+ sql """set enable_profile=true"""
+ // The join leg must receive its IN runtime filter before the split
+ // opens, so the rust converter sees it in the conjuncts.
+ sql """set runtime_filter_wait_infinitely=true"""
+ // TRuntimeFilterType.IN == 1: force the IN runtime-filter shape.
+ sql """set runtime_filter_type=1"""
+
+ // Reuse the framework's profile readiness polling and configured HTTP credentials.
+ def profileAction = new ProfileAction(context)
+ def profileTextOf = { String query, boolean rustExpected = true ->
+ sql(query)
+ def queryId = sql("select last_query_id()")[0][0]
+ // Negative and zero-counter checks must never inspect a partially published profile.
... 1169 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 12/35: [fix](arrow-flight) Deduplicate parallel result endpoints by backend (#68642)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7d0c1bf425e1692e87ed84b737ff1c8644ed77e3
Author: Gabriel <li...@selectdb.com>
AuthorDate: Tue Sep 29 21:57:10 2026 +0800
[fix](arrow-flight) Deduplicate parallel result endpoints by backend (#68642)
## Problem
With parallel result sinks, Nereids publishes one Flight endpoint per
pipeline instance even though instances on the same backend share a
query-id result buffer. Six result backends with eight instances each
therefore publish 48 endpoints for only six independent result streams.
## Fix
Publish one endpoint per backend in query-id mode, preserving first-seen
order and the existing instance-id behavior in non-parallel mode.
Backend IDs distinguish result owners even when Flight addresses are
shared.
Add five FE unit tests and a Flight regression covering unique tickets
and complete results for sequential/concurrent consumption with parallel
result sinks enabled and disabled.
## Validation
- Reproduced the original bug with three failing unit tests (including
48 endpoints instead of 6); all five tests pass after the fix.
- Compiled the changed FE class and tests with `javac --release 8`
against cached dependencies.
- FE Checkstyle and Groovy regression compilation checked locally.
- Full cluster regression is pending CI.
---
.../org/apache/doris/qe/NereidsCoordinator.java | 8 ++
.../qe/NereidsCoordinatorFlightEndpointsTest.java | 118 ++++++++++++++++++
.../test_flight_parallel_partitions.groovy | 134 +++++++++++++++++++++
3 files changed, 260 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
index 638128326c3..146df08108c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
@@ -74,8 +74,10 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Comparator;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.stream.Collectors;
/** NereidsCoordinator */
@@ -490,6 +492,7 @@ public class NereidsCoordinator extends Coordinator {
if (dataSink instanceof ResultSink || dataSink instanceof ResultFileSink) {
if (connectContext != null && !connectContext.isReturnResultFromLocal()) {
Preconditions.checkState(connectContext.getConnectType().equals(ConnectType.ARROW_FLIGHT_SQL));
+ Set<Long> resultBackendIds = new HashSet<>();
for (AssignedJob instance : topPlan.getInstanceJobs()) {
BackendWorker worker = (BackendWorker) instance.getAssignedWorker();
Backend backend = worker.getBackend();
@@ -498,6 +501,11 @@ public class NereidsCoordinator extends Coordinator {
}
TUniqueId finstId;
if (connectContext.getSessionVariable().enableParallelResultSink()) {
+ // Parallel instances on a BE share one query-id buffer, so their tickets
+ // cannot be published as independently consumable result partitions.
+ if (!resultBackendIds.add(worker.id())) {
+ continue;
+ }
finstId = getQueryId();
} else {
finstId = instance.instanceId();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/NereidsCoordinatorFlightEndpointsTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/NereidsCoordinatorFlightEndpointsTest.java
new file mode 100644
index 00000000000..6c6bb362190
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/NereidsCoordinatorFlightEndpointsTest.java
@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.qe;
+
+import org.apache.doris.nereids.trees.plans.distribute.PipelineDistributedPlan;
+import org.apache.doris.nereids.trees.plans.distribute.worker.BackendWorker;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJob;
+import org.apache.doris.planner.ResultSink;
+import org.apache.doris.service.arrowflight.results.FlightSqlEndpointsLocation;
+import org.apache.doris.system.Backend;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TUniqueId;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+
+public class NereidsCoordinatorFlightEndpointsTest {
+ private static final TUniqueId QUERY_ID = new TUniqueId(1, 2);
+
+ @Test
+ public void parallelInstancesShareOneEndpointPerBackend() throws Exception {
+ List<FlightSqlEndpointsLocation> endpoints = endpoints(true, false, 1, 8);
+ Assert.assertEquals(1, endpoints.size());
+ Assert.assertEquals(QUERY_ID, endpoints.get(0).getFinstId());
+ }
+
+ @Test
+ public void parallelEndpointsRetainAllResultBackendsInOrder() throws Exception {
+ List<FlightSqlEndpointsLocation> endpoints = endpoints(true, false, 6, 8);
+ Assert.assertEquals(6, endpoints.size());
+ for (int i = 0; i < endpoints.size(); i++) {
+ Assert.assertEquals(QUERY_ID, endpoints.get(i).getFinstId());
+ Assert.assertEquals(new TNetworkAddress("127.0.0.1", 8100 + i),
+ endpoints.get(i).getResultInternalServiceAddr());
+ }
+ }
+
+ @Test
+ public void sharedFlightAddressDoesNotMergeDifferentBackends() throws Exception {
+ List<FlightSqlEndpointsLocation> endpoints = endpoints(true, false, 2, 2);
+ Assert.assertEquals(2, endpoints.size());
+ Assert.assertEquals(endpoints.get(0).getResultFlightServerAddr(),
+ endpoints.get(1).getResultFlightServerAddr());
+ Assert.assertNotEquals(endpoints.get(0).getResultInternalServiceAddr(),
+ endpoints.get(1).getResultInternalServiceAddr());
+ }
+
+ @Test
+ public void nonParallelEndpointsKeepDistinctInstanceIds() throws Exception {
+ List<FlightSqlEndpointsLocation> endpoints = endpoints(false, false, 1, 2);
+ Assert.assertEquals(2, endpoints.size());
+ Assert.assertEquals(new TUniqueId(2, 0), endpoints.get(0).getFinstId());
+ Assert.assertEquals(new TUniqueId(2, 1), endpoints.get(1).getFinstId());
+ }
+
+ @Test
+ public void localResultDoesNotPublishFlightEndpoints() throws Exception {
+ Assert.assertTrue(endpoints(true, true, 2, 8).isEmpty());
+ }
+
+ private List<FlightSqlEndpointsLocation> endpoints(boolean parallel, boolean local,
+ int backendCount, int instancesPerBackend) throws Exception {
+ ConnectContext context = new ConnectContext();
+ context.connectType = ConnectContext.ConnectType.ARROW_FLIGHT_SQL;
+ context.setReturnResultFromLocal(local);
+ context.getSessionVariable().setEnableParallelResultSink(parallel);
+ CoordinatorContext coordinatorContext = Mockito.mock(CoordinatorContext.class);
+ setContextField(coordinatorContext, "connectContext", context);
+ setContextField(coordinatorContext, "dataSink", Mockito.mock(ResultSink.class));
+ PipelineDistributedPlan plan = Mockito.mock(PipelineDistributedPlan.class, Mockito.RETURNS_DEEP_STUBS);
+ List<AssignedJob> jobs = new ArrayList<>();
+ for (int i = 0; i < backendCount; i++) {
+ Backend backend = new Backend(i + 1, "127.0.0.1", 9000 + i);
+ backend.setBrpcPort(8100 + i);
+ // Backend identity must remain distinct even when Flight locations are shared.
+ backend.setArrowFlightSqlPort(8050);
+ for (int j = 0; j < instancesPerBackend; j++) {
+ AssignedJob job = Mockito.mock(AssignedJob.class);
+ Mockito.when(job.getAssignedWorker()).thenReturn(new BackendWorker(0, backend));
+ Mockito.when(job.instanceId()).thenReturn(new TUniqueId(2, i * instancesPerBackend + j));
+ jobs.add(job);
+ }
+ }
+ Mockito.when(plan.getInstanceJobs()).thenReturn(ImmutableList.copyOf(jobs));
+ Mockito.when(plan.getFragmentJob().getFragment().getOutputExprs()).thenReturn(new ArrayList<>());
+ NereidsCoordinator coordinator = Mockito.mock(NereidsCoordinator.class, Mockito.CALLS_REAL_METHODS);
+ Mockito.doReturn(QUERY_ID).when(coordinator).getQueryId();
+ coordinator.processTopSink(coordinatorContext, plan);
+ return context.getFlightSqlEndpointsLocations();
+ }
+
+ private void setContextField(CoordinatorContext context, String name, Object value) throws Exception {
+ Field field = CoordinatorContext.class.getDeclaredField(name);
+ field.setAccessible(true);
+ field.set(context, value);
+ }
+}
diff --git a/regression-test/suites/arrow_flight_sql_p0/test_flight_parallel_partitions.groovy b/regression-test/suites/arrow_flight_sql_p0/test_flight_parallel_partitions.groovy
new file mode 100644
index 00000000000..005341df0ee
--- /dev/null
+++ b/regression-test/suites/arrow_flight_sql_p0/test_flight_parallel_partitions.groovy
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.arrow.driver.jdbc.shaded.com.google.protobuf.Any
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CallOptions
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightClient
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.Location
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.sql.FlightSqlClient
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.sql.impl.FlightSql
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.memory.RootAllocator
+
+import java.util.concurrent.Callable
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+
+suite("test_flight_parallel_partitions", "arrow_flight_sql") {
+ def frontend = jdbc_sql_return_maparray("SHOW FRONTENDS").find {
+ it.IsMaster.toString().equalsIgnoreCase("true") && it.Alive.toString().equalsIgnoreCase("true")
+ }
+ assertNotNull(frontend)
+ assertTrue(frontend.ArrowFlightSqlPort.toString().toInteger() > 0)
+ def database = jdbc_sql("SELECT DATABASE()")[0][0]
+ def table = "${database}.flight_parallel_partition_source"
+ def allocator = new RootAllocator(Long.MAX_VALUE)
+ def feClient = FlightClient.builder(allocator,
+ Location.forGrpcInsecure(frontend.Host.toString(), frontend.ArrowFlightSqlPort.toString().toInteger())).build()
+ def client = new FlightSqlClient(feClient)
+ def auth
+ def readEndpoint = { endpoint, boolean collectRows ->
+ def rows = []
+ FlightClient.builder(allocator, endpoint.locations[0]).build().withCloseable { beClient ->
+ beClient.getStream(endpoint.ticket, auth, CallOptions.timeout(30, TimeUnit.SECONDS)).withCloseable { stream ->
+ while (stream.next()) {
+ if (collectRows) {
+ def vector = stream.root.getVector(0)
+ for (int i = 0; i < stream.root.rowCount; i++) {
+ rows.add(((Number) vector.getObject(i)).longValue())
+ }
+ }
+ }
+ }
+ }
+ rows
+ }
+ def executeSetting = { String statement ->
+ client.execute(statement, auth).endpoints.each { readEndpoint(it, false) }
+ }
+ try {
+ auth = feClient.authenticateBasicToken(context.config.otherConfigs.get("extArrowFlightSqlUser"),
+ context.config.otherConfigs.get("extArrowFlightSqlPassword")).get()
+ executeSetting("SET enable_sql_cache=false")
+ executeSetting("SET enable_nereids_distribute_planner=true")
+ executeSetting("SET parallel_pipeline_task_num=8")
+ executeSetting("SET query_timeout=60")
+ jdbc_sql("DROP TABLE IF EXISTS ${table}")
+ jdbc_sql("CREATE TABLE ${table} (id BIGINT NOT NULL) DISTRIBUTED BY HASH(id) BUCKETS 60 " +
+ "PROPERTIES(\"replication_num\"=\"1\")")
+ jdbc_sql("INSERT INTO ${table} SELECT number FROM numbers(\"number\"=\"60\")")
+ def resultBackendCount = jdbc_sql_return_maparray("SHOW TABLETS FROM ${table}")
+ .collect { it.BackendId }.unique().size()
+ [true, false].each { parallel ->
+ executeSetting("SET enable_parallel_result_sink=${parallel}")
+ [false, true].each { concurrent ->
+ // Each execution owns fresh tickets; consuming a ticket does not create a replayable partition.
+ def info = client.execute("SELECT id * 1000 + n AS sequence_id FROM ${table} " +
+ "LATERAL VIEW explode_numbers(1000) expanded AS n", auth)
+ assertTrue(!info.endpoints.isEmpty())
+ def tickets = info.endpoints.collect { Base64.encoder.encodeToString(it.ticket.bytes) }
+ assertEquals(tickets.size(), tickets.toSet().size(), "Duplicate Flight result tickets")
+ if (parallel) {
+ def resultAddresses = info.endpoints.collect { endpoint ->
+ def fields = Any.parseFrom(endpoint.ticket.bytes).unpack(FlightSql.TicketStatementQuery.class)
+ .statementHandle.toStringUtf8().split("&")
+ "${fields[1]}:${fields[2]}".toString()
+ }
+ // Instance parallelism must not publish multiple readers for the same BE result buffer.
+ assertEquals(info.endpoints.size(), resultAddresses.toSet().size())
+ if (resultBackendCount > 1) {
+ assertTrue(info.endpoints.size() > 1, "Expected multiple result backends")
+ }
+ } else {
+ assertEquals(1, info.endpoints.size())
+ }
+ def rows = []
+ if (concurrent) {
+ def executor = Executors.newFixedThreadPool(Math.min(8, info.endpoints.size()))
+ try {
+ def futures = info.endpoints.collect { endpoint ->
+ executor.submit({ readEndpoint(endpoint, true) } as Callable)
+ }
+ futures.each { rows.addAll(it.get(60, TimeUnit.SECONDS)) }
+ } finally {
+ executor.shutdownNow()
+ assertTrue(executor.awaitTermination(35, TimeUnit.SECONDS))
+ }
+ } else {
+ info.endpoints.each { rows.addAll(readEndpoint(it, true)) }
+ }
+ assertEquals(60000, rows.size())
+ assertEquals((0L..<60000L).toList(), rows.sort())
+ }
+ }
+ } finally {
+ try {
+ if (auth != null) {
+ feClient.closeSession(new org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CloseSessionRequest(), auth)
+ }
+ } finally {
+ try {
+ client.close()
+ } finally {
+ try {
+ allocator.close()
+ } finally {
+ jdbc_sql("DROP TABLE IF EXISTS ${table}")
+ }
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 24/35: branch-4.1: [fix](cloud) Respect recycle type when deleting partial update rowsets (#68594) (#68665)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 810c92fb83612570a8759ae56d6795253929bc14
Author: Yixuan Wang <wa...@selectdb.com>
AuthorDate: Wed Sep 30 22:00:18 2026 +0800
branch-4.1: [fix](cloud) Respect recycle type when deleting partial update rowsets (#68594) (#68665)
pick: https://github.com/apache/doris/pull/68594
---
cloud/src/recycler/recycler.cpp | 2 +-
cloud/test/recycler_test.cpp | 34 ++++++++++++++++++++++++++++++++++
2 files changed, 35 insertions(+), 1 deletion(-)
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index 7508dcf22d0..72a444f3da4 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -4432,7 +4432,7 @@ int InstanceRecycler::delete_rowset_data(
continue;
}
}
- if (rs.rowset_state() == RowsetStatePB::BEGIN_PARTIAL_UPDATE) {
+ if (rs.rowset_state() == RowsetStatePB::BEGIN_PARTIAL_UPDATE && !is_formal_rowset) {
// if rowset state is RowsetStatePB::BEGIN_PARTIAL_UPDATE, the number of segments data
// may be larger than num_segments field in RowsetMeta, so we need to delete the rowset's data by prefix
rowsets_delete_by_prefix.emplace_back(rs.resource_id(), tablet_id, rs.rowset_id_v2());
diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp
index 1df08daed36..df64727ae9c 100644
--- a/cloud/test/recycler_test.cpp
+++ b/cloud/test/recycler_test.cpp
@@ -7350,6 +7350,40 @@ TEST(RecyclerTest, delete_rowset_data) {
}
}
+TEST(RecyclerTest, delete_partial_update_rowset_data_by_recycling_state) {
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ constexpr auto resource_id = "partial_update_delete";
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ instance.add_obj_info()->set_id(resource_id);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
+ ASSERT_EQ(recycler.init(), 0);
+ auto accessor = recycler.accessor_map_.at(resource_id);
+
+ doris::TabletSchemaCloudPB schema;
+ schema.set_schema_version(1);
+ auto rowset = create_rowset(resource_id, 10001, 10002, 1, schema,
+ RowsetStatePB::BEGIN_PARTIAL_UPDATE);
+ const auto segment = segment_path(rowset.tablet_id(), rowset.rowset_id_v2(), 0);
+ const auto extra_segment = segment_path(rowset.tablet_id(), rowset.rowset_id_v2(), 1);
+ std::map<std::string, doris::RowsetMetaCloudPB> rowsets {{rowset.rowset_id_v2(), rowset}};
+
+ for (auto type : {RowsetRecyclingState::FORMAL_ROWSET, RowsetRecyclingState::TMP_ROWSET}) {
+ SCOPED_TRACE(static_cast<int>(type));
+ ASSERT_EQ(accessor->put_file(segment, ""), 0);
+ ASSERT_EQ(accessor->put_file(extra_segment, ""), 0);
+
+ RecyclerMetricsContext metrics_context;
+ ASSERT_EQ(recycler.delete_rowset_data(rowsets, type, metrics_context), 0);
+ EXPECT_EQ(accessor->exists(segment), 1);
+ EXPECT_EQ(accessor->exists(extra_segment),
+ type == RowsetRecyclingState::FORMAL_ROWSET ? 0 : 1);
+ }
+}
+
TEST(RecyclerTest, delete_rowset_data_without_delete_bitmap_meta) {
auto txn_kv = std::make_shared<MemTxnKv>();
ASSERT_EQ(txn_kv->init(), 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 08/35: branch-4.1: [fix](regexp) Apply extended regex fallback to non-constant patterns #68501 (#68591)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 09df202391a8849eb6870cf660c699100c7cfc41
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Sep 29 17:39:58 2026 +0800
branch-4.1: [fix](regexp) Apply extended regex fallback to non-constant patterns #68501 (#68591)
Cherry-picked from #68501
Co-authored-by: Jerry Hu <hu...@selectdb.com>
---
be/src/exprs/function/like.cpp | 131 +++++++++++----------
be/src/exprs/function/like.h | 17 +++
be/test/exprs/function/function_like_test.cpp | 104 ++++++++++++++--
.../test_regexp_extended_nonconstant_pattern.out | 28 +++++
...test_regexp_extended_nonconstant_pattern.groovy | 85 +++++++++++++
5 files changed, 298 insertions(+), 67 deletions(-)
diff --git a/be/src/exprs/function/like.cpp b/be/src/exprs/function/like.cpp
index d367de2b2c9..599a3347a9c 100644
--- a/be/src/exprs/function/like.cpp
+++ b/be/src/exprs/function/like.cpp
@@ -194,6 +194,7 @@ struct VectorEndsWithSearchState : public VectorPatternSearchState {
Status LikeSearchState::clone(LikeSearchState& cloned) const {
cloned.set_search_string(search_string);
cloned.enable_hyperscan_fallback = enable_hyperscan_fallback;
+ cloned.enable_extended_regex = enable_extended_regex;
std::string re_pattern;
FunctionLike::convert_like_pattern(this, pattern_str, &re_pattern);
@@ -431,10 +432,8 @@ Status FunctionLikeBase::constant_regex_fn_scalar(const LikeSearchState* state,
if (ret != HS_SUCCESS && ret != HS_SCAN_TERMINATED) {
return Status::RuntimeError(fmt::format("hyperscan error: {}", ret));
}
- } else if (state->boost_regex) { // use boost::regex for advanced features
- *result = boost::regex_search(val.data, val.data + val.size, *state->boost_regex);
- } else { // fallback to re2
- *result = RE2::PartialMatch(re2::StringPiece(val.data, val.size), *state->regex);
+ } else { // re2, or boost::regex for advanced features
+ RETURN_IF_ERROR(regex_search(state->regex.get(), state->boost_regex.get(), val, result));
}
return Status::OK();
@@ -442,15 +441,11 @@ Status FunctionLikeBase::constant_regex_fn_scalar(const LikeSearchState* state,
Status FunctionLikeBase::regexp_fn_scalar(const LikeSearchState* state, const StringRef& val,
const StringRef& pattern, unsigned char* result) {
- RE2::Options opts;
- opts.set_never_nl(false);
- opts.set_dot_nl(true);
- re2::RE2 re(re2::StringPiece(pattern.data, pattern.size), opts);
- if (re.ok()) {
- *result = RE2::PartialMatch(re2::StringPiece(val.data, val.size), re);
- } else {
- return Status::RuntimeError("Invalid pattern: {}", pattern.debug_string());
- }
+ std::unique_ptr<re2::RE2> regex;
+ std::unique_ptr<boost::regex> boost_regex;
+ RETURN_IF_ERROR(compile_regex(std::string_view(pattern.data, pattern.size),
+ state->enable_extended_regex, ®ex, &boost_regex));
+ RETURN_IF_ERROR(regex_search(regex.get(), boost_regex.get(), val, result));
return Status::OK();
}
@@ -469,17 +464,10 @@ Status FunctionLikeBase::constant_regex_fn(const LikeSearchState* state, const C
return Status::RuntimeError(fmt::format("hyperscan error: {}", ret));
}
}
- } else if (state->boost_regex) { // use boost::regex for advanced features
+ } else { // re2, or boost::regex for advanced features
for (size_t i = 0; i < sz; i++) {
- const auto& str_ref = val.get_data_at(i);
- *(result.data() + i) = boost::regex_search(str_ref.data, str_ref.data + str_ref.size,
- *state->boost_regex);
- }
- } else { // fallback to re2
- for (size_t i = 0; i < sz; i++) {
- const auto& str_ref = val.get_data_at(i);
- *(result.data() + i) =
- RE2::PartialMatch(re2::StringPiece(str_ref.data, str_ref.size), *state->regex);
+ RETURN_IF_ERROR(regex_search(state->regex.get(), state->boost_regex.get(),
+ val.get_data_at(i), result.data() + i));
}
}
@@ -507,29 +495,71 @@ Status FunctionLikeBase::regexp_fn(const LikeSearchState* state, const ColumnStr
hs_free_scratch(scratch);
hs_free_database(database);
- } else { // fallback to re2
+ } else { // fallback to re2, then boost::regex for advanced features
if (!state->enable_hyperscan_fallback) {
return hs_status;
}
- RE2::Options opts;
- opts.set_never_nl(false);
- opts.set_dot_nl(true);
- re2::RE2 re(re_pattern, opts);
- if (re.ok()) {
- auto sz = val.size();
- for (size_t i = 0; i < sz; i++) {
- const auto& str_ref = val.get_data_at(i);
- *(result.data() + i) =
- RE2::PartialMatch(re2::StringPiece(str_ref.data, str_ref.size), re);
- }
- } else {
- return Status::RuntimeError("Invalid pattern: {}", pattern.debug_string());
+ std::unique_ptr<re2::RE2> regex;
+ std::unique_ptr<boost::regex> boost_regex;
+ RETURN_IF_ERROR(
+ compile_regex(re_pattern, state->enable_extended_regex, ®ex, &boost_regex));
+ auto sz = val.size();
+ for (size_t i = 0; i < sz; i++) {
+ RETURN_IF_ERROR(regex_search(regex.get(), boost_regex.get(), val.get_data_at(i),
+ result.data() + i));
}
}
return Status::OK();
}
+Status FunctionLikeBase::compile_regex(std::string_view pattern, bool enable_extended_regex,
+ std::unique_ptr<re2::RE2>* regex,
+ std::unique_ptr<boost::regex>* boost_regex) {
+ RE2::Options opts;
+ opts.set_never_nl(false);
+ opts.set_dot_nl(true);
+ // A rejected pattern is reported through the returned Status (or handed to Boost.Regex), and
+ // non-constant patterns are compiled once per row, so keep RE2 from logging every rejection.
+ opts.set_log_errors(false);
+ *regex = std::make_unique<RE2>(re2::StringPiece(pattern.data(), pattern.size()), opts);
+ if ((*regex)->ok()) {
+ return Status::OK();
+ }
+ if (!enable_extended_regex) {
+ return Status::InternalError(
+ "Invalid regex expression: {}. Error: {}. If you need advanced regex features, "
+ "try setting enable_extended_regex=true",
+ pattern, (*regex)->error());
+ }
+
+ // RE2 failed, fallback to Boost.Regex
+ // This handles advanced regex features like zero-width assertions
+ regex->reset();
+ try {
+ *boost_regex =
+ std::make_unique<boost::regex>(pattern.data(), pattern.data() + pattern.size());
+ } catch (const boost::regex_error& e) {
+ return Status::InternalError("Invalid regex expression: {}. Error: {}", pattern, e.what());
+ }
+ return Status::OK();
+}
+
+Status FunctionLikeBase::regex_search(const re2::RE2* regex, const boost::regex* boost_regex,
+ const StringRef& val, unsigned char* result) {
+ if (boost_regex != nullptr) {
+ try {
+ *result = boost::regex_search(val.data, val.data + val.size, *boost_regex);
+ } catch (const boost::regex_error& e) {
+ return Status::InternalError("Failed to match regex expression: {}. Error: {}",
+ boost_regex->str(), e.what());
+ }
+ return Status::OK();
+ }
+ *result = RE2::PartialMatch(re2::StringPiece(val.data, val.size), *regex);
+ return Status::OK();
+}
+
// hyperscan compile expression to database and allocate scratch space
bool FunctionLikeBase::should_fallback_to_re2(std::string_view regexp) {
return is_hyperscan_regexp_expensive(regexp);
@@ -1022,6 +1052,7 @@ Status FunctionRegexpLike::open(FunctionContext* context,
state->is_like_pattern = false;
state->search_state.enable_hyperscan_fallback =
context->state()->query_options().enable_hyperscan_fallback;
+ state->search_state.enable_extended_regex = context->state()->enable_extended_regex();
state->function = regexp_fn;
state->scalar_function = regexp_fn_scalar;
if (context->is_col_constant(1)) {
@@ -1066,29 +1097,9 @@ Status FunctionRegexpLike::open(FunctionContext* context,
// reset hs_database to nullptr to indicate not use hyperscan
state->search_state.hs_database.reset();
state->search_state.hs_scratch.reset();
- RE2::Options opts;
- opts.set_never_nl(false);
- opts.set_dot_nl(true);
- state->search_state.regex = std::make_unique<RE2>(pattern_str, opts);
- if (!state->search_state.regex->ok()) {
- if (!context->state()->enable_extended_regex()) {
- return Status::InternalError(
- "Invalid regex expression: {}. Error: {}. If you need advanced "
- "regex features, try setting enable_extended_regex=true",
- pattern_str, state->search_state.regex->error());
- }
-
- // RE2 failed, fallback to Boost.Regex
- // This handles advanced regex features like zero-width assertions
- state->search_state.regex.reset();
- try {
- state->search_state.boost_regex =
- std::make_unique<boost::regex>(pattern_str);
- } catch (const boost::regex_error& e) {
- return Status::InternalError("Invalid regex expression: {}. Error: {}",
- pattern_str, e.what());
- }
- }
+ RETURN_IF_ERROR(compile_regex(
+ pattern_str, state->search_state.enable_extended_regex,
+ &state->search_state.regex, &state->search_state.boost_regex));
}
state->function = constant_regex_fn;
state->scalar_function = constant_regex_fn_scalar;
diff --git a/be/src/exprs/function/like.h b/be/src/exprs/function/like.h
index c56cb34793e..6b56b97a70d 100644
--- a/be/src/exprs/function/like.h
+++ b/be/src/exprs/function/like.h
@@ -185,6 +185,10 @@ struct LikeSearchState {
bool enable_hyperscan_fallback = true;
+ /// Whether a pattern rejected by RE2 may fall back to Boost.Regex, mirrors the
+ /// `enable_extended_regex` session variable.
+ bool enable_extended_regex = false;
+
/// Used for LIKE predicates if the pattern is a constant argument, and is either a
/// constant string or has a constant string at the beginning or end of the pattern.
/// This will be set in order to check for that pattern in the corresponding part of
@@ -366,6 +370,19 @@ protected:
static Status regexp_fn_scalar(const LikeSearchState* state, const StringRef& val,
const StringRef& pattern, unsigned char* result);
+ // Compile `pattern` with RE2. When RE2 rejects it and extended regex is enabled, fall back
+ // to Boost.Regex, which supports zero-width assertions such as `(?=...)` and `(?<=...)`.
+ // Exactly one of `regex` / `boost_regex` is set on success.
+ static Status compile_regex(std::string_view pattern, bool enable_extended_regex,
+ std::unique_ptr<re2::RE2>* regex,
+ std::unique_ptr<boost::regex>* boost_regex);
+
+ // Match `val` with whichever engine `compile_regex` produced. Boost.Regex can also fail while
+ // matching (e.g. the backtracking budget of a pathological pattern is exhausted), which is
+ // reported as a non-OK Status instead of an exception.
+ static Status regex_search(const re2::RE2* regex, const boost::regex* boost_regex,
+ const StringRef& val, unsigned char* result);
+
// hyperscan compile expression to database and allocate scratch space
static Status hs_prepare(FunctionContext* context, const char* expression,
hs_database_t** database, hs_scratch_t** scratch);
diff --git a/be/test/exprs/function/function_like_test.cpp b/be/test/exprs/function/function_like_test.cpp
index 1c0d40044a5..9430882e4a3 100644
--- a/be/test/exprs/function/function_like_test.cpp
+++ b/be/test/exprs/function/function_like_test.cpp
@@ -20,6 +20,7 @@
#include <string>
#include <vector>
+#include "core/assert_cast.h"
#include "core/block/block.h"
#include "core/column/column_const.h"
#include "core/column/column_string.h"
@@ -42,11 +43,27 @@ public:
using FunctionLikeBase::should_fallback_to_re2;
};
+// How the pattern column reaches the function: a plain column, a const column that is only
+// visible during execute(), or a const column already visible during open().
+enum class PatternMode { NON_CONST, CONST_AT_EXECUTE, CONST_AT_OPEN };
+
+static const char* pattern_mode_name(PatternMode mode) {
+ switch (mode) {
+ case PatternMode::NON_CONST:
+ return "non-constant pattern";
+ case PatternMode::CONST_AT_EXECUTE:
+ return "constant pattern during execute";
+ case PatternMode::CONST_AT_OPEN:
+ return "constant pattern during open";
+ }
+ return "";
+}
+
+// Runs `value <Function> pattern` for a single row with the given query options and returns the
+// match result through `result`; the Status of open()/execute_impl() is returned unchanged.
template <typename Function>
-Status execute_pattern_with_fallback_disabled(const std::string& value, const std::string& pattern,
- bool constant_known_at_open) {
- TQueryOptions query_options;
- query_options.__set_enable_hyperscan_fallback(false);
+Status execute_single_pattern(const TQueryOptions& query_options, const std::string& value,
+ const std::string& pattern, PatternMode mode, uint8_t* result) {
RuntimeState runtime_state(query_options, TQueryGlobals {});
auto string_type = std::make_shared<DataTypeString>();
@@ -57,10 +74,12 @@ Status execute_pattern_with_fallback_disabled(const std::string& value, const st
values->insert_data(value.data(), value.size());
auto patterns = ColumnString::create();
patterns->insert_data(pattern.data(), pattern.size());
- ColumnPtr pattern_column = ColumnConst::create(std::move(patterns), 1);
+ ColumnPtr pattern_column = mode == PatternMode::NON_CONST
+ ? ColumnPtr(std::move(patterns))
+ : ColumnConst::create(std::move(patterns), 1);
std::vector<std::shared_ptr<ColumnPtrWrapper>> constant_columns(2);
- if (constant_known_at_open) {
+ if (mode == PatternMode::CONST_AT_OPEN) {
constant_columns[1] = std::make_shared<ColumnPtrWrapper>(pattern_column);
}
context->set_constant_cols(constant_columns);
@@ -72,7 +91,31 @@ Status execute_pattern_with_fallback_disabled(const std::string& value, const st
block.insert({std::move(values), string_type, "value"});
block.insert({std::move(pattern_column), string_type, "pattern"});
block.insert({nullptr, std::make_shared<DataTypeUInt8>(), "result"});
- return function.execute_impl(context.get(), block, {0, 1}, 2, 1);
+ RETURN_IF_ERROR(function.execute_impl(context.get(), block, {0, 1}, 2, 1));
+
+ // execute_impl always installs a ColumnUInt8 at the result position.
+ *result = assert_cast<const ColumnUInt8&>(*block.get_by_position(2).column).get_element(0);
+ return Status::OK();
+}
+
+template <typename Function>
+Status execute_pattern_with_fallback_disabled(const std::string& value, const std::string& pattern,
+ bool constant_known_at_open) {
+ TQueryOptions query_options;
+ query_options.__set_enable_hyperscan_fallback(false);
+ uint8_t result = 0;
+ return execute_single_pattern<Function>(
+ query_options, value, pattern,
+ constant_known_at_open ? PatternMode::CONST_AT_OPEN : PatternMode::CONST_AT_EXECUTE,
+ &result);
+}
+
+Status execute_regexp_with_extended_regex(const std::string& value, const std::string& pattern,
+ bool enable_extended_regex, PatternMode mode,
+ uint8_t* result) {
+ TQueryOptions query_options;
+ query_options.__set_enable_extended_regex(enable_extended_regex);
+ return execute_single_pattern<FunctionRegexpLike>(query_options, value, pattern, mode, result);
}
template <typename Function>
@@ -314,6 +357,53 @@ TEST(FunctionLikeTest, regexp_empty_constant_pattern) {
check_constant_pattern_batch<FunctionRegexpLike>({"abc", ""}, "", {1, 1});
}
+TEST(FunctionLikeTest, regexp_extended_regex_all_pattern_modes) {
+ // Lookaround assertions are rejected by both Hyperscan and RE2, so they only work through
+ // the Boost.Regex fallback, which must be reachable no matter how the pattern is supplied.
+ struct Case {
+ std::string value;
+ std::string pattern;
+ uint8_t expected;
+ };
+ const std::vector<Case> cases = {{"foo123bar", "foo(?=123)", 1},
+ {"foo124bar", "foo(?=123)", 0},
+ {"foobar", "(?<=foo)bar", 1},
+ {"fobar", "(?<=foo)bar", 0}};
+
+ for (auto mode :
+ {PatternMode::NON_CONST, PatternMode::CONST_AT_EXECUTE, PatternMode::CONST_AT_OPEN}) {
+ SCOPED_TRACE(pattern_mode_name(mode));
+ for (const auto& c : cases) {
+ SCOPED_TRACE(c.value + " REGEXP " + c.pattern);
+ uint8_t result = 0;
+ auto status =
+ execute_regexp_with_extended_regex(c.value, c.pattern, true, mode, &result);
+ EXPECT_TRUE(status.ok()) << status.to_string();
+ EXPECT_EQ(result, c.expected);
+
+ status = execute_regexp_with_extended_regex(c.value, c.pattern, false, mode, &result);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("enable_extended_regex"), std::string::npos)
+ << status.to_string();
+ }
+
+ // A pattern that is invalid for Boost.Regex too still fails.
+ uint8_t result = 0;
+ auto status = execute_regexp_with_extended_regex("abc", "(", true, mode, &result);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("Invalid regex expression"), std::string::npos)
+ << status.to_string();
+
+ // Boost.Regex gives up on a pathological pattern while matching; that must surface as a
+ // Status, not as an exception escaping the function.
+ status = execute_regexp_with_extended_regex(std::string(60, 'a'), "(?=a)(a+)+b", true, mode,
+ &result);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("Failed to match regex expression"), std::string::npos)
+ << status.to_string();
+ }
+}
+
TEST(FunctionLikeTest, regexp_constant_substring_boundaries) {
check_constant_pattern_batch<FunctionRegexpLike>({"ab", "c", "", "abc", "", "xabcx"}, "abc",
{0, 0, 0, 1, 0, 1});
diff --git a/regression-test/data/query_p0/sql_functions/string_functions/test_regexp_extended_nonconstant_pattern.out b/regression-test/data/query_p0/sql_functions/string_functions/test_regexp_extended_nonconstant_pattern.out
new file mode 100644
index 00000000000..5511ea0c942
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/string_functions/test_regexp_extended_nonconstant_pattern.out
@@ -0,0 +1,28 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !constant_pattern --
+1 foobar true false
+2 fobar false false
+3 foo123bar false true
+4 foo124bar false false
+5 foobar true false
+6 \N \N \N
+7 foobar true false
+
+-- !column_pattern --
+1 foobar (?<=foo)bar true true true
+2 fobar (?<=foo)bar false false false
+3 foo123bar foo(?=123) true true true
+4 foo124bar foo(?=123) false false false
+5 foobar ^foo true true true
+6 \N foo(?=123) \N \N \N
+7 foobar \N \N \N \N
+
+-- !column_pattern_filter --
+1
+3
+5
+
+-- !column_pattern_not --
+2
+4
+
diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_regexp_extended_nonconstant_pattern.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_regexp_extended_nonconstant_pattern.groovy
new file mode 100644
index 00000000000..6f7fb88fde8
--- /dev/null
+++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_regexp_extended_nonconstant_pattern.groovy
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_regexp_extended_nonconstant_pattern") {
+ sql "DROP TABLE IF EXISTS test_regexp_extended_nonconstant_pattern"
+ sql """
+ CREATE TABLE test_regexp_extended_nonconstant_pattern (
+ id INT,
+ s VARCHAR(32),
+ p VARCHAR(64)
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO test_regexp_extended_nonconstant_pattern VALUES
+ (1, 'foobar', '(?<=foo)bar'),
+ (2, 'fobar', '(?<=foo)bar'),
+ (3, 'foo123bar', 'foo(?=123)'),
+ (4, 'foo124bar', 'foo(?=123)'),
+ (5, 'foobar', '^foo'),
+ (6, NULL, 'foo(?=123)'),
+ (7, 'foobar', NULL)
+ """
+
+ sql "SET enable_extended_regex = true"
+ // Lookaround patterns work the same whether they are constants or read from a column.
+ order_qt_constant_pattern """
+ SELECT id, s, s REGEXP '(?<=foo)bar', REGEXP(s, 'foo(?=123)')
+ FROM test_regexp_extended_nonconstant_pattern
+ """
+ order_qt_column_pattern """
+ SELECT id, s, p, s REGEXP p, REGEXP(s, p), s RLIKE p
+ FROM test_regexp_extended_nonconstant_pattern
+ """
+ order_qt_column_pattern_filter """
+ SELECT id FROM test_regexp_extended_nonconstant_pattern WHERE s REGEXP p
+ """
+ order_qt_column_pattern_not """
+ SELECT id FROM test_regexp_extended_nonconstant_pattern WHERE NOT s REGEXP p
+ """
+ // A column pattern that Boost.Regex rejects as well ('(' from substr) is still an error.
+ test {
+ sql """
+ SELECT id, s REGEXP substr(p, 1, 1) FROM test_regexp_extended_nonconstant_pattern
+ WHERE id = 1
+ """
+ exception "Invalid regex expression: ("
+ }
+ // Boost.Regex exhausts its backtracking budget on this column pattern; the query must fail
+ // with an error instead of crashing the backend.
+ test {
+ sql """
+ SELECT id FROM test_regexp_extended_nonconstant_pattern
+ WHERE repeat('a', 60) REGEXP concat(substr(p, 1, 0), '(?=a)(a+)+b')
+ """
+ exception "Failed to match regex expression"
+ }
+
+ sql "SET enable_extended_regex = false"
+ // Without the session variable, column patterns report the same error as constant ones.
+ test {
+ sql "SELECT id, s REGEXP p FROM test_regexp_extended_nonconstant_pattern ORDER BY id"
+ exception "try setting enable_extended_regex=true"
+ }
+ test {
+ sql "SELECT id, s REGEXP '(?<=foo)bar' FROM test_regexp_extended_nonconstant_pattern ORDER BY id"
+ exception "try setting enable_extended_regex=true"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 16/35: branch-4.1: [fix](auth) Treat only root@'%' as root in SET PASSWORD and ALTER USER (#68634)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit bd6fecbc979e5cb56cd72324cb452e8b1bbb52f3
Author: Calvin Kirs <gu...@selectdb.com>
AuthorDate: Tue Sep 29 23:35:06 2026 +0800
branch-4.1: [fix](auth) Treat only root@'%' as root in SET PASSWORD and ALTER USER (#68634)
https://github.com/apache/doris/pull/68559
---
.../trees/plans/commands/info/AlterUserInfo.java | 6 +-
.../trees/plans/commands/info/SetPassVarOp.java | 3 +-
.../doris/mysql/privilege/SystemRootUserTest.java | 108 +++++++++++++++++++++
.../suites/account_p0/test_system_user.groovy | 31 ++++++
4 files changed, 141 insertions(+), 7 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterUserInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterUserInfo.java
index bf0a1aa060b..34544f5fe53 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterUserInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterUserInfo.java
@@ -23,12 +23,10 @@ import org.apache.doris.analysis.TlsOptions;
import org.apache.doris.analysis.UserDesc;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.Env;
-import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
-import org.apache.doris.mysql.privilege.Auth;
import org.apache.doris.mysql.privilege.PasswordPolicy;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@@ -132,9 +130,7 @@ public class AlterUserInfo {
+ "actual number of type is " + ops.size());
}
- if (userDesc.getUserIdent().getQualifiedUser().equals(Auth.ROOT_USER)
- && !ClusterNamespace.getNameFromFullName(ConnectContext.get().getQualifiedUser())
- .equals(Auth.ROOT_USER)) {
+ if (userDesc.getUserIdent().isRootUser() && !ConnectContext.get().getCurrentUserIdentity().isRootUser()) {
throw new AnalysisException("Only root user can modify root user");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/SetPassVarOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/SetPassVarOp.java
index f142b9679a2..0453ffeb52c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/SetPassVarOp.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/SetPassVarOp.java
@@ -25,7 +25,6 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
-import org.apache.doris.mysql.privilege.Auth;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@@ -69,7 +68,7 @@ public class SetPassVarOp extends SetVarOp {
}
// 2. No user can set password for root expect for root user itself
- if (userIdent.getQualifiedUser().equals(Auth.ROOT_USER)) {
+ if (userIdent.isRootUser()) {
throw new AnalysisException("Can not set password for root user, except root itself");
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/SystemRootUserTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/SystemRootUserTest.java
new file mode 100644
index 00000000000..d60df7107d2
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/SystemRootUserTest.java
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.mysql.privilege;
+
+import org.apache.doris.analysis.UserIdentity;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.utframe.TestWithFeService;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Only the full identity root@'%' is the system root user. An account created as root@'<host>' is an
+ * ordinary account: a user with GRANT privilege manages it like any other account, and it has no more
+ * say over root@'%' than any other non-root user.
+ */
+public class SystemRootUserTest extends TestWithFeService {
+
+ private static final String HOST = "8.8.20.39";
+
+ @Override
+ protected void runBeforeAll() throws Exception {
+ run("CREATE USER 'root'@'" + HOST + "' IDENTIFIED BY 'Pwd_a1'");
+ run("GRANT GRANT_PRIV ON *.*.* TO 'root'@'" + HOST + "'");
+ run("CREATE USER 'grant_admin'@'%'");
+ run("GRANT GRANT_PRIV ON *.*.* TO 'grant_admin'@'%'");
+ }
+
+ @Override
+ protected void runBeforeEach() throws Exception {
+ useUser(Auth.ROOT_USER);
+ }
+
+ private void run(String sql) throws Exception {
+ ((Command) new NereidsParser().parseSingle(sql)).run(connectContext, null);
+ }
+
+ private void assertRejected(String sql, String expectedMessage) {
+ Exception e = Assertions.assertThrows(Exception.class, () -> run(sql));
+ Assertions.assertTrue(e.getMessage().contains(expectedMessage), e.getMessage());
+ }
+
+ private void assertPassword(UserIdentity user, String password) {
+ Assertions.assertDoesNotThrow(
+ () -> Env.getCurrentEnv().getAuth().checkPlainPasswordForUserIdentity(user, password, null));
+ }
+
+ private UserIdentity rootAtHost() {
+ UserIdentity user = new UserIdentity(Auth.ROOT_USER, HOST);
+ user.setIsAnalyzed();
+ return user;
+ }
+
+ @Test
+ public void testRootCanSetPasswordForRootAtSpecificHost() throws Exception {
+ run("SET PASSWORD FOR 'root'@'" + HOST + "' = PASSWORD('Pwd_f6')");
+ assertPassword(rootAtHost(), "Pwd_f6");
+ }
+
+ @Test
+ public void testGrantUserCanSetPasswordForRootAtSpecificHost() throws Exception {
+ useUser("grant_admin");
+ run("SET PASSWORD FOR 'root'@'" + HOST + "' = PASSWORD('Pwd_b2')");
+ assertPassword(rootAtHost(), "Pwd_b2");
+ }
+
+ @Test
+ public void testGrantUserCanAlterRootAtSpecificHost() throws Exception {
+ useUser("grant_admin");
+ run("ALTER USER 'root'@'" + HOST + "' IDENTIFIED BY 'Pwd_c3'");
+ assertPassword(rootAtHost(), "Pwd_c3");
+ }
+
+ @Test
+ public void testGrantUserStillCannotModifySystemRoot() throws Exception {
+ useUser("grant_admin");
+ assertRejected("SET PASSWORD FOR 'root'@'%' = PASSWORD('Pwd_d4')",
+ "Can not set password for root user, except root itself");
+ assertRejected("ALTER USER 'root'@'%' IDENTIFIED BY 'Pwd_d4'", "Only root user can modify root user");
+ assertPassword(UserIdentity.ROOT, "");
+ }
+
+ @Test
+ public void testRootAtSpecificHostCannotModifySystemRoot() throws Exception {
+ useUser(Auth.ROOT_USER, HOST);
+ assertRejected("SET PASSWORD FOR 'root'@'%' = PASSWORD('Pwd_e5')",
+ "Can not set password for root user, except root itself");
+ assertRejected("ALTER USER 'root'@'%' IDENTIFIED BY 'Pwd_e5'", "Only root user can modify root user");
+ assertPassword(UserIdentity.ROOT, "");
+ }
+}
diff --git a/regression-test/suites/account_p0/test_system_user.groovy b/regression-test/suites/account_p0/test_system_user.groovy
index d7b3ad213fb..829115680b4 100644
--- a/regression-test/suites/account_p0/test_system_user.groovy
+++ b/regression-test/suites/account_p0/test_system_user.groovy
@@ -100,4 +100,35 @@ suite("test_system_user","p0,auth") {
sql """
drop user `admin`@'8.8.8.8';
"""
+
+ // only root@'%' is the system root: root@'<host>' is managed like any other account, and a GRANT
+ // user that may manage it still can not modify root@'%'
+ String grantUser = "test_system_user_grant"
+ String pwd = 'C123_567p'
+ try_sql("DROP USER 'root'@'8.8.20.39'")
+ try_sql("DROP USER '${grantUser}'")
+ sql """CREATE USER 'root'@'8.8.20.39' IDENTIFIED BY 'Pwd_a1'"""
+ sql """SET PASSWORD FOR 'root'@'8.8.20.39' = PASSWORD('Pwd_b2')"""
+ sql """ALTER USER 'root'@'8.8.20.39' ACCOUNT_UNLOCK"""
+ sql """CREATE USER '${grantUser}' IDENTIFIED BY '${pwd}'"""
+ sql """GRANT GRANT_PRIV ON *.*.* TO ${grantUser}"""
+ if (isCloudMode()) {
+ def clusters = sql "SHOW CLUSTERS"
+ assertTrue(!clusters.isEmpty())
+ sql """GRANT USAGE_PRIV ON CLUSTER `${clusters[0][0]}` TO ${grantUser}"""
+ }
+ def tokens = context.config.jdbcUrl.split('/')
+ def url = tokens[0] + "//" + tokens[2] + "/" + "information_schema" + "?"
+ connect(grantUser, "${pwd}", url) {
+ sql """SET PASSWORD FOR 'root'@'8.8.20.39' = PASSWORD('Pwd_c3')"""
+ sql """ALTER USER 'root'@'8.8.20.39' IDENTIFIED BY 'Pwd_d4'"""
+ test {
+ sql """SET PASSWORD FOR 'root'@'%' = PASSWORD('Pwd_e5')"""
+ exception "Can not set password for root user"
+ }
+ test {
+ sql """ALTER USER 'root'@'%' IDENTIFIED BY 'Pwd_e5'"""
+ exception "Only root user can modify root user"
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 14/35: [fix](arrow-flight) Cancel queries when result streams close early (#68630)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit aa59d75da7f9c9367cfd8b33e2537c0a16c62317
Author: Gabriel <li...@selectdb.com>
AuthorDate: Tue Sep 29 22:02:05 2026 +0800
[fix](arrow-flight) Cancel queries when result streams close early (#68630)
## What problem does this PR solve?
Cancelling or closing an unfinished Arrow Flight stream stops the client
RPC but can leave result producers blocked until timeout. Propagate
stream aborts to the whole query and promptly release its Flight
buffers, while preserving normal EOF and valid unread results after
LIMIT_REACH or FINISHED.
Local reads use bounded buffer waits with explicit EOF; the reader owns
cancellation checks. Remote reads interrupt outstanding BRPC waits. A BE
reports an aborted result ID to the owning FE, which sends the existing
cancel_plan_fragment RPC to every participating BE. Lightweight
result-ID routing metadata survives local fragment and coordinator
completion without retaining the coordinator or its query queue slot;
session reset and execution-timeout expiry remove those routes. RPC
transport and application errors are checked, with bounded retries.
Cancellation bypasses the Arrow read pool.
On updated BEs, a query-to-buffer index removes all sibling Flight
buffers, even after their local QueryContext expires. Query cancellation
occurs outside buffer-map locks. A fetch racing with buffer removal
completes its RPC with an error.
## Validation
- 18 BE tests passed in 10 consecutive incremental ASAN runs (180
executions), including real Flight, Thrift and BRPC transport paths,
expired contexts, sibling cleanup, rejected Arrow work,
application-error retry, a legacy cancellation handler, and successful
fragment-stop preservation.
- Four FE tests passed for cancellation across backend addresses after
coordinator completion, application-error retry, expiry and session
cleanup. Changed FE sources and generated Thrift code compiled against
cached dependencies.
- FE Checkstyle, clang-format 16 and git diff whitespace checks passed.
The Groovy regression compiled against the regression framework.
- The cluster regression uses a distributed table, aborts only one
endpoint, checks that the query's pipelines disappear from every BE
within 20 seconds, then checks connection reuse and normal completion.
Full execution on rebuilt FE/BE binaries remains pending CI; local
validation used focused incremental builds.
## Upgrade behavior
Upgrade the serving FE and Flight-facing BE to enable the new
cancellation route. Result BEs can still be older: the FE uses their
existing query cancellation RPC rather than an unknown fetch flag. Older
result BEs retain their historical unread-buffer reclamation behavior
until upgraded; immediate sibling-buffer reclamation requires the
updated BE.
---
.../sink/writer/varrow_flight_result_writer.cpp | 21 +-
.../exec/sink/writer/varrow_flight_result_writer.h | 13 +-
be/src/runtime/result_buffer_mgr.cpp | 43 ++
be/src/runtime/result_buffer_mgr.h | 5 +
.../arrow_flight/arrow_flight_batch_reader.cpp | 154 +++++-
.../arrow_flight/arrow_flight_batch_reader.h | 23 +-
be/src/service/arrow_flight/flight_sql_service.cpp | 8 +-
be/src/service/internal_service.cpp | 9 +-
.../arrow_flight/arrow_flight_cancel_test.cpp | 567 +++++++++++++++++++++
.../main/java/org/apache/doris/qe/Coordinator.java | 5 +
.../org/apache/doris/qe/NereidsCoordinator.java | 6 +
.../java/org/apache/doris/qe/StmtExecutor.java | 6 +
.../apache/doris/service/FrontendServiceImpl.java | 6 +
.../arrowflight/FlightSqlQueryCancellation.java | 146 ++++++
.../arrowflight/results/FlightSqlChannel.java | 14 +-
.../FlightSqlQueryCancellationTest.java | 109 ++++
gensrc/thrift/FrontendService.thrift | 3 +
.../test_flight_cancel_cleanup.groovy | 128 +++++
18 files changed, 1227 insertions(+), 39 deletions(-)
diff --git a/be/src/exec/sink/writer/varrow_flight_result_writer.cpp b/be/src/exec/sink/writer/varrow_flight_result_writer.cpp
index fa20e42f0e7..67ea4b3810d 100644
--- a/be/src/exec/sink/writer/varrow_flight_result_writer.cpp
+++ b/be/src/exec/sink/writer/varrow_flight_result_writer.cpp
@@ -97,15 +97,29 @@ Status ArrowFlightResultBlockBuffer::get_schema(std::shared_ptr<arrow::Schema>*
print_id(_fragment_id), _status));
}
-Status ArrowFlightResultBlockBuffer::get_arrow_batch(std::shared_ptr<Block>* result) {
+void ArrowFlightResultBlockBuffer::cancel_query(const Status& reason) {
+ cancel(reason);
+ // The result buffer can be keyed by a fragment instance id rather than the query id.
+ // Keep only a weak query reference, and cancel outside the buffer/map locks.
+ if (auto query_ctx = _query_ctx.lock()) {
+ query_ctx->cancel(reason);
+ }
+}
+
+Status ArrowFlightResultBlockBuffer::get_arrow_batch(std::shared_ptr<Block>* result, bool* eos) {
+ *result = nullptr;
+ *eos = false;
std::unique_lock<std::mutex> l(_lock);
Defer defer {[&]() { _update_dependency(); }};
if (!_status.ok()) {
return _status;
}
- while (_result_batch_queue.empty() && _status.ok() && !_is_close) {
- _arrow_data_arrival.wait_for(l, std::chrono::milliseconds(20));
+ if (!_arrow_data_arrival.wait_for(l, std::chrono::milliseconds(20), [&] {
+ return !_result_batch_queue.empty() || !_status.ok() || _is_close;
+ })) {
+ // Let the reader check RPC cancellation without treating an empty wait as EOF.
+ return Status::OK();
}
if (!_status.ok()) {
@@ -126,6 +140,7 @@ Status ArrowFlightResultBlockBuffer::get_arrow_batch(std::shared_ptr<Block>* res
// normal path end
if (_is_close) {
+ *eos = true;
if (!_status.ok()) {
return _status;
}
diff --git a/be/src/exec/sink/writer/varrow_flight_result_writer.h b/be/src/exec/sink/writer/varrow_flight_result_writer.h
index 2d0420ed777..a82598658e7 100644
--- a/be/src/exec/sink/writer/varrow_flight_result_writer.h
+++ b/be/src/exec/sink/writer/varrow_flight_result_writer.h
@@ -20,6 +20,7 @@
#include "common/status.h"
#include "exec/sink/writer/result_writer.h"
#include "exprs/vexpr_fwd.h"
+#include "runtime/query_context.h"
#include "runtime/result_block_buffer.h"
#include "runtime/runtime_profile.h"
@@ -64,13 +65,19 @@ public:
: ResultBlockBuffer<GetArrowResultBatchCtx>(id, state, buffer_size),
_arrow_schema(schema),
_profile("ResultBlockBuffer " + print_id(_fragment_id)),
- _timezone_obj(state->timezone_obj()) {
+ _timezone_obj(state->timezone_obj()),
+ _query_id(state->query_id()),
+ _query_ctx(state->get_query_ctx() ? state->get_query_ctx()->weak_from_this()
+ : std::weak_ptr<QueryContext> {}) {
_serialize_batch_ns_timer = ADD_TIMER(&_profile, "SerializeBatchNsTime");
_uncompressed_bytes_counter = ADD_COUNTER(&_profile, "UncompressedBytes", TUnit::BYTES);
_compressed_bytes_counter = ADD_COUNTER(&_profile, "CompressedBytes", TUnit::BYTES);
}
~ArrowFlightResultBlockBuffer() override = default;
- Status get_arrow_batch(std::shared_ptr<Block>* result);
+ // A timed wait may return OK with no block and eos=false; the caller must retry.
+ Status get_arrow_batch(std::shared_ptr<Block>* result, bool* eos);
+ void cancel_query(const Status& reason);
+ const TUniqueId& query_id() const { return _query_id; }
void get_timezone(cctz::time_zone& timezone_obj) { timezone_obj = _timezone_obj; }
Status get_schema(std::shared_ptr<arrow::Schema>* arrow_schema);
@@ -83,6 +90,8 @@ private:
RuntimeProfile::Counter* _uncompressed_bytes_counter = nullptr;
RuntimeProfile::Counter* _compressed_bytes_counter = nullptr;
cctz::time_zone _timezone_obj;
+ const TUniqueId _query_id;
+ std::weak_ptr<QueryContext> _query_ctx;
};
class VArrowFlightResultWriter final : public ResultWriter {
diff --git a/be/src/runtime/result_buffer_mgr.cpp b/be/src/runtime/result_buffer_mgr.cpp
index 0e53128c0b3..c4a9a16ad7e 100644
--- a/be/src/runtime/result_buffer_mgr.cpp
+++ b/be/src/runtime/result_buffer_mgr.cpp
@@ -93,6 +93,9 @@ Status ResultBufferMgr::create_sender(const TUniqueId& unique_id, int buffer_siz
{
std::unique_lock<std::shared_mutex> wlock(_buffer_map_lock);
_buffer_map.insert(std::make_pair(unique_id, control_block));
+ if (arrow_flight) {
+ _arrow_flight_query_buffers[state->query_id()].insert(unique_id);
+ }
// ResultBlockBufferBase should destroy after max_timeout
// for exceed max_timeout FE will return timeout to client
// otherwise in some case may block all fragment handle threads
@@ -135,12 +138,52 @@ bool ResultBufferMgr::cancel(const TUniqueId& unique_id, const Status& reason) {
auto exist = _buffer_map.end() != iter;
if (exist) {
+ if (auto arrow_buffer =
+ std::dynamic_pointer_cast<ArrowFlightResultBlockBuffer>(iter->second)) {
+ auto query = _arrow_flight_query_buffers.find(arrow_buffer->query_id());
+ if (query != _arrow_flight_query_buffers.end()) {
+ query->second.erase(unique_id);
+ if (query->second.empty()) {
+ _arrow_flight_query_buffers.erase(query);
+ }
+ }
+ }
iter->second->cancel(reason);
_buffer_map.erase(iter);
}
return exist;
}
+void ResultBufferMgr::cancel_arrow_flight_query(const TUniqueId& buffer_id, const Status& reason) {
+ std::shared_ptr<ArrowFlightResultBlockBuffer> buffer;
+ if (find_buffer(buffer_id, buffer).ok()) {
+ cancel_arrow_flight_buffers(buffer->query_id(), reason);
+ }
+}
+
+void ResultBufferMgr::cancel_arrow_flight_buffers(const TUniqueId& query_id, const Status& reason) {
+ std::vector<std::shared_ptr<ArrowFlightResultBlockBuffer>> buffers;
+ {
+ std::unique_lock<std::shared_mutex> lock(_buffer_map_lock);
+ auto query = _arrow_flight_query_buffers.find(query_id);
+ if (query == _arrow_flight_query_buffers.end()) {
+ return;
+ }
+ for (const auto& buffer_id : query->second) {
+ auto it = _buffer_map.find(buffer_id);
+ DCHECK(it != _buffer_map.end());
+ buffers.push_back(std::static_pointer_cast<ArrowFlightResultBlockBuffer>(it->second));
+ _buffer_map.erase(it);
+ }
+ _arrow_flight_query_buffers.erase(query);
+ }
+ // Clear all sibling buffers even if their local query context has already expired.
+ // Never cancel a query while holding the result-buffer map lock.
+ for (const auto& buffer : buffers) {
+ buffer->cancel_query(reason);
+ }
+}
+
void ResultBufferMgr::cancel_at_time(time_t cancel_time, const TUniqueId& unique_id) {
std::lock_guard<std::mutex> l(_timeout_lock);
auto iter = _timeout_map.find(cancel_time);
diff --git a/be/src/runtime/result_buffer_mgr.h b/be/src/runtime/result_buffer_mgr.h
index 96cb6a5b7e6..e0170c88176 100644
--- a/be/src/runtime/result_buffer_mgr.h
+++ b/be/src/runtime/result_buffer_mgr.h
@@ -27,6 +27,7 @@
#include <mutex>
#include <shared_mutex>
#include <unordered_map>
+#include <unordered_set>
#include <vector>
#include "common/status.h"
@@ -71,6 +72,9 @@ public:
// cancel
bool cancel(const TUniqueId& unique_id, const Status& reason);
+ void cancel_arrow_flight_query(const TUniqueId& buffer_id, const Status& reason);
+ void cancel_arrow_flight_buffers(const TUniqueId& query_id, const Status& reason);
+
// cancel one query at a future time.
void cancel_at_time(time_t cancel_time, const TUniqueId& unique_id);
@@ -89,6 +93,7 @@ private:
std::shared_mutex _buffer_map_lock;
// buffer block map
BufferMap _buffer_map;
+ std::unordered_map<TUniqueId, std::unordered_set<TUniqueId>> _arrow_flight_query_buffers;
// lock for timeout map
std::mutex _timeout_lock;
diff --git a/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp b/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp
index 73761281799..7e11fe9a423 100644
--- a/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp
+++ b/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp
@@ -23,6 +23,8 @@
#include <arrow/type.h>
#include <gen_cpp/internal_service.pb.h>
+#include <condition_variable>
+
#include "core/block/block.h"
#include "format/arrow/arrow_block_convertor.h"
#include "format/arrow/arrow_row_batch.h"
@@ -35,9 +37,43 @@
#include "service/backend_options.h"
#include "util/brpc_client_cache.h"
#include "util/brpc_closure.h"
+#include "util/client_cache.h"
namespace doris::flight {
+namespace {
+// Poll cancellation on the RPC thread: no background thread may retain ServerCallContext.
+template <typename Response>
+class FlightReadCallback : public DummyBrpcCallback<Response> {
+public:
+ void call() override {
+ std::lock_guard lock(_mutex);
+ _done = true;
+ _cv.notify_all();
+ }
+
+ void wait(const std::function<bool()>& is_cancelled) {
+ std::unique_lock lock(_mutex);
+ while (!_done) {
+ if (is_cancelled()) {
+ lock.unlock();
+ brpc::StartCancel(this->call_id_);
+ this->join();
+ return;
+ }
+ _cv.wait_for(lock, std::chrono::milliseconds(20));
+ }
+ lock.unlock();
+ this->join();
+ }
+
+private:
+ std::mutex _mutex;
+ std::condition_variable _cv;
+ bool _done = false;
+};
+} // namespace
+
ArrowFlightBatchReaderBase::ArrowFlightBatchReaderBase(
const std::shared_ptr<QueryStatement>& statement)
: _statement(statement) {}
@@ -55,13 +91,79 @@ arrow::Status ArrowFlightBatchReaderBase::_return_invalid_status(const std::stri
return arrow::Status::Invalid(status_msg);
}
+bool ArrowFlightBatchReaderBase::is_cancelled() const {
+ return _closed.load() || (_is_cancelled && _is_cancelled());
+}
+
+void ArrowFlightBatchReaderBase::close(const Status& reason) {
+ if (_closed.exchange(true) || _eof.load()) {
+ return;
+ }
+ auto* env = ExecEnv::GetInstance();
+ if (_statement->result_addr.hostname == BackendOptions::get_localhost() &&
+ _statement->result_addr.port == config::brpc_port) {
+ env->result_mgr()->cancel_arrow_flight_query(_statement->query_id, reason);
+ }
+ // A result endpoint can outlive its local fragment. Find the owning FE by buffer ID,
+ // then use its query-wide cancellation route, including for older result BEs.
+ for (const auto& [address, info] : env->get_running_frontends()) {
+ Status status;
+ for (int attempt = 0; attempt < 2; ++attempt) {
+ FrontendServiceConnection client(env->frontend_client_cache(), address, 2000, &status);
+ if (status.ok()) {
+ try {
+ TStatus result;
+ client->cancelFlightQuery(result, _statement->query_id);
+ status = Status::create(result);
+ if (status.ok()) {
+ return;
+ }
+ if (result.status_code == TStatusCode::NOT_FOUND) {
+ break;
+ }
+ } catch (const apache::thrift::TException& e) {
+ status = Status::RpcError("Flight cancellation failed: {}", e.what());
+ // Discard a transport that may contain an incomplete response.
+ static_cast<void>(client.reopen(1000));
+ }
+ }
+ LOG(WARNING) << "Failed to cancel Flight result " << print_id(_statement->query_id)
+ << " through FE " << address << ": " << status;
+ }
+ }
+}
+
+arrow::Status ArrowFlightBatchReaderBase::Close() {
+ close(Status::Cancelled("Arrow Flight result stream closed before EOF"));
+ return arrow::Status::OK();
+}
+
+arrow::Status ArrowFlightBatchReaderBase::ReadNext(std::shared_ptr<arrow::RecordBatch>* out) {
+ *out = nullptr;
+ if (is_cancelled()) {
+ (void)Close();
+ return arrow::Status::Cancelled("Arrow Flight fetch cancelled");
+ }
+ auto status = [&]() -> arrow::Status {
+ RETURN_ARROW_STATUS_IF_CATCH_EXCEPTION(ReadNextImpl(out));
+ }();
+ if (!status.ok()) {
+ close(to_doris_status(status));
+ } else if (!*out) {
+ _eof = true;
+ }
+ return status;
+}
+
ArrowFlightBatchReaderBase::~ArrowFlightBatchReaderBase() {
+ // Transport errors can destroy a stream without another ReadNext/Close call.
+ (void)Close();
LOG(INFO) << fmt::format(
"ArrowFlightBatchReader finished, packet_seq={}, result_addr={}:{}, finistId={}, "
"convert_arrow_batch_timer={}, deserialize_block_timer={}, peak_memory_usage={}",
_packet_seq, _statement->result_addr.hostname, _statement->result_addr.port,
print_id(_statement->query_id), _convert_arrow_batch_timer, _deserialize_block_timer,
- _mem_tracker->peak_consumption());
+ _mem_tracker ? _mem_tracker->peak_consumption() : 0);
}
ArrowFlightBatchLocalReader::ArrowFlightBatchLocalReader(
@@ -74,7 +176,7 @@ ArrowFlightBatchLocalReader::ArrowFlightBatchLocalReader(
}
arrow::Result<std::shared_ptr<ArrowFlightBatchLocalReader>> ArrowFlightBatchLocalReader::Create(
- const std::shared_ptr<QueryStatement>& statement) {
+ const std::shared_ptr<QueryStatement>& statement, std::function<bool()> is_cancelled) {
DCHECK(statement->result_addr.hostname == BackendOptions::get_localhost());
std::shared_ptr<ArrowFlightResultBlockBuffer> arrow_buffer;
RETURN_ARROW_STATUS_IF_ERROR(
@@ -86,6 +188,7 @@ arrow::Result<std::shared_ptr<ArrowFlightBatchLocalReader>> ArrowFlightBatchLoca
std::shared_ptr<MemTrackerLimiter> mem_tracker = arrow_buffer->mem_tracker();
std::shared_ptr<ArrowFlightBatchLocalReader> result(
new ArrowFlightBatchLocalReader(statement, schema, mem_tracker));
+ result->_is_cancelled = std::move(is_cancelled);
arrow_buffer->get_timezone(result->_timezone_obj);
return result;
}
@@ -99,10 +202,16 @@ arrow::Status ArrowFlightBatchLocalReader::ReadNextImpl(std::shared_ptr<arrow::R
RETURN_ARROW_STATUS_IF_ERROR(
ExecEnv::GetInstance()->result_mgr()->find_buffer(tid, arrow_buffer));
std::shared_ptr<Block> result;
- auto st = arrow_buffer->get_arrow_batch(&result);
- st.prepend("ArrowFlightBatchLocalReader fetch arrow data failed");
- ARROW_RETURN_NOT_OK(to_arrow_status(st));
- if (result == nullptr) {
+ bool eos = false;
+ while (!result && !eos) {
+ if (is_cancelled()) {
+ return arrow::Status::Cancelled("Arrow Flight fetch cancelled");
+ }
+ auto st = arrow_buffer->get_arrow_batch(&result, &eos);
+ st.prepend("ArrowFlightBatchLocalReader fetch arrow data failed");
+ ARROW_RETURN_NOT_OK(to_arrow_status(st));
+ }
+ if (eos) {
// eof, normal path end
return arrow::Status::OK();
}
@@ -110,8 +219,8 @@ arrow::Status ArrowFlightBatchLocalReader::ReadNextImpl(std::shared_ptr<arrow::R
{
// convert one batch
SCOPED_ATOMIC_TIMER(&_convert_arrow_batch_timer);
- st = ArrowFlightArrowBlockConvertor(_schema, _timezone_obj)
- .convert_to_arrow(*result, arrow::default_memory_pool(), out);
+ auto st = ArrowFlightArrowBlockConvertor(_schema, _timezone_obj)
+ .convert_to_arrow(*result, arrow::default_memory_pool(), out);
st.prepend("ArrowFlightBatchLocalReader convert block to arrow batch failed");
ARROW_RETURN_NOT_OK(to_arrow_status(st));
}
@@ -124,10 +233,6 @@ arrow::Status ArrowFlightBatchLocalReader::ReadNextImpl(std::shared_ptr<arrow::R
return arrow::Status::OK();
}
-arrow::Status ArrowFlightBatchLocalReader::ReadNext(std::shared_ptr<arrow::RecordBatch>* out) {
- RETURN_ARROW_STATUS_IF_CATCH_EXCEPTION(ReadNextImpl(out));
-}
-
ArrowFlightBatchRemoteReader::ArrowFlightBatchRemoteReader(
const std::shared_ptr<QueryStatement>& statement,
const std::shared_ptr<PBackendService_Stub>& stub)
@@ -138,7 +243,7 @@ ArrowFlightBatchRemoteReader::ArrowFlightBatchRemoteReader(
}
arrow::Result<std::shared_ptr<ArrowFlightBatchRemoteReader>> ArrowFlightBatchRemoteReader::Create(
- const std::shared_ptr<QueryStatement>& statement) {
+ const std::shared_ptr<QueryStatement>& statement, std::function<bool()> is_cancelled) {
std::shared_ptr<PBackendService_Stub> stub =
ExecEnv::GetInstance()->brpc_internal_client_cache()->get_client(
statement->result_addr);
@@ -153,6 +258,7 @@ arrow::Result<std::shared_ptr<ArrowFlightBatchRemoteReader>> ArrowFlightBatchRem
std::shared_ptr<ArrowFlightBatchRemoteReader> result(
new ArrowFlightBatchRemoteReader(statement, stub));
+ result->_is_cancelled = std::move(is_cancelled);
ARROW_RETURN_NOT_OK(result->init_schema());
return result;
}
@@ -163,17 +269,20 @@ arrow::Status ArrowFlightBatchRemoteReader::_fetch_schema() {
auto* pfinst_id = request->mutable_finst_id();
pfinst_id->set_hi(_statement->query_id.hi);
pfinst_id->set_lo(_statement->query_id.lo);
- auto callback = DummyBrpcCallback<PFetchArrowFlightSchemaResult>::create_shared();
+ auto callback = std::make_shared<FlightReadCallback<PFetchArrowFlightSchemaResult>>();
auto closure = AutoReleaseClosure<
PFetchArrowFlightSchemaRequest,
- DummyBrpcCallback<PFetchArrowFlightSchemaResult>>::create_unique(request, callback);
+ FlightReadCallback<PFetchArrowFlightSchemaResult>>::create_unique(request, callback);
callback->cntl_->set_timeout_ms(config::arrow_flight_reader_brpc_controller_timeout_ms);
callback->cntl_->ignore_eovercrowded();
_brpc_stub->fetch_arrow_flight_schema(closure->cntl_.get(), closure->request_.get(),
closure->response_.get(), closure.get());
closure.release();
- callback->join();
+ callback->wait([this] { return is_cancelled(); });
+ if (is_cancelled()) {
+ return arrow::Status::Cancelled("Arrow Flight fetch cancelled");
+ }
if (callback->cntl_->Failed()) {
if (!ExecEnv::GetInstance()->brpc_internal_client_cache()->available(
@@ -212,17 +321,20 @@ arrow::Status ArrowFlightBatchRemoteReader::_fetch_data() {
auto* pfinst_id = request->mutable_finst_id();
pfinst_id->set_hi(_statement->query_id.hi);
pfinst_id->set_lo(_statement->query_id.lo);
- auto callback = DummyBrpcCallback<PFetchArrowDataResult>::create_shared();
+ auto callback = std::make_shared<FlightReadCallback<PFetchArrowDataResult>>();
auto closure = AutoReleaseClosure<
PFetchArrowDataRequest,
- DummyBrpcCallback<PFetchArrowDataResult>>::create_unique(request, callback);
+ FlightReadCallback<PFetchArrowDataResult>>::create_unique(request, callback);
callback->cntl_->set_timeout_ms(config::arrow_flight_reader_brpc_controller_timeout_ms);
callback->cntl_->ignore_eovercrowded();
_brpc_stub->fetch_arrow_data(closure->cntl_.get(), closure->request_.get(),
closure->response_.get(), closure.get());
closure.release();
- callback->join();
+ callback->wait([this] { return is_cancelled(); });
+ if (is_cancelled()) {
+ return arrow::Status::Cancelled("Arrow Flight fetch cancelled");
+ }
if (callback->cntl_->Failed()) {
if (!ExecEnv::GetInstance()->brpc_internal_client_cache()->available(
@@ -317,8 +429,4 @@ arrow::Status ArrowFlightBatchRemoteReader::ReadNextImpl(std::shared_ptr<arrow::
return arrow::Status::OK();
}
-arrow::Status ArrowFlightBatchRemoteReader::ReadNext(std::shared_ptr<arrow::RecordBatch>* out) {
- RETURN_ARROW_STATUS_IF_CATCH_EXCEPTION(ReadNextImpl(out));
-}
-
} // namespace doris::flight
diff --git a/be/src/service/arrow_flight/arrow_flight_batch_reader.h b/be/src/service/arrow_flight/arrow_flight_batch_reader.h
index f72a35a8479..d8dc6618ebe 100644
--- a/be/src/service/arrow_flight/arrow_flight_batch_reader.h
+++ b/be/src/service/arrow_flight/arrow_flight_batch_reader.h
@@ -20,6 +20,7 @@
#include <cctz/time_zone.h>
#include <gen_cpp/Types_types.h>
+#include <functional>
#include <memory>
#include <utility>
@@ -48,10 +49,19 @@ class ArrowFlightBatchReaderBase : public arrow::RecordBatchReader {
public:
// RecordBatchReader force override
[[nodiscard]] std::shared_ptr<arrow::Schema> schema() const override;
+ arrow::Status Close() override;
+ arrow::Status ReadNext(std::shared_ptr<arrow::RecordBatch>* out) override;
protected:
ArrowFlightBatchReaderBase(const std::shared_ptr<QueryStatement>& statement);
~ArrowFlightBatchReaderBase() override;
+ virtual arrow::Status ReadNextImpl(std::shared_ptr<arrow::RecordBatch>* out) = 0;
+ bool is_cancelled() const;
+ void close(const Status& reason);
+ std::function<bool()> _is_cancelled;
+ std::atomic<bool> _closed {false};
+ std::atomic<bool> _eof {false};
+
arrow::Status _return_invalid_status(const std::string& msg);
std::shared_ptr<QueryStatement> _statement;
@@ -67,34 +77,33 @@ protected:
class ArrowFlightBatchLocalReader : public ArrowFlightBatchReaderBase {
public:
static arrow::Result<std::shared_ptr<ArrowFlightBatchLocalReader>> Create(
- const std::shared_ptr<QueryStatement>& statement);
-
- arrow::Status ReadNext(std::shared_ptr<arrow::RecordBatch>* out) override;
+ const std::shared_ptr<QueryStatement>& statement,
+ std::function<bool()> is_cancelled = {});
private:
ArrowFlightBatchLocalReader(const std::shared_ptr<QueryStatement>& statement,
const std::shared_ptr<arrow::Schema>& schema,
const std::shared_ptr<MemTrackerLimiter>& mem_tracker);
- arrow::Status ReadNextImpl(std::shared_ptr<arrow::RecordBatch>* out);
+ arrow::Status ReadNextImpl(std::shared_ptr<arrow::RecordBatch>* out) override;
};
class ArrowFlightBatchRemoteReader : public ArrowFlightBatchReaderBase {
public:
static arrow::Result<std::shared_ptr<ArrowFlightBatchRemoteReader>> Create(
- const std::shared_ptr<QueryStatement>& statement);
+ const std::shared_ptr<QueryStatement>& statement,
+ std::function<bool()> is_cancelled = {});
// create arrow RecordBatchReader must initialize the schema.
// so when creating arrow RecordBatchReader, fetch result data once,
// which will return Block and some necessary information, and extract arrow schema from Block.
arrow::Status init_schema();
- arrow::Status ReadNext(std::shared_ptr<arrow::RecordBatch>* out) override;
private:
ArrowFlightBatchRemoteReader(const std::shared_ptr<QueryStatement>& statement,
const std::shared_ptr<PBackendService_Stub>& stub);
- arrow::Status ReadNextImpl(std::shared_ptr<arrow::RecordBatch>* out);
+ arrow::Status ReadNextImpl(std::shared_ptr<arrow::RecordBatch>* out) override;
arrow::Status _fetch_schema();
arrow::Status _fetch_data();
diff --git a/be/src/service/arrow_flight/flight_sql_service.cpp b/be/src/service/arrow_flight/flight_sql_service.cpp
index 26129559b2f..34acc0d6008 100644
--- a/be/src/service/arrow_flight/flight_sql_service.cpp
+++ b/be/src/service/arrow_flight/flight_sql_service.cpp
@@ -87,11 +87,15 @@ public:
if (statement->result_addr.hostname == BackendOptions::get_localhost() &&
statement->result_addr.port == config::brpc_port) {
std::shared_ptr<ArrowFlightBatchLocalReader> reader;
- ARROW_ASSIGN_OR_RAISE(reader, ArrowFlightBatchLocalReader::Create(statement));
+ ARROW_ASSIGN_OR_RAISE(
+ reader, ArrowFlightBatchLocalReader::Create(
+ statement, [&context] { return context.is_cancelled(); }));
return std::make_unique<arrow::flight::RecordBatchStream>(reader);
} else {
std::shared_ptr<ArrowFlightBatchRemoteReader> reader;
- ARROW_ASSIGN_OR_RAISE(reader, ArrowFlightBatchRemoteReader::Create(statement));
+ ARROW_ASSIGN_OR_RAISE(
+ reader, ArrowFlightBatchRemoteReader::Create(
+ statement, [&context] { return context.is_cancelled(); }));
return std::make_unique<arrow::flight::RecordBatchStream>(reader);
}
}
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index 30a042f82f4..887140a97aa 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -661,6 +661,11 @@ void PInternalService::cancel_plan_fragment(google::protobuf::RpcController* /*c
LOG(INFO) << fmt::format("Cancel query {}, reason: {}", print_id(query_id),
actual_cancel_status.to_string());
_exec_env->fragment_mgr()->cancel_query(query_id, actual_cancel_status);
+ // LIMIT_REACH/FINISHED can leave valid unread results; only an abort may discard them.
+ if (!actual_cancel_status.ok() && !actual_cancel_status.is<ErrorCode::LIMIT_REACH>() &&
+ !actual_cancel_status.is<ErrorCode::FINISHED>()) {
+ _exec_env->result_mgr()->cancel_arrow_flight_buffers(query_id, actual_cancel_status);
+ }
// TODO: the logic seems useless, cancel only return Status::OK. remove it
st.to_protobuf(result->mutable_status());
@@ -694,12 +699,14 @@ void PInternalService::fetch_arrow_data(google::protobuf::RpcController* control
PFetchArrowDataResult* result,
google::protobuf::Closure* done) {
bool ret = _arrow_flight_work_pool.try_offer([request, result, done]() {
- auto ctx = GetArrowResultBatchCtx::create_shared(result, done);
TUniqueId unique_id = UniqueId(request->finst_id()).to_thrift(); // query_id or instance_id
+ auto ctx = GetArrowResultBatchCtx::create_shared(result, done);
std::shared_ptr<ArrowFlightResultBlockBuffer> arrow_buffer;
auto st = ExecEnv::GetInstance()->result_mgr()->find_buffer(unique_id, arrow_buffer);
if (!st.ok()) {
LOG(WARNING) << "Result buffer not found! Query ID: " << print_id(unique_id);
+ // A cancellation can remove the buffer before an already queued fetch runs.
+ ctx->on_failure(st);
return;
}
if (st = arrow_buffer->get_batch(ctx); !st.ok()) {
diff --git a/be/test/service/arrow_flight/arrow_flight_cancel_test.cpp b/be/test/service/arrow_flight/arrow_flight_cancel_test.cpp
new file mode 100644
index 00000000000..2e8d343bc70
--- /dev/null
+++ b/be/test/service/arrow_flight/arrow_flight_cancel_test.cpp
@@ -0,0 +1,567 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/flight/client.h>
+#include <arrow/flight/sql/server.h>
+#include <brpc/server.h>
+#include <gtest/gtest.h>
+#include <thrift/protocol/TBinaryProtocol.h>
+#include <thrift/server/TThreadedServer.h>
+#include <thrift/transport/TBufferTransports.h>
+#include <thrift/transport/TServerSocket.h>
+
+#include <future>
+#include <thread>
+
+#include "exec/pipeline/dependency.h"
+#include "load/channel/load_stream_mgr.h"
+#include "runtime/result_buffer_mgr.h"
+#include "service/arrow_flight/arrow_flight_batch_reader.h"
+#include "service/arrow_flight/flight_sql_service.h"
+#include "service/backend_options.h"
+#include "service/internal_service.h"
+#include "testutil/column_helper.h"
+#include "testutil/mock/mock_runtime_state.h"
+#include "util/brpc_client_cache.h"
+#include "util/client_cache.h"
+#include "util/dns_cache.h"
+
+namespace doris::flight {
+
+class ArrowFlightCancelTest : public testing::Test {
+protected:
+ void SetUp() override {
+ _previous_mgr = std::exchange(ExecEnv::GetInstance()->_result_mgr, &_mgr);
+ _state._batch_size = 1;
+ _id.hi = 1;
+ _id.lo = 2;
+ auto schema = arrow::schema({arrow::field("value", arrow::int64())});
+ std::shared_ptr<ResultBlockBufferBase> buffer;
+ ASSERT_TRUE(_mgr.create_sender(_id, 16, &buffer, &_state, true, schema).ok());
+ _buffer = std::dynamic_pointer_cast<ArrowFlightResultBlockBuffer>(buffer);
+ _dep = Dependency::create_shared(0, 0, "Result", true);
+ _buffer->set_dependency(_state.fragment_instance_id(), _dep);
+ TNetworkAddress address;
+ address.hostname = BackendOptions::get_localhost();
+ address.port = config::brpc_port;
+ _statement = std::make_shared<QueryStatement>(_id, address, "select value");
+ }
+ void TearDown() override { ExecEnv::GetInstance()->_result_mgr = _previous_mgr; }
+ void fill_buffer() {
+ auto block = std::make_shared<Block>(ColumnHelper::create_block<DataTypeInt64>({1, 2}));
+ ASSERT_TRUE(_buffer->add_batch(&_state, block).ok());
+ ASSERT_FALSE(_dep->ready());
+ }
+ bool registered() {
+ std::shared_ptr<ArrowFlightResultBlockBuffer> found;
+ return _mgr.find_buffer(_id, found).ok();
+ }
+ MockRuntimeState _state;
+ ResultBufferMgr _mgr;
+ ResultBufferMgr* _previous_mgr = nullptr;
+ TUniqueId _id;
+ std::shared_ptr<ArrowFlightResultBlockBuffer> _buffer;
+ std::shared_ptr<Dependency> _dep;
+ std::shared_ptr<QueryStatement> _statement;
+};
+
+TEST_F(ArrowFlightCancelTest, EarlyCloseReleasesBackpressuredBuffer) {
+ fill_buffer();
+ auto result = ArrowFlightBatchLocalReader::Create(_statement);
+ ASSERT_TRUE(result.ok()) << result.status();
+ auto reader = *result;
+ ASSERT_TRUE(reader->Close().ok());
+ EXPECT_FALSE(registered());
+ EXPECT_TRUE(_dep->ready());
+ EXPECT_TRUE(_state.get_query_ctx()->is_cancelled());
+ std::shared_ptr<Block> block;
+ bool eos = false;
+ EXPECT_FALSE(_buffer->get_arrow_batch(&block, &eos).ok());
+ EXPECT_TRUE(reader->Close().ok());
+}
+
+TEST_F(ArrowFlightCancelTest, DestructionReleasesAbandonedBuffer) {
+ fill_buffer();
+ {
+ auto reader = ArrowFlightBatchLocalReader::Create(_statement);
+ ASSERT_TRUE(reader.ok()) << reader.status();
+ }
+ EXPECT_FALSE(registered());
+ EXPECT_TRUE(_dep->ready());
+ EXPECT_TRUE(_state.get_query_ctx()->is_cancelled());
+}
+
+TEST_F(ArrowFlightCancelTest, CloseReleasesSiblingBuffersAfterContextExpires) {
+ TUniqueId sibling_id = _id;
+ ++sibling_id.lo;
+ std::shared_ptr<ResultBlockBufferBase> sibling;
+ ASSERT_TRUE(_mgr.create_sender(sibling_id, 16, &sibling, &_state, true,
+ arrow::schema({arrow::field("value", arrow::int64())}))
+ .ok());
+ auto arrow_sibling = std::dynamic_pointer_cast<ArrowFlightResultBlockBuffer>(sibling);
+ auto block = std::make_shared<Block>(ColumnHelper::create_block<DataTypeInt64>({1, 2}));
+ ASSERT_TRUE(arrow_sibling->add_batch(&_state, block).ok());
+ auto reader = ArrowFlightBatchLocalReader::Create(_statement);
+ ASSERT_TRUE(reader.ok()) << reader.status();
+ _state._query_ctx_uptr.reset();
+ _state._query_ctx = nullptr;
+ ASSERT_TRUE((*reader)->Close().ok());
+ EXPECT_FALSE(registered());
+ std::shared_ptr<ArrowFlightResultBlockBuffer> found;
+ EXPECT_FALSE(_mgr.find_buffer(sibling_id, found).ok());
+ EXPECT_TRUE(arrow_sibling->_result_batch_queue.empty());
+}
+
+TEST_F(ArrowFlightCancelTest, NormalEofIsNotCancellation) {
+ bool fully_closed = false;
+ ASSERT_TRUE(_buffer->close(_state.fragment_instance_id(), Status::OK(), 0, fully_closed).ok());
+ auto result = ArrowFlightBatchLocalReader::Create(_statement);
+ ASSERT_TRUE(result.ok()) << result.status();
+ auto reader = *result;
+ std::shared_ptr<arrow::RecordBatch> batch;
+ ASSERT_TRUE(reader->ReadNext(&batch).ok());
+ EXPECT_EQ(batch, nullptr);
+ EXPECT_TRUE(reader->Close().ok());
+ std::shared_ptr<Block> block;
+ bool eos = false;
+ EXPECT_TRUE(_buffer->get_arrow_batch(&block, &eos).ok());
+ EXPECT_TRUE(eos);
+ EXPECT_FALSE(_state.get_query_ctx()->is_cancelled());
+}
+
+TEST_F(ArrowFlightCancelTest, CancellationInterruptsEmptyBufferWait) {
+ std::atomic<bool> cancelled = false;
+ std::promise<void> checked;
+ std::atomic<int> checks = 0;
+ auto result = ArrowFlightBatchLocalReader::Create(_statement, [&] {
+ if (checks.fetch_add(1) == 3) {
+ checked.set_value();
+ }
+ return cancelled.load();
+ });
+ ASSERT_TRUE(result.ok()) << result.status();
+ auto reader = *result;
+ auto fetch = std::async(std::launch::async, [&] {
+ std::shared_ptr<arrow::RecordBatch> batch;
+ return reader->ReadNext(&batch);
+ });
+ EXPECT_EQ(checked.get_future().wait_for(std::chrono::seconds(2)), std::future_status::ready);
+ cancelled = true;
+ const auto ready = fetch.wait_for(std::chrono::seconds(2));
+ // Ensure a broken implementation fails instead of hanging the test runner.
+ if (ready != std::future_status::ready) {
+ _buffer->cancel(Status::Cancelled("test cleanup"));
+ }
+ EXPECT_EQ(ready, std::future_status::ready);
+ EXPECT_FALSE(fetch.get().ok());
+ EXPECT_FALSE(registered());
+ EXPECT_TRUE(_state.get_query_ctx()->is_cancelled());
+}
+
+TEST_F(ArrowFlightCancelTest, EmptyBufferReadReturnsForRetry) {
+ auto fetch = std::async(std::launch::async, [&] {
+ auto block = std::make_shared<Block>();
+ bool eos = true;
+ auto status = _buffer->get_arrow_batch(&block, &eos);
+ EXPECT_EQ(block, nullptr);
+ EXPECT_FALSE(eos);
+ return status;
+ });
+ const auto ready = fetch.wait_for(std::chrono::seconds(2));
+ // Bound the test even if an empty buffer incorrectly waits until query completion.
+ if (ready != std::future_status::ready) {
+ _buffer->cancel(Status::Cancelled("test cleanup"));
+ }
+ EXPECT_EQ(ready, std::future_status::ready);
+ EXPECT_TRUE(fetch.get().ok());
+}
+
+TEST_F(ArrowFlightCancelTest, EmptyWaitsDoNotFinishReader) {
+ std::promise<void> retried;
+ std::atomic<int> checks = 0;
+ auto result = ArrowFlightBatchLocalReader::Create(_statement, [&] {
+ if (checks.fetch_add(1) == 3) {
+ retried.set_value();
+ }
+ return false;
+ });
+ ASSERT_TRUE(result.ok()) << result.status();
+ auto reader = *result;
+ std::shared_ptr<arrow::RecordBatch> batch;
+ auto fetch = std::async(std::launch::async, [&] { return reader->ReadNext(&batch); });
+ EXPECT_EQ(retried.get_future().wait_for(std::chrono::seconds(2)), std::future_status::ready);
+ EXPECT_EQ(fetch.wait_for(std::chrono::seconds(0)), std::future_status::timeout);
+ auto block = std::make_shared<Block>(ColumnHelper::create_block<DataTypeInt64>({1, 2}));
+ EXPECT_TRUE(_buffer->add_batch(&_state, block).ok());
+ bool fully_closed = false;
+ EXPECT_TRUE(_buffer->close(_state.fragment_instance_id(), Status::OK(), 2, fully_closed).ok());
+ const auto ready = fetch.wait_for(std::chrono::seconds(2));
+ if (ready != std::future_status::ready) {
+ _buffer->cancel(Status::Cancelled("test cleanup"));
+ }
+ EXPECT_EQ(ready, std::future_status::ready);
+ ASSERT_TRUE(fetch.get().ok());
+ ASSERT_NE(batch, nullptr);
+ EXPECT_EQ(batch->num_rows(), 2);
+ ASSERT_TRUE(reader->ReadNext(&batch).ok());
+ EXPECT_EQ(batch, nullptr);
+ EXPECT_TRUE(reader->Close().ok());
+ EXPECT_FALSE(_state.get_query_ctx()->is_cancelled());
+}
+
+TEST_F(ArrowFlightCancelTest, ConversionFailureCancelsQuery) {
+ auto block = std::make_shared<Block>(ColumnHelper::create_block<DataTypeString>({"bad"}));
+ ASSERT_TRUE(_buffer->add_batch(&_state, block).ok());
+ auto result = ArrowFlightBatchLocalReader::Create(_statement);
+ ASSERT_TRUE(result.ok()) << result.status();
+ std::shared_ptr<arrow::RecordBatch> batch;
+ EXPECT_FALSE((*result)->ReadNext(&batch).ok());
+ EXPECT_FALSE(registered());
+ EXPECT_TRUE(_state.get_query_ctx()->is_cancelled());
+}
+
+TEST_F(ArrowFlightCancelTest, RealFlightRpcCancellationCleansUpQuery) {
+ auto created = FlightSqlServer::create();
+ ASSERT_TRUE(created.ok()) << created.status();
+ auto server = *created;
+ ASSERT_TRUE(server->init(0).ok());
+ auto location = arrow::flight::Location::ForGrpcTcp("127.0.0.1", server->port());
+ ASSERT_TRUE(location.ok()) << location.status();
+ auto client_options = arrow::flight::FlightClientOptions::Defaults();
+ client_options.generic_options.emplace_back("grpc.enable_http_proxy", 0);
+ auto connected = arrow::flight::FlightClient::Connect(*location, client_options);
+ ASSERT_TRUE(connected.ok()) << connected.status();
+ auto client = std::move(*connected);
+ auto handle = arrow::flight::sql::CreateStatementQueryTicket(
+ print_id(_id) + "&" + BackendOptions::get_localhost() + "&" +
+ std::to_string(config::brpc_port) + "&");
+ ASSERT_TRUE(handle.ok()) << handle.status();
+ arrow::flight::FlightCallOptions options;
+ options.timeout = std::chrono::seconds(5);
+ auto fetched = client->DoGet(options, arrow::flight::Ticket {*handle});
+ ASSERT_TRUE(fetched.ok()) << fetched.status();
+ auto reader = std::move(*fetched);
+ reader->Cancel();
+ for (int i = 0; i < 200 && (registered() || !_state.get_query_ctx()->is_cancelled()); ++i) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ }
+ EXPECT_FALSE(registered());
+ EXPECT_TRUE(_state.get_query_ctx()->is_cancelled());
+ _buffer->cancel(Status::Cancelled("test cleanup"));
+ reader.reset();
+ EXPECT_TRUE(client->Close().ok());
+ EXPECT_TRUE(server->join().ok());
+}
+
+class FlightCancelCoordinator : public FrontendServiceNull {
+public:
+ void cancelFlightQuery(TStatus& result, const TUniqueId& result_id) override {
+ if (++calls <= rejections) {
+ result.__set_status_code(TStatusCode::CANCELLED);
+ return;
+ }
+ if (result_id != buffer_id) {
+ result.__set_status_code(TStatusCode::NOT_FOUND);
+ return;
+ }
+ PCancelPlanFragmentRequest request;
+ request.mutable_finst_id()->set_hi(result_id.hi);
+ request.mutable_finst_id()->set_lo(result_id.lo);
+ request.mutable_query_id()->set_hi(query_id.hi);
+ request.mutable_query_id()->set_lo(query_id.lo);
+ Status::Cancelled("Flight stream aborted").to_protobuf(request.mutable_cancel_status());
+ PCancelPlanFragmentResult response;
+ brpc::Controller controller;
+ controller.set_timeout_ms(1000);
+ stub->cancel_plan_fragment(&controller, &request, &response, nullptr);
+ if (controller.Failed()) {
+ Status::RpcError(controller.ErrorText()).to_thrift(&result);
+ } else {
+ Status::create(response.status()).to_thrift(&result);
+ }
+ }
+ TUniqueId buffer_id;
+ TUniqueId query_id;
+ std::shared_ptr<PBackendService_Stub> stub;
+ std::atomic<int> calls {0};
+ int rejections = 0;
+};
+
+class FlightCancelServerReady : public apache::thrift::server::TServerEventHandler {
+public:
+ void preServe() override { ready.set_value(); }
+ std::promise<void> ready;
+};
+
+class FlightCancelResultService : public PInternalService {
+public:
+ explicit FlightCancelResultService(ExecEnv* env) : PInternalService(env) {}
+
+ void cancel_plan_fragment(google::protobuf::RpcController* controller,
+ const PCancelPlanFragmentRequest* request,
+ PCancelPlanFragmentResult* result,
+ google::protobuf::Closure* done) override {
+ if (!legacy_cancel) {
+ PInternalService::cancel_plan_fragment(controller, request, result, done);
+ return;
+ }
+ brpc::ClosureGuard guard(done);
+ // Emulate the pre-change handler: only the existing query cancellation RPC is understood.
+ _exec_env->fragment_mgr()->cancel_query(UniqueId(request->query_id()).to_thrift(),
+ Status::create(request->cancel_status()));
+ Status::OK().to_protobuf(result->mutable_status());
+ }
+ bool legacy_cancel = false;
+};
+
+class ArrowFlightRemoteCancelTest : public ArrowFlightCancelTest {
+protected:
+ void SetUp() override {
+ ArrowFlightCancelTest::SetUp();
+ static DNSCache dns_cache;
+ _previous_dns_cache = std::exchange(ExecEnv::GetInstance()->_dns_cache, &dns_cache);
+ _previous_cache = std::exchange(ExecEnv::GetInstance()->_internal_client_cache, &_cache);
+ auto heavy = std::exchange(config::brpc_heavy_work_pool_threads, 1);
+ auto light = std::exchange(config::brpc_light_work_pool_threads, 1);
+ auto flight = std::exchange(config::brpc_arrow_flight_work_pool_threads, 2);
+ _previous_load_mgr = std::move(ExecEnv::GetInstance()->_load_stream_mgr);
+ ExecEnv::GetInstance()->_load_stream_mgr = std::make_unique<LoadStreamMgr>(1);
+ _service = std::make_unique<FlightCancelResultService>(ExecEnv::GetInstance());
+ config::brpc_heavy_work_pool_threads = heavy;
+ config::brpc_light_work_pool_threads = light;
+ config::brpc_arrow_flight_work_pool_threads = flight;
+ ASSERT_EQ(_server.AddService(_service.get(), brpc::SERVER_DOESNT_OWN_SERVICE), 0);
+ ASSERT_EQ(_server.Start("127.0.0.1:0", nullptr), 0);
+ _statement->result_addr.hostname = "127.0.0.1";
+ _statement->result_addr.port = _server.listen_address().port;
+ _fragment_mgr =
+ std::make_unique<MockFragmentManager>(_cancel_status, ExecEnv::GetInstance());
+ _fragment_mgr->stop();
+ _previous_fragment_mgr =
+ std::exchange(ExecEnv::GetInstance()->_fragment_mgr, _fragment_mgr.get());
+ _frontend_cache = std::make_unique<ClientCache<FrontendServiceClient>>();
+ _previous_frontend_cache = std::exchange(ExecEnv::GetInstance()->_frontend_client_cache,
+ _frontend_cache.get());
+ _coordinator = std::make_shared<FlightCancelCoordinator>();
+ _coordinator->buffer_id = _id;
+ _coordinator->query_id = _state.query_id();
+ _coordinator->stub = _cache.get_client(_statement->result_addr);
+ auto processor = std::make_shared<FrontendServiceProcessor>(_coordinator);
+ auto socket = std::make_shared<apache::thrift::transport::TServerSocket>("127.0.0.1", 0);
+ _fe_server = std::make_unique<apache::thrift::server::TThreadedServer>(
+ processor, socket,
+ std::make_shared<apache::thrift::transport::TBufferedTransportFactory>(),
+ std::make_shared<apache::thrift::protocol::TBinaryProtocolFactory>());
+ auto ready = std::make_shared<FlightCancelServerReady>();
+ _fe_server->setServerEventHandler(ready);
+ _fe_thread = std::thread([this] { _fe_server->serve(); });
+ ASSERT_EQ(ready->ready.get_future().wait_for(std::chrono::seconds(5)),
+ std::future_status::ready);
+ TFrontendInfo frontend;
+ frontend.coordinator_address.hostname = "127.0.0.1";
+ frontend.coordinator_address.port = socket->getPort();
+ _previous_frontends = ExecEnv::GetInstance()->_frontends;
+ ExecEnv::GetInstance()->update_frontends({frontend});
+ }
+ void TearDown() override {
+ _frontend_cache.reset();
+ if (_fe_server) {
+ _fe_server->stop();
+ _fe_thread.join();
+ }
+ ExecEnv::GetInstance()->_dns_cache = _previous_dns_cache;
+ ExecEnv::GetInstance()->_frontend_client_cache = _previous_frontend_cache;
+ ExecEnv::GetInstance()->_frontends = _previous_frontends;
+ ExecEnv::GetInstance()->_fragment_mgr = _previous_fragment_mgr;
+ _buffer->cancel(Status::Cancelled("test cleanup"));
+ _server.Stop(0);
+ _server.Join();
+ _service.reset();
+ ExecEnv::GetInstance()->_load_stream_mgr = std::move(_previous_load_mgr);
+ ExecEnv::GetInstance()->_internal_client_cache = _previous_cache;
+ ArrowFlightCancelTest::TearDown();
+ }
+ BrpcClientCache<PBackendService_Stub> _cache;
+ BrpcClientCache<PBackendService_Stub>* _previous_cache = nullptr;
+ std::unique_ptr<LoadStreamMgr> _previous_load_mgr;
+ std::unique_ptr<FlightCancelResultService> _service;
+ brpc::Server _server;
+ DNSCache* _previous_dns_cache = nullptr;
+ Status _cancel_status;
+ std::unique_ptr<MockFragmentManager> _fragment_mgr;
+ FragmentMgr* _previous_fragment_mgr = nullptr;
+ std::unique_ptr<ClientCache<FrontendServiceClient>> _frontend_cache;
+ ClientCache<FrontendServiceClient>* _previous_frontend_cache = nullptr;
+ std::shared_ptr<FlightCancelCoordinator> _coordinator;
+ std::unique_ptr<apache::thrift::server::TThreadedServer> _fe_server;
+ std::thread _fe_thread;
+ std::map<TNetworkAddress, FrontendInfo> _previous_frontends;
+};
+
+TEST_F(ArrowFlightRemoteCancelTest, CloseReachesResultBackend) {
+ fill_buffer();
+ auto result = ArrowFlightBatchRemoteReader::Create(_statement);
+ ASSERT_TRUE(result.ok()) << result.status();
+ EXPECT_TRUE((*result)->Close().ok());
+ EXPECT_FALSE(registered());
+ EXPECT_TRUE(_dep->ready());
+ EXPECT_TRUE(_state.get_query_ctx()->is_cancelled());
+ EXPECT_TRUE((*result)->Close().ok());
+}
+
+TEST_F(ArrowFlightRemoteCancelTest, CancelBypassesRejectedArrowWork) {
+ fill_buffer();
+ auto reader = ArrowFlightBatchRemoteReader::Create(_statement);
+ ASSERT_TRUE(reader.ok()) << reader.status();
+ _service->_arrow_flight_work_pool.shutdown();
+ EXPECT_FALSE(_service->_arrow_flight_work_pool.try_offer([] {}));
+ ASSERT_TRUE((*reader)->Close().ok());
+ EXPECT_FALSE(registered());
+ EXPECT_TRUE(_dep->ready());
+ EXPECT_TRUE(_state.get_query_ctx()->is_cancelled());
+ EXPECT_EQ(_coordinator->calls, 1);
+}
+
+TEST_F(ArrowFlightRemoteCancelTest, RetryCoordinatorApplicationError) {
+ fill_buffer();
+ auto reader = ArrowFlightBatchRemoteReader::Create(_statement);
+ ASSERT_TRUE(reader.ok()) << reader.status();
+ _coordinator->rejections = 1;
+ ASSERT_TRUE((*reader)->Close().ok());
+ EXPECT_EQ(_coordinator->calls, 2);
+ EXPECT_FALSE(registered());
+ EXPECT_TRUE(_state.get_query_ctx()->is_cancelled());
+}
+
+TEST_F(ArrowFlightRemoteCancelTest, OlderResultBackendReceivesQueryCancellation) {
+ fill_buffer();
+ auto reader = ArrowFlightBatchRemoteReader::Create(_statement);
+ ASSERT_TRUE(reader.ok()) << reader.status();
+ _service->legacy_cancel = true;
+ ASSERT_TRUE((*reader)->Close().ok());
+ EXPECT_EQ(_coordinator->calls, 1);
+ EXPECT_FALSE(_cancel_status.ok());
+ // Older result BEs still own their historical buffer cleanup policy.
+ EXPECT_TRUE(registered());
+}
+
+TEST_F(ArrowFlightRemoteCancelTest, FinishedLocalEndpointStillReportsAbort) {
+ auto statement = std::make_shared<QueryStatement>(*_statement);
+ statement->result_addr.hostname = BackendOptions::get_localhost();
+ statement->result_addr.port = config::brpc_port;
+ fill_buffer();
+ bool fully_closed = false;
+ ASSERT_TRUE(_buffer->close(_state.fragment_instance_id(), Status::OK(), 2, fully_closed).ok());
+ auto reader = ArrowFlightBatchLocalReader::Create(statement);
+ ASSERT_TRUE(reader.ok()) << reader.status();
+ std::shared_ptr<arrow::RecordBatch> batch;
+ ASSERT_TRUE((*reader)->ReadNext(&batch).ok());
+ ASSERT_NE(batch, nullptr);
+ _state._query_ctx_uptr.reset();
+ _state._query_ctx = nullptr;
+ ASSERT_TRUE((*reader)->Close().ok());
+ EXPECT_EQ(_coordinator->calls, 1);
+ EXPECT_FALSE(_cancel_status.ok());
+ EXPECT_FALSE(registered());
+}
+
+TEST_F(ArrowFlightRemoteCancelTest, CancellationInterruptsPendingBrpcFetch) {
+ std::atomic<bool> cancelled = false;
+ auto result =
+ ArrowFlightBatchRemoteReader::Create(_statement, [&] { return cancelled.load(); });
+ ASSERT_TRUE(result.ok()) << result.status();
+ auto fetch = std::async(std::launch::async, [&] {
+ std::shared_ptr<arrow::RecordBatch> batch;
+ return (*result)->ReadNext(&batch);
+ });
+ // Wait until the server has queued the fetch; cancelling before that misses the blocking path.
+ bool waiting = false;
+ for (int i = 0; i < 200; ++i) {
+ {
+ std::lock_guard lock(_buffer->_lock);
+ waiting = !_buffer->_waiting_rpc.empty();
+ }
+ if (waiting) {
+ break;
+ }
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ }
+ EXPECT_TRUE(waiting);
+ cancelled = true;
+ auto ready = fetch.wait_for(std::chrono::seconds(3));
+ if (ready != std::future_status::ready) {
+ _buffer->cancel(Status::Cancelled("test cleanup"));
+ }
+ EXPECT_EQ(ready, std::future_status::ready);
+ EXPECT_FALSE(fetch.get().ok());
+ EXPECT_FALSE(registered());
+ EXPECT_TRUE(_state.get_query_ctx()->is_cancelled());
+}
+
+TEST_F(ArrowFlightRemoteCancelTest, FetchAfterCancellationStillCompletesRpc) {
+ auto reader = ArrowFlightBatchRemoteReader::Create(_statement);
+ ASSERT_TRUE(reader.ok()) << reader.status();
+ ASSERT_TRUE((*reader)->Close().ok());
+ auto stub = _cache.get_client(_statement->result_addr);
+ ASSERT_NE(stub, nullptr);
+ PFetchArrowDataRequest request;
+ request.mutable_finst_id()->set_hi(_id.hi);
+ request.mutable_finst_id()->set_lo(_id.lo);
+ PFetchArrowDataResult response;
+ brpc::Controller controller;
+ controller.set_timeout_ms(1000);
+ stub->fetch_arrow_data(&controller, &request, &response, nullptr);
+ EXPECT_FALSE(controller.Failed()) << controller.ErrorText();
+ EXPECT_TRUE(response.has_status());
+ EXPECT_FALSE(Status::create(response.status()).ok());
+}
+
+TEST_F(ArrowFlightRemoteCancelTest, SuccessfulFragmentStopPreservesUnreadResults) {
+ fill_buffer();
+ auto stub = _cache.get_client(_statement->result_addr);
+ for (const auto& reason : {Status::Error<ErrorCode::LIMIT_REACH>("limit reached"),
+ Status::Error<ErrorCode::FINISHED>("finished")}) {
+ PCancelPlanFragmentRequest request;
+ request.mutable_finst_id()->set_hi(_id.hi);
+ request.mutable_finst_id()->set_lo(_id.lo);
+ request.mutable_query_id()->set_hi(_state.query_id().hi);
+ request.mutable_query_id()->set_lo(_state.query_id().lo);
+ reason.to_protobuf(request.mutable_cancel_status());
+ PCancelPlanFragmentResult response;
+ brpc::Controller controller;
+ controller.set_timeout_ms(1000);
+ stub->cancel_plan_fragment(&controller, &request, &response, nullptr);
+ ASSERT_FALSE(controller.Failed()) << controller.ErrorText();
+ EXPECT_TRUE(Status::create(response.status()).ok());
+ EXPECT_TRUE(registered());
+ EXPECT_FALSE(_buffer->_result_batch_queue.empty());
+ }
+}
+
+TEST_F(ArrowFlightRemoteCancelTest, NormalEofDoesNotCancelQuery) {
+ bool fully_closed = false;
+ ASSERT_TRUE(_buffer->close(_state.fragment_instance_id(), Status::OK(), 0, fully_closed).ok());
+ auto result = ArrowFlightBatchRemoteReader::Create(_statement);
+ ASSERT_TRUE(result.ok()) << result.status();
+ std::shared_ptr<arrow::RecordBatch> batch;
+ EXPECT_TRUE((*result)->ReadNext(&batch).ok());
+ EXPECT_EQ(batch, nullptr);
+ EXPECT_TRUE((*result)->Close().ok());
+ EXPECT_FALSE(_state.get_query_ctx()->is_cancelled());
+}
+
+} // namespace doris::flight
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index c582dccf7f9..c2019d05d26 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -3828,6 +3828,11 @@ public class Coordinator implements CoordInterface {
return result;
}
+ public List<TNetworkAddress> getBackendBrpcAddresses() {
+ return beToPipelineExecCtxs.values().stream().map(ctx -> ctx.brpcAddr.deepCopy())
+ .collect(Collectors.toList());
+ }
+
@Override
public List<TNetworkAddress> getInvolvedBackends() {
List<TNetworkAddress> backendAddresses = Lists.newArrayList();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
index 146df08108c..2dc02abc697 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
@@ -359,6 +359,12 @@ public class NereidsCoordinator extends Coordinator {
return coordinatorContext.asLoadProcessor().loadContext.getErrorTabletInfos();
}
+ @Override
+ public List<TNetworkAddress> getBackendBrpcAddresses() {
+ return executionTask.getChildrenTasks().values().stream()
+ .map(task -> task.getBackend().getBrpcAddress().deepCopy()).collect(Collectors.toList());
+ }
+
@Override
public List<TNetworkAddress> getInvolvedBackends() {
return Utils.fastToImmutableList(coordinatorContext.backends.get().keySet());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index d8e228e65e8..1919141ec00 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -1458,6 +1458,12 @@ public class StmtExecutor {
if (context.getConnectType().equals(ConnectType.ARROW_FLIGHT_SQL)) {
Preconditions.checkState(!context.isReturnResultFromLocal());
profile.getSummaryProfile().setTempStartTime();
+ if (coordBase == coord) {
+ context.getFlightSqlChannel().registerRemoteQuery(coord.getQueryId(),
+ context.getFlightSqlEndpointsLocations().stream()
+ .map(endpoint -> endpoint.getFinstId()).distinct().collect(Collectors.toList()),
+ coord.getBackendBrpcAddresses(), coord.getQueryOptions().getExecutionTimeout());
+ }
// The client pulls the results from the BE later (DoGet). Only an external-table
// scan in batch mode still needs the coordinator after this point: the BE fetches
// its splits lazily from the split source the coordinator holds, so closing the
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 54101ad5640..14f38a680e4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -141,6 +141,7 @@ import org.apache.doris.qe.VariableMgr;
import org.apache.doris.resource.BackendSelection;
import org.apache.doris.resource.BackendSelectionManager;
import org.apache.doris.service.arrowflight.FlightSqlConnectProcessor;
+import org.apache.doris.service.arrowflight.FlightSqlQueryCancellation;
import org.apache.doris.statistics.AnalysisManager;
import org.apache.doris.statistics.ColStatsData;
import org.apache.doris.statistics.ColumnStatistic;
@@ -1092,6 +1093,11 @@ public class FrontendServiceImpl implements FrontendService.Iface {
return QeProcessorImpl.INSTANCE.reportExecStatus(params, getClientAddr());
}
+ @Override
+ public TStatus cancelFlightQuery(TUniqueId resultId) throws TException {
+ return FlightSqlQueryCancellation.INSTANCE.cancel(resultId);
+ }
+
@Override
public TFetchSplitBatchResult fetchSplitBatch(TFetchSplitBatchRequest request) throws TException {
TFetchSplitBatchResult result = new TFetchSplitBatchResult();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQueryCancellation.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQueryCancellation.java
new file mode 100644
index 00000000000..4fb006e774c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQueryCancellation.java
@@ -0,0 +1,146 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service.arrowflight;
+
+import org.apache.doris.common.Status;
+import org.apache.doris.proto.InternalService.PCancelPlanFragmentResult;
+import org.apache.doris.rpc.BackendServiceProxy;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TStatus;
+import org.apache.doris.thrift.TStatusCode;
+import org.apache.doris.thrift.TUniqueId;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.Expiry;
+import com.github.benmanes.caffeine.cache.Scheduler;
+import com.github.benmanes.caffeine.cache.Ticker;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+public final class FlightSqlQueryCancellation {
+ private static final Logger LOG = LogManager.getLogger(FlightSqlQueryCancellation.class);
+ public static final FlightSqlQueryCancellation INSTANCE = new FlightSqlQueryCancellation(Ticker.systemTicker());
+ private final Cache<TUniqueId, Route> results;
+
+ FlightSqlQueryCancellation(Ticker ticker) {
+ results = Caffeine.newBuilder().ticker(ticker).scheduler(Scheduler.systemScheduler())
+ .expireAfter(new Expiry<TUniqueId, Route>() {
+ @Override
+ public long expireAfterCreate(TUniqueId key, Route route, long now) {
+ return route.ttlNanos;
+ }
+
+ @Override
+ public long expireAfterUpdate(TUniqueId key, Route route, long now, long duration) {
+ return route.ttlNanos;
+ }
+
+ @Override
+ public long expireAfterRead(TUniqueId key, Route route, long now, long duration) {
+ return duration;
+ }
+ }).build();
+ }
+
+ public void register(TUniqueId queryId, List<TUniqueId> resultIds,
+ List<TNetworkAddress> backends, int timeoutSeconds) {
+ if (backends.isEmpty()) {
+ throw new IllegalArgumentException("Flight query has no cancellation backends");
+ }
+ // Keep only cancellation addresses, not the coordinator, scan state, or query queue slot.
+ // Ordinary Flight coordinators are unregistered when GetFlightInfo returns.
+ Route route = new Route(queryId.deepCopy(),
+ resultIds.stream().map(TUniqueId::deepCopy).distinct().collect(Collectors.toList()),
+ backends.stream().map(TNetworkAddress::deepCopy).distinct().collect(Collectors.toList()),
+ TimeUnit.SECONDS.toNanos(Math.max(0L, timeoutSeconds) + 5));
+ for (TUniqueId resultId : route.resultIds) {
+ results.put(resultId, route);
+ }
+ }
+
+ public void unregister(List<TUniqueId> resultIds) {
+ results.invalidateAll(resultIds);
+ }
+
+ public TStatus cancel(TUniqueId resultId) {
+ Route route = results.getIfPresent(resultId);
+ if (route == null) {
+ return new TStatus(TStatusCode.NOT_FOUND);
+ }
+ Status reason = new Status(TStatusCode.CANCELLED, "Arrow Flight stream closed before EOF");
+ TStatus status = new TStatus(TStatusCode.OK);
+ List<Future<PCancelPlanFragmentResult>> futures = new ArrayList<>();
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1);
+ for (TNetworkAddress backend : route.backends) {
+ try {
+ // The existing RPC is understood by older result BEs and bypasses the Flight read pool.
+ futures.add(BackendServiceProxy.getInstance()
+ .cancelPipelineXPlanFragmentAsync(backend, route.queryId, reason));
+ } catch (Exception e) {
+ LOG.warn("Failed to send Flight cancellation for {} to {}", route.queryId, backend, e);
+ status = new TStatus(TStatusCode.INTERNAL_ERROR);
+ }
+ }
+ for (Future<PCancelPlanFragmentResult> future : futures) {
+ try {
+ PCancelPlanFragmentResult response = future.get(Math.max(0L, deadline - System.nanoTime()),
+ TimeUnit.NANOSECONDS);
+ if (!response.hasStatus()) {
+ status = new TStatus(TStatusCode.INTERNAL_ERROR);
+ } else if (response.getStatus().getStatusCode() != TStatusCode.OK.getValue()) {
+ status = new TStatus(new Status(response.getStatus()).getErrorCode());
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return new TStatus(TStatusCode.CANCELLED);
+ } catch (Exception e) {
+ LOG.warn("Failed to complete Flight cancellation for {}", route.queryId, e);
+ status = new TStatus(TStatusCode.INTERNAL_ERROR);
+ }
+ }
+ if (status.getStatusCode() == TStatusCode.OK) {
+ // An unsuccessful attempt must remain routable for the caller's bounded retry.
+ for (TUniqueId id : route.resultIds) {
+ results.asMap().remove(id, route);
+ }
+ }
+ return status;
+ }
+
+ private static final class Route {
+ private final TUniqueId queryId;
+ private final List<TUniqueId> resultIds;
+ private final List<TNetworkAddress> backends;
+ private final long ttlNanos;
+
+ private Route(TUniqueId queryId, List<TUniqueId> resultIds,
+ List<TNetworkAddress> backends, long ttlNanos) {
+ this.queryId = queryId;
+ this.resultIds = resultIds;
+ this.backends = backends;
+ this.ttlNanos = ttlNanos;
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java
index 2781994dfa1..6fe8369bbe4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java
@@ -21,6 +21,9 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.ResultSet;
import org.apache.doris.qe.ResultSetMetaData;
+import org.apache.doris.service.arrowflight.FlightSqlQueryCancellation;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TUniqueId;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
@@ -44,6 +47,7 @@ import java.util.concurrent.TimeUnit;
public class FlightSqlChannel {
private final Cache<String, FlightSqlResultCacheEntry> resultCache;
private final BufferAllocator allocator;
+ private final List<TUniqueId> remoteResultIds = new ArrayList<>();
public FlightSqlChannel() {
// The Stmt result is not picked up by the Client within 10 minutes and will be deleted.
@@ -162,8 +166,16 @@ public class FlightSqlChannel {
return allocator.getAllocatedMemory();
}
- public void reset() {
+ public synchronized void registerRemoteQuery(TUniqueId queryId, List<TUniqueId> resultIds,
+ List<TNetworkAddress> backends, int timeoutSeconds) {
+ FlightSqlQueryCancellation.INSTANCE.register(queryId, resultIds, backends, timeoutSeconds);
+ remoteResultIds.addAll(resultIds);
+ }
+
+ public synchronized void reset() {
resultCache.invalidateAll();
+ FlightSqlQueryCancellation.INSTANCE.unregister(remoteResultIds);
+ remoteResultIds.clear();
}
public void close() {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlQueryCancellationTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlQueryCancellationTest.java
new file mode 100644
index 00000000000..b6a8d9362cf
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlQueryCancellationTest.java
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service.arrowflight;
+
+import org.apache.doris.proto.InternalService.PCancelPlanFragmentResult;
+import org.apache.doris.proto.Types.PStatus;
+import org.apache.doris.rpc.BackendServiceProxy;
+import org.apache.doris.service.arrowflight.results.FlightSqlChannel;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TStatusCode;
+import org.apache.doris.thrift.TUniqueId;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.Futures;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class FlightSqlQueryCancellationTest {
+ private static final TUniqueId QUERY = new TUniqueId(10L, 20L);
+ private static final TUniqueId FIRST = new TUniqueId(10L, 21L);
+ private static final TUniqueId SECOND = new TUniqueId(10L, 22L);
+ private static final TNetworkAddress FINISHED_BE = new TNetworkAddress("127.0.0.1", 8001);
+ private static final TNetworkAddress ACTIVE_BE = new TNetworkAddress("127.0.0.1", 8002);
+
+ private static PCancelPlanFragmentResult response(TStatusCode status) {
+ return PCancelPlanFragmentResult.newBuilder()
+ .setStatus(PStatus.newBuilder().setStatusCode(status.getValue())).build();
+ }
+
+ @Test
+ public void cancelReachesAllBackendsWithoutLiveCoordinatorOrResultFragment() throws Exception {
+ FlightSqlQueryCancellation routes = new FlightSqlQueryCancellation(() -> 0L);
+ // Use Guava collection factories because FE tests compile against the Java 8 API.
+ // The selected endpoint and its coordinator have finished; only routing metadata survives.
+ routes.register(QUERY, ImmutableList.of(FIRST, SECOND), ImmutableList.of(FINISHED_BE, ACTIVE_BE), 120);
+ BackendServiceProxy proxy = Mockito.mock(BackendServiceProxy.class);
+ Mockito.when(proxy.cancelPipelineXPlanFragmentAsync(Mockito.any(), Mockito.eq(QUERY), Mockito.any()))
+ .thenReturn(Futures.immediateFuture(response(TStatusCode.OK)));
+ try (MockedStatic<BackendServiceProxy> mocked = Mockito.mockStatic(BackendServiceProxy.class)) {
+ mocked.when(BackendServiceProxy::getInstance).thenReturn(proxy);
+ Assert.assertEquals(TStatusCode.OK, routes.cancel(FIRST).getStatusCode());
+ Mockito.verify(proxy).cancelPipelineXPlanFragmentAsync(Mockito.eq(FINISHED_BE),
+ Mockito.eq(QUERY), Mockito.any());
+ Mockito.verify(proxy).cancelPipelineXPlanFragmentAsync(Mockito.eq(ACTIVE_BE),
+ Mockito.eq(QUERY), Mockito.any());
+ }
+ Assert.assertEquals(TStatusCode.NOT_FOUND, routes.cancel(SECOND).getStatusCode());
+ }
+
+ @Test
+ public void applicationErrorKeepsRouteForRetry() throws Exception {
+ FlightSqlQueryCancellation routes = new FlightSqlQueryCancellation(() -> 0L);
+ routes.register(QUERY, ImmutableList.of(FIRST), ImmutableList.of(ACTIVE_BE), 120);
+ BackendServiceProxy proxy = Mockito.mock(BackendServiceProxy.class);
+ Mockito.when(proxy.cancelPipelineXPlanFragmentAsync(Mockito.any(), Mockito.eq(QUERY), Mockito.any()))
+ .thenReturn(Futures.immediateFuture(response(TStatusCode.CANCELLED)),
+ Futures.immediateFuture(response(TStatusCode.OK)));
+ try (MockedStatic<BackendServiceProxy> mocked = Mockito.mockStatic(BackendServiceProxy.class)) {
+ mocked.when(BackendServiceProxy::getInstance).thenReturn(proxy);
+ Assert.assertEquals(TStatusCode.CANCELLED, routes.cancel(FIRST).getStatusCode());
+ Assert.assertEquals(TStatusCode.OK, routes.cancel(FIRST).getStatusCode());
+ }
+ }
+
+ @Test
+ public void routesExpireAfterExecutionTimeout() {
+ AtomicLong clock = new AtomicLong();
+ FlightSqlQueryCancellation routes = new FlightSqlQueryCancellation(clock::get);
+ routes.register(QUERY, ImmutableList.of(FIRST, SECOND), ImmutableList.of(ACTIVE_BE), 120);
+ clock.set(TimeUnit.SECONDS.toNanos(126));
+ Assert.assertEquals(TStatusCode.NOT_FOUND, routes.cancel(FIRST).getStatusCode());
+ Assert.assertEquals(TStatusCode.NOT_FOUND, routes.cancel(SECOND).getStatusCode());
+ }
+
+ @Test
+ public void channelCleanupRemovesItsRoutes() {
+ FlightSqlChannel channel = new FlightSqlChannel();
+ try {
+ channel.registerRemoteQuery(QUERY, ImmutableList.of(FIRST, SECOND), ImmutableList.of(ACTIVE_BE), 120);
+ channel.reset();
+ Assert.assertEquals(TStatusCode.NOT_FOUND,
+ FlightSqlQueryCancellation.INSTANCE.cancel(FIRST).getStatusCode());
+ Assert.assertEquals(TStatusCode.NOT_FOUND,
+ FlightSqlQueryCancellation.INSTANCE.cancel(SECOND).getStatusCode());
+ } finally {
+ channel.close();
+ }
+ }
+}
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index 2f5a305a239..afccc6ce002 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -1935,4 +1935,7 @@ service FrontendService {
TGetOlapTableMetaResult getOlapTableMeta(1: TGetOlapTableMetaRequest request)
Status.TStatus syncCloudTabletStats(1: TSyncCloudTabletStatsRequest request)
+
+ // Result buffers can outlive their local fragments; abort through the owning FE's query-level route.
+ Status.TStatus cancelFlightQuery(1: Types.TUniqueId result_id)
}
diff --git a/regression-test/suites/arrow_flight_sql_p0/test_flight_cancel_cleanup.groovy b/regression-test/suites/arrow_flight_sql_p0/test_flight_cancel_cleanup.groovy
new file mode 100644
index 00000000000..aff57aeac4d
--- /dev/null
+++ b/regression-test/suites/arrow_flight_sql_p0/test_flight_cancel_cleanup.groovy
@@ -0,0 +1,128 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.arrow.driver.jdbc.shaded.com.google.protobuf.Any
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightClient
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.Location
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.sql.FlightSqlClient
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.sql.impl.FlightSql
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.memory.RootAllocator
+
+suite("test_flight_cancel_cleanup", "arrow_flight_sql") {
+ def frontend = jdbc_sql_return_maparray("SHOW FRONTENDS").find {
+ it.IsMaster.toString().equalsIgnoreCase("true") && it.Alive.toString().equalsIgnoreCase("true")
+ }
+ assertNotNull(frontend)
+ assertTrue(frontend.ArrowFlightSqlPort.toString().toInteger() > 0)
+ def backends = jdbc_sql_return_maparray("SHOW BACKENDS").findAll {
+ it.Alive.toString().equalsIgnoreCase("true")
+ }
+ assertTrue(!backends.isEmpty())
+ def database = jdbc_sql("SELECT DATABASE()")[0][0]
+ def table = "${database}.flight_cancel_cleanup_source"
+ def allocator = new RootAllocator(Long.MAX_VALUE)
+ def feClient = FlightClient.builder(allocator,
+ Location.forGrpcInsecure(frontend.Host.toString(), frontend.ArrowFlightSqlPort.toString().toInteger())).build()
+ def client = new FlightSqlClient(feClient)
+ def auth
+ def consume = { String query ->
+ def info = client.execute(query, auth)
+ def rows = 0
+ info.endpoints.each { endpoint ->
+ FlightClient.builder(allocator, endpoint.locations[0]).build().withCloseable { beClient ->
+ beClient.getStream(endpoint.ticket, auth).withCloseable { stream ->
+ while (stream.next()) {
+ rows += stream.root.rowCount
+ }
+ }
+ }
+ }
+ rows
+ }
+ try {
+ auth = feClient.authenticateBasicToken(context.config.otherConfigs.get("extArrowFlightSqlUser"),
+ context.config.otherConfigs.get("extArrowFlightSqlPassword")).get()
+ consume("SET enable_sql_cache=false")
+ // The ticket is a query id in parallel mode, so inspect this query instead of global task counts.
+ consume("SET enable_parallel_result_sink=true")
+ consume("SET query_timeout=120")
+ jdbc_sql("DROP TABLE IF EXISTS ${table}")
+ jdbc_sql("CREATE TABLE ${table} (id BIGINT) DISTRIBUTED BY HASH(id) BUCKETS 8 " +
+ "PROPERTIES(\"replication_num\"=\"1\")")
+ jdbc_sql("INSERT INTO ${table} SELECT number FROM numbers(\"number\"=\"1024\")")
+ def tabletBackends = jdbc_sql_return_maparray("SHOW TABLETS FROM ${table}")
+ .collect { it.BackendId }.unique().size()
+ [false, true].each { explicitCancel ->
+ def info = client.execute("SELECT id, n FROM ${table} " +
+ "LATERAL VIEW explode_numbers(100000000) expanded AS n", auth)
+ if (tabletBackends > 1) {
+ assertTrue(info.endpoints.size() > 1, "Expected distributed Flight result endpoints")
+ }
+ def ids = info.endpoints.collect { endpoint ->
+ Any.parseFrom(endpoint.ticket.bytes).unpack(FlightSql.TicketStatementQuery.class)
+ .statementHandle.toStringUtf8().split("&")[0]
+ }.unique()
+ // Abort only one endpoint; the cancellation must reach every participating BE.
+ [info.endpoints[0]].each { endpoint ->
+ FlightClient.builder(allocator, endpoint.locations[0]).build().withCloseable { beClient ->
+ beClient.getStream(endpoint.ticket, auth).withCloseable { stream ->
+ assertTrue(stream.next())
+ assertTrue(stream.root.rowCount > 0)
+ if (explicitCancel) {
+ stream.cancel("test cancellation", null)
+ }
+ // Closing after one batch must also cancel the unfinished producer.
+ }
+ }
+ }
+ def deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(20)
+ def remaining = []
+ while (true) {
+ remaining = []
+ backends.each { backend ->
+ ids.each { id ->
+ def body = new URL("http://${backend.Host}:${backend.HttpPort}/api/query_pipeline_tasks/${id}")
+ .getText(connectTimeout: 3000, readTimeout: 3000)
+ if (!body.contains("not found")) {
+ remaining.add("${id}: ${body}")
+ }
+ }
+ }
+ if (remaining.isEmpty() || System.nanoTime() >= deadline) {
+ break
+ }
+ sleep(100)
+ }
+ assertTrue(remaining.isEmpty(), "Cancelled Flight query retained pipelines: ${remaining}")
+ assertEquals(1, consume("SELECT 1 AS healthy"))
+ }
+ assertEquals(10, consume("SELECT number FROM numbers(\"number\"=\"10\")"))
+ } finally {
+ try {
+ if (auth != null) {
+ feClient.closeSession(new org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CloseSessionRequest(), auth)
+ }
+ } finally {
+ try {
+ client.close()
+ allocator.close()
+ } finally {
+ jdbc_sql("DROP TABLE IF EXISTS ${table}")
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 26/35: branch-4.1: [fix](auth) Reject privilege object names with more than three parts in GRANT/REVOKE #68660 (#68682)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 62b9b6130f4087b01c7d9fac7fed49afb338e4b9
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Wed Sep 30 22:04:20 2026 +0800
branch-4.1: [fix](auth) Reject privilege object names with more than three parts in GRANT/REVOKE #68660 (#68682)
Cherry-picked from #68660
Co-authored-by: Calvin Kirs <gu...@selectdb.com>
---
.../org/apache/doris/mysql/privilege/Auth.java | 10 ++--
.../doris/nereids/parser/LogicalPlanBuilder.java | 68 +++++++---------------
.../commands/RevokeTablePrivilegeCommand.java | 13 ++---
.../commands/GrantTablePrivilegeCommandTest.java | 29 +++++++++
.../commands/RevokeTablePrivilegeCommandTest.java | 29 +++++++++
.../test_grant_revoke_object_name.groovy | 51 ++++++++++++++++
6 files changed, 139 insertions(+), 61 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java
index ce3faecc1ca..f41d82013a8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java
@@ -912,12 +912,10 @@ public class Auth implements Writable {
// revoke table
public void revokeTablePrivilegeCommand(RevokeTablePrivilegeCommand command) throws DdlException {
- if (command.getTablePattern() != null) {
- PrivBitSet privs = PrivBitSet.of(command.getPrivileges());
- revokeInternal(command.getUserIdentity().orElse(null), command.getRole().orElse(null),
- command.getTablePattern(), privs, command.getColPrivileges(),
- true /* err on non exist */, false /* is replay */);
- }
+ PrivBitSet privs = PrivBitSet.of(command.getPrivileges());
+ revokeInternal(command.getUserIdentity().orElse(null), command.getRole().orElse(null),
+ command.getTablePattern(), privs, command.getColPrivileges(),
+ true /* err on non exist */, false /* is replay */);
}
public void replayRevoke(PrivInfo info) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 43f762fe982..492b299c7dc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -9125,35 +9125,28 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
return new TransactionRollbackCommand();
}
+ /**
+ * The object of a table privilege is db, db.tbl or ctl.db.tbl; TablePattern.analyze() checks its wildcards.
+ */
+ private TablePattern parsePrivilegeTablePattern(DorisParser.MultipartIdentifierOrAsteriskContext ctx) {
+ List<String> parts = visitMultipartIdentifierOrAsterisk(ctx);
+ switch (parts.size()) {
+ case 1:
+ return new TablePattern(parts.get(0), "");
+ case 2:
+ return new TablePattern(parts.get(0), parts.get(1));
+ case 3:
+ return new TablePattern(parts.get(0), parts.get(1), parts.get(2));
+ default:
+ throw new ParseException("Privilege object name should be db, db.tbl or ctl.db.tbl, but got: "
+ + String.join(".", parts), ctx);
+ }
+ }
+
@Override
public LogicalPlan visitGrantTablePrivilege(DorisParser.GrantTablePrivilegeContext ctx) {
List<AccessPrivilegeWithCols> accessPrivilegeWithCols = visitPrivilegeList(ctx.privilegeList());
-
- List<String> parts = visitMultipartIdentifierOrAsterisk(ctx.multipartIdentifierOrAsterisk());
- int size = parts.size();
-
- if (size < 1) {
- throw new AnalysisException("grant table privilege statement missing parameters");
- }
-
- TablePattern tablePattern = null;
- if (size == 1) {
- String db = parts.get(size - 1);
- tablePattern = new TablePattern(db, "");
- }
-
- if (size == 2) {
- String db = parts.get(size - 2);
- String tbl = parts.get(size - 1);
- tablePattern = new TablePattern(db, tbl);
- }
-
- if (size == 3) {
- String ctl = parts.get(size - 3);
- String db = parts.get(size - 2);
- String tbl = parts.get(size - 1);
- tablePattern = new TablePattern(ctl, db, tbl);
- }
+ TablePattern tablePattern = parsePrivilegeTablePattern(ctx.multipartIdentifierOrAsterisk());
Optional<UserIdentity> userIdentity = Optional.empty();
Optional<String> role = Optional.empty();
@@ -9299,28 +9292,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
@Override
public LogicalPlan visitRevokeTablePrivilege(DorisParser.RevokeTablePrivilegeContext ctx) {
List<AccessPrivilegeWithCols> accessPrivilegeWithCols = visitPrivilegeList(ctx.privilegeList());
-
- List<String> parts = visitMultipartIdentifierOrAsterisk(ctx.multipartIdentifierOrAsterisk());
- int size = parts.size();
-
- TablePattern tablePattern = null;
- if (size == 1) {
- String db = parts.get(size - 1);
- tablePattern = new TablePattern(db, "");
- }
-
- if (size == 2) {
- String db = parts.get(size - 2);
- String tbl = parts.get(size - 1);
- tablePattern = new TablePattern(db, tbl);
- }
-
- if (size == 3) {
- String ctl = parts.get(size - 3);
- String db = parts.get(size - 2);
- String tbl = parts.get(size - 1);
- tablePattern = new TablePattern(ctl, db, tbl);
- }
+ TablePattern tablePattern = parsePrivilegeTablePattern(ctx.multipartIdentifierOrAsterisk());
Optional<UserIdentity> userIdentity = Optional.empty();
Optional<String> role = Optional.empty();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RevokeTablePrivilegeCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RevokeTablePrivilegeCommand.java
index 1748d6bd57b..5213df7cf01 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RevokeTablePrivilegeCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RevokeTablePrivilegeCommand.java
@@ -38,6 +38,7 @@ import org.apache.commons.collections4.MapUtils;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
@@ -59,10 +60,10 @@ public class RevokeTablePrivilegeCommand extends Command implements ForwardWithS
public RevokeTablePrivilegeCommand(List<AccessPrivilegeWithCols> accessPrivileges, TablePattern tablePattern,
Optional<UserIdentity> userIdentity, Optional<String> role) {
super(PlanType.REVOKE_TABLE_PRIVILEGE_COMMAND);
- this.accessPrivileges = accessPrivileges;
- this.tablePattern = tablePattern;
- this.userIdentity = userIdentity;
- this.role = role;
+ this.accessPrivileges = Objects.requireNonNull(accessPrivileges, "accessPrivileges is null");
+ this.tablePattern = Objects.requireNonNull(tablePattern, "tablePattern is null");
+ this.userIdentity = Objects.requireNonNull(userIdentity, "userIdentity is null");
+ this.role = Objects.requireNonNull(role, "role is null");
}
@Override
@@ -100,9 +101,7 @@ public class RevokeTablePrivilegeCommand extends Command implements ForwardWithS
}
// Revoke operation obey the same rule as Grant operation. reuse the same method
- if (tablePattern != null) {
- GrantTablePrivilegeCommand.checkTablePrivileges(privileges, tablePattern, colPrivileges);
- }
+ GrantTablePrivilegeCommand.checkTablePrivileges(privileges, tablePattern, colPrivileges);
}
@Override
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/GrantTablePrivilegeCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/GrantTablePrivilegeCommandTest.java
index 65f80e75732..9683ed5b161 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/GrantTablePrivilegeCommandTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/GrantTablePrivilegeCommandTest.java
@@ -23,6 +23,7 @@ import org.apache.doris.catalog.AccessPrivilege;
import org.apache.doris.catalog.AccessPrivilegeWithCols;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
+import org.apache.doris.nereids.exceptions.ParseException;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.utframe.TestWithFeService;
@@ -191,4 +192,32 @@ public class GrantTablePrivilegeCommandTest extends TestWithFeService {
LogicalPlan plan = new NereidsParser().parseSingle(sql);
((Command) plan).run(connectContext, null);
}
+
+ @Test
+ public void testObjectName() {
+ NereidsParser nereidsParser = new NereidsParser();
+ String[][] cases = {
+ {"GRANT SELECT_PRIV ON test TO 'jack'", "test.*"},
+ {"GRANT SELECT_PRIV ON test.test_table TO 'jack'", "test.test_table"},
+ {"GRANT SELECT_PRIV ON internal.test.test_table TO 'jack'", "internal.test.test_table"},
+ };
+ for (String[] c : cases) {
+ LogicalPlan plan = nereidsParser.parseSingle(c[0]);
+ Assertions.assertTrue(plan instanceof GrantTablePrivilegeCommand, c[0]);
+ Assertions.assertEquals(c[1], ((GrantTablePrivilegeCommand) plan).getTablePattern().toString(), c[0]);
+ }
+ }
+
+ @Test
+ public void testObjectNameWithTooManyParts() {
+ NereidsParser nereidsParser = new NereidsParser();
+ for (String name : new String[] {"a.b.c.d", "*.*.*.*", "a.b.c.d.e"}) {
+ String sql = "GRANT SELECT_PRIV ON " + name + " TO 'jack'";
+ ParseException exception = Assertions.assertThrows(ParseException.class,
+ () -> nereidsParser.parseSingle(sql), sql);
+ Assertions.assertTrue(exception.getMessage().contains(
+ "Privilege object name should be db, db.tbl or ctl.db.tbl, but got: " + name),
+ exception.getMessage());
+ }
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/RevokeTablePrivilegeCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/RevokeTablePrivilegeCommandTest.java
index 9c04d120eb1..06738995913 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/RevokeTablePrivilegeCommandTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/RevokeTablePrivilegeCommandTest.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.plans.commands;
import org.apache.doris.analysis.TablePattern;
import org.apache.doris.catalog.AccessPrivilege;
import org.apache.doris.catalog.AccessPrivilegeWithCols;
+import org.apache.doris.nereids.exceptions.ParseException;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.utframe.TestWithFeService;
@@ -95,4 +96,32 @@ public class RevokeTablePrivilegeCommandTest extends TestWithFeService {
Assertions.assertTrue(revokeplan2 instanceof RevokeTablePrivilegeCommand);
Assertions.assertDoesNotThrow(() -> ((RevokeTablePrivilegeCommand) revokeplan2).run(connectContext, null));
}
+
+ @Test
+ public void testObjectName() {
+ NereidsParser nereidsParser = new NereidsParser();
+ String[][] cases = {
+ {"REVOKE SELECT_PRIV ON test FROM 'jack'", "test.*"},
+ {"REVOKE SELECT_PRIV ON test.test_table FROM 'jack'", "test.test_table"},
+ {"REVOKE SELECT_PRIV ON internal.test.test_table FROM 'jack'", "internal.test.test_table"},
+ };
+ for (String[] c : cases) {
+ LogicalPlan plan = nereidsParser.parseSingle(c[0]);
+ Assertions.assertTrue(plan instanceof RevokeTablePrivilegeCommand, c[0]);
+ Assertions.assertEquals(c[1], ((RevokeTablePrivilegeCommand) plan).getTablePattern().toString(), c[0]);
+ }
+ }
+
+ @Test
+ public void testObjectNameWithTooManyParts() {
+ NereidsParser nereidsParser = new NereidsParser();
+ for (String name : new String[] {"a.b.c.d", "*.*.*.*", "a.b.c.d.e"}) {
+ String sql = "REVOKE SELECT_PRIV ON " + name + " FROM 'jack'";
+ ParseException exception = Assertions.assertThrows(ParseException.class,
+ () -> nereidsParser.parseSingle(sql), sql);
+ Assertions.assertTrue(exception.getMessage().contains(
+ "Privilege object name should be db, db.tbl or ctl.db.tbl, but got: " + name),
+ exception.getMessage());
+ }
+ }
}
diff --git a/regression-test/suites/account_p0/test_grant_revoke_object_name.groovy b/regression-test/suites/account_p0/test_grant_revoke_object_name.groovy
new file mode 100644
index 00000000000..6801a383dd0
--- /dev/null
+++ b/regression-test/suites/account_p0/test_grant_revoke_object_name.groovy
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_grant_revoke_object_name") {
+ sql """DROP USER IF EXISTS 'test_grant_revoke_object_name_user'"""
+ sql """DROP ROLE IF EXISTS test_grant_revoke_object_name_role"""
+ sql """DROP DATABASE IF EXISTS test_grant_revoke_object_name_db"""
+ sql """CREATE DATABASE test_grant_revoke_object_name_db"""
+ sql """CREATE USER 'test_grant_revoke_object_name_user'"""
+ sql """CREATE ROLE test_grant_revoke_object_name_role"""
+
+ // db, db.tbl and ctl.db.tbl are the only object name shapes a table privilege takes
+ sql """GRANT SELECT_PRIV ON test_grant_revoke_object_name_db TO 'test_grant_revoke_object_name_user'"""
+ sql """REVOKE SELECT_PRIV ON test_grant_revoke_object_name_db FROM 'test_grant_revoke_object_name_user'"""
+ sql """GRANT SELECT_PRIV ON test_grant_revoke_object_name_db.* TO 'test_grant_revoke_object_name_user'"""
+ sql """REVOKE SELECT_PRIV ON test_grant_revoke_object_name_db.* FROM 'test_grant_revoke_object_name_user'"""
+ sql """GRANT SELECT_PRIV ON internal.test_grant_revoke_object_name_db.* TO 'test_grant_revoke_object_name_user'"""
+ sql """REVOKE SELECT_PRIV ON internal.test_grant_revoke_object_name_db.* FROM 'test_grant_revoke_object_name_user'"""
+
+ // four or more parts are rejected while parsing, for both a user and a role
+ test {
+ sql """GRANT SELECT_PRIV ON a.b.c.d TO 'test_grant_revoke_object_name_user'"""
+ exception "Privilege object name should be db, db.tbl or ctl.db.tbl, but got: a.b.c.d"
+ }
+ test {
+ sql """REVOKE SELECT_PRIV ON a.b.c.d FROM 'test_grant_revoke_object_name_user'"""
+ exception "Privilege object name should be db, db.tbl or ctl.db.tbl, but got: a.b.c.d"
+ }
+ test {
+ sql """GRANT SELECT_PRIV ON internal.test_grant_revoke_object_name_db.t.c TO ROLE test_grant_revoke_object_name_role"""
+ exception "Privilege object name should be db, db.tbl or ctl.db.tbl, but got: internal.test_grant_revoke_object_name_db.t.c"
+ }
+ test {
+ sql """REVOKE SELECT_PRIV ON *.*.*.* FROM ROLE test_grant_revoke_object_name_role"""
+ exception "Privilege object name should be db, db.tbl or ctl.db.tbl, but got: *.*.*.*"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 20/35: branch-4.1: [fix](meta cache) Invalidate external row count cache after metadata changes #64160 (#68196)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 37e65c49e084bad8e4864964dcde78c7d01472e1
Author: 924060929 <la...@selectdb.com>
AuthorDate: Wed Sep 30 21:04:19 2026 +0800
branch-4.1: [fix](meta cache) Invalidate external row count cache after metadata changes #64160 (#68196)
Cherry-picked from #64160
---
.../main/java/org/apache/doris/catalog/Env.java | 2 +-
.../org/apache/doris/catalog/RefreshManager.java | 182 +++-
.../org/apache/doris/datasource/CatalogMgr.java | 96 +-
.../apache/doris/datasource/ExternalCatalog.java | 290 ++++-
.../apache/doris/datasource/ExternalDatabase.java | 69 +-
.../doris/datasource/ExternalMetaCacheMgr.java | 237 +++-
.../doris/datasource/ExternalRowCountCache.java | 350 +++++-
.../org/apache/doris/datasource/ExternalTable.java | 2 +-
.../doris/datasource/hive/HMSExternalCatalog.java | 71 +-
.../doris/datasource/hive/HiveMetadataOps.java | 29 +-
.../datasource/hive/event/AddPartitionEvent.java | 3 +-
.../datasource/hive/event/AlterDatabaseEvent.java | 17 +-
.../datasource/hive/event/AlterPartitionEvent.java | 6 +-
.../datasource/hive/event/CreateDatabaseEvent.java | 3 +-
.../datasource/hive/event/DropDatabaseEvent.java | 2 +-
.../datasource/hive/event/DropPartitionEvent.java | 2 +-
.../doris/datasource/hive/event/InsertEvent.java | 4 +-
.../datasource/iceberg/IcebergExternalCatalog.java | 15 +-
.../datasource/iceberg/IcebergMetadataOps.java | 26 +-
.../doris/datasource/jdbc/JdbcExternalCatalog.java | 31 +-
.../datasource/lance/LanceExternalCatalog.java | 4 +
.../maxcompute/MaxComputeMetadataOps.java | 22 +-
.../doris/datasource/metacache/MetaCache.java | 16 +
.../metacache/paimon/PaimonTableLoader.java | 9 +
.../datasource/operations/ExternalMetadataOps.java | 33 +-
.../datasource/paimon/PaimonExternalCatalog.java | 31 +-
.../datasource/paimon/PaimonExternalMetaCache.java | 4 +-
.../doris/datasource/paimon/PaimonMetadataOps.java | 52 +-
.../insert/BaseExternalTableInsertExecutor.java | 9 +-
.../plans/commands/insert/HiveInsertExecutor.java | 40 +-
.../java/org/apache/doris/persist/DropDbInfo.java | 12 +
.../java/org/apache/doris/persist/DropInfo.java | 12 +
.../java/org/apache/doris/persist/EditLog.java | 2 +-
.../doris/catalog/RefreshManagerReplayTest.java | 23 +
.../apache/doris/catalog/RefreshManagerTest.java | 511 +++++++++
.../apache/doris/datasource/CatalogMgrTest.java | 1127 +++++++++++++++++++-
.../ExternalMetaCacheRouteResolverTest.java | 444 +++++++-
.../datasource/ExternalRowCountCacheTest.java | 557 +++++++++-
.../org/apache/doris/datasource/MetaCacheTest.java | 9 +
.../doris/datasource/hive/HiveMetadataOpsTest.java | 120 +++
.../datasource/jdbc/JdbcExternalCatalogTest.java | 10 +
.../lance/LanceCatalogLifecycleTest.java | 63 ++
.../paimon/PaimonExternalMetaCacheTest.java | 130 ++-
.../datasource/paimon/PaimonMetadataOpsTest.java | 16 +
.../commands/insert/HiveInsertExecutorTest.java | 343 ++++++
.../doris/persist/DropAndRecoverInfoTest.java | 13 +
.../org/apache/doris/persist/DropDbInfoTest.java | 13 +
47 files changed, 4770 insertions(+), 292 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 3d95469223c..9eccdb1374b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3580,7 +3580,7 @@ public class Env {
} else {
ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(info.getCtlName());
if (externalCatalog != null) {
- externalCatalog.replayDropDb(info.getDbName());
+ externalCatalog.replayDropDb(info.getDbName(), info.getResolvedDbName());
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
index 952acc3efb2..bc8495c78a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
@@ -18,12 +18,14 @@
package org.apache.doris.catalog;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.CatalogLog;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalDatabase;
+import org.apache.doris.datasource.ExternalMetaCacheMgr;
import org.apache.doris.datasource.ExternalObjectLog;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
@@ -95,7 +97,7 @@ public class RefreshManager {
}
DatabaseIf db = catalog.getDbOrDdlException(dbName);
// Local DB-object eviction also resets metadata; only an explicit refresh retires access.
- invalidateLanceTableAccess(catalog);
+ invalidateLanceTableAccess((ExternalDatabase) db);
refreshDbInternal((ExternalDatabase) db);
ExternalObjectLog log = ExternalObjectLog.createForRefreshDb(catalog.getId(), db.getFullName());
@@ -110,7 +112,7 @@ public class RefreshManager {
LOG.warn("failed to find catalog when replaying refresh db: {}", log.debugForRefreshDb());
return;
}
- invalidateLanceTableAccess(catalog);
+ invalidateLanceTableAccess(catalog, log);
Optional<ExternalDatabase<? extends ExternalTable>> db;
if (!Strings.isNullOrEmpty(log.getDbName())) {
db = catalog.getDbForReplay(log.getDbName());
@@ -119,22 +121,41 @@ public class RefreshManager {
}
if (!db.isPresent()) {
- LOG.warn("failed to find db when replaying refresh db: {}", log.debugForRefreshDb());
- invalidatePaimonCatalogForUnresolvedReplay(catalog);
+ invalidateColdReplayDatabase(catalog, log);
} else {
refreshDbInternal(db.get());
}
});
}
- private void invalidateLanceTableAccess(CatalogIf catalog) {
- // Access entries outlive the bounded DB/table object caches. Replay must invalidate by
- // catalog identity before its cache-only object lookup can return early, including ID logs.
- if (catalog instanceof LanceExternalCatalog) {
+ private void invalidateLanceTableAccess(ExternalDatabase db) {
+ if (db.getCatalog() instanceof LanceExternalCatalog) {
+ ExternalCatalog catalog = (ExternalCatalog) db.getCatalog();
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(catalog.getId(), db.getId());
((LanceExternalCatalog) catalog).invalidateTableAccessCache();
}
}
+ private void invalidateLanceTableAccess(ExternalCatalog catalog, ExternalObjectLog log) {
+ if (!(catalog instanceof LanceExternalCatalog)) {
+ return;
+ }
+ // Access entries outlive the bounded DB/table object caches. Fence the retained identity
+ // before retiring access, even if the following cache-only object lookup misses.
+ ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr();
+ if (!Strings.isNullOrEmpty(log.getDbName()) && !Strings.isNullOrEmpty(log.getTableName())) {
+ cacheMgr.invalidateRowCountCache(catalog.getId(), log.getDbName(), log.getTableName());
+ } else {
+ Optional<Pair<String, Long>> identity = catalog.getDbIdentityForReplay(log.getDbName(), log.getDbId());
+ if (identity.isPresent()) {
+ cacheMgr.invalidateRowCountCache(catalog.getId(), identity.get().second);
+ } else {
+ cacheMgr.invalidateRowCountCache(catalog.getId());
+ }
+ }
+ ((LanceExternalCatalog) catalog).invalidateTableAccessCache();
+ }
+
private void refreshDbInternal(ExternalDatabase db) {
db.resetMetaToUninitialized();
LOG.info("refresh database {} in catalog {}", db.getFullName(), db.getCatalog().getName());
@@ -174,6 +195,22 @@ public class RefreshManager {
Env.getCurrentEnv().getEditLog().logRefreshExternalTable(log);
}
+ public void refreshTableAfterCommit(ExternalTable table) {
+ long updateTime = System.currentTimeMillis();
+ try {
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table);
+ refreshTableInternal((ExternalDatabase) table.getDatabase(), table, updateTime);
+ } catch (RuntimeException e) {
+ // The external transaction is already committed. Still notify follower FEs even if a
+ // cache layer failed, so peers do not keep serving the pre-commit state.
+ LOG.warn("Failed to refresh table cache after committing external insert for {}",
+ table.getNameWithFullQualifiers(), e);
+ }
+ ExternalObjectLog log = ExternalObjectLog.createForRefreshTable(
+ table.getCatalog().getId(), table.getDatabase().getFullName(), table.getName(), updateTime);
+ Env.getCurrentEnv().getEditLog().logRefreshExternalTable(log);
+ }
+
public void replayRefreshTable(ExternalObjectLog log) {
replayRefreshSafely("refresh table " + log.getCatalogId(), () -> {
ExternalCatalog catalog = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr()
@@ -182,7 +219,7 @@ public class RefreshManager {
LOG.warn("failed to find catalog when replaying refresh table: {}", log.debugForRefreshTable());
return;
}
- invalidateLanceTableAccess(catalog);
+ invalidateLanceTableAccess(catalog, log);
Optional<ExternalDatabase<? extends ExternalTable>> db;
if (!Strings.isNullOrEmpty(log.getDbName())) {
db = catalog.getDbForReplay(log.getDbName());
@@ -191,8 +228,7 @@ public class RefreshManager {
}
// See comment in refreshDbInternal for why db and table may be null.
if (!db.isPresent()) {
- LOG.warn("failed to find db when replaying refresh table: {}", log.debugForRefreshTable());
- invalidatePaimonCatalogForUnresolvedReplay(catalog);
+ invalidateColdReplayDatabase(catalog, log);
return;
}
Optional<? extends ExternalTable> table;
@@ -210,7 +246,19 @@ public class RefreshManager {
&& !db.get().hasLocalTableName(log.getTableName())) {
db.get().retireAllTableObjectsWithoutEngineInvalidation();
}
- invalidatePaimonCatalogForUnresolvedReplay(catalog);
+ // The independent row-count cache can outlive the table object, so fence the
+ // canonical database scope before acknowledging the committed refresh.
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateRowCountCache(catalog.getId(), db.get().getId());
+ if (!Strings.isNullOrEmpty(log.getTableName())) {
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableByNameOrWider(
+ catalog.getId(), db.get().getFullName(), log.getTableName());
+ } else {
+ // Legacy ID-only records cannot recover a cold table's engine key.
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(
+ catalog.getId(), db.get().getId(), db.get().getFullName());
+ }
+ invalidatePaimonCatalogForReplay(catalog);
return;
}
if (!Strings.isNullOrEmpty(log.getNewTableName())) {
@@ -223,11 +271,33 @@ public class RefreshManager {
if (catalog instanceof HMSExternalCatalog
&& ((modifiedPartNames != null && !modifiedPartNames.isEmpty())
|| (newPartNames != null && !newPartNames.isEmpty()))) {
- // Partition-level cache invalidation, only for hive catalog
- HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
- .hive(catalog.getId());
- cache.refreshAffectedPartitionsCache(
- (HMSExternalTable) table.get(), modifiedPartNames, newPartNames);
+ // Partition-level cache invalidation, only for hive catalog. Fence the held table
+ // before the hive(...) lookup can lazily initialize the group and throw; otherwise
+ // a failed acquisition skips the row-count fence and the full-table fallback.
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table.get());
+ try {
+ HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
+ .hive(catalog.getId());
+ cache.refreshAffectedPartitionsCache((HMSExternalTable) table.get(), modifiedPartNames,
+ newPartNames);
+ // The held HMS table also caches the pre-insert table parameters.
+ table.get().unsetObjectCreated();
+ // Close the admission window the opening fence left open: a load admitted after
+ // it can publish the pre-insert value from the still-resident file list.
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table.get());
+ } catch (RuntimeException e) {
+ // The insert is already committed. A partial or failed selective refresh must not
+ // leave stale Hive partition/file entries, so fall back to the same conservative
+ // full-table invalidation the leader uses.
+ LOG.warn("failed to refresh affected partitions when replaying refresh table for {}",
+ table.get().getNameWithFullQualifiers(), e);
+ try {
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(table.get());
+ } finally {
+ table.get().unsetObjectCreated();
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table.get());
+ }
+ }
if (table.get() instanceof HMSExternalTable && log.getLastUpdateTime() > 0) {
((HMSExternalTable) table.get()).setUpdateTime(log.getLastUpdateTime());
}
@@ -253,19 +323,34 @@ public class RefreshManager {
if (!(catalog instanceof ExternalCatalog)) {
throw new DdlException("Only support refresh ExternalCatalog Tables");
}
+ if (catalog instanceof HMSExternalCatalog
+ && ((HMSExternalCatalog) catalog).isPartitionEventTargetExcluded(dbName, tableName)) {
+ return;
+ }
+ // Whole-table events are already committed remotely. Fence the row count by cached identity
+ // before any database/table reload can fail and make the not-found path return.
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateRowCountCache(catalog.getId(), dbName, tableName);
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
+ // Cold database: widen so independently resident engine entries are retired too.
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName);
return;
}
TableIf table = db.getTableNullable(tableName);
if (table == null) {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName);
return;
}
refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, updateTime);
}
public void refreshTableInternal(ExternalDatabase db, ExternalTable table, long updateTime) {
+ // The table's metadata can become visible to a new load as soon as it is reset.
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table);
table.unsetObjectCreated();
// Iceberg partition evolution can change partition specs across FEs.
// Clear related-table validation cache to avoid stale partitioned/unpartitioned judgment.
@@ -293,13 +378,51 @@ public class RefreshManager {
}
}
- private void invalidatePaimonCatalogForUnresolvedReplay(ExternalCatalog catalog) {
+ private void invalidatePaimonCatalogForReplay(ExternalCatalog catalog) {
if (catalog instanceof PaimonExternalCatalog) {
Env.getCurrentEnv().getExtMetaCacheMgr()
.invalidateCatalogByEngine(catalog.getId(), PaimonExternalMetaCache.ENGINE);
}
}
+ private void invalidateColdReplayDatabase(ExternalCatalog catalog, ExternalObjectLog log) {
+ Optional<Pair<String, Long>> identity = catalog.getDbIdentityForReplay(log.getDbName(), log.getDbId());
+ if (identity.isPresent()) {
+ // The object may be evicted while its canonical name and deterministic ID are still
+ // known. Retire only that database's engine entries and independent row counts.
+ LOG.debug("database object is cold when replaying refresh in catalog {}, db {}",
+ catalog.getName(), identity.get().first);
+ // A mode-2 names refresh can rebind the logged Foo to a new FOO. The current
+ // identity cannot fence an SDK-only Foo handle or Foo's independent row counts.
+ boolean reboundPaimonName = catalog instanceof PaimonExternalCatalog
+ && catalog.getLowerCaseDatabaseNames() == 2
+ && !Strings.isNullOrEmpty(log.getDbName())
+ && !identity.get().first.equals(log.getDbName());
+ ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr();
+ if (reboundPaimonName) {
+ cacheMgr.invalidateRowCountCache(catalog.getId());
+ }
+ try {
+ cacheMgr.invalidateDb(catalog.getId(), identity.get().second, identity.get().first);
+ } finally {
+ if (reboundPaimonName) {
+ try {
+ invalidatePaimonCatalogForReplay(catalog);
+ } finally {
+ // Close the load window opened after the first fence.
+ cacheMgr.invalidateRowCountCache(catalog.getId());
+ }
+ }
+ }
+ } else {
+ // A lost mode-2 mapping cannot identify the database safely; widen the fence.
+ LOG.warn("failed to resolve database identity when replaying refresh: {}",
+ log.debugForRefreshDb());
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(catalog.getId());
+ invalidatePaimonCatalogForReplay(catalog);
+ }
+ }
+
// Refresh partition
public void refreshPartitions(String catalogName, String dbName, String tableName,
List<String> partitionNames, long updateTime, boolean ignoreIfNotExists)
@@ -314,8 +437,29 @@ public class RefreshManager {
if (!(catalog instanceof ExternalCatalog)) {
throw new DdlException("Only support ExternalCatalog");
}
+ if (catalog instanceof HMSExternalCatalog
+ && ((HMSExternalCatalog) catalog).isPartitionEventTargetExcluded(dbName, tableName)) {
+ return;
+ }
+ // Partition events are already committed remotely. Fence the row count by cached identity
+ // before any database/table reload can fail and make the ignored-not-found path return.
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateRowCountCache(catalog.getId(), dbName, tableName);
+ try {
+ refreshPartitionsAfterFence(catalog, dbName, tableName, partitionNames, updateTime,
+ ignoreIfNotExists);
+ } finally {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateRowCountCache(catalog.getId(), dbName, tableName);
+ }
+ }
+
+ private void refreshPartitionsAfterFence(CatalogIf catalog, String dbName, String tableName,
+ List<String> partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException {
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName);
if (!ignoreIfNotExists) {
throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
}
@@ -324,6 +468,8 @@ public class RefreshManager {
TableIf table = db.getTableNullable(tableName);
if (table == null) {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName);
if (!ignoreIfNotExists) {
throw new DdlException("Table " + tableName + " does not exist in db " + dbName);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 227214e5962..918dac65e40 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -158,8 +158,13 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
}
String catalogName = catalog.getName();
Env.getCurrentEnv().getRefreshManager().removeFromRefreshMap(catalogId);
+ // Remove both lock-free mappings first: a row-count load admitted after the fence below
+ // could otherwise still resolve this catalog through them and alias a same-name
+ // replacement, because RowCountKey equality uses only tableId. Fence last, while this
+ // write lock still excludes a same-name CREATE.
idToCatalog.remove(catalogId);
nameToCatalog.remove(catalogName);
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(catalogId);
return new RemovedCatalog(catalog, catalogName);
}
@@ -476,7 +481,11 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
}
log.setNewProps(loggedProperties);
replayAlterCatalogProps(log, oldProperties, false);
- Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_PROPS, log);
+ if (!(catalog instanceof ExternalCatalog)) {
+ // External catalogs journal the committed change from the fenced helper once publication
+ // has happened; non-external catalogs only reach this point on success.
+ Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_PROPS, log);
+ }
} finally {
writeUnlock();
}
@@ -931,15 +940,38 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
Integer[] sec = {metadataRefreshIntervalSec, metadataRefreshIntervalSec};
Env.getCurrentEnv().getRefreshManager().addToRefreshMap(catalogId, sec);
}
- externalCatalog.modifyCatalogProps(newProps);
// The commit reset the catalog's execution context and closed its SDK resources. Cached
// base generations and projections are bound to the replaced context; retire them now so
// the next statement loads a generation the planning fences accept, instead of retrying
- // against an unplannable cached generation until managed refresh.
+ // against an unplannable cached generation until managed refresh. The properties are
+ // published before the reset's throwable cleanup, so retirement must run either way.
Env currentEnv = Env.getCurrentEnv();
ExternalMetaCacheMgr cacheMgr = currentEnv == null ? null : currentEnv.getExtMetaCacheMgr();
- if (cacheMgr != null) {
- cacheMgr.onCatalogOperationalContextChanged(externalCatalog.getId());
+ try {
+ if (cacheMgr != null) {
+ // Close the old row-count generation before publishing the new catalog context.
+ // The completion fence below also retires values loaded during the reset.
+ cacheMgr.invalidateRowCountCache(externalCatalog.getId());
+ }
+ externalCatalog.modifyCatalogProps(newProps);
+ } catch (RuntimeException e) {
+ if (!isReplay) {
+ throw e;
+ }
+ // A follower must not terminate because a local connector cleanup failed while applying
+ // an already-durable ALTER record. The property publication and the derived-state
+ // transitions above are failure-safe, so the record is considered applied.
+ LOG.warn("Failed to complete local cleanup while replaying ALTER CATALOG for {}: {}",
+ externalCatalog.getName(), e.getMessage(), e);
+ } finally {
+ if (cacheMgr != null) {
+ cacheMgr.onCatalogOperationalContextChanged(externalCatalog.getId());
+ }
+ if (!isReplay && currentEnv != null) {
+ // Publication has happened once modifyCatalogProps is reached; journal it even if the
+ // reset's own cleanup throws. Pre-publication validation failures never reach here.
+ currentEnv.getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_PROPS, log);
+ }
}
}
@@ -1039,7 +1071,11 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
if (!(catalog instanceof ExternalCatalog)) {
throw new DdlException("Only support drop ExternalCatalog databases");
}
- ((HMSExternalCatalog) catalog).unregisterDatabase(dbName);
+ HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog;
+ if (hmsCatalog.isDatabaseEventTargetExcluded(dbName)) {
+ return;
+ }
+ hmsCatalog.unregisterDatabase(dbName);
}
public void registerExternalDatabaseFromEvent(String dbName, String catalogName)
@@ -1054,7 +1090,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog;
long metadataLoadEpoch = hmsCatalog.acquireMetadataLoadEpoch();
- long dbId = Util.genIdByName(catalogName, dbName);
+ long dbId = Util.genIdByName(catalogName, hmsCatalog.localDatabaseNameFromRemote(dbName));
// -1L means it will be dropped later, ignore
if (dbId == ExternalMetaIdMgr.META_ID_FOR_NOT_EXISTS) {
return;
@@ -1075,8 +1111,29 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
if (!(catalog instanceof ExternalCatalog)) {
throw new DdlException("Only support ExternalCatalog");
}
+ if (catalog instanceof HMSExternalCatalog
+ && ((HMSExternalCatalog) catalog).isPartitionEventTargetExcluded(dbName, tableName)) {
+ return;
+ }
+ // Partition events are already committed remotely. Fence the row count by cached identity
+ // before any database/table reload can fail and make the ignored-not-found path return.
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateRowCountCache(catalog.getId(), dbName, tableName);
+ try {
+ addExternalPartitionsAfterFence(catalog, dbName, tableName, partitionNames, updateTime,
+ ignoreIfNotExists);
+ } finally {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateRowCountCache(catalog.getId(), dbName, tableName);
+ }
+ }
+
+ private void addExternalPartitionsAfterFence(CatalogIf catalog, String dbName, String tableName,
+ List<String> partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException {
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName);
if (!ignoreIfNotExists) {
throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
}
@@ -1085,6 +1142,8 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
TableIf table = db.getTableNullable(tableName);
if (table == null) {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName);
if (!ignoreIfNotExists) {
throw new DdlException("Table " + tableName + " does not exist in db " + dbName);
}
@@ -1118,8 +1177,29 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
if (!(catalog instanceof ExternalCatalog)) {
throw new DdlException("Only support ExternalCatalog");
}
+ if (catalog instanceof HMSExternalCatalog
+ && ((HMSExternalCatalog) catalog).isPartitionEventTargetExcluded(dbName, tableName)) {
+ return;
+ }
+ // Partition events are already committed remotely. Fence the row count by cached identity
+ // before any database/table reload can fail and make the ignored-not-found path return.
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateRowCountCache(catalog.getId(), dbName, tableName);
+ try {
+ dropExternalPartitionsAfterFence(catalog, dbName, tableName, partitionNames, updateTime,
+ ignoreIfNotExists);
+ } finally {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateRowCountCache(catalog.getId(), dbName, tableName);
+ }
+ }
+
+ private void dropExternalPartitionsAfterFence(CatalogIf catalog, String dbName, String tableName,
+ List<String> partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException {
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName);
if (!ignoreIfNotExists) {
throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
}
@@ -1128,6 +1208,8 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
TableIf table = db.getTableNullable(tableName);
if (table == null) {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName);
if (!ignoreIfNotExists) {
throw new DdlException("Table " + tableName + " does not exist in db " + dbName);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index fc1a3ad9c6e..df73f543184 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -72,6 +72,7 @@ import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.qe.GlobalVariable;
import org.apache.doris.transaction.TransactionManager;
+import com.github.benmanes.caffeine.cache.RemovalCause;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@@ -187,6 +188,7 @@ public abstract class ExternalCatalog
protected MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache;
private ThreadLocal<Boolean> invalidateEngineCacheOnDatabaseRemoval =
ThreadLocal.withInitial(() -> true);
+ private transient ThreadLocal<Boolean> invalidatingAllMetaCache = ThreadLocal.withInitial(() -> false);
protected ExecutionAuthenticator executionAuthenticator;
protected ThreadPoolExecutor threadPoolWithPreAuth;
// Map lowercase database names to actual remote database names for case-insensitive lookup
@@ -427,13 +429,12 @@ public abstract class ExternalCatalog
Math.max(Config.max_meta_object_cache_num, 1),
ignored -> getFilteredDatabaseNames(),
this::updateLowerCaseToDatabaseName,
- (remoteName, localName) -> lowerCaseToDatabaseName.put(remoteName.toLowerCase(), remoteName),
- localName -> lowerCaseToDatabaseName.remove(localName.toLowerCase()),
+ (remoteName, localName) -> lowerCaseToDatabaseName.put(foldDatabaseName(remoteName), remoteName),
+ localName -> lowerCaseToDatabaseName.remove(foldDatabaseName(localName)),
localDbName -> Optional.ofNullable(
buildDbForInit(null, localDbName, Util.genIdByName(name, localDbName), logType,
true)),
- (key, value, cause) -> value.ifPresent(
- v -> v.resetMetaToUninitialized(invalidateEngineCacheOnDatabaseRemoval.get())),
+ (key, value, cause) -> handleDatabaseMetaCacheRemoval(value, cause),
this::acquireMetadataLoadEpoch,
this::isMetadataLoadEpochCurrent);
}
@@ -580,20 +581,11 @@ public abstract class ExternalCatalog
allDatabases = allDatabases.stream()
.filter(dbName -> isDatabaseAllowedByFilter(
- dbName, includeDatabaseMap, excludeDatabaseMap, false))
+ dbName, includeDatabaseMap, excludeDatabaseMap, getLowerCaseDatabaseNames() != 0))
.collect(Collectors.toList());
for (String remoteDbName : allDatabases) {
- String localDbName = fromRemoteDatabaseName(remoteDbName);
- // Apply lower_case_database_names mode to local name
- int dbNameMode = getLowerCaseDatabaseNames();
- if (dbNameMode == 1) {
- localDbName = localDbName.toLowerCase();
- } else if (dbNameMode == 2) {
- // Mode 2: preserve original remote case for display
- localDbName = remoteDbName;
- }
- remoteToLocalPairs.add(Pair.of(remoteDbName, localDbName));
+ remoteToLocalPairs.add(Pair.of(remoteDbName, localDatabaseNameFromRemote(remoteDbName)));
}
// Check for conflicts when lower_case_meta_names = true or lower_case_database_names = 2
@@ -603,7 +595,7 @@ public abstract class ExternalCatalog
// Collect lowercased local names and their remote counterparts
for (Pair<String, String> pair : remoteToLocalPairs) {
- String lowerCaseLocalName = pair.second.toLowerCase();
+ String lowerCaseLocalName = foldDatabaseName(pair.second);
lowerCaseToRemoteNames.computeIfAbsent(lowerCaseLocalName, k -> Lists.newArrayList()).add(pair.first);
}
@@ -626,12 +618,24 @@ public abstract class ExternalCatalog
return remoteToLocalPairs;
}
- protected boolean isDatabaseAllowedByFilter(String dbName) {
- return isDatabaseAllowedByFilter(dbName, getIncludeDatabaseMap(), getExcludeDatabaseMap(), false);
+ /** Use the same local identity for database discovery and HMS create events. */
+ protected final String localDatabaseNameFromRemote(String remoteDbName) {
+ String localDbName = fromRemoteDatabaseName(remoteDbName);
+ int mode = getLowerCaseDatabaseNames();
+ if (mode == 1) {
+ return foldDatabaseName(localDbName);
+ }
+ // Mode 2 preserves remote spelling for display and deterministic database IDs.
+ return mode == 2 ? remoteDbName : localDbName;
+ }
+
+ private static String foldDatabaseName(String dbName) {
+ return dbName.toLowerCase(Locale.ROOT);
}
- protected boolean isDatabaseAllowedByFilterIgnoringCase(String dbName) {
- return isDatabaseAllowedByFilter(dbName, getIncludeDatabaseMap(), getExcludeDatabaseMap(), true);
+ protected boolean isDatabaseAllowedByFilter(String dbName) {
+ return isDatabaseAllowedByFilter(dbName, getIncludeDatabaseMap(), getExcludeDatabaseMap(),
+ getLowerCaseDatabaseNames() != 0);
}
private boolean isDatabaseAllowedByFilter(String dbName, Map<String, Boolean> includeDatabaseMap,
@@ -650,14 +654,14 @@ public abstract class ExternalCatalog
if (!ignoreCase) {
return databaseMap.containsKey(dbName);
}
- String normalizedDbName = dbName.toLowerCase(Locale.ROOT);
+ String normalizedDbName = foldDatabaseName(dbName);
return databaseMap.keySet().stream()
- .anyMatch(configuredName -> configuredName.toLowerCase(Locale.ROOT).equals(normalizedDbName));
+ .anyMatch(configuredName -> foldDatabaseName(configuredName).equals(normalizedDbName));
}
private void updateLowerCaseToDatabaseName(List<Pair<String, String>> names) {
Map<String, String> updated = Maps.newConcurrentMap();
- names.forEach(pair -> updated.put(pair.key().toLowerCase(), pair.key()));
+ names.forEach(pair -> updated.put(foldDatabaseName(pair.key()), pair.key()));
lowerCaseToDatabaseName = updated;
}
@@ -714,9 +718,12 @@ public abstract class ExternalCatalog
*/
public void onRefreshCache(boolean invalidCache) {
setLastUpdateTime(System.currentTimeMillis());
- refreshMetaCacheOnly(invalidCache);
- if (invalidCache) {
- Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(id);
+ try {
+ refreshMetaCacheOnly(invalidCache);
+ } finally {
+ if (invalidCache) {
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(id);
+ }
}
}
@@ -724,16 +731,29 @@ public abstract class ExternalCatalog
* Refresh meta cache only (database level cache), without invalidating catalog level cache.
*/
private void refreshMetaCacheOnly(boolean invalidCache) {
- if (metaCache != null) {
- // A catalog-wide engine invalidation below supersedes every database invalidation.
- // The legacy cache uses a synchronous removal listener, so this thread-local scope
- // prevents one full SDK-cache scan per cached database without affecting concurrent
- // expiry callbacks on other threads.
- invalidateEngineCacheOnDatabaseRemoval.set(!invalidCache);
- try {
- metaCache.invalidateAll();
- } finally {
- invalidateEngineCacheOnDatabaseRemoval.remove();
+ Runnable objectInvalidation;
+ ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr();
+ synchronized (this) {
+ if (metaCache == null) {
+ return;
+ }
+ // Publish the names/object generation transition together, before allowing another
+ // catalog initialization. The old cache removal callbacks can be much slower.
+ cacheMgr.invalidateRowCountCache(id);
+ metaCache.invalidateNames();
+ objectInvalidation = metaCache.retireObjects();
+ }
+ // A catalog-wide engine invalidation below supersedes every database invalidation.
+ // Removal listeners are synchronous, but run them after releasing the catalog monitor.
+ invalidateEngineCacheOnDatabaseRemoval.set(!invalidCache);
+ invalidatingAllMetaCache.set(true);
+ try {
+ objectInvalidation.run();
+ } finally {
+ invalidatingAllMetaCache.remove();
+ invalidateEngineCacheOnDatabaseRemoval.remove();
+ if (!invalidCache) {
+ cacheMgr.invalidateRowCountCache(id);
}
}
}
@@ -924,7 +944,7 @@ public abstract class ExternalCatalog
* @return
*/
public Optional<ExternalDatabase<? extends ExternalTable>> getDbForReplay(long dbId) {
- if (!isInitialized()) {
+ if (!isInitialized() || metaCache == null) {
return Optional.empty();
}
return metaCache.getMetaObjById(dbId);
@@ -941,7 +961,7 @@ public abstract class ExternalCatalog
LOG.debug("getDbForReplay from metacache, db: {}.{}, catalog id: {}, is catalog init: {}",
this.name, dbName, this.id, isInitialized());
}
- if (!isInitialized()) {
+ if (!isInitialized() || metaCache == null) {
return Optional.empty();
}
@@ -954,6 +974,41 @@ public abstract class ExternalCatalog
return metaCache.tryGetMetaObj(localDbName);
}
+ /** Resolve a replay log's database identity without reloading an evicted database object. */
+ public Optional<Pair<String, Long>> getDbIdentityForReplay(String dbName, long dbId) {
+ if (!isInitialized() || metaCache == null) {
+ return Optional.empty();
+ }
+ if (dbName != null && !dbName.isEmpty()) {
+ String localName = getLocalDatabaseName(dbName, true);
+ return localName == null ? Optional.empty()
+ : Optional.of(Pair.of(localName, Util.genIdByName(name, localName)));
+ }
+ return metaCache.getNameByIdIfPresent(dbId).map(localName -> Pair.of(localName, dbId));
+ }
+
+ /** A DROP must not follow a mode-2 mapping that has rebound to a case-only replacement. */
+ private Optional<Pair<String, Long>> getDbIdentityForDrop(String dbName) {
+ if (!isInitialized() || metaCache == null) {
+ return Optional.empty();
+ }
+ String localName = getLocalDatabaseName(dbName, true);
+ if (localName == null) {
+ return Optional.empty();
+ }
+ if (getLowerCaseDatabaseNames() == 2 && !localName.equals(dbName)) {
+ long historicalId = Util.genIdByName(name, dbName);
+ return metaCache.getNameByIdIfPresent(historicalId)
+ .map(historicalName -> Pair.of(historicalName, historicalId));
+ }
+ return Optional.of(Pair.of(localName, Util.genIdByName(name, localName)));
+ }
+
+ /** Cache-only DROP lookup; normal replay lookups may follow a current case-insensitive alias. */
+ public Optional<ExternalDatabase<? extends ExternalTable>> getDbForDropReplay(String dbName) {
+ return getDbIdentityForDrop(dbName).flatMap(identity -> metaCache.tryGetMetaObj(identity.first));
+ }
+
/**
* Build a database instance.
* If checkExists is true, it will check if the database exists in the remote system.
@@ -968,7 +1023,7 @@ public abstract class ExternalCatalog
long dbId, InitCatalogLog.Type logType, boolean checkExists) {
// Step 1: Map local database name if not already provided
if (localDbName == null && remoteDbName != null) {
- localDbName = fromRemoteDatabaseName(remoteDbName);
+ localDbName = localDatabaseNameFromRemote(remoteDbName);
}
// Step 2:
@@ -1059,6 +1114,7 @@ public abstract class ExternalCatalog
objectCreated = false;
metadataLoadEpoch = new AtomicLong();
invalidateEngineCacheOnDatabaseRemoval = ThreadLocal.withInitial(() -> true);
+ invalidatingAllMetaCache = ThreadLocal.withInitial(() -> false);
// TODO: This code is to compatible with older version of metadata.
// Could only remove after all users upgrate to the new version.
if (logType == null) {
@@ -1131,14 +1187,15 @@ public abstract class ExternalCatalog
throw new DdlException("Drop database is not supported for catalog: " + getName());
}
try {
- if (!metadataOps.dropDb(dbName, ifExists, force)) {
+ Optional<String> resolvedDbName = metadataOps.dropDbWithResolvedName(dbName, ifExists, force);
+ if (!resolvedDbName.isPresent()) {
// No remote drop happened (for example DROP DATABASE IF EXISTS on a missing
// database). Do not journal the no-op, otherwise every follower would replay a
// post-drop hook that retires caches for a database that was never touched.
LOG.info("skip drop database {}.{} because the database does not exist", getName(), dbName);
return;
}
- DropDbInfo info = new DropDbInfo(getName(), dbName);
+ DropDbInfo info = new DropDbInfo(getName(), resolvedDbName.get(), resolvedDbName.get());
Env.getCurrentEnv().getEditLog().logDropDb(info);
} catch (Exception e) {
LOG.warn("Failed to drop database {} in catalog {}", dbName, getName(), e);
@@ -1152,6 +1209,18 @@ public abstract class ExternalCatalog
}
}
+ public void replayDropDb(String dbName, String resolvedDbName) {
+ if (resolvedDbName != null) {
+ replayDropDb(resolvedDbName);
+ } else if (getLowerCaseDatabaseNames() == 2) {
+ // An old name-only record cannot distinguish a historical DROP from an alias that
+ // targeted a case-only replacement. The dropped name must not remain visible either.
+ retireUnresolvedDatabaseGeneration(true);
+ } else {
+ replayDropDb(dbName);
+ }
+ }
+
@Override
public boolean createTable(CreateTableInfo createTableInfo) throws UserException {
makeSureInitialized();
@@ -1223,7 +1292,7 @@ public abstract class ExternalCatalog
}
try {
metadataOps.dropTable(dorisTable, ifExists);
- DropInfo info = new DropInfo(getName(), dbName, tableName);
+ DropInfo info = new DropInfo(getName(), db.getFullName(), dorisTable.getName(), db.getFullName());
Env.getCurrentEnv().getEditLog().logDropTable(info);
} catch (Exception e) {
LOG.warn("Failed to drop a table", e);
@@ -1237,6 +1306,18 @@ public abstract class ExternalCatalog
}
}
+ public void replayDropTable(String dbName, String tblName, String resolvedDbName) {
+ if (resolvedDbName != null) {
+ replayDropTable(resolvedDbName, tblName);
+ } else if (getLowerCaseDatabaseNames() == 2) {
+ // Old logs carry only caller spelling, which can identify either side of a case-only
+ // database replacement. Never guess which cached database was actually dropped.
+ retireUnresolvedDatabaseGeneration();
+ } else {
+ replayDropTable(dbName, tblName);
+ }
+ }
+
/**
* Unregisters a database from the catalog.
* Internally, remove the database meta from cache
@@ -1247,10 +1328,62 @@ public abstract class ExternalCatalog
if (LOG.isDebugEnabled()) {
LOG.debug("unregister database [{}]", dbName);
}
- if (isInitialized()) {
- metaCache.invalidate(dbName, Util.genIdByName(name, dbName));
+ if (!isInitialized()) {
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(getId(), dbName);
+ return;
+ }
+ Optional<Pair<String, Long>> identity = getDbIdentityForDrop(dbName);
+ if (!identity.isPresent()) {
+ // A mode-2 remote-to-local mapping can disappear (for example after a names refresh)
+ // while the resident database object survives. The canonical key is then unknown, so
+ // treat the scope as unknown: retire every cached database object and flush the engine
+ // caches and row counts catalog-wide instead of evicting the wrong local key.
+ retireAllDatabaseObjectsWithoutEngineInvalidation();
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(getId());
+ return;
+ }
+ unregisterDatabase(identity.get().first, identity.get().second);
+ }
+
+ /** A no-op remote DROP may retire matching resident history, but never unrelated databases. */
+ public void retireCachedDatabaseForNoOp(String dbName) {
+ if (!isInitialized() || metaCache == null) {
+ return;
+ }
+ for (Pair<String, Long> identity : metaCache.getCachedIdentitiesMatching(
+ dbName, getLowerCaseDatabaseNames() != 0)) {
+ unregisterDatabase(identity.first, identity.second);
+ }
+ }
+
+ private void unregisterDatabase(String localDbName, long dbId) {
+ // A cold database has no removal callback. Fence its deterministic table IDs before
+ // removing the name slot, so a same-name replacement cannot use an old completed count.
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(getId(), dbId);
+ // The explicit routed invalidation below owns this DROP. Suppress the object removal
+ // callback's routed work so a warm database does not flush the SDK cache twice.
+ invalidateEngineCacheOnDatabaseRemoval.set(false);
+ try {
+ metaCache.invalidate(localDbName, dbId);
+ } finally {
+ invalidateEngineCacheOnDatabaseRemoval.remove();
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(getId(), dbId, localDbName);
}
- Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(getId(), dbName);
+ }
+
+ boolean shouldInvalidateRowCountOnDatabaseRemoval() {
+ return invalidateEngineCacheOnDatabaseRemoval.get() && !invalidatingAllMetaCache.get();
+ }
+
+ boolean shouldInvalidateRoutedCacheOnDatabaseRemoval() {
+ return invalidateEngineCacheOnDatabaseRemoval.get();
+ }
+
+ void handleDatabaseMetaCacheRemoval(Optional<ExternalDatabase<? extends ExternalTable>> value,
+ RemovalCause cause) {
+ value.ifPresent(v -> v.resetMetaToUninitialized(
+ shouldInvalidateRoutedCacheOnDatabaseRemoval(),
+ !cause.wasEvicted() && shouldInvalidateRowCountOnDatabaseRemoval()));
}
/**
@@ -1263,11 +1396,60 @@ public abstract class ExternalCatalog
if (metaCache == null) {
return;
}
+ // A mode-2 mapping loss can hide an old object while its row count remains reachable.
+ // Close that window before swapping the database-object cache generation.
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(getId());
invalidateEngineCacheOnDatabaseRemoval.set(false);
try {
metaCache.invalidateObjects();
} finally {
invalidateEngineCacheOnDatabaseRemoval.remove();
+ // The removal callbacks above suppress every per-database row-count fence, so publish
+ // one catalog-wide fence here; otherwise a recreated same-name object can reuse a stale
+ // count through its deterministic table id.
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(getId());
+ }
+ }
+
+ /**
+ * Best-effort cleanup for a drop event whose canonical database object cannot be resolved
+ * (for example a mode-2 name mapping was lost). Retires the hidden database-object generation
+ * and flushes the catalog-wide engine state so a same-name recreation cannot reuse the old
+ * incarnation, engine entries, or row counts.
+ */
+ public void retireUnresolvedDatabaseGeneration() {
+ retireUnresolvedDatabaseGeneration(false);
+ }
+
+ private void retireUnresolvedDatabaseGeneration(boolean invalidateNames) {
+ try {
+ if (invalidateNames && metaCache != null) {
+ // A name refresh can expose a new incarnation before the object generation swaps.
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(getId());
+ metaCache.invalidateNames();
+ }
+ retireAllDatabaseObjectsWithoutEngineInvalidation();
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(getId());
+ } catch (Exception e) {
+ LOG.warn("Failed to retire unresolved database objects for catalog {}: {}",
+ getName(), e.getMessage(), e);
+ }
+ }
+
+ /** Retire a cold replay database narrowly when its canonical name survived object eviction. */
+ public void invalidateColdDatabaseForReplay(String dbName) {
+ try {
+ Optional<Pair<String, Long>> identity = getDbIdentityForDrop(dbName);
+ if (identity.isPresent()) {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateDb(getId(), identity.get().second, identity.get().first);
+ } else {
+ retireUnresolvedDatabaseGeneration();
+ }
+ } catch (Exception e) {
+ // The remote DROP is already committed; local cleanup cannot suppress its edit log.
+ LOG.warn("Failed to invalidate cold database {} in catalog {}: {}",
+ dbName, getName(), e.getMessage(), e);
}
}
@@ -1306,7 +1488,6 @@ public abstract class ExternalCatalog
}
tbls.add(tbl);
}
- LOG.info("debug get include table map: {}", includeTableMap);
return includeTableMap;
}
@@ -1365,14 +1546,14 @@ public abstract class ExternalCatalog
if (mode == 1) {
// Mode 1: Store as lowercase
- finalName = dbName.toLowerCase();
+ finalName = foldDatabaseName(dbName);
} else if (mode == 2) {
// Mode 2: Case-insensitive comparison
- finalName = lowerCaseToDatabaseName.get(dbName.toLowerCase());
+ finalName = lowerCaseToDatabaseName.get(foldDatabaseName(dbName));
if (finalName == null && !isReplay) {
try {
metaCache.refreshNames();
- finalName = lowerCaseToDatabaseName.get(dbName.toLowerCase());
+ finalName = lowerCaseToDatabaseName.get(foldDatabaseName(dbName));
} catch (Exception e) {
if (Thread.currentThread().isInterrupted()
&& e instanceof java.util.concurrent.CompletionException
@@ -1489,7 +1670,16 @@ public abstract class ExternalCatalog
@Override
public void notifyPropertiesUpdated(Map<String, String> updatedProps) {
- CatalogIf.super.notifyPropertiesUpdated(updatedProps);
+ try {
+ CatalogIf.super.notifyPropertiesUpdated(updatedProps);
+ } finally {
+ // The committed ALTER already published the properties; the property-specific cache-group
+ // removal must still run when the reset's own client cleanup throws.
+ invalidatePropertySpecificCacheGroups(updatedProps);
+ }
+ }
+
+ private void invalidatePropertySpecificCacheGroups(Map<String, String> updatedProps) {
String schemaCacheTtl = updatedProps.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null);
ExternalMetaCacheMgr extMetaCacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr();
if (java.util.Objects.nonNull(schemaCacheTtl)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index 2e1950971d8..d66aff16bf4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -120,16 +120,25 @@ public abstract class ExternalDatabase<T extends ExternalTable>
}
public void resetMetaToUninitialized() {
- resetMetaToUninitialized(true);
+ resetMetaToUninitialized(true, true);
}
public void resetMetaToUninitialized(boolean invalidateEngineCache) {
+ resetMetaToUninitialized(invalidateEngineCache, invalidateEngineCache);
+ }
+
+ public void resetMetaToUninitialized(boolean invalidateEngineCache, boolean invalidateRowCountCache) {
if (LOG.isDebugEnabled()) {
LOG.debug("resetToUninitialized db name {}, id {}, isInitializing: {}, initialized: {}",
this.name, this.id, isInitializing, initialized, new Exception());
}
MetaCache<T> cacheToInvalidate = null;
Runnable objectInvalidation = null;
+ if (invalidateRowCountCache) {
+ // Fence readers before the table-object generation is retired. The final fence below
+ // also removes values loaded during the reset and routed engine invalidation.
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(extCatalog.getId(), getId());
+ }
try {
synchronized (this) {
metadataLoadEpoch.updateAndGet(epoch -> epoch.next(epoch.catalogEpoch));
@@ -146,8 +155,20 @@ public abstract class ExternalDatabase<T extends ExternalTable>
objectInvalidation.run();
}
}
- if (invalidateEngineCache) {
- Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(this);
+ try {
+ if (invalidateEngineCache) {
+ // Route through the typed overload: connector-specific caches (for example Paimon's
+ // table loader) are keyed by the database object and are not fully covered by the
+ // name-based scan in invalidateDb(long, String).
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(this);
+ }
+ } finally {
+ if (invalidateRowCountCache) {
+ // Independent of the routed invalidation: a connector cache failure (for example
+ // Paimon's CacheException) must not skip the row-count fence.
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateRowCountCache(extCatalog.getId(), getId());
+ }
}
}
@@ -396,6 +417,13 @@ public abstract class ExternalDatabase<T extends ExternalTable>
return getLocalTableName(tableName, true) != null;
}
+ /** Resolve a cold table's deterministic ID without loading its evicted object. */
+ public OptionalLong getTableIdForReplay(String tableName) {
+ String localName = getLocalTableName(tableName, true);
+ return localName == null ? OptionalLong.empty()
+ : OptionalLong.of(Util.genIdByName(extCatalog.getName(), name, localName));
+ }
+
@Override
public void readLock() {
this.rwLock.readLock().lock();
@@ -619,15 +647,36 @@ public abstract class ExternalDatabase<T extends ExternalTable>
// Check whether the table still exists in the local replay cache.
ExternalTable dorisTable = getTableForReplay(tableName).orElse(null);
if (dorisTable == null) {
+ // A mode-2 table-name mapping can disappear while the old object stays resident in
+ // MetaCache.metaObjCache; getTableForReplay then misses only because the mapping is
+ // absent. Retire the hidden table-object generation so a same-name recreation cannot
+ // reuse the prior incarnation, then widen the engine and row-count fence.
+ if (!hasLocalTableName(tableName)) {
+ retireAllTableObjectsWithoutEngineInvalidation();
+ }
+ // The table object cache is much smaller than the row-count and Hive engine caches. A drop
+ // or rename must still retire stale entries when the table object was evicted, and the
+ // event carries the caller spelling, so widen to the canonical scope.
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateTableByNameOrWider(extCatalog.getId(), getFullName(), tableName);
return false;
}
- // clear the cache related to this table.
+ // Fence the held table's row count before releasing its local name/id slot: a query does not
+ // take this database's write lock, so a same-name replacement admitted right after the removal
+ // could otherwise observe the retired row-count entry through the deterministic table id.
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(dorisTable);
+ // Local retirement is unconditional: a routed engine failure must not leave the table visible.
if (isInitialized()) {
metaCache.invalidate(dorisTable.getName(),
Util.genIdByName(extCatalog.getName(), name, dorisTable.getName()));
}
-
- Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(dorisTable);
+ // Routed engine invalidation plus completion fencing, isolated from the local retirement.
+ try {
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(dorisTable);
+ } catch (RuntimeException e) {
+ LOG.warn("failed to invalidate engine caches after unregistering table {}.{}", name,
+ dorisTable.getName(), e);
+ }
return true;
}
@@ -639,7 +688,13 @@ public abstract class ExternalDatabase<T extends ExternalTable>
*/
public void retireAllTableObjectsWithoutEngineInvalidation() {
if (metaCache != null) {
- metaCache.invalidateObjects();
+ ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr();
+ cacheMgr.invalidateRowCountCache(extCatalog.getId(), getId());
+ try {
+ metaCache.invalidateObjects();
+ } finally {
+ cacheMgr.invalidateRowCountCache(extCatalog.getId(), getId());
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
index a7980d57871..300f2c6c3bf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
@@ -19,6 +19,7 @@ package org.apache.doris.datasource;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.Config;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.datasource.doris.DorisExternalMetaCache;
import org.apache.doris.datasource.hive.HiveExternalMetaCache;
@@ -51,6 +52,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.OptionalLong;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.locks.Lock;
@@ -380,10 +382,15 @@ public class ExternalMetaCacheMgr {
}
public void invalidateCatalog(long catalogId) {
- invalidateLanceTableAccess(catalogId);
- routeCatalogEngines(catalogId, cache -> safeInvalidate(
- cache, catalogId, "invalidateCatalog",
- () -> cache.invalidateCatalogEntries(catalogId)));
+ rowCountCache.invalidateCatalog(catalogId);
+ try {
+ invalidateLanceTableAccess(catalogId);
+ routeCatalogEngines(catalogId, cache -> safeInvalidate(
+ cache, catalogId, "invalidateCatalog",
+ () -> cache.invalidateCatalogEntries(catalogId)));
+ } finally {
+ rowCountCache.invalidateCatalog(catalogId);
+ }
}
public void invalidateCatalogByEngine(long catalogId, String engine) {
@@ -433,9 +440,16 @@ public class ExternalMetaCacheMgr {
* retained runtime, its group) and lets the next statement load a coherent generation.
*/
public void onCatalogOperationalContextChanged(long catalogId) {
- routeCatalogEngines(catalogId, cache -> safeInvalidate(
- cache, catalogId, "onCatalogOperationalContextChanged",
- () -> cache.invalidateCatalogEntries(catalogId)));
+ try {
+ routeCatalogEngines(catalogId, cache -> safeInvalidate(
+ cache, catalogId, "onCatalogOperationalContextChanged",
+ () -> cache.invalidateCatalogEntries(catalogId)));
+ } finally {
+ // The committed property change reset the catalog against a new target. Database objects
+ // already evicted from the smaller metadata cache never run removal callbacks, so publish
+ // one catalog-wide row-count fence to cover those cold scopes as well.
+ rowCountCache.invalidateCatalog(catalogId);
+ }
}
public void removeCatalogPermanently(long catalogId) {
@@ -454,7 +468,13 @@ public class ExternalMetaCacheMgr {
}
}
} finally {
- lifecycleLock.unlock();
+ // DROP never reuses the catalog ID. The earlier row-count fence protects readers;
+ // release its generation after permanent engine cleanup, not on rename.
+ try {
+ rowCountCache.releaseCatalog(catalogId);
+ } finally {
+ lifecycleLock.unlock();
+ }
}
}
@@ -464,11 +484,19 @@ public class ExternalMetaCacheMgr {
Lock lifecycleLock = catalogLifecycleLocks.get(catalogId);
lifecycleLock.lock();
try {
- catalog.rollBackCatalogProps(oldProperties);
- routeCatalogEngines(catalogId, cache -> safeInvalidate(
- cache, catalogId, "rollbackCatalogProperties",
- () -> cache.invalidateCatalog(catalogId)));
+ try {
+ catalog.rollBackCatalogProps(oldProperties);
+ catalog.resetToUninitialized(false);
+ } finally {
+ // Retire any group initialized from the rejected candidate even when the reset's
+ // own cleanup (JDBC/Trino/Lance close) throws; otherwise an already-initialized
+ // group keeps taking the fast path against the old target.
+ routeCatalogEngines(catalogId, cache -> safeInvalidate(
+ cache, catalogId, "rollbackCatalogProperties",
+ () -> cache.invalidateCatalog(catalogId)));
+ }
} finally {
+ rowCountCache.invalidateCatalog(catalogId);
lifecycleLock.unlock();
}
}
@@ -486,8 +514,49 @@ public class ExternalMetaCacheMgr {
}
public void invalidateDb(long catalogId, String dbName) {
- routeCatalogEngines(catalogId, cache -> safeInvalidate(
- cache, catalogId, "invalidateDb", () -> cache.invalidateDb(catalogId, dbName)));
+ Optional<ExternalDatabase<? extends ExternalTable>> db = getCachedDb(catalogId, dbName);
+ if (db.isPresent()) {
+ invalidateDb(catalogId, db.get().getId(), db.get().getFullName());
+ return;
+ }
+ Optional<Pair<String, Long>> identity = getDbIdentityForReplay(catalogId, dbName);
+ if (identity.isPresent()) {
+ invalidateDb(catalogId, identity.get().second, identity.get().first);
+ } else {
+ invalidateDb(catalogId, dbName, OptionalLong.empty(), true);
+ }
+ }
+
+ public void invalidateDb(long catalogId, long dbId, String dbName) {
+ invalidateDb(catalogId, dbName, OptionalLong.of(dbId), true);
+ }
+
+ void invalidateDb(long catalogId, long dbId, String dbName, boolean invalidateRowCountCache) {
+ invalidateDb(catalogId, dbName, OptionalLong.of(dbId), invalidateRowCountCache);
+ }
+
+ private void invalidateDb(long catalogId, String dbName, OptionalLong dbId, boolean invalidateRowCountCache) {
+ if (invalidateRowCountCache) {
+ invalidateDatabaseRowCount(catalogId, dbId);
+ }
+ try {
+ routeCatalogEngines(catalogId, cache -> safeInvalidate(
+ cache, catalogId, "invalidateDb", () -> cache.invalidateDb(catalogId, dbName)));
+ } finally {
+ if (invalidateRowCountCache) {
+ invalidateDatabaseRowCount(catalogId, dbId);
+ }
+ }
+ }
+
+ private void invalidateDatabaseRowCount(long catalogId, OptionalLong dbId) {
+ if (dbId.isPresent()) {
+ rowCountCache.invalidateDb(catalogId, dbId.getAsLong());
+ } else {
+ // The database object cache is smaller than the row-count cache. If the object has
+ // already been evicted, retire the catalog scope rather than hashing caller spelling.
+ rowCountCache.invalidateCatalog(catalogId);
+ }
}
public void invalidateDb(ExternalDatabase<?> database) {
@@ -497,16 +566,30 @@ public class ExternalMetaCacheMgr {
}
public void invalidateTable(long catalogId, String dbName, String tableName) {
- invalidateLanceTableAccess(catalogId);
- routeCatalogEngines(catalogId, cache -> safeInvalidate(
- cache, catalogId, "invalidateTable",
- () -> cache.invalidateTable(catalogId, dbName, tableName)));
+ Runnable rowCountFence = () -> rowCountCache.invalidateCatalog(catalogId);
+ try {
+ rowCountFence = resolveTableRowCountFence(catalogId, dbName, getCachedDb(catalogId, dbName), tableName);
+ rowCountFence.run();
+ invalidateLanceTableAccess(catalogId);
+ routeCatalogEngines(catalogId, cache -> safeInvalidate(
+ cache, catalogId, "invalidateTable",
+ () -> cache.invalidateTable(catalogId, dbName, tableName)));
+ } finally {
+ rowCountFence.run();
+ }
}
public void invalidateTableByEngine(long catalogId, String engine, String dbName, String tableName) {
- routeSpecifiedEngine(engine, cache -> safeInvalidate(
- cache, catalogId, "invalidateTableByEngine",
- () -> cache.invalidateTable(catalogId, dbName, tableName)));
+ Runnable rowCountFence = () -> rowCountCache.invalidateCatalog(catalogId);
+ try {
+ rowCountFence = resolveTableRowCountFence(catalogId, dbName, getCachedDb(catalogId, dbName), tableName);
+ rowCountFence.run();
+ routeSpecifiedEngine(engine, cache -> safeInvalidate(
+ cache, catalogId, "invalidateTableByEngine",
+ () -> cache.invalidateTable(catalogId, dbName, tableName)));
+ } finally {
+ rowCountFence.run();
+ }
}
private void invalidateLanceTableAccess(long catalogId) {
@@ -520,9 +603,55 @@ public class ExternalMetaCacheMgr {
public void invalidatePartitions(long catalogId,
String dbName, String tableName, List<String> partitions) {
- routeCatalogEngines(catalogId, cache -> safeInvalidate(
- cache, catalogId, "invalidatePartitions",
- () -> cache.invalidatePartitions(catalogId, dbName, tableName, partitions)));
+ Runnable rowCountFence = () -> rowCountCache.invalidateCatalog(catalogId);
+ try {
+ rowCountFence = resolveTableRowCountFence(catalogId, dbName, getCachedDb(catalogId, dbName), tableName);
+ rowCountFence.run();
+ routeCatalogEngines(catalogId, cache -> safeInvalidate(
+ cache, catalogId, "invalidatePartitions",
+ () -> cache.invalidatePartitions(catalogId, dbName, tableName, partitions)));
+ } finally {
+ rowCountFence.run();
+ }
+ }
+
+ private Runnable resolveTableRowCountFence(long catalogId, String dbName,
+ Optional<ExternalDatabase<? extends ExternalTable>> db, String tableName) {
+ if (db.isPresent()) {
+ Optional<? extends ExternalTable> table = db.get().getTableForReplay(tableName);
+ if (table.isPresent()) {
+ long tableCatalogId = table.get().getCatalog().getId();
+ long tableDbId = table.get().getDb().getId();
+ long tableId = table.get().getId();
+ return () -> rowCountCache.invalidateTable(tableCatalogId, tableDbId, tableId);
+ }
+ long dbId = db.get().getId();
+ OptionalLong tableId = db.get().getTableIdForReplay(tableName);
+ if (tableId.isPresent()) {
+ return () -> rowCountCache.invalidateTable(catalogId, dbId, tableId.getAsLong());
+ }
+ return () -> rowCountCache.invalidateDb(catalogId, dbId);
+ }
+ Optional<Pair<String, Long>> identity = getDbIdentityForReplay(catalogId, dbName);
+ if (identity.isPresent()) {
+ long dbId = identity.get().second;
+ return () -> rowCountCache.invalidateDb(catalogId, dbId);
+ }
+ return () -> rowCountCache.invalidateCatalog(catalogId);
+ }
+
+ private Optional<Pair<String, Long>> getDbIdentityForReplay(long catalogId, String dbName) {
+ CatalogIf<?> catalog = getCatalog(catalogId);
+ return catalog instanceof ExternalCatalog
+ ? ((ExternalCatalog) catalog).getDbIdentityForReplay(dbName, 0L) : Optional.empty();
+ }
+
+ private Optional<ExternalDatabase<? extends ExternalTable>> getCachedDb(long catalogId, String dbName) {
+ CatalogIf<?> catalog = getCatalog(catalogId);
+ if (!(catalog instanceof ExternalCatalog)) {
+ return Optional.empty();
+ }
+ return ((ExternalCatalog) catalog).getDbForReplay(dbName);
}
public List<CatalogMetaCacheStats> getCatalogCacheStats(long catalogId) {
@@ -689,17 +818,67 @@ public class ExternalMetaCacheMgr {
public void invalidateTableCache(ExternalTable dorisTable) {
long catalogId = dorisTable.getCatalog().getId();
+ // Engine eviction can expose a new snapshot before the completion fence below.
+ invalidateRowCountCache(dorisTable);
// Typed table invalidation bypasses the name-based invalidateTable() entry point, so the
// Lance access-cache retirement that used to happen there has to be repeated here.
- invalidateLanceTableAccess(catalogId);
- routeCatalogEngines(catalogId, cache -> safeInvalidate(
- cache, catalogId, "invalidateTable", () -> cache.invalidateTable(dorisTable)));
+ try {
+ invalidateLanceTableAccess(catalogId);
+ routeCatalogEngines(catalogId, cache -> safeInvalidate(
+ cache, catalogId, "invalidateTable", () -> cache.invalidateTable(dorisTable)));
+ } finally {
+ invalidateRowCountCache(dorisTable);
+ }
if (LOG.isDebugEnabled()) {
LOG.debug("invalid table cache for {}.{} in catalog {}", dorisTable.getRemoteDbName(),
dorisTable.getRemoteName(), dorisTable.getCatalog().getName());
}
}
+ /**
+ * Best-effort invalidation for a metadata event that carries the caller's DB/table spelling.
+ * Resolves canonical local identity, then fences engine caches and row counts at the narrowest
+ * scope that still covers the event; widens to the canonical database or catalog scope when the
+ * cached object has already been evicted, so caller spelling can never miss a canonical key.
+ */
+ public void invalidateTableByNameOrWider(long catalogId, String dbName, String tableName) {
+ // A cold object miss may route through engine invalidation before its final fence.
+ invalidateRowCountCache(catalogId, dbName, tableName);
+ Optional<ExternalDatabase<? extends ExternalTable>> db = getCachedDb(catalogId, dbName);
+ if (!db.isPresent()) {
+ Optional<Pair<String, Long>> identity = getDbIdentityForReplay(catalogId, dbName);
+ if (identity.isPresent()) {
+ invalidateDb(catalogId, identity.get().second, identity.get().first);
+ } else {
+ invalidateCatalog(catalogId);
+ }
+ return;
+ }
+ Optional<? extends ExternalTable> table = db.get().getTableForReplay(tableName);
+ if (table.isPresent()) {
+ invalidateTableCache(table.get());
+ } else {
+ invalidateDb(catalogId, db.get().getId(), db.get().getFullName());
+ }
+ }
+
+ public void invalidateRowCountCache(ExternalTable table) {
+ rowCountCache.invalidateTable(table.getCatalog().getId(), table.getDb().getId(), table.getId());
+ }
+
+ public void invalidateRowCountCache(long catalogId) {
+ rowCountCache.invalidateCatalog(catalogId);
+ }
+
+ public void invalidateRowCountCache(long catalogId, long dbId) {
+ rowCountCache.invalidateDb(catalogId, dbId);
+ }
+
+ public void invalidateRowCountCache(long catalogId, String dbName, String tableName) {
+ Optional<ExternalDatabase<? extends ExternalTable>> db = getCachedDb(catalogId, dbName);
+ resolveTableRowCountFence(catalogId, dbName, db, tableName).run();
+ }
+
public LegacyMetaCacheFactory legacyMetaCacheFactory() {
return legacyMetaCacheFactory;
}
@@ -720,6 +899,10 @@ public class ExternalMetaCacheMgr {
bindCatalogPreparers();
}
+ void replaceRowCountCacheForTest(ExternalRowCountCache cache) {
+ rowCountCache = cache;
+ }
+
/**
* Fallback implementation of {@link AbstractExternalMetaCache} for engines that do not
* provide dedicated cache entries.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
index f32ba5ae20c..f27543c2f27 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
@@ -24,22 +24,63 @@ import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.BasicAsyncCacheLoader;
import org.apache.doris.statistics.util.StatisticsUtil;
+import com.github.benmanes.caffeine.cache.AsyncCacheLoader;
import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.Ticker;
import lombok.Getter;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.Objects;
import java.util.Optional;
import java.util.OptionalLong;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Supplier;
public class ExternalRowCountCache {
private static final Logger LOG = LogManager.getLogger(ExternalRowCountCache.class);
private final AsyncLoadingCache<RowCountKey, Optional<Long>> rowCountCache;
+ private final ConcurrentHashMap<LoadKey, Set<LoadFence>> inFlightLoads = new ConcurrentHashMap<>();
+ // A catalog refresh changes this generation in O(1). Old cache entries remain bounded by
+ // Caffeine's size/expiry policy but are never addressable by a new metadata generation.
+ // A held table may issue a late read after permanent DROP releases its catalog generation.
+ // Keep even those post-DROP admissions bounded; eviction forces a fresh unique generation.
+ private final Cache<Long, AtomicLong> catalogGenerations;
+ // Database generations are bounded independently of database CREATE/DROP churn. Eviction
+ // merely forces a new, globally unique generation and makes the old entries unreachable.
+ private final Cache<DatabaseKey, Long> dbGenerations;
+ // Generations are globally unique: two catalog IDs with the same table ID must not alias
+ // when one catalog has retired its old cache generation.
+ private final AtomicLong nextCatalogGeneration = new AtomicLong();
+ // Serialize future publication with explicit invalidation. Invalidation marks matching in-flight loads
+ // before removing their cache entries, so a refresh that finishes later cannot republish stale data.
+ private final ReentrantReadWriteLock publicationLock = new ReentrantReadWriteLock();
public ExternalRowCountCache(ExecutorService executor) {
+ this(executor, null);
+ }
+
+ ExternalRowCountCache(ExecutorService executor, Ticker ticker) {
+ this(executor, ticker, new RowCountCacheLoader());
+ }
+
+ ExternalRowCountCache(ExecutorService executor, Ticker ticker, RowCountCacheLoader loader) {
+ this(executor, ticker, loader, Config.max_external_table_row_count_cache_num);
+ }
+
+ ExternalRowCountCache(ExecutorService executor, Ticker ticker, RowCountCacheLoader loader,
+ long maxGenerationEntries) {
+ catalogGenerations = Caffeine.newBuilder().maximumSize(maxGenerationEntries).build();
+ dbGenerations = Caffeine.newBuilder().maximumSize(maxGenerationEntries).build();
// 1. set expireAfterWrite to 1 day, avoid too many entries
// 2. set refreshAfterWrite to 10min(default), so that the cache will be refreshed after 10min
CacheFactory rowCountCacheFactory = new CacheFactory(
@@ -47,8 +88,9 @@ public class ExternalRowCountCache {
OptionalLong.of(Config.external_cache_refresh_time_minutes * 60),
Config.max_external_table_row_count_cache_num,
false,
- null);
- rowCountCache = rowCountCacheFactory.buildAsyncCache(new RowCountCacheLoader(), executor);
+ ticker);
+ rowCountCache = rowCountCacheFactory.buildAsyncCache(
+ new InvalidationAwareLoader(loader), executor);
}
@Getter
@@ -56,11 +98,19 @@ public class ExternalRowCountCache {
private final long catalogId;
private final long dbId;
private final long tableId;
+ private final long catalogGeneration;
+ private final long dbGeneration;
public RowCountKey(long catalogId, long dbId, long tableId) {
+ this(catalogId, dbId, tableId, 0, 0);
+ }
+
+ private RowCountKey(long catalogId, long dbId, long tableId, long catalogGeneration, long dbGeneration) {
this.catalogId = catalogId;
this.dbId = dbId;
this.tableId = tableId;
+ this.catalogGeneration = catalogGeneration;
+ this.dbGeneration = dbGeneration;
}
@Override
@@ -71,12 +121,15 @@ public class ExternalRowCountCache {
if (!(obj instanceof RowCountKey)) {
return false;
}
- return ((RowCountKey) obj).tableId == this.tableId;
+ RowCountKey other = (RowCountKey) obj;
+ return other.tableId == tableId && other.catalogGeneration == catalogGeneration
+ && other.dbGeneration == dbGeneration;
}
@Override
public int hashCode() {
- return (int) tableId;
+ int hash = 31 * Long.hashCode(tableId) + Long.hashCode(catalogGeneration);
+ return 31 * hash + Long.hashCode(dbGeneration);
}
}
@@ -87,6 +140,174 @@ public class ExternalRowCountCache {
}
}
+ private final class InvalidationAwareLoader implements AsyncCacheLoader<RowCountKey, Optional<Long>> {
+ private final RowCountCacheLoader delegate;
+
+ private InvalidationAwareLoader(RowCountCacheLoader delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public CompletableFuture<Optional<Long>> asyncLoad(RowCountKey key, Executor executor) {
+ return loadWithInvalidationFence(key, executor, () -> delegate.doLoad(key));
+ }
+ }
+
+ private static final class LoadFence {
+ private boolean invalidated;
+ }
+
+ // A cache generation keeps the existing table-ID identity within one catalog incarnation.
+ // In-flight loads need the complete scope so database/table invalidation can fence a
+ // same-tableId replacement without touching another catalog's load.
+ private static final class LoadKey {
+ private final long catalogId;
+ private final long dbId;
+ private final long tableId;
+
+ private LoadKey(RowCountKey key) {
+ catalogId = key.catalogId;
+ dbId = key.dbId;
+ tableId = key.tableId;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof LoadKey)) {
+ return false;
+ }
+ LoadKey other = (LoadKey) obj;
+ return catalogId == other.catalogId && dbId == other.dbId && tableId == other.tableId;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(catalogId, dbId, tableId);
+ }
+ }
+
+ private static final class DatabaseKey {
+ private final long catalogId;
+ private final long dbId;
+
+ private DatabaseKey(long catalogId, long dbId) {
+ this.catalogId = catalogId;
+ this.dbId = dbId;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof DatabaseKey)) {
+ return false;
+ }
+ DatabaseKey other = (DatabaseKey) obj;
+ return catalogId == other.catalogId && dbId == other.dbId;
+ }
+
+ @Override
+ public int hashCode() {
+ return 31 * Long.hashCode(catalogId) + Long.hashCode(dbId);
+ }
+ }
+
+ private CompletableFuture<Optional<Long>> loadWithInvalidationFence(
+ RowCountKey key, Executor executor, Supplier<Optional<Long>> loader) {
+ LoadFence fence = new LoadFence();
+ LoadKey loadKey = new LoadKey(key);
+ publicationLock.readLock().lock();
+ try {
+ inFlightLoads.compute(loadKey, (ignored, fences) -> {
+ Set<LoadFence> currentFences = fences == null ? ConcurrentHashMap.newKeySet() : fences;
+ currentFences.add(fence);
+ return currentFences;
+ });
+ } finally {
+ publicationLock.readLock().unlock();
+ }
+
+ CompletableFuture<Optional<Long>> publishedFuture = new CompletableFuture<>();
+ CompletableFuture<Optional<Long>> loadFuture;
+ try {
+ loadFuture = CompletableFuture.supplyAsync(loader, executor);
+ } catch (RuntimeException e) {
+ publicationLock.readLock().lock();
+ try {
+ removeInFlightLoad(loadKey, fence);
+ } finally {
+ publicationLock.readLock().unlock();
+ }
+ throw e;
+ }
+ loadFuture.whenComplete((value, throwable) -> {
+ publicationLock.readLock().lock();
+ try {
+ if (throwable != null) {
+ publishedFuture.completeExceptionally(throwable);
+ } else if (fence.invalidated) {
+ publishedFuture.complete(null);
+ } else {
+ publishedFuture.complete(value);
+ }
+ } finally {
+ removeInFlightLoad(loadKey, fence);
+ publicationLock.readLock().unlock();
+ }
+ });
+ return publishedFuture;
+ }
+
+ private void removeInFlightLoad(LoadKey key, LoadFence fence) {
+ inFlightLoads.computeIfPresent(key, (ignored, fences) -> {
+ fences.remove(fence);
+ return fences.isEmpty() ? null : fences;
+ });
+ }
+
+ int getInFlightLoadCountForTest() {
+ publicationLock.readLock().lock();
+ try {
+ return inFlightLoads.values().stream().mapToInt(Set::size).sum();
+ } finally {
+ publicationLock.readLock().unlock();
+ }
+ }
+
+ void refreshForTest(long catalogId, long dbId, long tableId) {
+ rowCountCache.synchronous().refresh(new RowCountKey(
+ catalogId, dbId, tableId, currentCatalogGeneration(catalogId), currentDbGeneration(catalogId, dbId)));
+ }
+
+ private long currentCatalogGeneration(long catalogId) {
+ return catalogGenerations.get(catalogId,
+ ignored -> new AtomicLong(nextCatalogGeneration.incrementAndGet())).get();
+ }
+
+ private boolean isCatalogGenerationCurrent(long catalogId, long generation) {
+ AtomicLong current = catalogGenerations.getIfPresent(catalogId);
+ return current != null && current.get() == generation;
+ }
+
+ private long currentDbGeneration(long catalogId, long dbId) {
+ return dbGenerations.get(new DatabaseKey(catalogId, dbId),
+ ignored -> nextCatalogGeneration.incrementAndGet());
+ }
+
+ private boolean isDbGenerationCurrent(long catalogId, long dbId, long generation) {
+ Long current = dbGenerations.getIfPresent(new DatabaseKey(catalogId, dbId));
+ return current != null && current == generation;
+ }
+
+ int getCatalogGenerationCountForTest() {
+ catalogGenerations.cleanUp();
+ return catalogGenerations.asMap().size();
+ }
+
static Optional<Long> loadRowCount(RowCountKey rowCountKey, boolean fillMetaCache) {
try {
ExternalTable table = (ExternalTable) StatisticsUtil.findTable(
@@ -106,9 +327,7 @@ public class ExternalRowCountCache {
// so we can't try to load the row count until the cache expire.
// Throw an exception here will cause too much stack log in fe.out.
// So we return null when exception happen.
- // Null may raise NPE in caller, but that is expected.
- // We catch that NPE and return a default value -1 without keep the value in cache,
- // so we can trigger the load function to fetch row count again next time in this exception case.
+ // The caller treats null as an uncached UNKNOWN result, so the next read can retry.
return null;
}
}
@@ -120,19 +339,35 @@ public class ExternalRowCountCache {
* @return Cached row count or -1 if not exist
*/
public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean fillMetaCache) {
- RowCountKey key = new RowCountKey(catalogId, dbId, tableId);
try {
- CompletableFuture<Optional<Long>> f = fillMetaCache
- ? rowCountCache.get(key, (rowCountKey, executor) -> CompletableFuture.supplyAsync(
- () -> loadRowCount(rowCountKey, true), executor))
- : rowCountCache.get(key);
+ CompletableFuture<Optional<Long>> f;
+ long generation;
+ long dbGeneration;
+ publicationLock.readLock().lock();
+ try {
+ generation = currentCatalogGeneration(catalogId);
+ dbGeneration = currentDbGeneration(catalogId, dbId);
+ RowCountKey key = new RowCountKey(catalogId, dbId, tableId, generation, dbGeneration);
+ f = fillMetaCache
+ ? rowCountCache.get(key, (rowCountKey, executor) -> loadWithInvalidationFence(
+ rowCountKey, executor, () -> loadRowCount(rowCountKey, true)))
+ : rowCountCache.get(key);
+ } finally {
+ publicationLock.readLock().unlock();
+ }
// Get row count synchronously by default.
if (ConnectContext.get() == null
|| ConnectContext.get().getSessionVariable().fetchHiveRowCountSync) {
- return f.get().orElse(TableIf.UNKNOWN_ROW_COUNT);
+ Optional<Long> value = f.get();
+ return value != null && isCatalogGenerationCurrent(catalogId, generation)
+ && isDbGenerationCurrent(catalogId, dbId, dbGeneration)
+ ? value.orElse(TableIf.UNKNOWN_ROW_COUNT) : TableIf.UNKNOWN_ROW_COUNT;
} else {
if (f.isDone()) {
- return f.get().orElse(TableIf.UNKNOWN_ROW_COUNT);
+ Optional<Long> value = f.get();
+ return value != null && isCatalogGenerationCurrent(catalogId, generation)
+ && isDbGenerationCurrent(catalogId, dbId, dbGeneration)
+ ? value.orElse(TableIf.UNKNOWN_ROW_COUNT) : TableIf.UNKNOWN_ROW_COUNT;
}
LOG.info("Row count for table {}.{}.{} is still processing.", catalogId, dbId, tableId);
}
@@ -148,13 +383,33 @@ public class ExternalRowCountCache {
* @return Cached row count or -1 if not exist
*/
public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) {
- RowCountKey key = new RowCountKey(catalogId, dbId, tableId);
try {
- CompletableFuture<Optional<Long>> f = rowCountCache.getIfPresent(key);
+ CompletableFuture<Optional<Long>> f;
+ long generation;
+ long dbGeneration;
+ publicationLock.readLock().lock();
+ try {
+ AtomicLong current = catalogGenerations.getIfPresent(catalogId);
+ if (current == null) {
+ return -1;
+ }
+ generation = current.get();
+ Long currentDb = dbGenerations.getIfPresent(new DatabaseKey(catalogId, dbId));
+ if (currentDb == null) {
+ return -1;
+ }
+ dbGeneration = currentDb;
+ RowCountKey key = new RowCountKey(catalogId, dbId, tableId, generation, dbGeneration);
+ f = rowCountCache.getIfPresent(key);
+ } finally {
+ publicationLock.readLock().unlock();
+ }
if (f == null) {
return -1;
- } else if (f.isDone()) {
- return f.get().orElse(-1L);
+ } else if (f.isDone() && isCatalogGenerationCurrent(catalogId, generation)
+ && isDbGenerationCurrent(catalogId, dbId, dbGeneration)) {
+ Optional<Long> value = f.get();
+ return value == null ? -1L : value.orElse(-1L);
}
} catch (Exception e) {
LOG.warn("Unexpected exception while returning row count if present", e);
@@ -162,4 +417,63 @@ public class ExternalRowCountCache {
return -1;
}
+ // Catalog invalidation is a constant-time generation fence. Old entries remain bounded by
+ // Caffeine and cannot be returned or reused after the generation changes.
+ void invalidateCatalog(long catalogId) {
+ publicationLock.writeLock().lock();
+ try {
+ AtomicLong current = catalogGenerations.getIfPresent(catalogId);
+ if (current != null) {
+ current.set(nextCatalogGeneration.incrementAndGet());
+ }
+ } finally {
+ publicationLock.writeLock().unlock();
+ }
+ }
+
+ /** Catalog IDs are never reused after DROP; old readers still fail the generation check. */
+ void releaseCatalog(long catalogId) {
+ publicationLock.writeLock().lock();
+ try {
+ catalogGenerations.invalidate(catalogId);
+ } finally {
+ publicationLock.writeLock().unlock();
+ }
+ }
+
+ void invalidateDb(long catalogId, long dbId) {
+ publicationLock.writeLock().lock();
+ try {
+ DatabaseKey key = new DatabaseKey(catalogId, dbId);
+ Long current = dbGenerations.getIfPresent(key);
+ if (current != null) {
+ dbGenerations.put(key, nextCatalogGeneration.incrementAndGet());
+ }
+ } finally {
+ publicationLock.writeLock().unlock();
+ }
+ }
+
+ void invalidateTable(long catalogId, long dbId, long tableId) {
+ publicationLock.writeLock().lock();
+ try {
+ AtomicLong current = catalogGenerations.getIfPresent(catalogId);
+ if (current == null) {
+ return;
+ }
+ Long currentDb = dbGenerations.getIfPresent(new DatabaseKey(catalogId, dbId));
+ if (currentDb == null) {
+ return;
+ }
+ RowCountKey key = new RowCountKey(catalogId, dbId, tableId, current.get(), currentDb);
+ Set<LoadFence> fences = inFlightLoads.get(new LoadKey(key));
+ if (fences != null) {
+ fences.forEach(fence -> fence.invalidated = true);
+ }
+ rowCountCache.synchronous().invalidate(key);
+ } finally {
+ publicationLock.writeLock().unlock();
+ }
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index 3e194a72539..2a5f5240e60 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -413,7 +413,7 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable {
throw new NotImplementedException("implement in sub class");
}
- public void unsetObjectCreated() {
+ public synchronized void unsetObjectCreated() {
this.objectCreated = false;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
index 68bfeb2034e..41fa33a94d3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
@@ -93,6 +93,9 @@ public class HMSExternalCatalog extends ExternalCatalog {
private volatile AbstractHiveProperties hmsProperties;
private AtomicLong runtimeGeneration = new AtomicLong();
+ private transient volatile String eventIncludeTableList;
+ private transient volatile Map<String, List<String>> eventIncludeTableMap;
+
public long getRuntimeGeneration() {
return runtimeGeneration.get();
}
@@ -307,8 +310,8 @@ public class HMSExternalCatalog extends ExternalCatalog {
LOG.debug("create database [{}]", dbName);
}
- // HMS notification events normalize database names to lowercase before reaching this boundary.
- if (!isDatabaseAllowedByFilterIgnoringCase(dbName)) {
+ // The event caller passes the original remote spelling used by database-list filters.
+ if (isDatabaseEventTargetExcluded(dbName)) {
return true;
}
@@ -320,21 +323,59 @@ public class HMSExternalCatalog extends ExternalCatalog {
return false;
}
- @Override
- public void notifyPropertiesUpdated(Map<String, String> updatedProps) {
- super.notifyPropertiesUpdated(updatedProps);
- String fileMetaCacheTtl = updatedProps.getOrDefault(FILE_META_CACHE_TTL_SECOND, null);
- String partitionCacheTtl = updatedProps.getOrDefault(PARTITION_CACHE_TTL_SECOND, null);
- if (Objects.nonNull(fileMetaCacheTtl) || Objects.nonNull(partitionCacheTtl)) {
- Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HiveExternalMetaCache.ENGINE);
+ /** HMS notifications are unfiltered; excluded targets must not change this catalog's caches. */
+ public boolean isDatabaseEventTargetExcluded(String dbName) {
+ return !isDatabaseAllowedByFilter(dbName);
+ }
+
+ public boolean isPartitionEventTargetExcluded(String dbName, String tableName) {
+ if (isDatabaseEventTargetExcluded(dbName)) {
+ return true;
}
- if (updatedProps.keySet().stream()
- .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, HudiExternalMetaCache.ENGINE))) {
- Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HudiExternalMetaCache.ENGINE);
+ List<String> includedTables = getEventIncludeTableMap().get(dbName);
+ return includedTables != null && !includedTables.isEmpty()
+ && includedTables.stream().noneMatch(name -> name.equalsIgnoreCase(tableName));
+ }
+
+ private Map<String, List<String>> getEventIncludeTableMap() {
+ String configuredList = catalogProperty.getOrDefault(INCLUDE_TABLE_LIST, "");
+ Map<String, List<String>> snapshot = eventIncludeTableMap;
+ if (snapshot == null || !configuredList.equals(eventIncludeTableList)) {
+ synchronized (this) {
+ configuredList = catalogProperty.getOrDefault(INCLUDE_TABLE_LIST, "");
+ snapshot = eventIncludeTableMap;
+ if (snapshot == null || !configuredList.equals(eventIncludeTableList)) {
+ snapshot = getIncludeTableMap();
+ eventIncludeTableMap = snapshot;
+ eventIncludeTableList = configuredList;
+ }
+ }
}
- if (updatedProps.keySet().stream()
- .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, IcebergExternalMetaCache.ENGINE))) {
- Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE);
+ return snapshot;
+ }
+
+ @Override
+ public void notifyPropertiesUpdated(Map<String, String> updatedProps) {
+ try {
+ super.notifyPropertiesUpdated(updatedProps);
+ } finally {
+ // The committed ALTER already published the properties; the engine-specific groups below
+ // must still be retired when the generic reset cleanup throws.
+ String fileMetaCacheTtl = updatedProps.getOrDefault(FILE_META_CACHE_TTL_SECOND, null);
+ String partitionCacheTtl = updatedProps.getOrDefault(PARTITION_CACHE_TTL_SECOND, null);
+ if (Objects.nonNull(fileMetaCacheTtl) || Objects.nonNull(partitionCacheTtl)) {
+ Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HiveExternalMetaCache.ENGINE);
+ }
+ if (updatedProps.keySet().stream()
+ .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, HudiExternalMetaCache.ENGINE))) {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .removeCatalogByEngine(getId(), HudiExternalMetaCache.ENGINE);
+ }
+ if (updatedProps.keySet().stream()
+ .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, IcebergExternalMetaCache.ENGINE))) {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE);
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
index bb607fb0b1d..b6126db8407 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
@@ -131,14 +131,20 @@ public class HiveMetadataOps implements ExternalMetadataOps {
@Override
public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws DdlException {
+ return dropDbImplWithResolvedName(dbName, ifExists, force).isPresent();
+ }
+
+ @Override
+ public Optional<String> dropDbImplWithResolvedName(String dbName, boolean ifExists, boolean force)
+ throws DdlException {
ExternalDatabase dorisDb = catalog.getDbNullable(dbName);
if (dorisDb == null) {
if (ifExists) {
LOG.info("drop database[{}] which does not exist", dbName);
- return false;
+ return Optional.empty();
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_DROP_EXISTS, dbName);
- return false;
+ return Optional.empty();
}
}
try {
@@ -161,7 +167,7 @@ public class HiveMetadataOps implements ExternalMetadataOps {
}
}
client.dropDatabase(dorisDb.getRemoteName());
- return true;
+ return Optional.of(dorisDb.getFullName());
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
@@ -172,6 +178,11 @@ public class HiveMetadataOps implements ExternalMetadataOps {
catalog.unregisterDatabase(dbName);
}
+ @Override
+ public void afterDropDbNoOp(String dbName) {
+ catalog.retireCachedDatabaseForNoOp(dbName);
+ }
+
@Override
public boolean createTableImpl(CreateTableInfo createTableInfo) throws UserException {
String dbName = createTableInfo.getDbName();
@@ -320,9 +331,13 @@ public class HiveMetadataOps implements ExternalMetadataOps {
@Override
public void afterDropTable(String dbName, String tblName) {
- Optional<ExternalDatabase<?>> db = catalog.getDbForReplay(dbName);
+ Optional<ExternalDatabase<?>> db = catalog.getDbForDropReplay(dbName);
if (db.isPresent()) {
db.get().unregisterTable(tblName);
+ } else {
+ // A cold object can retain a known canonical name; only a lost mode-2 mapping needs
+ // catalog-wide retirement of hidden objects and engine entries.
+ catalog.invalidateColdDatabaseForReplay(dbName);
}
LOG.info("after drop table {}.{}.{}, is db exists: {}",
getCatalog().getName(), dbName, tblName, db.isPresent());
@@ -348,8 +363,14 @@ public class HiveMetadataOps implements ExternalMetadataOps {
if (tbl.isPresent()) {
Env.getCurrentEnv().getRefreshManager()
.refreshTableInternal(db.get(), (ExternalTable) tbl.get(), updateTime);
+ return;
}
}
+ // The table or database object is cold. The event carries the caller's spelling, which may
+ // not match canonical local cache keys under lower_case_*_names=2, so widen to whatever
+ // scope still covers the event instead of routing the raw name.
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .invalidateTableByNameOrWider(catalog.getId(), dbName, tblName);
} catch (Exception e) {
LOG.warn("exception when calling afterTruncateTable for db: {}, table: {}, error: {}",
dbName, tblName, e.getMessage(), e);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
index e582c3d2662..dd5e75e4fed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
@@ -106,7 +106,8 @@ public class AddPartitionEvent extends MetastorePartitionEvent {
return;
}
Env.getCurrentEnv().getCatalogMgr()
- .addExternalPartitions(catalogName, dbName, hmsTbl.getTableName(), partitionNames, eventTime, true);
+ .addExternalPartitions(catalogName, hmsTbl.getDbName(), hmsTbl.getTableName(),
+ partitionNames, eventTime, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
getMsgWithEventInfo("Failed to process event"), e);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
index 8f293260058..00bc3f3ac39 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
@@ -22,6 +22,7 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.common.DdlException;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.hive.HMSExternalCatalog;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -81,14 +82,24 @@ public class AlterDatabaseEvent extends MetastoreEvent {
if (!(catalog instanceof ExternalCatalog)) {
throw new DdlException("Only support ExternalCatalog Databases");
}
- if (catalog.getDbNullable(dbAfter.getName()) != null) {
+ HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog;
+ boolean beforeExcluded = hmsCatalog.isDatabaseEventTargetExcluded(dbBefore.getName());
+ boolean afterExcluded = hmsCatalog.isDatabaseEventTargetExcluded(dbAfter.getName());
+ if (beforeExcluded && afterExcluded) {
+ return;
+ }
+ if (!afterExcluded && catalog.getDbNullable(dbAfter.getName()) != null) {
logInfo("AlterExternalDatabase canceled, because dbAfter has exist, "
+ "catalogName:[{}],dbName:[{}]",
catalogName, dbAfter.getName());
return;
}
- Env.getCurrentEnv().getCatalogMgr().unregisterExternalDatabase(dbBefore.getName(), catalogName);
- Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbAfter.getName(), catalogName);
+ if (!beforeExcluded) {
+ Env.getCurrentEnv().getCatalogMgr().unregisterExternalDatabase(dbBefore.getName(), catalogName);
+ }
+ if (!afterExcluded) {
+ Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbAfter.getName(), catalogName);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
index d9898f68d98..2a91c6c724b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
@@ -113,14 +113,14 @@ public class AlterPartitionEvent extends MetastorePartitionEvent {
catalogName, dbName, tblName, partitionNameBefore, partitionNameAfter);
if (isRename) {
Env.getCurrentEnv().getCatalogMgr()
- .dropExternalPartitions(catalogName, dbName, tblName,
+ .dropExternalPartitions(catalogName, hmsTbl.getDbName(), tblName,
Lists.newArrayList(partitionNameBefore), eventTime, true);
Env.getCurrentEnv().getCatalogMgr()
- .addExternalPartitions(catalogName, dbName, tblName,
+ .addExternalPartitions(catalogName, hmsTbl.getDbName(), tblName,
Lists.newArrayList(partitionNameAfter), eventTime, true);
} else {
Env.getCurrentEnv().getRefreshManager()
- .refreshPartitions(catalogName, dbName, hmsTbl.getTableName(),
+ .refreshPartitions(catalogName, hmsTbl.getDbName(), hmsTbl.getTableName(),
Lists.newArrayList(partitionNameAfter), eventTime, true);
}
} catch (DdlException e) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
index 2d81377f4b6..2b7474a8c3c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
@@ -55,7 +55,8 @@ public class CreateDatabaseEvent extends MetastoreEvent {
protected void process() throws MetastoreNotificationException {
try {
logInfo("catalogName:[{}],dbName:[{}]", catalogName, dbName);
- Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbName, catalogName);
+ Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(
+ event == null ? dbName : event.getDbName(), catalogName);
} catch (DdlException e) {
throw new MetastoreNotificationException(
getMsgWithEventInfo("Failed to process event"), e);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
index 6ab089232b9..e48574601f4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
@@ -55,7 +55,7 @@ public class DropDatabaseEvent extends MetastoreEvent {
try {
logInfo("catalogName:[{}],dbName:[{}]", catalogName, dbName);
Env.getCurrentEnv().getCatalogMgr()
- .unregisterExternalDatabase(dbName, catalogName);
+ .unregisterExternalDatabase(event == null ? dbName : event.getDbName(), catalogName);
} catch (DdlException e) {
throw new MetastoreNotificationException(
getMsgWithEventInfo("Failed to process event"), e);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
index 737ad8f28b9..9a7e7ab671a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
@@ -105,7 +105,7 @@ public class DropPartitionEvent extends MetastorePartitionEvent {
return;
}
Env.getCurrentEnv().getCatalogMgr()
- .dropExternalPartitions(catalogName, dbName, hmsTbl.getTableName(),
+ .dropExternalPartitions(catalogName, hmsTbl.getDbName(), hmsTbl.getTableName(),
partitionNames, eventTime, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
index 4b4abd0264d..b51282be977 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
@@ -71,8 +71,8 @@ public class InsertEvent extends MetastoreTableEvent {
* the file cache of this table,
* but <a href="https://github.com/apache/doris/pull/17932">this PR</a> has fixed it.
*/
- Env.getCurrentEnv().getRefreshManager().refreshExternalTableFromEvent(catalogName, dbName, tblName,
- eventTime);
+ Env.getCurrentEnv().getRefreshManager().refreshExternalTableFromEvent(
+ catalogName, event == null ? dbName : event.getDbName(), tblName, eventTime);
} catch (DdlException e) {
throw new MetastoreNotificationException(
getMsgWithEventInfo("Failed to process event"), e);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
index 03ef1b899ea..618102e2d05 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
@@ -159,11 +159,16 @@ public abstract class IcebergExternalCatalog extends ExternalCatalog {
@Override
public void notifyPropertiesUpdated(Map<String, String> updatedProps) {
- super.notifyPropertiesUpdated(updatedProps);
- if (updatedProps.keySet().stream()
- .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, IcebergExternalMetaCache.ENGINE))) {
- Env.getCurrentEnv().getExtMetaCacheMgr()
- .removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE);
+ try {
+ super.notifyPropertiesUpdated(updatedProps);
+ } finally {
+ // The committed ALTER already published the properties; retire the engine group even
+ // when the generic reset cleanup throws.
+ if (updatedProps.keySet().stream()
+ .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, IcebergExternalMetaCache.ENGINE))) {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE);
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index dca41bb9b4c..ccc693f4ad2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -276,6 +276,12 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
@Override
public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws DdlException {
+ return dropDbImplWithResolvedName(dbName, ifExists, force).isPresent();
+ }
+
+ @Override
+ public Optional<String> dropDbImplWithResolvedName(String dbName, boolean ifExists, boolean force)
+ throws DdlException {
try {
return executeCatalogOperation(() -> performDropDb(dbName, ifExists, force));
} catch (Exception e) {
@@ -284,15 +290,15 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
}
}
- private boolean performDropDb(String dbName, boolean ifExists, boolean force) throws DdlException {
+ private Optional<String> performDropDb(String dbName, boolean ifExists, boolean force) throws DdlException {
ExternalDatabase dorisDb = getDatabaseWithinCatalogGeneration(dbName);
if (dorisDb == null) {
if (ifExists) {
LOG.info("drop database[{}] which does not exist", dbName);
- return false;
+ return Optional.empty();
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_DROP_EXISTS, dbName);
- return false;
+ return Optional.empty();
}
}
if (force) {
@@ -317,11 +323,11 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
// The remote namespace is already gone, but the resolved local database object
// still has to be unregistered, so treat the drop as handled.
LOG.info("drop database[{}] force which does not exist", dbName);
- return true;
+ return Optional.of(dorisDb.getFullName());
}
}
nsCatalog.dropNamespace(getNamespace(dorisDb.getRemoteName()));
- return true;
+ return Optional.of(dorisDb.getFullName());
}
@Override
@@ -329,6 +335,11 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
dorisCatalog.unregisterDatabase(dbName);
}
+ @Override
+ public void afterDropDbNoOp(String dbName) {
+ dorisCatalog.retireCachedDatabaseForNoOp(dbName);
+ }
+
@Override
public boolean createTableImpl(CreateTableInfo createTableInfo) throws UserException {
try {
@@ -446,9 +457,12 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
@Override
public void afterDropTable(String dbName, String tblName) {
- Optional<ExternalDatabase<?>> db = dorisCatalog.getDbForReplay(dbName);
+ Optional<ExternalDatabase<?>> db = dorisCatalog.getDbForDropReplay(dbName);
if (db.isPresent()) {
db.get().unregisterTable(tblName);
+ } else {
+ // Retire a cold known DB narrowly; a lost mode-2 mapping needs catalog-wide retirement.
+ dorisCatalog.invalidateColdDatabaseForReplay(dbName);
}
LOG.info("after drop table {}.{}.{}. is db exists: {}",
dorisCatalog.getName(), dbName, tblName, db.isPresent());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
index fe48adffff7..a418ec9f0f0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
@@ -120,6 +120,22 @@ public class JdbcExternalCatalog extends ExternalCatalog {
ExternalFunctionRules.check(catalogProperty.getProperties().getOrDefault(JdbcResource.FUNCTION_RULES, ""));
}
+ @Override
+ public boolean validatePropertiesBeforeUpdate(
+ Map<String, String> currentProperties, Map<String, String> updatedProperties) throws DdlException {
+ // The identifier mapping is rebuilt during reset, after property publication. Reject an
+ // invalid candidate here so a failed ALTER cannot leave invalid durable properties.
+ if (updatedProperties.containsKey(META_NAMES_MAPPING)
+ || updatedProperties.containsKey(LOWER_CASE_META_NAMES)) {
+ String mapping = updatedProperties.getOrDefault(META_NAMES_MAPPING, getMetaNamesMapping());
+ String lowerCase = updatedProperties.getOrDefault(LOWER_CASE_META_NAMES, getLowerCaseMetaNames());
+ new JdbcIdentifierMapping(
+ Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase(),
+ Boolean.parseBoolean(lowerCase), mapping);
+ }
+ return false;
+ }
+
/**
* Enforce the mandatory driver URL rule on user-facing CREATE and ALTER CATALOG validation.
* Catalog replay does not call {@link #checkProperties()}, so existing catalogs remain compatible.
@@ -149,11 +165,16 @@ public class JdbcExternalCatalog extends ExternalCatalog {
@Override
public void resetToUninitialized(boolean invalidCache) {
- super.resetToUninitialized(invalidCache);
- this.identifierMapping = new JdbcIdentifierMapping(
- (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()),
- Boolean.parseBoolean(getLowerCaseMetaNames()),
- getMetaNamesMapping());
+ try {
+ super.resetToUninitialized(invalidCache);
+ } finally {
+ // The committed ALTER has already published the new properties; rebuild the derived name
+ // mapping even when the reset's client cleanup (closeClient) throws.
+ this.identifierMapping = new JdbcIdentifierMapping(
+ (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()),
+ Boolean.parseBoolean(getLowerCaseMetaNames()),
+ getMetaNamesMapping());
+ }
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java
index 179697d436f..187d2b19121 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java
@@ -18,6 +18,7 @@
package org.apache.doris.datasource.lance;
import org.apache.doris.analysis.TableSnapshot;
+import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.datasource.CatalogProperty;
@@ -285,6 +286,7 @@ public class LanceExternalCatalog extends ExternalCatalog {
@Override
public void unregisterDatabase(String dbName) {
// Dropping a namespace is a semantic change, unlike routine local DB-object eviction.
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(getId());
invalidateTableAccessCache();
super.unregisterDatabase(dbName);
}
@@ -299,6 +301,8 @@ public class LanceExternalCatalog extends ExternalCatalog {
@Override
public void onRefreshCache(boolean invalidCache) {
if (invalidCache) {
+ // A new Session can resolve a replacement dataset before the catalog reset below.
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(getId());
refreshSessionCache();
}
super.onRefreshCache(invalidCache);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java
index a7c5ba2c7f2..3df2e3bee4d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java
@@ -128,14 +128,20 @@ public class MaxComputeMetadataOps implements ExternalMetadataOps {
@Override
public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws DdlException {
+ return dropDbImplWithResolvedName(dbName, ifExists, force).isPresent();
+ }
+
+ @Override
+ public Optional<String> dropDbImplWithResolvedName(String dbName, boolean ifExists, boolean force)
+ throws DdlException {
ExternalDatabase<?> dorisDb = dorisCatalog.getDbNullable(dbName);
if (dorisDb == null) {
if (ifExists) {
LOG.info("drop database[{}] which does not exist", dbName);
- return false;
+ return Optional.empty();
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_DROP_EXISTS, dbName);
- return false;
+ return Optional.empty();
}
}
if (force) {
@@ -153,7 +159,7 @@ public class MaxComputeMetadataOps implements ExternalMetadataOps {
}
}
dorisCatalog.getMcStructureHelper().dropDb(odps, dbName, ifExists);
- return true;
+ return Optional.of(dorisDb.getFullName());
}
@Override
@@ -161,6 +167,11 @@ public class MaxComputeMetadataOps implements ExternalMetadataOps {
dorisCatalog.unregisterDatabase(dbName);
}
+ @Override
+ public void afterDropDbNoOp(String dbName) {
+ dorisCatalog.retireCachedDatabaseForNoOp(dbName);
+ }
+
// ==================== Create Table ====================
@Override
@@ -290,9 +301,12 @@ public class MaxComputeMetadataOps implements ExternalMetadataOps {
@Override
public void afterDropTable(String dbName, String tblName) {
- Optional<ExternalDatabase<?>> db = dorisCatalog.getDbForReplay(dbName);
+ Optional<ExternalDatabase<?>> db = dorisCatalog.getDbForDropReplay(dbName);
if (db.isPresent()) {
db.get().unregisterTable(tblName);
+ } else {
+ // Retire a cold known DB narrowly; a lost mode-2 mapping needs catalog-wide retirement.
+ dorisCatalog.invalidateColdDatabaseForReplay(dbName);
}
LOG.info("after drop table {}.{}.{}, is db exists: {}",
dorisCatalog.getName(), dbName, tblName, db.isPresent());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
index 3d82518b24e..cf4ea6b66fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
@@ -31,6 +31,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
@@ -553,6 +554,21 @@ public class MetaCache<T> {
return name == null ? Optional.empty() : getMetaObj(name, id);
}
+ /** Return the retained canonical name without loading an evicted metadata object. */
+ public Optional<String> getNameByIdIfPresent(long id) {
+ return Optional.ofNullable(withMetaObjLifecycleReadLock(() -> idToName.get(id)));
+ }
+
+ /** Snapshot resident object identities matching a DROP alias under the catalog's name mode. */
+ public List<Pair<String, Long>> getCachedIdentitiesMatching(String name, boolean ignoreCase) {
+ String foldedName = ignoreCase ? name.toLowerCase(Locale.ROOT) : name;
+ return withMetaObjLifecycleReadLock(() -> idToName.entrySet().stream()
+ .filter(entry -> ignoreCase ? entry.getValue().toLowerCase(Locale.ROOT).equals(foldedName)
+ : entry.getValue().equals(name))
+ .map(entry -> Pair.of(entry.getValue(), entry.getKey()))
+ .collect(Collectors.toList()));
+ }
+
public void updateCache(String remoteName, String localName, T obj, long id) {
updateCache(remoteName, localName, obj, id, namesLoadEpochSupplier.getAsLong());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java
index 902ac2fcf5f..c3e9f165f9a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java
@@ -52,6 +52,15 @@ public final class PaimonTableLoader {
}
}
+ public void invalidateDatabase(long catalogId, String localDbName) {
+ try {
+ catalog(catalogId).invalidatePaimonDatabaseByLocalName(localDbName);
+ } catch (Exception e) {
+ throw new CacheException("failed to invalidate paimon database %s.%s: %s",
+ e, catalogId, localDbName, e.getMessage());
+ }
+ }
+
public void invalidateCatalog(long catalogId) {
try {
catalog(catalogId).invalidatePaimonCatalog();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
index acc443ced20..853831c02a8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
@@ -34,6 +34,7 @@ import org.apache.iceberg.view.View;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
/**
* all external metadata operations use this interface
@@ -79,34 +80,44 @@ public interface ExternalMetadataOps {
* @throws DdlException
*/
default boolean dropDb(String dbName, boolean ifExists, boolean force) throws DdlException {
- if (!dropDbImpl(dbName, ifExists, force)) {
+ return dropDbWithResolvedName(dbName, ifExists, force).isPresent();
+ }
+
+ default Optional<String> dropDbWithResolvedName(String dbName, boolean ifExists, boolean force)
+ throws DdlException {
+ Optional<String> resolvedDbName = dropDbImplWithResolvedName(dbName, ifExists, force);
+ if (!resolvedDbName.isPresent()) {
// No remote mutation happened, so do not run the post-drop hook or journal the
// operation. A retained local incarnation may still need cleanup (for example a lost
// case-insensitive name mapping), so give the implementation a separate hook.
afterDropDbNoOp(dbName);
- return false;
+ return Optional.empty();
}
- afterDropDb(dbName);
- return true;
+ afterDropDb(resolvedDbName.get());
+ return resolvedDbName;
}
/**
- * @return whether the remote database was dropped. Returns {@code false} when the call was a
- * no-op (for example {@code IF EXISTS} on a database that does not exist).
+ * @return the local name of the database actually dropped, or empty when the remote operation
+ * was a no-op (for example {@code IF EXISTS} on a missing database).
*/
+ default Optional<String> dropDbImplWithResolvedName(String dbName, boolean ifExists, boolean force)
+ throws DdlException {
+ return dropDbImpl(dbName, ifExists, force) ? Optional.of(dbName) : Optional.empty();
+ }
+
+ /** Retained for existing connector implementations on release branches. */
boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws DdlException;
void afterDropDb(String dbName);
/**
* Cleanup hook for a drop that did not mutate the remote metastore (for example
- * {@code DROP DATABASE IF EXISTS} on a database that does not exist). The default preserves the
- * pre-existing local cleanup for connectors whose {@code afterDropDb} only unregisters the
- * database, so a retained incarnation is still retired. Paimon overrides it to do targeted
- * retirement without its broad engine-cache flush.
+ * {@code DROP DATABASE IF EXISTS} on a database that does not exist). Remote state did not
+ * change, so a connector must opt in to any cache-only cleanup. Calling
+ * {@code afterDropDb} here could turn an absent name into catalog-wide invalidation.
*/
default void afterDropDbNoOp(String dbName) {
- afterDropDb(dbName);
}
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
index 80a9dfc0e2c..52788a751d0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
@@ -173,6 +173,21 @@ public class PaimonExternalCatalog extends ExternalCatalog {
}));
}
+ public synchronized void invalidatePaimonDatabaseByLocalName(String localDbName) throws Exception {
+ if (!isInitialized()) {
+ return;
+ }
+ withSdkCatalogCacheWriteLock(() -> executionAuthenticator.execute(() -> {
+ boolean caseSensitive = catalog.caseSensitive();
+ invalidateCachedPaimonTables(identifier -> {
+ String remoteName = identifier.getDatabaseName();
+ String candidateLocalName = localDatabaseNameFromRemote(remoteName);
+ return identifierPartEquals(candidateLocalName, localDbName, caseSensitive);
+ });
+ return null;
+ }));
+ }
+
public synchronized void invalidatePaimonCatalog() throws Exception {
if (!isInitialized()) {
return;
@@ -469,11 +484,17 @@ public class PaimonExternalCatalog extends ExternalCatalog {
@Override
public void notifyPropertiesUpdated(Map<String, String> updatedProps) {
- super.notifyPropertiesUpdated(updatedProps);
- if (updatedProps.keySet().stream()
- .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, PaimonExternalMetaCache.ENGINE)
- || AbstractPaimonProperties.isTableOptionProperty(key))) {
- Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), PaimonExternalMetaCache.ENGINE);
+ try {
+ super.notifyPropertiesUpdated(updatedProps);
+ } finally {
+ // The committed ALTER already published the properties; retire the engine group even
+ // when the generic reset cleanup throws.
+ if (updatedProps.keySet().stream()
+ .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, PaimonExternalMetaCache.ENGINE)
+ || AbstractPaimonProperties.isTableOptionProperty(key))) {
+ Env.getCurrentEnv().getExtMetaCacheMgr()
+ .removeCatalogByEngine(getId(), PaimonExternalMetaCache.ENGINE);
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java
index 1f937156dc4..4f8c525f392 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java
@@ -150,7 +150,9 @@ public class PaimonExternalMetaCache extends AbstractExternalMetaCache {
@Override
public void invalidateDb(long catalogId, String dbName) {
try {
- invalidatePaimonTables(catalogId, nameMapping -> matchDb(nameMapping, dbName));
+ // Match the SDK's own entries independently of Doris tableEntry: direct metadata
+ // paths can populate the SDK before any Doris table handle exists.
+ tableLoader.invalidateDatabase(catalogId, dbName);
} finally {
super.invalidateDb(catalogId, dbName);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java
index 718ffa1d992..dab26f3acc9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java
@@ -147,6 +147,12 @@ public class PaimonMetadataOps implements ExternalMetadataOps {
@Override
public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws DdlException {
+ return dropDbImplWithResolvedName(dbName, ifExists, force).isPresent();
+ }
+
+ @Override
+ public Optional<String> dropDbImplWithResolvedName(String dbName, boolean ifExists, boolean force)
+ throws DdlException {
try {
return executionAuthenticator.execute(() -> performDropDb(dbName, ifExists, force));
} catch (Exception e) {
@@ -155,17 +161,17 @@ public class PaimonMetadataOps implements ExternalMetadataOps {
}
}
- private boolean performDropDb(String dbName, boolean ifExists, boolean force) throws DdlException {
+ private Optional<String> performDropDb(String dbName, boolean ifExists, boolean force) throws DdlException {
ExternalDatabase dorisDb = dorisCatalog.getDbNullable(dbName);
if (dorisDb == null) {
if (ifExists) {
LOG.info("drop database[{}] which does not exist", dbName);
// Database does not exist and IF EXISTS is specified; treat as no-op.
- return false;
+ return Optional.empty();
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_DROP_EXISTS, dbName);
// ErrorReport.reportDdlException is expected to throw DdlException.
- return false;
+ return Optional.empty();
}
}
@@ -186,30 +192,15 @@ public class PaimonMetadataOps implements ExternalMetadataOps {
} catch (DatabaseNotEmptyException e) {
throw new RuntimeException("database " + dbName + " is not empty! please check!");
}
- return true;
+ return Optional.of(dorisDb.getFullName());
}
@Override
public void afterDropDb(String dbName) {
- Optional<ExternalDatabase<? extends ExternalTable>> db = dorisCatalog.getDbForReplay(dbName);
try {
- if (db.isPresent()) {
- // getDbForReplay normalizes case-insensitive database names (lower_case_database_names
- // mode 1/2), so an alternate-case DROP DATABASE can resolve the cached database while
- // an exact-key eviction with the caller's spelling would miss it. Evict by the resolved
- // canonical local key so the removal listener still performs the one typed SDK
- // invalidation; do not add a second typed scan under the catalog write fence.
- dorisCatalog.unregisterDatabase(db.get().getFullName());
- return;
- }
- // The cached database could not be resolved (for example a mode-2 case mapping was removed
- // by a names refresh before replay). Exact-key eviction can miss the canonical local key,
- // so also retire the remaining legacy database objects; otherwise a same-name recreation
- // could reuse the stale object and its nested table-name cache. The catalog-wide engine
- // flush below covers the SDK side, so the per-database engine callbacks are suppressed.
+ // The DROP owns historical-name resolution. Ordinary replay lookup also serves CREATE
+ // and REFRESH, where a case-insensitive alias must still resolve the current database.
dorisCatalog.unregisterDatabase(dbName);
- dorisCatalog.retireAllDatabaseObjectsWithoutEngineInvalidation();
- invalidatePaimonCatalogForUnresolvedReplay();
} catch (Exception e) {
// The remote drop is already committed and ExternalCatalog.dropDb still has to journal
// it; keep the post-drop cache cleanup best-effort so the log and its replay are not
@@ -222,13 +213,9 @@ public class PaimonMetadataOps implements ExternalMetadataOps {
@Override
public void afterDropDbNoOp(String dbName) {
try {
- // A no-op IF EXISTS drop can still leave a retained canonical database object when a
- // case-insensitive (mode 2) name mapping was lost before the statement. Retire the legacy
- // database objects so a same-name recreation cannot reuse the stale incarnation. The SDK
- // catalog is intentionally not flushed because the remote metastore was not mutated.
- dorisCatalog.retireAllDatabaseObjectsWithoutEngineInvalidation();
+ dorisCatalog.retireCachedDatabaseForNoOp(dbName);
} catch (Exception e) {
- LOG.warn("Failed to retire legacy database objects after no-op drop of {}: {}",
+ LOG.warn("Failed to retire cached database after no-op drop of {}: {}",
dbName, e.getMessage(), e);
}
}
@@ -394,7 +381,7 @@ public class PaimonMetadataOps implements ExternalMetadataOps {
@Override
public void afterDropTable(String dbName, String tblName) {
- Optional<ExternalDatabase<?>> db = dorisCatalog.getDbForReplay(dbName);
+ Optional<ExternalDatabase<?>> db = dorisCatalog.getDbForDropReplay(dbName);
try {
if (db.isPresent()) {
boolean invalidated = db.get().unregisterTableForReplay(tblName);
@@ -408,12 +395,9 @@ public class PaimonMetadataOps implements ExternalMetadataOps {
invalidatePaimonCatalogForUnresolvedReplay();
}
} else {
- // The database itself could not be resolved (for example a mode-2 mapping was lost
- // before replay). Retire any retained legacy database object first so a same-name
- // recreation cannot reuse its stale nested table-name cache, then flush the engine
- // group; a failure in either is best-effort so the drop log is still written.
- dorisCatalog.retireAllDatabaseObjectsWithoutEngineInvalidation();
- invalidatePaimonCatalogForUnresolvedReplay();
+ // A cold DB with a retained canonical mapping has a narrow invalidation target.
+ // Only a genuinely lost mapping requires catalog-wide hidden-object retirement.
+ dorisCatalog.invalidateColdDatabaseForReplay(dbName);
}
} catch (Exception e) {
// The remote drop is already committed and ExternalCatalog.dropTable still has to
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java
index e597732871b..288bcb32f5f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java
@@ -136,12 +136,9 @@ public abstract class BaseExternalTableInsertExecutor extends AbstractInsertExec
* Default: full table refresh.
*/
protected void doAfterCommit() throws DdlException {
- // Default: full table refresh
- Env.getCurrentEnv().getRefreshManager().handleRefreshTable(
- catalogName,
- table.getDatabase().getFullName(),
- table.getName(),
- true);
+ // The transaction is already committed and this executor still owns the exact table
+ // identity. Do not re-resolve it through an evictable metadata cache before fencing it.
+ Env.getCurrentEnv().getRefreshManager().refreshTableAfterCommit((ExternalTable) table);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java
index 760a2c0551d..46b0dd93618 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java
@@ -83,19 +83,45 @@ public class HiveInsertExecutor extends BaseExternalTableInsertExecutor {
protected void doAfterCommit() throws DdlException {
HMSExternalTable hmsTable = (HMSExternalTable) table;
+ // The transaction is already committed. Fence the row-count cache by the held table
+ // identity before any fallible cache work (including isPartitionedTable reinitialization),
+ // so an evicted or partially reloaded table cannot retain the pre-insert count.
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(hmsTable);
+
// For partitioned tables, do selective partition refresh
// For non-partitioned tables, do full table cache invalidation
List<String> modifiedPartNames = Lists.newArrayList();
List<String> newPartNames = Lists.newArrayList();
- if (hmsTable.isPartitionedTable() && partitionUpdates != null && !partitionUpdates.isEmpty()) {
- HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
- .hive(hmsTable.getCatalog().getId());
- cache.refreshAffectedPartitions(hmsTable, partitionUpdates, modifiedPartNames, newPartNames);
- } else {
- // Non-partitioned table or no partition updates, do full table refresh
- Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(hmsTable);
+ try {
+ if (hmsTable.isPartitionedTable() && partitionUpdates != null && !partitionUpdates.isEmpty()) {
+ HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
+ .hive(hmsTable.getCatalog().getId());
+ cache.refreshAffectedPartitions(hmsTable, partitionUpdates, modifiedPartNames, newPartNames);
+ } else {
+ // Non-partitioned table or no partition updates, do full table refresh
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(hmsTable);
+ }
+ } catch (RuntimeException e) {
+ // The transaction is already committed. Fall back to a full invalidation and a
+ // full-refresh edit log instead of reporting a failed insert or skipping peers.
+ LOG.warn("Failed to refresh caches after committing Hive insert for {}",
+ hmsTable.getNameWithFullQualifiers(), e);
+ modifiedPartNames.clear();
+ newPartNames.clear();
+ try {
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(hmsTable);
+ } catch (RuntimeException fallbackException) {
+ LOG.warn("Failed to invalidate table cache after committing Hive insert for {}",
+ hmsTable.getNameWithFullQualifiers(), fallbackException);
+ }
}
+ // Engine invalidation does not retire the held HMS table parameters. A row-count load
+ // admitted during either refresh path may still read pre-commit numRows/totalSize, so
+ // rebuild that source and close the admission window for both selective and full paths.
+ hmsTable.unsetObjectCreated();
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(hmsTable);
+
// Write edit log to notify other FEs
long updateTime = System.currentTimeMillis();
hmsTable.setUpdateTime(updateTime);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java
index 25e8ab74d1e..9ceb4251119 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java
@@ -34,6 +34,8 @@ public class DropDbInfo implements Writable, GsonPostProcessable {
private String ctlName;
@SerializedName(value = "dbName")
private String dbName;
+ @SerializedName(value = "resolvedDbName")
+ private String resolvedDbName;
@SerializedName(value = "forceDrop")
private boolean forceDrop = false;
@SerializedName(value = "recycleTime")
@@ -49,6 +51,12 @@ public class DropDbInfo implements Writable, GsonPostProcessable {
this.dbName = dbName;
}
+ // New external logs retain the database actually resolved by the leader. Older logs lack this field.
+ public DropDbInfo(String ctlName, String dbName, String resolvedDbName) {
+ this(ctlName, dbName);
+ this.resolvedDbName = resolvedDbName;
+ }
+
// for internal table
public DropDbInfo(String dbName, boolean forceDrop, long recycleTime) {
this.dbName = dbName;
@@ -64,6 +72,10 @@ public class DropDbInfo implements Writable, GsonPostProcessable {
return dbName;
}
+ public String getResolvedDbName() {
+ return resolvedDbName;
+ }
+
public boolean isForceDrop() {
return forceDrop;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
index c2082ecc226..872e4c471d2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
@@ -34,6 +34,8 @@ public class DropInfo implements Writable {
private String ctl;
@SerializedName(value = "db")
private String db;
+ @SerializedName(value = "resolvedDb")
+ private String resolvedDb;
@SerializedName(value = "dbId")
private long dbId;
@SerializedName(value = "tableId")
@@ -61,6 +63,12 @@ public class DropInfo implements Writable {
this.tableName = tbl;
}
+ // New external logs retain the database actually resolved by the leader. Older logs lack this field.
+ public DropInfo(String ctl, String db, String tbl, String resolvedDb) {
+ this(ctl, db, tbl);
+ this.resolvedDb = resolvedDb;
+ }
+
// for internal table
public DropInfo(long dbId, long tableId, String tableName, boolean isView, boolean forceDrop,
long recycleTime) {
@@ -89,6 +97,10 @@ public class DropInfo implements Writable {
return db;
}
+ public String getResolvedDb() {
+ return resolvedDb;
+ }
+
public long getDbId() {
return this.dbId;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index d2760bb7179..ce7ab3219cf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -395,7 +395,7 @@ public class EditLog {
ExternalCatalog ctl = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr()
.getCatalog(info.getCtl());
if (ctl != null) {
- ctl.replayDropTable(info.getDb(), info.getTableName());
+ ctl.replayDropTable(info.getDb(), info.getTableName(), info.getResolvedDb());
}
}
break;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerReplayTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerReplayTest.java
index 85a78719714..c69381ac5a3 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerReplayTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerReplayTest.java
@@ -87,6 +87,29 @@ public class RefreshManagerReplayTest {
}
}
+ @Test
+ public void testReplayRefreshColdTableRetiresResidentEngineEntries() {
+ long catalogId = 4L;
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ @SuppressWarnings("unchecked")
+ ExternalDatabase<ExternalTable> db = Mockito.mock(ExternalDatabase.class);
+ Mockito.doReturn(Optional.of(db)).when(catalog).getDbForReplay("db");
+ Mockito.doReturn(Optional.empty()).when(db).getTableForReplay("table");
+ Mockito.when(db.getFullName()).thenReturn("db");
+ Mockito.when(db.getId()).thenReturn(5L);
+ Mockito.when(db.hasLocalTableName("table")).thenReturn(true);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = mockEnv(catalogId, catalog);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ new RefreshManager().replayRefreshTable(
+ ExternalObjectLog.createForRefreshTable(catalogId, "db", "table", 0L));
+ }
+ Mockito.verify(cacheMgr).invalidateTableByNameOrWider(catalogId, "db", "table");
+ }
+
private Env mockEnv(long catalogId, ExternalCatalog catalog) {
CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java
new file mode 100644
index 00000000000..f19a1a4c4ac
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java
@@ -0,0 +1,511 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.datasource.CatalogMgr;
+import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.ExternalDatabase;
+import org.apache.doris.datasource.ExternalMetaCacheMgr;
+import org.apache.doris.datasource.ExternalObjectLog;
+import org.apache.doris.datasource.ExternalTable;
+import org.apache.doris.datasource.hive.HMSExternalCatalog;
+import org.apache.doris.datasource.hive.HMSExternalTable;
+import org.apache.doris.datasource.hive.HiveExternalMetaCache;
+import org.apache.doris.datasource.lance.LanceExternalCatalog;
+import org.apache.doris.datasource.paimon.PaimonExternalCatalog;
+import org.apache.doris.datasource.paimon.PaimonExternalMetaCache;
+import org.apache.doris.persist.EditLog;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.InOrder;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.Optional;
+
+public class RefreshManagerTest {
+
+ @Test
+ @SuppressWarnings("unchecked")
+ void testColdKnownDatabaseReplayKeepsUnrelatedHotDatabase() {
+ long catalogId = 81L;
+ long coldDbId = 82L;
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getDbForReplay("cold_db")).thenReturn(Optional.empty());
+ Mockito.when(catalog.getDbIdentityForReplay("cold_db", 0L))
+ .thenReturn(Optional.of(Pair.of("CanonicalDb", coldDbId)));
+ ExternalDatabase<?> hotDb = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(catalog.getDbForReplay("hot_db"))
+ .thenReturn((Optional) Optional.of(hotDb));
+
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ new RefreshManager().replayRefreshDb(
+ ExternalObjectLog.createForRefreshDb(catalogId, "cold_db"));
+ new RefreshManager().replayRefreshTable(
+ ExternalObjectLog.createForRefreshTable(catalogId, "cold_db", "tbl", 1L));
+ }
+
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateDb(catalogId, coldDbId, "CanonicalDb");
+ Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(catalogId);
+ Mockito.verify(hotDb, Mockito.never()).resetMetaToUninitialized();
+ }
+
+ @Test
+ void testColdReboundPaimonReplayFencesOldRowCountsAroundSdkInvalidation() {
+ long catalogId = 83L;
+ long replacementDbId = 84L;
+ PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getLowerCaseDatabaseNames()).thenReturn(2);
+ Mockito.when(catalog.getDbForReplay("Foo")).thenReturn(Optional.empty());
+ Mockito.when(catalog.getDbIdentityForReplay("Foo", 0L))
+ .thenReturn(Optional.of(Pair.of("FOO", replacementDbId)));
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ new RefreshManager().replayRefreshDb(ExternalObjectLog.createForRefreshDb(catalogId, "Foo"));
+ }
+
+ InOrder order = Mockito.inOrder(cacheMgr);
+ order.verify(cacheMgr).invalidateRowCountCache(catalogId);
+ order.verify(cacheMgr).invalidateDb(catalogId, replacementDbId, "FOO");
+ order.verify(cacheMgr).invalidateCatalogByEngine(catalogId, PaimonExternalMetaCache.ENGINE);
+ order.verify(cacheMgr).invalidateRowCountCache(catalogId);
+ }
+
+ @Test
+ void testColdDatabaseReplayInvalidatesCatalogRowCount() {
+ long catalogId = 51L;
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.empty());
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ new RefreshManager().replayRefreshDb(ExternalObjectLog.createForRefreshDb(catalogId, "db1"));
+ }
+
+ Mockito.verify(cacheMgr).invalidateCatalog(catalogId);
+ }
+
+ @Test
+ void testLanceDatabaseReplayFencesRowCountBeforeRetiringAccess() {
+ long catalogId = 52L;
+ long dbId = 53L;
+ LanceExternalCatalog catalog = Mockito.mock(LanceExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getDbIdentityForReplay("db1", 0L))
+ .thenReturn(Optional.of(Pair.of("db1", dbId)));
+ Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.empty());
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.doAnswer(invocation -> {
+ Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, dbId);
+ return null;
+ }).when(catalog).invalidateTableAccessCache();
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ new RefreshManager().replayRefreshDb(ExternalObjectLog.createForRefreshDb(catalogId, "db1"));
+ }
+
+ Mockito.verify(catalog).invalidateTableAccessCache();
+ Mockito.verify(cacheMgr).invalidateDb(catalogId, dbId, "db1");
+ }
+
+ @Test
+ void testLanceLeaderDatabaseRefreshFencesRowCountBeforeRetiringAccess() throws Exception {
+ long catalogId = 56L;
+ long dbId = 57L;
+ LanceExternalCatalog catalog = Mockito.mock(LanceExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getName()).thenReturn("lance");
+ Mockito.doReturn(db).when(catalog).getDbOrDdlException("db1");
+ Mockito.when(db.getId()).thenReturn(dbId);
+ Mockito.doReturn(catalog).when(db).getCatalog();
+ Mockito.when(db.getFullName()).thenReturn("db1");
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.doReturn(catalog).when(catalogMgr).getCatalog("lance");
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.doAnswer(invocation -> {
+ Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, dbId);
+ return null;
+ }).when(catalog).invalidateTableAccessCache();
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class));
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ new RefreshManager().handleRefreshDb("lance", "db1");
+ }
+
+ Mockito.verify(catalog).invalidateTableAccessCache();
+ Mockito.verify(db).resetMetaToUninitialized();
+ }
+
+ @Test
+ void testLanceTableReplayFencesRowCountBeforeRetiringAccess() {
+ long catalogId = 54L;
+ long dbId = 55L;
+ LanceExternalCatalog catalog = Mockito.mock(LanceExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getDbIdentityForReplay("db1", 0L))
+ .thenReturn(Optional.of(Pair.of("db1", dbId)));
+ Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.empty());
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.doAnswer(invocation -> {
+ Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, "db1", "tbl1");
+ return null;
+ }).when(catalog).invalidateTableAccessCache();
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ new RefreshManager().replayRefreshTable(
+ ExternalObjectLog.createForRefreshTable(catalogId, "db1", "tbl1", 1L));
+ }
+
+ Mockito.verify(catalog).invalidateTableAccessCache();
+ Mockito.verify(cacheMgr).invalidateDb(catalogId, dbId, "db1");
+ }
+
+ @Test
+ void testPartitionReplayInvalidatesRowCountBeforeCacheFailure() {
+ long catalogId = 52L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.of(db));
+ Mockito.doReturn(Optional.of(table)).when(db).getTableForReplay("tbl1");
+
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ HiveExternalMetaCache hiveCache = Mockito.mock(HiveExternalMetaCache.class);
+ Mockito.when(cacheMgr.hive(catalogId)).thenReturn(hiveCache);
+ Mockito.doThrow(new IllegalStateException("partition cache failure"))
+ .when(hiveCache).refreshAffectedPartitionsCache(
+ Mockito.eq(table), Mockito.anyList(), Mockito.anyList());
+ Mockito.doThrow(new IllegalStateException("full cache failure"))
+ .when(cacheMgr).invalidateTableCache(table);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ ExternalObjectLog log = ExternalObjectLog.createForRefreshPartitions(
+ catalogId, "db1", "tbl1",
+ java.util.Collections.singletonList("p=1"), java.util.Collections.emptyList(), 1L);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ // Replay cleanup failures are logged and swallowed so a committed refresh does not take
+ // the FE down; the row-count fence must still have been published first.
+ Assertions.assertDoesNotThrow(() -> new RefreshManager().replayRefreshTable(log));
+ }
+
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(table);
+ Mockito.verify(table).unsetObjectCreated();
+ }
+
+ @Test
+ void testPartitionReplayReloadsHeldHiveTableBeforeClosingRowCountFence() {
+ long catalogId = 80L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.of(db));
+ Mockito.doReturn(Optional.of(table)).when(db).getTableForReplay("tbl1");
+
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ HiveExternalMetaCache hiveCache = Mockito.mock(HiveExternalMetaCache.class);
+ Mockito.when(cacheMgr.hive(catalogId)).thenReturn(hiveCache);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ ExternalObjectLog log = ExternalObjectLog.createForRefreshPartitions(
+ catalogId, "db1", "tbl1",
+ java.util.Collections.singletonList("p=1"), java.util.Collections.emptyList(), 1L);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ new RefreshManager().replayRefreshTable(log);
+ }
+
+ InOrder order = Mockito.inOrder(hiveCache, table, cacheMgr);
+ order.verify(hiveCache).refreshAffectedPartitionsCache(Mockito.eq(table), Mockito.anyList(),
+ Mockito.anyList());
+ order.verify(table).unsetObjectCreated();
+ order.verify(cacheMgr).invalidateRowCountCache(table);
+ }
+
+ @Test
+ void testAlterPartitionInvalidatesRowCountBeforeCacheFailure() throws Exception {
+ long catalogId = 53L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.doReturn(db).when(catalog).getDbNullable("db1");
+ Mockito.doReturn(table).when(db).getTableNullable("tbl1");
+ Mockito.when(table.getCatalog()).thenReturn(catalog);
+
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.doReturn(catalog).when(catalogMgr).getCatalog("hms");
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(cacheMgr.hive(catalogId)).thenThrow(new IllegalStateException("partition cache failure"));
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ Assertions.assertThrows(IllegalStateException.class,
+ () -> new RefreshManager().refreshPartitions(
+ "hms", "db1", "tbl1", java.util.Collections.singletonList("p=1"), 1L, true));
+ }
+
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId, "db1", "tbl1");
+ }
+
+ @Test
+ void testColdAlterPartitionEventFencesRowCountBeforeIgnoredTableMiss() throws Exception {
+ long catalogId = 54L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.doReturn(db).when(catalog).getDbNullable("db1");
+ Mockito.doReturn(null).when(db).getTableNullable("tbl1");
+
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.doReturn(catalog).when(catalogMgr).getCatalog("hms");
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ new RefreshManager().refreshPartitions(
+ "hms", "db1", "tbl1", java.util.Collections.singletonList("p=1"), 1L, true);
+ }
+
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId, "db1", "tbl1");
+ Mockito.verify(cacheMgr).invalidateTableByNameOrWider(catalogId, "db1", "tbl1");
+ Mockito.verify(cacheMgr, Mockito.never()).hive(catalogId);
+ }
+
+ @Test
+ void testSuccessfulPartitionRefreshClosesRowCountAdmissionWindow() throws Exception {
+ long catalogId = 69L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.doReturn(db).when(catalog).getDbNullable("db1");
+ Mockito.doReturn(table).when(db).getTableNullable("tbl1");
+ Mockito.when(table.getCatalog()).thenReturn(catalog);
+
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.doReturn(catalog).when(catalogMgr).getCatalog("hms");
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ HiveExternalMetaCache hiveCache = Mockito.mock(HiveExternalMetaCache.class);
+ Mockito.when(cacheMgr.hive(catalogId)).thenReturn(hiveCache);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ new RefreshManager().refreshPartitions(
+ "hms", "db1", "tbl1", java.util.Collections.singletonList("p=1"), 1L, true);
+ }
+
+ // One fence before the selective invalidation, one after it closes the admission window.
+ Mockito.verify(cacheMgr, Mockito.times(2))
+ .invalidateRowCountCache(catalogId, "db1", "tbl1");
+ }
+
+ @Test
+ void testCommittedRefreshUsesHeldTableAndStillLogsAfterCacheFailure() {
+ long catalogId = 55L;
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ ExternalTable table = Mockito.mock(ExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(db.getFullName()).thenReturn("db1");
+ Mockito.when(table.getCatalog()).thenReturn(catalog);
+ Mockito.when(table.getDatabase()).thenReturn(db);
+ Mockito.when(table.getName()).thenReturn("tbl1");
+ Mockito.when(table.getNameWithFullQualifiers()).thenReturn("ctl.db1.tbl1");
+
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.doThrow(new IllegalStateException("table cache failure"))
+ .when(cacheMgr).invalidateTableCache(table);
+ EditLog editLog = Mockito.mock(EditLog.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(env.getEditLog()).thenReturn(editLog);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ Assertions.assertDoesNotThrow(() -> new RefreshManager().refreshTableAfterCommit(table));
+ }
+
+ InOrder order = Mockito.inOrder(cacheMgr, editLog);
+ order.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(table);
+ order.verify(cacheMgr).invalidateTableCache(table);
+ order.verify(editLog).logRefreshExternalTable(Mockito.argThat(log ->
+ log.getCatalogId() == catalogId
+ && "db1".equals(log.getDbName())
+ && "tbl1".equals(log.getTableName())));
+ }
+
+ @Test
+ void testFullRefreshFencesHeldTableBeforeMetadataReset() {
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ ExternalTable table = Mockito.mock(ExternalTable.class);
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ Mockito.when(db.getCatalog()).thenReturn(catalog);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ new RefreshManager().refreshTableInternal(db, table, 0L);
+ }
+ InOrder order = Mockito.inOrder(cacheMgr, table);
+ order.verify(cacheMgr).invalidateRowCountCache(table);
+ order.verify(table).unsetObjectCreated();
+ order.verify(cacheMgr).invalidateTableCache(table);
+ }
+
+ @Test
+ void testWholeTableEventFencesRowCountBeforeIgnoredTableMiss() throws Exception {
+ long catalogId = 56L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.doReturn(db).when(catalog).getDbNullable("db1");
+ Mockito.doReturn(null).when(db).getTableNullable("tbl1");
+
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.doReturn(catalog).when(catalogMgr).getCatalog("hms");
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ new RefreshManager().refreshExternalTableFromEvent("hms", "db1", "tbl1", 1L);
+ }
+
+ Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, "db1", "tbl1");
+ Mockito.verify(cacheMgr).invalidateTableByNameOrWider(catalogId, "db1", "tbl1");
+ Mockito.verify(cacheMgr, Mockito.never()).hive(catalogId);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ void testColdRefreshTableReplayWidensToCanonicalDatabaseScope() {
+ long catalogId = 57L;
+ long dbId = 58L;
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getDbForReplay("db1")).thenReturn((Optional) Optional.of(db));
+ Mockito.when(db.getId()).thenReturn(dbId);
+ Mockito.when(db.getFullName()).thenReturn("MixedDb");
+ Mockito.doReturn(Optional.empty()).when(db).getTableForReplay("tbl1");
+
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ ExternalObjectLog log = ExternalObjectLog.createForRefreshTable(catalogId, "db1", "tbl1", 1L);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ new RefreshManager().replayRefreshTable(log);
+ }
+
+ Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, dbId);
+ }
+
+ @Test
+ void testColdRefreshDbReplayWidensToCatalogScope() {
+ long catalogId = 59L;
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.empty());
+
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ ExternalObjectLog log = ExternalObjectLog.createForRefreshTable(catalogId, "db1", "tbl1", 1L);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ new RefreshManager().replayRefreshTable(log);
+ }
+
+ Mockito.verify(cacheMgr).invalidateCatalog(catalogId);
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java
index a35716aa95d..0fa144a6ef2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java
@@ -19,14 +19,34 @@ package org.apache.doris.datasource;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.JdbcResource;
+import org.apache.doris.catalog.RefreshManager;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.Pair;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.hive.HMSExternalCatalog;
+import org.apache.doris.datasource.hive.HMSExternalTable;
+import org.apache.doris.datasource.hive.HiveExternalMetaCache;
+import org.apache.doris.datasource.jdbc.JdbcExternalCatalog;
+import org.apache.doris.datasource.metacache.MetaCache;
+import org.apache.doris.datasource.operations.ExternalMetadataOps;
import org.apache.doris.datasource.paimon.PaimonExternalCatalog;
import org.apache.doris.datasource.property.metastore.AbstractPaimonProperties;
+import org.apache.doris.nereids.exceptions.NotSupportedException;
+import org.apache.doris.persist.DropDbInfo;
+import org.apache.doris.persist.DropInfo;
+import org.apache.doris.persist.EditLog;
+import org.apache.doris.statistics.query.QueryStats;
+import com.github.benmanes.caffeine.cache.RemovalCause;
import com.google.common.collect.ImmutableMap;
+import mockit.Mock;
+import mockit.MockUp;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
@@ -34,16 +54,177 @@ import java.lang.reflect.Field;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalLong;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
public class CatalogMgrTest {
+ @Test
+ void testDatabaseCapacityEvictionDoesNotFenceAllRowCounts() {
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(49L);
+ @SuppressWarnings("unchecked")
+ ExternalDatabase<? extends ExternalTable> db = Mockito.mock(ExternalDatabase.class);
+
+ catalog.handleDatabaseMetaCacheRemoval(Optional.of(db), RemovalCause.SIZE);
+ Mockito.verify(db).resetMetaToUninitialized(true, false);
+
+ catalog.handleDatabaseMetaCacheRemoval(Optional.of(db), RemovalCause.EXPLICIT);
+ Mockito.verify(db).resetMetaToUninitialized(true, true);
+ }
+
+ @Test
+ void testReplayDatabaseIdentitySurvivesObjectEviction() throws Exception {
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(83L,
+ ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2"));
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ catalog.installMetaCache(metaCache);
+ Field mappingField = ExternalCatalog.class.getDeclaredField("lowerCaseToDatabaseName");
+ mappingField.setAccessible(true);
+ @SuppressWarnings("unchecked")
+ Map<String, String> mapping = (Map<String, String>) mappingField.get(catalog);
+ mapping.put("mixeddb", "MixedDb");
+
+ @SuppressWarnings("unchecked")
+ ExternalDatabase<? extends ExternalTable> warmDb = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(metaCache.tryGetMetaObj("MixedDb")).thenReturn(Optional.of(warmDb));
+ Assertions.assertSame(warmDb, catalog.getDbForReplay("mixeddb").orElseThrow(AssertionError::new));
+ Assertions.assertEquals(Pair.of("MixedDb", Util.genIdByName("testing_catalog", "MixedDb")),
+ catalog.getDbIdentityForReplay("mixeddb", 0L).orElseThrow(AssertionError::new));
+ Assertions.assertEquals(Pair.of("MixedDb", Util.genIdByName("testing_catalog", "MixedDb")),
+ catalog.getDbIdentityForReplay("MixedDb", 0L).orElseThrow(AssertionError::new));
+ Mockito.verify(metaCache, Mockito.never()).getMetaObj(Mockito.anyString(), Mockito.anyLong());
+ Mockito.when(metaCache.getNameByIdIfPresent(84L)).thenReturn(Optional.of("MixedDb"));
+ Assertions.assertEquals(Pair.of("MixedDb", 84L),
+ catalog.getDbIdentityForReplay(null, 84L).orElseThrow(AssertionError::new));
+ }
+
+ @Test
+ void testColdKnownDropInvalidatesOnlyItsDatabase() {
+ long catalogId = 85L;
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId);
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ catalog.installMetaCache(metaCache);
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalog.invalidateColdDatabaseForReplay("cold_db");
+ }
+
+ Mockito.verify(cacheMgr).invalidateDb(catalogId,
+ Util.genIdByName("testing_catalog", "cold_db"), "cold_db");
+ Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(catalogId);
+ }
+
+ @Test
+ void testModeOneDatabaseEventCreateAndRenameUseDiscoveryIdentity() throws Exception {
+ long catalogId = 86L;
+ TestingHmsEventCatalog catalog = new TestingHmsEventCatalog(catalogId,
+ ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "1"));
+ CatalogMgr catalogMgr = new CatalogMgr();
+ addNamedCatalog(catalogMgr, catalog);
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(catalog.metaCache.updateCache(Mockito.anyString(), Mockito.anyString(),
+ Mockito.any(), Mockito.anyLong(), Mockito.anyLong())).thenReturn(true);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalogMgr.registerExternalDatabaseFromEvent("Foo", "hms");
+ catalogMgr.unregisterExternalDatabase("Foo", "hms");
+ // ALTER DATABASE rename registers the new remote spelling through the same event route.
+ catalogMgr.registerExternalDatabaseFromEvent("Bar", "hms");
+ catalogMgr.unregisterExternalDatabase("Bar", "hms");
+ }
+
+ long fooId = Util.genIdByName("hms", "foo");
+ long barId = Util.genIdByName("hms", "bar");
+ ArgumentCaptor<ExternalDatabase> dbCaptor = ArgumentCaptor.forClass(ExternalDatabase.class);
+ Mockito.verify(catalog.metaCache).updateCache(Mockito.eq("Foo"), Mockito.eq("foo"),
+ dbCaptor.capture(), Mockito.eq(fooId), Mockito.anyLong());
+ Assertions.assertEquals("Foo", dbCaptor.getValue().getRemoteName());
+ Assertions.assertEquals("foo", dbCaptor.getValue().getFullName());
+ Mockito.verify(catalog.metaCache).invalidate("foo", fooId);
+ Mockito.verify(catalog.metaCache).updateCache(Mockito.eq("Bar"), Mockito.eq("bar"),
+ dbCaptor.capture(), Mockito.eq(barId), Mockito.anyLong());
+ Mockito.verify(catalog.metaCache).invalidate("bar", barId);
+ Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, fooId);
+ Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, barId);
+ }
+
+ @Test
+ void testModeOneDatabaseEventLookupAndDropUseRootLocale() throws Exception {
+ Locale previousLocale = Locale.getDefault();
+ Locale.setDefault(Locale.forLanguageTag("tr-TR"));
+ try {
+ long catalogId = 88L;
+ long dbId = Util.genIdByName("hms", "i");
+ TestingHmsEventCatalog catalog = new TestingHmsEventCatalog(catalogId,
+ ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "1"));
+ CatalogMgr catalogMgr = new CatalogMgr();
+ addNamedCatalog(catalogMgr, catalog);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(catalog.metaCache.updateCache(Mockito.anyString(), Mockito.anyString(),
+ Mockito.any(), Mockito.anyLong(), Mockito.anyLong())).thenReturn(true);
+ @SuppressWarnings("unchecked")
+ ExternalDatabase<? extends ExternalTable> db = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(catalog.metaCache.getMetaObj("i", dbId)).thenReturn(Optional.of(db));
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalogMgr.registerExternalDatabaseFromEvent("I", "hms");
+ Assertions.assertSame(db, catalog.getDbNullable("I"));
+ catalogMgr.unregisterExternalDatabase("I", "hms");
+ }
+
+ Mockito.verify(catalog.metaCache).updateCache(Mockito.eq("I"), Mockito.eq("i"),
+ Mockito.any(), Mockito.eq(dbId), Mockito.anyLong());
+ Mockito.verify(catalog.metaCache).invalidate("i", dbId);
+ Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, dbId);
+ } finally {
+ Locale.setDefault(previousLocale);
+ }
+ }
+
+ @Test
+ void testModeTwoDatabaseLookupUsesRootLocale() throws Exception {
+ Locale previousLocale = Locale.getDefault();
+ Locale.setDefault(Locale.forLanguageTag("tr-TR"));
+ try {
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(89L,
+ ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2"));
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ catalog.installMetaCache(metaCache);
+ Field mappingField = ExternalCatalog.class.getDeclaredField("lowerCaseToDatabaseName");
+ mappingField.setAccessible(true);
+ @SuppressWarnings("unchecked")
+ Map<String, String> mapping = (Map<String, String>) mappingField.get(catalog);
+ mapping.put("i", "I");
+
+ Assertions.assertEquals(Pair.of("I", Util.genIdByName("testing_catalog", "I")),
+ catalog.getDbIdentityForReplay("I", 0L).orElseThrow(AssertionError::new));
+ } finally {
+ Locale.setDefault(previousLocale);
+ }
+ }
+
private static void addCatalog(CatalogMgr catalogMgr, ExternalCatalog catalog) throws Exception {
Field idToCatalogField = CatalogMgr.class.getDeclaredField("idToCatalog");
idToCatalogField.setAccessible(true);
@@ -54,6 +235,17 @@ public class CatalogMgrTest {
idToCatalog.put(catalog.getId(), catalog);
}
+ private static void addNamedCatalog(CatalogMgr catalogMgr, ExternalCatalog catalog) throws Exception {
+ addCatalog(catalogMgr, catalog);
+ Field nameToCatalogField = CatalogMgr.class.getDeclaredField("nameToCatalog");
+ nameToCatalogField.setAccessible(true);
+ @SuppressWarnings("unchecked")
+ ConcurrentMap<String, CatalogIf<? extends DatabaseIf<? extends TableIf>>> nameToCatalog =
+ (ConcurrentMap<String, CatalogIf<? extends DatabaseIf<? extends TableIf>>>)
+ nameToCatalogField.get(catalogMgr);
+ nameToCatalog.put(catalog.getName(), catalog);
+ }
+
@Test
void testAlterCatalogRollsBackUncheckedValidationFailure() throws Exception {
CatalogMgr catalogMgr = new CatalogMgr();
@@ -80,6 +272,41 @@ public class CatalogMgrTest {
Mockito.verify(catalog, Mockito.never()).modifyCatalogProps(newProperties);
}
+ @Test
+ void testMalformedJdbcMappingAlterDoesNotPublishOrJournal() throws Exception {
+ Map<String, String> properties = new HashMap<>(ImmutableMap.of(
+ "type", "jdbc",
+ JdbcResource.DRIVER_URL, "driver.jar",
+ JdbcResource.JDBC_URL, "jdbc:oracle:thin:@127.0.0.1:1521:XE",
+ JdbcResource.DRIVER_CLASS, "oracle.jdbc.driver.OracleDriver"));
+ JdbcExternalCatalog catalog = new JdbcExternalCatalog(98L, "jdbc", null, properties, "");
+ CatalogMgr catalogMgr = new CatalogMgr();
+ addCatalog(catalogMgr, catalog);
+ Map<String, String> oldProperties = new HashMap<>(catalog.getProperties());
+ CatalogLog log = new CatalogLog();
+ log.setCatalogId(catalog.getId());
+ log.setNewProps(ImmutableMap.of(ExternalCatalog.META_NAMES_MAPPING, "not-json"));
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ EditLog editLog = Mockito.mock(EditLog.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(env.getEditLog()).thenReturn(editLog);
+ Mockito.when(cacheMgr.withCatalogLifecycleLock(Mockito.eq(catalog.getId()), Mockito.any()))
+ .thenAnswer(invocation -> {
+ java.util.function.Supplier<?> action = invocation.getArgument(1);
+ return action.get();
+ });
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ Assertions.assertThrows(DdlException.class,
+ () -> catalogMgr.replayAlterCatalogProps(log, oldProperties, false));
+ }
+
+ Assertions.assertEquals(oldProperties, catalog.getProperties());
+ Mockito.verify(editLog, Mockito.never()).logCatalogLog(Mockito.anyShort(), Mockito.any());
+ }
+
@Test
void testDetachedValidationNeverPublishesCandidateToConcurrentInitialization() throws Exception {
CatalogMgr catalogMgr = new CatalogMgr();
@@ -138,6 +365,7 @@ public class CatalogMgrTest {
Env env = Mockito.mock(Env.class);
ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class));
Mockito.when(cacheMgr.withCatalogLifecycleLock(Mockito.eq(catalogId), Mockito.any()))
.thenAnswer(invocation -> {
java.util.function.Supplier<?> action = invocation.getArgument(1);
@@ -219,8 +447,862 @@ public class CatalogMgrTest {
}
Mockito.verify(catalog, Mockito.never()).tryModifyCatalogProps(Mockito.any());
- Mockito.verify(catalog).modifyCatalogProps(newProperties);
+ InOrder order = Mockito.inOrder(cacheMgr, catalog);
+ order.verify(cacheMgr).invalidateRowCountCache(catalogId);
+ order.verify(catalog).modifyCatalogProps(newProperties);
+ order.verify(cacheMgr).onCatalogOperationalContextChanged(catalogId);
+ }
+
+ @Test
+ void testUnsupportedAddPartitionEventStillInvalidatesRowCount() throws Exception {
+ CatalogMgr catalogMgr = new CatalogMgr();
+ long catalogId = 46L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getName()).thenReturn("hms");
+ Mockito.doReturn(db).when(catalog).getDbNullable("db1");
+ Mockito.doReturn(table).when(db).getTableNullable("tbl1");
+ Mockito.when(table.getPartitionColumnTypes(Mockito.any()))
+ .thenThrow(new NotSupportedException("unsupported table"));
+ addNamedCatalog(catalogMgr, catalog);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalogMgr.addExternalPartitions(
+ "hms", "db1", "tbl1", Collections.singletonList("p=1"), 1L, false);
+ }
+
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId, "db1", "tbl1");
+ Mockito.verify(cacheMgr, Mockito.never()).invalidateTableByNameOrWider(Mockito.anyLong(),
+ Mockito.anyString(), Mockito.anyString());
+ Mockito.verify(cacheMgr, Mockito.never()).hive(catalogId);
+ }
+
+ @Test
+ void testAddPartitionCacheAcquisitionFailureClosesRowCountFence() throws Exception {
+ CatalogMgr catalogMgr = new CatalogMgr();
+ long catalogId = 48L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getName()).thenReturn("hms");
+ Mockito.doReturn(db).when(catalog).getDbNullable("db1");
+ Mockito.doReturn(table).when(db).getTableNullable("tbl1");
+ Mockito.when(table.getPartitionColumnTypes(Mockito.any())).thenReturn(Collections.emptyList());
+ addNamedCatalog(catalogMgr, catalog);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(cacheMgr.hive(catalogId)).thenThrow(new IllegalStateException("cache acquisition failed"));
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ Assertions.assertThrows(IllegalStateException.class,
+ () -> catalogMgr.addExternalPartitions(
+ "hms", "db1", "tbl1", Collections.singletonList("p=1"), 1L, false));
+ }
+
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId, "db1", "tbl1");
+ }
+
+ @Test
+ void testDropPartitionEventInvalidatesRowCountBeforeCacheFailure() throws Exception {
+ CatalogMgr catalogMgr = new CatalogMgr();
+ long catalogId = 49L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getName()).thenReturn("hms");
+ Mockito.doReturn(db).when(catalog).getDbNullable("db1");
+ Mockito.doReturn(table).when(db).getTableNullable("tbl1");
+ addNamedCatalog(catalogMgr, catalog);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ HiveExternalMetaCache hiveCache = Mockito.mock(HiveExternalMetaCache.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(cacheMgr.hive(catalogId)).thenReturn(hiveCache);
+ Mockito.doThrow(new IllegalStateException("partition cache failure"))
+ .when(hiveCache).dropPartitionsCache(
+ Mockito.eq(table), Mockito.anyList(), Mockito.eq(true));
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ Assertions.assertThrows(IllegalStateException.class,
+ () -> catalogMgr.dropExternalPartitions(
+ "hms", "db1", "tbl1", Collections.singletonList("p=1"), 1L, false));
+ }
+
+ // The opening fence and the failure-safe closing fence both run.
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId, "db1", "tbl1");
+ }
+
+ @Test
+ void testSuccessfulPartitionEventClosesRowCountAdmissionWindow() throws Exception {
+ CatalogMgr catalogMgr = new CatalogMgr();
+ long catalogId = 62L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getName()).thenReturn("hms");
+ Mockito.doReturn(db).when(catalog).getDbNullable("db1");
+ Mockito.doReturn(table).when(db).getTableNullable("tbl1");
+ Mockito.when(table.getPartitionColumnTypes(Mockito.any())).thenReturn(Collections.emptyList());
+ addNamedCatalog(catalogMgr, catalog);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ HiveExternalMetaCache hiveCache = Mockito.mock(HiveExternalMetaCache.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(cacheMgr.hive(catalogId)).thenReturn(hiveCache);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalogMgr.addExternalPartitions(
+ "hms", "db1", "tbl1", Collections.singletonList("p=1"), 1L, false);
+ }
+
+ // One fence before the selective update, one after it closes the admission window.
+ Mockito.verify(cacheMgr, Mockito.times(2))
+ .invalidateRowCountCache(catalogId, "db1", "tbl1");
+ }
+
+ @Test
+ void testColdPartitionEventsFenceRowCountBeforeIgnoredTableMiss() throws Exception {
+ CatalogMgr catalogMgr = new CatalogMgr();
+ long catalogId = 50L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getName()).thenReturn("hms");
+ Mockito.doReturn(db).when(catalog).getDbNullable("db1");
+ Mockito.doReturn(null).when(db).getTableNullable("tbl1");
+ addNamedCatalog(catalogMgr, catalog);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalogMgr.addExternalPartitions(
+ "hms", "db1", "tbl1", Collections.singletonList("p=1"), 1L, true);
+ catalogMgr.dropExternalPartitions(
+ "hms", "db1", "tbl1", Collections.singletonList("p=1"), 1L, true);
+ }
+
+ Mockito.verify(cacheMgr, Mockito.times(4))
+ .invalidateRowCountCache(catalogId, "db1", "tbl1");
+ Mockito.verify(cacheMgr, Mockito.times(2))
+ .invalidateTableByNameOrWider(catalogId, "db1", "tbl1");
+ Mockito.verify(cacheMgr, Mockito.never()).hive(catalogId);
+ }
+
+ @Test
+ void testCatalogRefreshUsesSingleRowCountInvalidationOwner() {
+ long catalogId = 50L;
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId);
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ Mockito.when(metaCache.retireObjects()).thenReturn(() -> { });
+ catalog.installMetaCache(metaCache);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalog.onRefreshCache(true);
+ }
+
+ InOrder order = Mockito.inOrder(cacheMgr, metaCache);
+ order.verify(cacheMgr).invalidateRowCountCache(catalogId);
+ order.verify(metaCache).invalidateNames();
+ order.verify(metaCache).retireObjects();
+ order.verify(cacheMgr).invalidateCatalog(catalogId);
+ Mockito.verify(cacheMgr, Mockito.never()).getRowCountCache();
+ }
+
+ @Test
+ void testCatalogRefreshRunsRemovalCallbacksOutsideCatalogMonitor() throws Exception {
+ long catalogId = 99L;
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId);
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ catalog.installMetaCache(metaCache);
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ CountDownLatch monitorAcquired = new CountDownLatch(1);
+ Mockito.when(metaCache.retireObjects()).thenReturn(() -> {
+ executor.submit(() -> {
+ synchronized (catalog) {
+ monitorAcquired.countDown();
+ }
+ });
+ try {
+ Assertions.assertTrue(monitorAcquired.await(5, TimeUnit.SECONDS));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ });
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalog.onRefreshCache(false);
+ } finally {
+ executor.shutdownNow();
+ }
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId);
+ }
+
+ @Test
+ void testMetadataOnlyRefreshStillInvalidatesRowCount() {
+ long catalogId = 51L;
+ long dbId = 52L;
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId);
+ ExternalDatabase<ExternalTable> db = new ExternalDatabase<ExternalTable>(
+ catalog, dbId, "db1", "db1", InitDatabaseLog.Type.TEST) {
+ @Override
+ protected ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
+ ExternalCatalog externalCatalog, ExternalDatabase externalDatabase) {
+ return null;
+ }
+ };
+ ExecutorService removalExecutor = Executors.newSingleThreadExecutor();
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = new MetaCache<>(
+ "databaseCache",
+ removalExecutor,
+ OptionalLong.empty(),
+ OptionalLong.empty(),
+ 10,
+ key -> Collections.emptyList(),
+ key -> Optional.empty(),
+ (key, value, cause) -> catalog.handleDatabaseMetaCacheRemoval(value, cause));
+ metaCache.updateCache("db1", "db1", db, dbId);
+ catalog.installMetaCache(metaCache);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try {
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalog.onRefreshCache(false);
+ }
+ } finally {
+ removalExecutor.shutdownNow();
+ }
+
+ Mockito.verify(cacheMgr).invalidateDb(db);
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId);
+ Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(Mockito.anyLong());
+ }
+
+ @Test
+ void testUnregisterDatabaseRemovesLocalEntryWhenEngineInvalidationFails() {
+ long catalogId = 47L;
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId);
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ catalog.installMetaCache(metaCache);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.doThrow(new IllegalStateException("engine invalidation failed"))
+ .when(cacheMgr).invalidateDb(catalogId,
+ Util.genIdByName("testing_catalog", "CanonicalDb"), "CanonicalDb");
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ Assertions.assertThrows(IllegalStateException.class,
+ () -> catalog.unregisterDatabase("CanonicalDb"));
+ }
+
+ // The local metadata entry is removed before the engine invalidation runs, so a failing
+ // engine flush cannot leave the dropped database resident.
+ Mockito.verify(metaCache).invalidate("CanonicalDb",
+ Util.genIdByName("testing_catalog", "CanonicalDb"));
+ Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId,
+ Util.genIdByName("testing_catalog", "CanonicalDb"));
+ }
+
+ @Test
+ void testResidentDatabaseRemovalInvalidatesLocalAndRoutedCache() {
+ long catalogId = 53L;
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId);
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ catalog.installMetaCache(metaCache);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalog.unregisterDatabase("CanonicalDb");
+ }
+
+ Mockito.verify(metaCache).invalidate("CanonicalDb",
+ Util.genIdByName("testing_catalog", "CanonicalDb"));
+ Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId,
+ Util.genIdByName("testing_catalog", "CanonicalDb"));
+ Mockito.verify(cacheMgr).invalidateDb(catalogId,
+ Util.genIdByName("testing_catalog", "CanonicalDb"), "CanonicalDb");
+ }
+
+ @Test
+ void testColdDatabaseDropFencesBeforeRemovingNameSlot() throws Exception {
+ long catalogId = 86L;
+ long dbId = Util.genIdByName("testing_catalog", "cold_db");
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId);
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ catalog.installMetaCache(metaCache);
+ CountDownLatch nameRemovalStarted = new CountDownLatch(1);
+ CountDownLatch releaseNameRemoval = new CountDownLatch(1);
+ Mockito.doAnswer(invocation -> {
+ nameRemovalStarted.countDown();
+ Assertions.assertTrue(releaseNameRemoval.await(10, TimeUnit.SECONDS));
+ return null;
+ }).when(metaCache).invalidate("cold_db", dbId);
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ new MockUp<Env>() {
+ @Mock
+ Env getCurrentEnv() {
+ return env;
+ }
+ };
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ Future<?> drop = executor.submit(() -> {
+ catalog.unregisterDatabase("cold_db");
+ });
+ Assertions.assertTrue(nameRemovalStarted.await(10, TimeUnit.SECONDS));
+ Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, dbId);
+ Mockito.verify(cacheMgr, Mockito.never()).invalidateDb(catalogId, dbId, "cold_db");
+ releaseNameRemoval.countDown();
+ drop.get(10, TimeUnit.SECONDS);
+ Mockito.verify(cacheMgr).invalidateDb(catalogId, dbId, "cold_db");
+ } finally {
+ releaseNameRemoval.countDown();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testUnresolvedModeTwoDatabaseRemovalRetiresCatalogScope() throws Exception {
+ long catalogId = 60L;
+ Map<String, String> properties = ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2");
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId, properties);
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ catalog.installMetaCache(metaCache);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ // The mode-2 mapping is gone, so the canonical local key cannot be resolved.
+ catalog.unregisterDatabase("dropped_db");
+ }
+
+ // Unknown scope: retire every cached database object and flush engine caches/row counts
+ // catalog-wide instead of evicting a lowercased local key that may not exist.
+ Mockito.verify(metaCache).invalidateObjects();
+ Mockito.verify(cacheMgr).invalidateCatalog(catalogId);
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId);
+ Mockito.verify(cacheMgr, Mockito.never()).invalidateDb(Mockito.anyLong(), Mockito.anyString());
+ }
+
+ @Test
+ void testReboundModeTwoDatabaseDropDoesNotRetireReplacement() throws Exception {
+ long catalogId = 89L;
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId,
+ ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2"));
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ catalog.installMetaCache(metaCache);
+ Field mappingField = ExternalCatalog.class.getDeclaredField("lowerCaseToDatabaseName");
+ mappingField.setAccessible(true);
+ @SuppressWarnings("unchecked")
+ Map<String, String> mapping = (Map<String, String>) mappingField.get(catalog);
+ mapping.put("foo", "FOO");
+ long oldId = Util.genIdByName("testing_catalog", "Foo");
+ Mockito.when(metaCache.getNameByIdIfPresent(oldId)).thenReturn(Optional.of("Foo"));
+ @SuppressWarnings("unchecked")
+ ExternalDatabase<? extends ExternalTable> oldDb = Mockito.mock(ExternalDatabase.class);
+ @SuppressWarnings("unchecked")
+ ExternalDatabase<? extends ExternalTable> replacementDb = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(metaCache.tryGetMetaObj("Foo")).thenReturn(Optional.of(oldDb));
+ Mockito.when(metaCache.tryGetMetaObj("FOO")).thenReturn(Optional.of(replacementDb));
+ Assertions.assertSame(replacementDb, catalog.getDbForReplay("Foo").orElseThrow(AssertionError::new));
+ Assertions.assertSame(oldDb, catalog.getDbForDropReplay("Foo").orElseThrow(AssertionError::new));
+ Assertions.assertSame(replacementDb, catalog.getDbForDropReplay("FOO").orElseThrow(AssertionError::new));
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalog.unregisterDatabase("Foo");
+ }
+
+ Mockito.verify(metaCache).invalidate("Foo", oldId);
+ Mockito.verify(metaCache, Mockito.never()).invalidate("FOO",
+ Util.genIdByName("testing_catalog", "FOO"));
+ Mockito.verify(cacheMgr).invalidateDb(catalogId, oldId, "Foo");
+ }
+
+ @Test
+ void testColdReboundModeTwoDropUsesHistoricalIdentity() throws Exception {
+ long catalogId = 90L;
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId,
+ ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2"));
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ catalog.installMetaCache(metaCache);
+ Field mappingField = ExternalCatalog.class.getDeclaredField("lowerCaseToDatabaseName");
+ mappingField.setAccessible(true);
+ @SuppressWarnings("unchecked")
+ Map<String, String> mapping = (Map<String, String>) mappingField.get(catalog);
+ mapping.put("foo", "FOO");
+ long oldId = Util.genIdByName("testing_catalog", "Foo");
+ Mockito.when(metaCache.getNameByIdIfPresent(oldId)).thenReturn(Optional.of("Foo"));
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalog.invalidateColdDatabaseForReplay("Foo");
+ }
+
+ Mockito.verify(cacheMgr).invalidateDb(catalogId, oldId, "Foo");
+ Mockito.verify(cacheMgr, Mockito.never()).invalidateDb(catalogId,
+ Util.genIdByName("testing_catalog", "FOO"), "FOO");
+ }
+
+ @Test
+ void testModeTwoDropReplayUsesResolvedTargetAndLegacyLogRetiresBothCandidates() throws Exception {
+ long catalogId = 91L;
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId,
+ ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2"));
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ catalog.installMetaCache(metaCache);
+ ExternalMetadataOps metadataOps = Mockito.mock(ExternalMetadataOps.class);
+ catalog.metadataOps = metadataOps;
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ // Both histories carry caller spelling Foo. New logs preserve the resolved target.
+ catalog.replayDropTable("Foo", "t", "FOO");
+ catalog.replayDropDb("Foo", "FOO");
+ catalog.replayDropTable("Foo", "t", "Foo");
+ catalog.replayDropDb("Foo", "Foo");
+ // An old name-only log cannot distinguish those histories.
+ catalog.replayDropTable("Foo", "t", null);
+ catalog.replayDropDb("Foo", null);
+ }
+
+ Mockito.verify(metadataOps).afterDropTable("FOO", "t");
+ Mockito.verify(metadataOps).afterDropDb("FOO");
+ Mockito.verify(metadataOps).afterDropTable("Foo", "t");
+ Mockito.verify(metadataOps).afterDropDb("Foo");
+ Mockito.verify(metaCache, Mockito.times(2)).invalidateObjects();
+ Mockito.verify(metaCache).invalidateNames();
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateCatalog(catalogId);
+ }
+
+ @Test
+ void testLegacyModeTwoDatabaseDropRefreshesPopulatedNames() {
+ long catalogId = 94L;
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId,
+ ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2"));
+ AtomicReference<List<Pair<String, String>>> remoteNames = new AtomicReference<>(
+ Collections.singletonList(Pair.of("Foo", "Foo")));
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = new MetaCache<>(
+ "databaseCache", executor, OptionalLong.empty(), OptionalLong.empty(), 10,
+ ignored -> remoteNames.get(), ignored -> Optional.empty(), (key, value, cause) -> { });
+ catalog.installMetaCache(metaCache);
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ Assertions.assertEquals(Collections.singletonList("Foo"), catalog.getDbNames());
+ remoteNames.set(Collections.emptyList());
+
+ catalog.replayDropDb("Foo", null);
+
+ Assertions.assertTrue(catalog.getDbNames().isEmpty());
+ Mockito.verify(cacheMgr).invalidateCatalog(catalogId);
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testNoOpDatabaseDropKeepsUnrelatedCachedDatabase() {
+ long catalogId = 95L;
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId,
+ ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2"));
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = new MetaCache<>(
+ "databaseCache", executor, OptionalLong.empty(), OptionalLong.empty(), 10,
+ ignored -> Collections.emptyList(), ignored -> Optional.empty(), (key, value, cause) -> { });
+ long oldId = Util.genIdByName("testing_catalog", "Foo");
+ long unrelatedId = Util.genIdByName("testing_catalog", "sales");
+ long turkishId = Util.genIdByName("testing_catalog", "I");
+ ExternalDatabase<?> oldDb = Mockito.mock(ExternalDatabase.class);
+ ExternalDatabase<?> unrelatedDb = Mockito.mock(ExternalDatabase.class);
+ ExternalDatabase<?> turkishDb = Mockito.mock(ExternalDatabase.class);
+ metaCache.addObjForTest(oldId, "Foo", oldDb);
+ metaCache.addObjForTest(unrelatedId, "sales", unrelatedDb);
+ metaCache.addObjForTest(turkishId, "I", turkishDb);
+ catalog.installMetaCache(metaCache);
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalog.retireCachedDatabaseForNoOp("absent");
+ Assertions.assertSame(unrelatedDb, metaCache.tryGetMetaObj("sales").orElseThrow(AssertionError::new));
+ // Locale.ROOT distinguishes I (i) from dotless i. A no-op DROP of the latter
+ // must not retire the resident I database.
+ catalog.retireCachedDatabaseForNoOp("ı");
+ Assertions.assertSame(turkishDb, metaCache.tryGetMetaObj("I").orElseThrow(AssertionError::new));
+ Mockito.verifyNoInteractions(cacheMgr);
+
+ // A lost mode-2 mapping still permits narrow retirement of a retained historical ID.
+ catalog.retireCachedDatabaseForNoOp("foo");
+ Assertions.assertFalse(metaCache.tryGetMetaObj("Foo").isPresent());
+ Assertions.assertSame(unrelatedDb, metaCache.tryGetMetaObj("sales").orElseThrow(AssertionError::new));
+ Assertions.assertSame(turkishDb, metaCache.tryGetMetaObj("I").orElseThrow(AssertionError::new));
+ Mockito.verify(cacheMgr).invalidateDb(catalogId, oldId, "Foo");
+ Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(catalogId);
+ Mockito.verify(cacheMgr, Mockito.never()).invalidateRowCountCache(catalogId, unrelatedId);
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testFilteredPartitionEventsLeaveOtherDatabaseCachesUntouched() throws Exception {
+ long catalogId = 96L;
+ HMSExternalCatalog catalog = new HMSExternalCatalog(catalogId, "hms", null,
+ ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2",
+ ExternalCatalog.EXCLUDE_DATABASE_LIST, "archived",
+ ExternalCatalog.INCLUDE_TABLE_LIST, "sales.hot"), "");
+ CatalogMgr catalogMgr = new CatalogMgr();
+ addNamedCatalog(catalogMgr, catalog);
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ Assertions.assertFalse(catalog.isPartitionEventTargetExcluded("sales", "hot"));
+ for (String[] target : new String[][] {{"archived", "t"}, {"sales", "cold"}}) {
+ catalogMgr.addExternalPartitions("hms", target[0], target[1],
+ Collections.singletonList("p=1"), 1L, true);
+ catalogMgr.dropExternalPartitions("hms", target[0], target[1],
+ Collections.singletonList("p=1"), 1L, true);
+ new RefreshManager().refreshPartitions("hms", target[0], target[1],
+ Collections.singletonList("p=1"), 1L, true);
+ new RefreshManager().refreshExternalTableFromEvent("hms", target[0], target[1], 1L);
+ }
+ catalogMgr.unregisterExternalDatabase("archived", "hms");
+ Mockito.verifyNoInteractions(cacheMgr);
+ }
+ }
+
+ @Test
+ void testHmsEventFilterMatchesListedCaseAndRefreshesSnapshot() throws Exception {
+ HMSExternalCatalog catalog = new HMSExternalCatalog(97L, "hms", null,
+ new HashMap<>(ImmutableMap.of(ExternalCatalog.EXCLUDE_DATABASE_LIST, "SALES",
+ ExternalCatalog.INCLUDE_TABLE_LIST, "SALES.cold")), "");
+ // Listing compares the remote spelling exactly: neither filter hides sales.hot.
+ Assertions.assertFalse(catalog.isDatabaseEventTargetExcluded("sales"));
+ Assertions.assertFalse(catalog.isPartitionEventTargetExcluded("sales", "hot"));
+ Assertions.assertTrue(catalog.isDatabaseEventTargetExcluded("SALES"));
+ catalog.getCatalogProperty().addProperty(ExternalCatalog.INCLUDE_TABLE_LIST, "sales.cold");
+ Assertions.assertTrue(catalog.isPartitionEventTargetExcluded("sales", "hot"));
+ }
+
+ @Test
+ void testHmsEventFilterMatchesNormalizedDatabaseNames() {
+ for (String mode : new String[] {"1", "2"}) {
+ HMSExternalCatalog catalog = new HMSExternalCatalog(97L, "hms", null,
+ new HashMap<>(ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, mode,
+ ExternalCatalog.INCLUDE_DATABASE_LIST, "AllowedDb",
+ ExternalCatalog.EXCLUDE_DATABASE_LIST, "ExcludedDb")), "");
+ Assertions.assertFalse(catalog.isDatabaseEventTargetExcluded("alloweddb"));
+ Assertions.assertTrue(catalog.isDatabaseEventTargetExcluded("excludeddb"));
+ }
+ }
+
+ @Test
+ void testLiveDropLogsResolvedDatabaseInsteadOfCallerAlias() throws Exception {
+ ExternalDatabase<?> resolvedDb = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(resolvedDb.getFullName()).thenReturn("FOO");
+ ExternalTable resolvedTable = Mockito.mock(ExternalTable.class);
+ Mockito.when(resolvedTable.getName()).thenReturn("t");
+ Mockito.doReturn(resolvedTable).when(resolvedDb).getTableNullable("t");
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(92L,
+ ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2")) {
+ @Override
+ public ExternalDatabase<? extends ExternalTable> getDbNullable(String dbName) {
+ return resolvedDb;
+ }
+ };
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ catalog.installMetaCache(metaCache);
+ Field mappingField = ExternalCatalog.class.getDeclaredField("lowerCaseToDatabaseName");
+ mappingField.setAccessible(true);
+ @SuppressWarnings("unchecked")
+ Map<String, String> mapping = (Map<String, String>) mappingField.get(catalog);
+ mapping.put("foo", "FOO");
+ ExternalMetadataOps metadataOps = Mockito.mock(ExternalMetadataOps.class, Mockito.CALLS_REAL_METHODS);
+ Mockito.when(metadataOps.dropDbImplWithResolvedName("Foo", false, false))
+ .thenReturn(Optional.of("FOO"));
+ catalog.metadataOps = metadataOps;
+ Env env = Mockito.mock(Env.class);
+ EditLog editLog = Mockito.mock(EditLog.class);
+ Mockito.when(env.getEditLog()).thenReturn(editLog);
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalog.dropDb("Foo", false, false);
+ catalog.dropTable("Foo", "t", false, false, false, false, false);
+ }
+
+ Mockito.verify(metadataOps).afterDropDb("FOO");
+ ArgumentCaptor<DropDbInfo> log = ArgumentCaptor.forClass(DropDbInfo.class);
+ Mockito.verify(editLog).logDropDb(log.capture());
+ Assertions.assertEquals("FOO", log.getValue().getDbName());
+ Assertions.assertEquals("FOO", log.getValue().getResolvedDbName());
+ ArgumentCaptor<DropInfo> tableLog = ArgumentCaptor.forClass(DropInfo.class);
+ Mockito.verify(editLog).logDropTable(tableLog.capture());
+ Assertions.assertEquals("FOO", tableLog.getValue().getDb());
+ Assertions.assertEquals("FOO", tableLog.getValue().getResolvedDb());
+ }
+
+ @Test
+ void testUnresolvedDatabaseDropRetiresGenerationAndFencesCatalog() {
+ long catalogId = 72L;
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId);
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ catalog.installMetaCache(metaCache);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalog.retireUnresolvedDatabaseGeneration();
+ }
+
+ // Retire the hidden database generation and fence the catalog scope (engine + row counts),
+ // even though the per-database removal callbacks suppress their own fences.
+ Mockito.verify(metaCache).invalidateObjects();
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId);
+ Mockito.verify(cacheMgr).invalidateCatalog(catalogId);
+ }
+
+ @Test
+ void testUnresolvedDatabaseFencesBeforeObjectGenerationSwap() throws Exception {
+ long catalogId = 87L;
+ TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId);
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache = Mockito.mock(MetaCache.class);
+ catalog.installMetaCache(metaCache);
+ CountDownLatch swapStarted = new CountDownLatch(1);
+ CountDownLatch releaseSwap = new CountDownLatch(1);
+ Mockito.doAnswer(invocation -> {
+ swapStarted.countDown();
+ Assertions.assertTrue(releaseSwap.await(10, TimeUnit.SECONDS));
+ return null;
+ }).when(metaCache).invalidateObjects();
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ new MockUp<Env>() {
+ @Mock
+ Env getCurrentEnv() {
+ return env;
+ }
+ };
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ Future<?> retirement = executor.submit(() -> {
+ catalog.retireAllDatabaseObjectsWithoutEngineInvalidation();
+ });
+ Assertions.assertTrue(swapStarted.await(10, TimeUnit.SECONDS));
+ Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId);
+ releaseSwap.countDown();
+ retirement.get(10, TimeUnit.SECONDS);
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId);
+ } finally {
+ releaseSwap.countDown();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testUnresolvedTableNameRetiresHiddenTableGeneration() throws Exception {
+ long catalogId = 73L;
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getLowerCaseTableNames()).thenReturn(2);
+ ExternalDatabase<ExternalTable> db = new ExternalDatabase<ExternalTable>(
+ catalog, 74L, "db1", "db1", InitDatabaseLog.Type.TEST) {
+ @Override
+ protected ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
+ ExternalCatalog externalCatalog, ExternalDatabase externalDatabase) {
+ return null;
+ }
+ };
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalTable> tableCache = Mockito.mock(MetaCache.class);
+ Field metaCacheField = ExternalDatabase.class.getDeclaredField("metaCache");
+ metaCacheField.setAccessible(true);
+ metaCacheField.set(db, tableCache);
+
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ Assertions.assertFalse(db.unregisterTableForReplay("Tbl"));
+ }
+
+ // The mode-2 mapping is gone: retire the hidden table generation and widen the fence.
+ Mockito.verify(tableCache).invalidateObjects();
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId, 74L);
+ Mockito.verify(cacheMgr).invalidateTableByNameOrWider(catalogId, "db1", "Tbl");
+ }
+
+ @Test
+ void testCommittedAlterStillRetiresAndLogsWhenResetCleanupThrows() throws Exception {
+ long catalogId = 58L;
+ CatalogMgr catalogMgr = new CatalogMgr();
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getName()).thenReturn("catalog_name");
+ Map<String, String> oldProperties = ImmutableMap.of("s3.access_key", "old", "warehouse", "s3://bucket");
+ Map<String, String> committedProperties = ImmutableMap.of("s3.access_key", "new", "warehouse", "s3://bucket");
+ Mockito.when(catalog.getProperties()).thenReturn(oldProperties, committedProperties);
+ Mockito.doThrow(new RuntimeException("close failed"))
+ .when(catalog).modifyCatalogProps(Mockito.any());
+ addNamedCatalog(catalogMgr, catalog);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ EditLog editLog = Mockito.mock(EditLog.class);
+ RefreshManager refreshManager = Mockito.mock(RefreshManager.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(env.getEditLog()).thenReturn(editLog);
+ Mockito.when(env.getRefreshManager()).thenReturn(refreshManager);
+ Mockito.when(cacheMgr.withCatalogLifecycleLock(Mockito.eq(catalogId), Mockito.any()))
+ .thenAnswer(invocation -> {
+ java.util.function.Supplier<?> action = invocation.getArgument(1);
+ return action.get();
+ });
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ Assertions.assertThrows(RuntimeException.class,
+ () -> catalogMgr.alterCatalogProps("catalog_name", ImmutableMap.of("s3.access_key", "new")));
+ }
+
+ // The reset published the new properties before its throwable cleanup: retirement and
+ // durability must still happen.
Mockito.verify(cacheMgr).onCatalogOperationalContextChanged(catalogId);
+ Mockito.verify(editLog).logCatalogLog(Mockito.anyShort(), Mockito.any());
+ }
+
+ @Test
+ void testReplayAlterCatalogPropsSurvivesResetCleanupFailure() throws Exception {
+ long catalogId = 61L;
+ CatalogMgr catalogMgr = new CatalogMgr();
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getName()).thenReturn("catalog_name");
+ Mockito.doThrow(new RuntimeException("close failed during replay"))
+ .when(catalog).modifyCatalogProps(Mockito.any());
+ addCatalog(catalogMgr, catalog);
+
+ CatalogLog log = new CatalogLog();
+ log.setCatalogId(catalogId);
+ log.setNewProps(ImmutableMap.of("s3.access_key", "new"));
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ EditLog editLog = Mockito.mock(EditLog.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(env.getEditLog()).thenReturn(editLog);
+ Mockito.when(cacheMgr.withCatalogLifecycleLock(Mockito.eq(catalogId), Mockito.any()))
+ .thenAnswer(invocation -> {
+ java.util.function.Supplier<?> action = invocation.getArgument(1);
+ return action.get();
+ });
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ // A follower must not terminate when local connector cleanup fails while applying an
+ // already-durable ALTER record.
+ Assertions.assertDoesNotThrow(() -> catalogMgr.replayAlterCatalogProps(log, null, true));
+ }
+
+ Mockito.verify(cacheMgr).onCatalogOperationalContextChanged(catalogId);
+ // Replay never journals.
+ Mockito.verify(editLog, Mockito.never()).logCatalogLog(Mockito.anyShort(), Mockito.any());
+ }
+
+ @Test
+ void testDropCatalogFencesRowCountBeforeCleanup() throws Exception {
+ long catalogId = 57L;
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getName()).thenReturn("dropme");
+ CatalogMgr catalogMgr = new CatalogMgr();
+ addNamedCatalog(catalogMgr, catalog);
+
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ EditLog editLog = Mockito.mock(EditLog.class);
+ RefreshManager refreshManager = Mockito.mock(RefreshManager.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(env.getEditLog()).thenReturn(editLog);
+ Mockito.when(env.getRefreshManager()).thenReturn(refreshManager);
+ Mockito.when(env.getQueryStats()).thenReturn(Mockito.mock(QueryStats.class));
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalogMgr.dropCatalog("dropme", false);
+ }
+
+ InOrder order = Mockito.inOrder(cacheMgr, catalog);
+ order.verify(cacheMgr).invalidateRowCountCache(catalogId);
+ order.verify(catalog).onClose();
+ Assertions.assertNull(catalogMgr.getCatalog("dropme"));
}
private static class LatchingValidationCatalog extends ExternalCatalog {
@@ -274,4 +1356,47 @@ public class CatalogMgrTest {
// This test isolates edit-log property restoration from environment-owned cache services.
}
}
+
+ private static class TestingUnregisterCatalog extends ExternalCatalog {
+ TestingUnregisterCatalog(long id) {
+ this(id, Collections.emptyMap());
+ }
+
+ TestingUnregisterCatalog(long id, Map<String, String> properties) {
+ super(id, "testing_catalog", InitCatalogLog.Type.TEST, "");
+ catalogProperty = new CatalogProperty(null, properties);
+ }
+
+ void installMetaCache(MetaCache<ExternalDatabase<? extends ExternalTable>> cache) {
+ metaCache = cache;
+ initialized = true;
+ }
+
+ @Override
+ protected List<String> listTableNamesFromRemote(SessionContext ctx, String dbName) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public boolean tableExist(SessionContext ctx, String dbName, String tblName) {
+ return false;
+ }
+
+ @Override
+ protected void initLocalObjectsImpl() {
+ }
+ }
+
+ private static class TestingHmsEventCatalog extends HMSExternalCatalog {
+ @SuppressWarnings("unchecked")
+ TestingHmsEventCatalog(long id, Map<String, String> properties) {
+ super(id, "hms", null, properties, "");
+ metaCache = Mockito.mock(MetaCache.class);
+ initialized = true;
+ }
+
+ @Override
+ protected void initLocalObjectsImpl() {
+ }
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java
index 111a8173257..598edb8995f 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java
@@ -20,6 +20,7 @@ package org.apache.doris.datasource;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.Pair;
import org.apache.doris.datasource.doris.RemoteDorisExternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.iceberg.IcebergHMSExternalCatalog;
@@ -29,6 +30,7 @@ import org.apache.doris.datasource.metacache.MetaCacheEntry;
import org.apache.doris.datasource.metacache.MetaCacheEntryStats;
import org.apache.doris.datasource.paimon.PaimonExternalCatalog;
+import com.google.common.util.concurrent.Uninterruptibles;
import mockit.Mock;
import mockit.MockUp;
import org.junit.Assert;
@@ -39,6 +41,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
@@ -304,12 +307,40 @@ public class ExternalMetaCacheRouteResolverTest {
mockCurrentCatalog(catalogId, catalog);
hive.initializedCatalogIds.add(catalogId);
Map<String, String> oldProperties = Collections.singletonMap("generation", "old");
-
- metaCacheMgr.rollbackCatalogProperties(catalog, oldProperties);
-
- Mockito.verify(catalog).rollBackCatalogProps(oldProperties);
- Assert.assertFalse(hive.isCatalogInitialized(catalogId));
- Assert.assertEquals(1, hive.invalidateCatalogCalls);
+ CountDownLatch loadStarted = new CountDownLatch(1);
+ CountDownLatch allowLoadToFinish = new CountDownLatch(1);
+ ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() {
+ @Override
+ protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey rowCountKey) {
+ loadStarted.countDown();
+ Assert.assertTrue(Uninterruptibles.awaitUninterruptibly(
+ allowLoadToFinish, 30, TimeUnit.SECONDS));
+ return Optional.of(100L);
+ }
+ };
+ ExecutorService loaderExecutor = Executors.newSingleThreadExecutor();
+ metaCacheMgr.replaceRowCountCacheForTest(new ExternalRowCountCache(loaderExecutor, null, loader));
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future<Long> load = caller.submit(
+ () -> metaCacheMgr.getRowCountCache().getCachedRowCount(catalogId, 2L, 3L, false));
+ Assert.assertTrue(loadStarted.await(30, TimeUnit.SECONDS));
+
+ metaCacheMgr.rollbackCatalogProperties(catalog, oldProperties);
+ allowLoadToFinish.countDown();
+
+ Assert.assertEquals(TableIf.UNKNOWN_ROW_COUNT, (long) load.get(30, TimeUnit.SECONDS));
+ Assert.assertEquals(TableIf.UNKNOWN_ROW_COUNT,
+ metaCacheMgr.getRowCountCache().getCachedRowCountIfPresent(catalogId, 2L, 3L));
+ Mockito.verify(catalog).rollBackCatalogProps(oldProperties);
+ Mockito.verify(catalog).resetToUninitialized(false);
+ Assert.assertFalse(hive.isCatalogInitialized(catalogId));
+ Assert.assertEquals(1, hive.invalidateCatalogCalls);
+ } finally {
+ allowLoadToFinish.countDown();
+ caller.shutdownNow();
+ loaderExecutor.shutdownNow();
+ }
}
@Test
@@ -410,6 +441,407 @@ public class ExternalMetaCacheRouteResolverTest {
Assert.assertEquals(0, paimon.invalidateCatalogCalls);
}
+ @Test
+ public void testEngineSpecificTableInvalidationAlsoFencesRowCount() throws Exception {
+ RecordingExternalMetaCache hive = new RecordingExternalMetaCache(
+ "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog);
+ ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive);
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ metaCacheMgr.replaceRowCountCacheForTest(rowCountCache);
+ long catalogId = 17L;
+ long dbId = 18L;
+ long tableId = 19L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ ExternalTable table = Mockito.mock(ExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.of(db));
+ Mockito.when(db.getId()).thenReturn(dbId);
+ Mockito.doReturn(Optional.of(table)).when(db).getTableForReplay("tbl1");
+ Mockito.when(table.getCatalog()).thenReturn(catalog);
+ Mockito.when(table.getDb()).thenReturn(db);
+ Mockito.when(table.getId()).thenReturn(tableId);
+ mockCurrentCatalog(catalogId, catalog);
+ hive.initializedCatalogIds.add(catalogId);
+
+ metaCacheMgr.invalidateTableByEngine(catalogId, "hive", "db1", "tbl1");
+
+ Assert.assertEquals(1, hive.invalidateTableCalls);
+ Mockito.verify(rowCountCache, Mockito.times(2)).invalidateTable(catalogId, dbId, tableId);
+ }
+
+ @Test
+ public void testDatabaseInvalidationCanPreserveSingleBulkRowCountOwner() throws Exception {
+ RecordingExternalMetaCache hive = new RecordingExternalMetaCache(
+ "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog);
+ RecordingExternalMetaCache hudi = new RecordingExternalMetaCache(
+ "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache(
+ "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive, hudi, iceberg);
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ metaCacheMgr.replaceRowCountCacheForTest(rowCountCache);
+ long catalogId = 20L;
+ long dbId = 21L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ mockCurrentCatalog(catalogId, catalog);
+ hive.initializedCatalogIds.add(catalogId);
+ hudi.initializedCatalogIds.add(catalogId);
+ iceberg.initializedCatalogIds.add(catalogId);
+
+ metaCacheMgr.invalidateDb(catalogId, dbId, "db1", false);
+
+ Assert.assertEquals(1, hive.invalidateDbCalls);
+ Assert.assertEquals(1, hudi.invalidateDbCalls);
+ Assert.assertEquals(1, iceberg.invalidateDbCalls);
+ Mockito.verifyNoInteractions(rowCountCache);
+ }
+
+ @Test
+ public void testOperationalContextChangeFencesRowCountForEvictedDatabases() {
+ long catalogId = 24L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ mockCurrentCatalog(catalogId, catalog);
+ ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true);
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ metaCacheMgr.replaceRowCountCacheForTest(rowCountCache);
+
+ metaCacheMgr.onCatalogOperationalContextChanged(catalogId);
+
+ Mockito.verify(rowCountCache).invalidateCatalog(catalogId);
+ }
+
+ @Test
+ public void testPermanentCatalogRemovalReleasesRowCountGenerationButRenameDoesNot() {
+ long catalogId = 29L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ mockCurrentCatalog(catalogId, catalog);
+ ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true);
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ metaCacheMgr.replaceRowCountCacheForTest(rowCountCache);
+
+ metaCacheMgr.removeCatalog(catalogId);
+ Mockito.verify(rowCountCache, Mockito.never()).releaseCatalog(catalogId);
+ metaCacheMgr.removeCatalogPermanently(catalogId);
+ Mockito.verify(rowCountCache).releaseCatalog(catalogId);
+ }
+
+ @Test
+ public void testRollbackRetiresEngineCachesWhenResetCleanupThrows() throws Exception {
+ ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true);
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ metaCacheMgr.replaceRowCountCacheForTest(rowCountCache);
+ long catalogId = 25L;
+ HMSExternalCatalog catalog = Mockito.spy(
+ new HMSExternalCatalog(catalogId, "hms", null, Collections.emptyMap(), ""));
+ mockCurrentCatalog(catalogId, catalog);
+ ExternalMetaCache hive = metaCacheMgr.hive(catalogId);
+ Assert.assertTrue(hive.isCatalogInitialized(catalogId));
+ Mockito.doThrow(new RuntimeException("close failed")).when(catalog).resetToUninitialized(false);
+
+ Assert.assertThrows(RuntimeException.class,
+ () -> metaCacheMgr.rollbackCatalogProperties(catalog, Collections.emptyMap()));
+
+ Assert.assertFalse(hive.isCatalogInitialized(catalogId));
+ Mockito.verify(rowCountCache).invalidateCatalog(catalogId);
+ }
+
+ @Test
+ public void testNameBasedInvalidationWidensToCanonicalDatabaseWhenTableCold() throws Exception {
+ long catalogId = 26L;
+ long dbId = 27L;
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ RecordingExternalMetaCache hive = new RecordingExternalMetaCache(
+ "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog) {
+ @Override
+ public void invalidateDb(long id, String name) {
+ // Observe the opening fence while routed invalidation is still running.
+ Mockito.verify(rowCountCache, Mockito.times(2)).invalidateDb(catalogId, dbId);
+ super.invalidateDb(id, name);
+ }
+ };
+ RecordingExternalMetaCache hudi = new RecordingExternalMetaCache(
+ "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache(
+ "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive, hudi, iceberg);
+ metaCacheMgr.replaceRowCountCacheForTest(rowCountCache);
+ HMSExternalCatalog catalog = Mockito.spy(
+ new HMSExternalCatalog(catalogId, "hms", null, Collections.emptyMap(), ""));
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(db.getId()).thenReturn(dbId);
+ Mockito.when(db.getFullName()).thenReturn("MixedDb");
+ Mockito.doReturn(Optional.empty()).when(db).getTableForReplay("mixedtbl");
+ Mockito.doReturn(Optional.of(db)).when(catalog).getDbForReplay("mixeddb");
+ mockCurrentCatalog(catalogId, catalog);
+ hive.initializedCatalogIds.add(catalogId);
+ hudi.initializedCatalogIds.add(catalogId);
+ iceberg.initializedCatalogIds.add(catalogId);
+
+ metaCacheMgr.invalidateTableByNameOrWider(catalogId, "mixeddb", "mixedtbl");
+
+ Assert.assertEquals(1, hive.invalidateDbCalls);
+ Mockito.verify(rowCountCache, Mockito.times(3)).invalidateDb(catalogId, dbId);
+ }
+
+ @Test
+ public void testTypedTableInvalidationFencesBeforeEngineEviction() throws Exception {
+ long catalogId = 90L;
+ long dbId = 91L;
+ long tableId = 92L;
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ RecordingExternalMetaCache hive = new RecordingExternalMetaCache(
+ "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog) {
+ @Override
+ public void invalidateTable(long id, String dbName, String tableName) {
+ Mockito.verify(rowCountCache).invalidateTable(catalogId, dbId, tableId);
+ super.invalidateTable(id, dbName, tableName);
+ }
+ };
+ RecordingExternalMetaCache hudi = new RecordingExternalMetaCache(
+ "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache(
+ "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ ExternalMetaCacheMgr cacheMgr = newManagerWithCaches(hive, hudi, iceberg);
+ cacheMgr.replaceRowCountCacheForTest(rowCountCache);
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(db.getId()).thenReturn(dbId);
+ ExternalTable table = Mockito.mock(ExternalTable.class);
+ Mockito.when(table.getCatalog()).thenReturn(catalog);
+ Mockito.when(table.getDb()).thenReturn(db);
+ Mockito.when(table.getId()).thenReturn(tableId);
+ Mockito.when(table.getDbName()).thenReturn("db");
+ Mockito.when(table.getName()).thenReturn("tbl");
+ mockCurrentCatalog(catalogId, catalog);
+ hive.initializedCatalogIds.add(catalogId);
+
+ cacheMgr.invalidateTableCache(table);
+
+ Assert.assertEquals(1, hive.invalidateTableCalls);
+ Mockito.verify(rowCountCache, Mockito.times(2)).invalidateTable(catalogId, dbId, tableId);
+ }
+
+ @Test
+ public void testExplicitDatabaseInvalidationFencesBeforeEngineEviction() throws Exception {
+ long catalogId = 93L;
+ long dbId = 94L;
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ RecordingExternalMetaCache hive = new RecordingExternalMetaCache(
+ "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog) {
+ @Override
+ public void invalidateDb(long id, String name) {
+ Mockito.verify(rowCountCache).invalidateDb(catalogId, dbId);
+ super.invalidateDb(id, name);
+ }
+ };
+ RecordingExternalMetaCache hudi = new RecordingExternalMetaCache(
+ "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache(
+ "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ ExternalMetaCacheMgr cacheMgr = newManagerWithCaches(hive, hudi, iceberg);
+ cacheMgr.replaceRowCountCacheForTest(rowCountCache);
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ mockCurrentCatalog(catalogId, catalog);
+ hive.initializedCatalogIds.add(catalogId);
+
+ cacheMgr.invalidateDb(catalogId, dbId, "db");
+
+ Assert.assertEquals(1, hive.invalidateDbCalls);
+ Mockito.verify(rowCountCache, Mockito.times(2)).invalidateDb(catalogId, dbId);
+ }
+
+ @Test
+ public void testCatalogInvalidationFencesBeforeEngineEviction() throws Exception {
+ long catalogId = 98L;
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ RecordingExternalMetaCache hive = new RecordingExternalMetaCache(
+ "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog) {
+ @Override
+ public void invalidateCatalogEntries(long id) {
+ Mockito.verify(rowCountCache).invalidateCatalog(catalogId);
+ super.invalidateCatalogEntries(id);
+ }
+ };
+ RecordingExternalMetaCache hudi = new RecordingExternalMetaCache(
+ "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache(
+ "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ ExternalMetaCacheMgr cacheMgr = newManagerWithCaches(hive, hudi, iceberg);
+ cacheMgr.replaceRowCountCacheForTest(rowCountCache);
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ mockCurrentCatalog(catalogId, catalog);
+ hive.initializedCatalogIds.add(catalogId);
+
+ cacheMgr.invalidateCatalog(catalogId);
+
+ Assert.assertEquals(1, hive.invalidateCatalogEntriesCalls);
+ Mockito.verify(rowCountCache, Mockito.times(2)).invalidateCatalog(catalogId);
+ }
+
+ @Test
+ public void testEngineSpecificTableInvalidationFencesBeforeEviction() throws Exception {
+ long catalogId = 95L;
+ long dbId = 96L;
+ long tableId = 97L;
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ RecordingExternalMetaCache hive = new RecordingExternalMetaCache(
+ "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog) {
+ @Override
+ public void invalidateTable(long id, String dbName, String tableName) {
+ Mockito.verify(rowCountCache).invalidateTable(catalogId, dbId, tableId);
+ super.invalidateTable(id, dbName, tableName);
+ }
+ };
+ ExternalMetaCacheMgr cacheMgr = newManagerWithCaches(hive);
+ cacheMgr.replaceRowCountCacheForTest(rowCountCache);
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ ExternalTable table = Mockito.mock(ExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.doReturn(Optional.of(db)).when(catalog).getDbForReplay("db");
+ Mockito.doReturn(Optional.of(table)).when(db).getTableForReplay("tbl");
+ Mockito.when(table.getCatalog()).thenReturn(catalog);
+ Mockito.when(table.getDb()).thenReturn(db);
+ Mockito.when(table.getId()).thenReturn(tableId);
+ Mockito.when(db.getId()).thenReturn(dbId);
+ mockCurrentCatalog(catalogId, catalog);
+ hive.initializedCatalogIds.add(catalogId);
+
+ cacheMgr.invalidateTableByEngine(catalogId, "hive", "db", "tbl");
+
+ Assert.assertEquals(1, hive.invalidateTableCalls);
+ Mockito.verify(rowCountCache, Mockito.times(2)).invalidateTable(catalogId, dbId, tableId);
+ }
+
+ @Test
+ public void testColdKnownDatabaseTableRoutesDoNotFenceUnrelatedCounts() throws Exception {
+ long catalogId = 101L;
+ long dbId = 102L;
+ RecordingExternalMetaCache hive = new RecordingExternalMetaCache(
+ "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog);
+ RecordingExternalMetaCache hudi = new RecordingExternalMetaCache(
+ "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache(
+ "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ ExternalMetaCacheMgr cacheMgr = newManagerWithCaches(hive, hudi, iceberg);
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ cacheMgr.replaceRowCountCacheForTest(rowCountCache);
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ Mockito.doReturn(Optional.empty()).when(catalog).getDbForReplay("cold_db");
+ Mockito.when(catalog.getDbIdentityForReplay("cold_db", 0L))
+ .thenReturn(Optional.of(Pair.of("ColdDb", dbId)));
+ mockCurrentCatalog(catalogId, catalog);
+ hive.initializedCatalogIds.add(catalogId);
+
+ cacheMgr.invalidateTable(catalogId, "cold_db", "tbl");
+ cacheMgr.invalidateTableByEngine(catalogId, "hive", "cold_db", "tbl");
+ cacheMgr.invalidatePartitions(catalogId, "cold_db", "tbl", Collections.singletonList("p"));
+
+ Mockito.verify(rowCountCache, Mockito.times(6)).invalidateDb(catalogId, dbId);
+ Mockito.verify(rowCountCache, Mockito.never()).invalidateCatalog(catalogId);
+ }
+
+ @Test
+ public void testNameBasedInvalidationWidensToCatalogWhenDatabaseCold() throws Exception {
+ RecordingExternalMetaCache hive = new RecordingExternalMetaCache(
+ "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog);
+ RecordingExternalMetaCache hudi = new RecordingExternalMetaCache(
+ "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache(
+ "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive, hudi, iceberg);
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ metaCacheMgr.replaceRowCountCacheForTest(rowCountCache);
+ long catalogId = 28L;
+ HMSExternalCatalog catalog = Mockito.spy(
+ new HMSExternalCatalog(catalogId, "hms", null, Collections.emptyMap(), ""));
+ Mockito.doReturn(Optional.empty()).when(catalog).getDbForReplay("colddb");
+ mockCurrentCatalog(catalogId, catalog);
+ hive.initializedCatalogIds.add(catalogId);
+ hudi.initializedCatalogIds.add(catalogId);
+ iceberg.initializedCatalogIds.add(catalogId);
+
+ metaCacheMgr.invalidateTableByNameOrWider(catalogId, "colddb", "tbl");
+
+ Assert.assertEquals(1, hive.invalidateCatalogEntriesCalls);
+ Mockito.verify(rowCountCache, Mockito.times(3)).invalidateCatalog(catalogId);
+ }
+
+ @Test
+ public void testNameBasedInvalidationUsesKnownColdDatabaseIdentity() throws Exception {
+ RecordingExternalMetaCache hive = new RecordingExternalMetaCache(
+ "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog);
+ RecordingExternalMetaCache hudi = new RecordingExternalMetaCache(
+ "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache(
+ "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog);
+ ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive, hudi, iceberg);
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ metaCacheMgr.replaceRowCountCacheForTest(rowCountCache);
+ long catalogId = 29L;
+ long dbId = 30L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ Mockito.when(catalog.getDbForReplay("coldDb")).thenReturn(Optional.empty());
+ Mockito.when(catalog.getDbIdentityForReplay("coldDb", 0L))
+ .thenReturn(Optional.of(org.apache.doris.common.Pair.of("CanonicalDb", dbId)));
+ mockCurrentCatalog(catalogId, catalog);
+ hive.initializedCatalogIds.add(catalogId);
+ hudi.initializedCatalogIds.add(catalogId);
+ iceberg.initializedCatalogIds.add(catalogId);
+
+ metaCacheMgr.invalidateTableByNameOrWider(catalogId, "coldDb", "tbl");
+ metaCacheMgr.invalidateRowCountCache(catalogId, "coldDb", "tbl");
+
+ Assert.assertEquals(1, hive.invalidateDbCalls);
+ Assert.assertEquals(0, hive.invalidateCatalogEntriesCalls);
+ Mockito.verify(rowCountCache, Mockito.times(4)).invalidateDb(catalogId, dbId);
+ Mockito.verify(rowCountCache, Mockito.never()).invalidateCatalog(catalogId);
+ }
+
+ @Test
+ public void testNameBasedRowCountFenceUsesRetainedColdTableIdentity() {
+ long catalogId = 22L;
+ long dbId = 23L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.of(db));
+ Mockito.when(db.getId()).thenReturn(dbId);
+ Mockito.doReturn(Optional.empty()).when(db).getTableForReplay("tbl1");
+ Mockito.when(db.getTableIdForReplay("tbl1")).thenReturn(java.util.OptionalLong.of(24L));
+ mockCurrentCatalog(catalogId, catalog);
+
+ ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true);
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ metaCacheMgr.replaceRowCountCacheForTest(rowCountCache);
+
+ metaCacheMgr.invalidateRowCountCache(catalogId, "db1", "tbl1");
+
+ Mockito.verify(rowCountCache).invalidateTable(catalogId, dbId, 24L);
+ Mockito.verify(rowCountCache, Mockito.never()).invalidateDb(catalogId, dbId);
+ }
+
+ @Test
+ public void testNameBasedRowCountFenceWidensWhenColdTableNameIsLost() {
+ long catalogId = 22L;
+ long dbId = 23L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.of(db));
+ Mockito.when(db.getId()).thenReturn(dbId);
+ Mockito.doReturn(Optional.empty()).when(db).getTableForReplay("tbl1");
+ mockCurrentCatalog(catalogId, catalog);
+
+ ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true);
+ ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class);
+ metaCacheMgr.replaceRowCountCacheForTest(rowCountCache);
+
+ metaCacheMgr.invalidateRowCountCache(catalogId, "db1", "tbl1");
+
+ Mockito.verify(rowCountCache).invalidateDb(catalogId, dbId);
+ }
+
@SuppressWarnings("unchecked")
private ExternalMetaCacheMgr newManagerWithCaches(RecordingExternalMetaCache... caches) throws Exception {
ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java
index 075806aa82b..4d7451c64b0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java
@@ -17,22 +17,205 @@
package org.apache.doris.datasource;
+import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.ThreadPoolManager;
+import org.apache.doris.datasource.metacache.MetaCache;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.Uninterruptibles;
import mockit.Mock;
import mockit.MockUp;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
+import java.lang.reflect.Field;
import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
public class ExternalRowCountCacheTest {
+ @Test
+ public void testCatalogCacheResetSuppressesPerDatabaseRowCountScan() {
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(1L);
+ ExternalMetaCacheMgr metaCacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(metaCacheMgr);
+ new MockUp<Env>() {
+ @Mock
+ Env getCurrentEnv() {
+ return env;
+ }
+ };
+ ExternalDatabase<ExternalTable> db = new ExternalDatabase<ExternalTable>(
+ catalog, 2L, "db", "db", InitDatabaseLog.Type.TEST) {
+ @Override
+ protected ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
+ ExternalCatalog externalCatalog, ExternalDatabase externalDatabase) {
+ return null;
+ }
+ };
+
+ // A catalog-wide refresh suppresses the per-database engine flush, so no per-database
+ // row-count fence may run; the catalog scope is fenced once by the caller instead.
+ db.resetMetaToUninitialized(true, false);
+ Mockito.verify(metaCacheMgr).invalidateDb(db);
+ Mockito.verify(metaCacheMgr, Mockito.never()).invalidateRowCountCache(1L, 2L);
+
+ // An ordinary removal fences both the engine cache and this database's row counts.
+ db.resetMetaToUninitialized();
+ org.mockito.InOrder order = Mockito.inOrder(metaCacheMgr);
+ order.verify(metaCacheMgr).invalidateRowCountCache(1L, 2L);
+ order.verify(metaCacheMgr).invalidateDb(db);
+ order.verify(metaCacheMgr).invalidateRowCountCache(1L, 2L);
+ }
+
+ @Test
+ public void testDatabaseRemovalFencesRowCountWhenEngineInvalidationFails() {
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(1L);
+ ExternalMetaCacheMgr metaCacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(metaCacheMgr);
+ new MockUp<Env>() {
+ @Mock
+ Env getCurrentEnv() {
+ return env;
+ }
+ };
+ ExternalDatabase<ExternalTable> db = new ExternalDatabase<ExternalTable>(
+ catalog, 2L, "db", "db", InitDatabaseLog.Type.TEST) {
+ @Override
+ protected ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
+ ExternalCatalog externalCatalog, ExternalDatabase externalDatabase) {
+ return null;
+ }
+ };
+ Mockito.doThrow(new IllegalStateException("engine invalidation failed"))
+ .when(metaCacheMgr).invalidateDb(db);
+
+ Assertions.assertThrows(IllegalStateException.class, () -> db.resetMetaToUninitialized(true, true));
+
+ // The independent row-count fence must still run when the routed invalidation throws.
+ Mockito.verify(metaCacheMgr, Mockito.times(2)).invalidateRowCountCache(1L, 2L);
+ }
+
+ @Test
+ public void testDatabaseRefreshFencesRowCountBeforeRetiringTableObjects() throws Exception {
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(1L);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ new MockUp<Env>() {
+ @Mock
+ Env getCurrentEnv() {
+ return env;
+ }
+ };
+ ExternalDatabase<ExternalTable> db = new ExternalDatabase<ExternalTable>(
+ catalog, 2L, "db", "db", InitDatabaseLog.Type.TEST) {
+ @Override
+ protected ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
+ ExternalCatalog externalCatalog, ExternalDatabase externalDatabase) {
+ return null;
+ }
+ };
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalTable> tableCache = Mockito.mock(MetaCache.class);
+ CountDownLatch generationSwap = new CountDownLatch(1);
+ CountDownLatch completeSwap = new CountDownLatch(1);
+ Mockito.when(tableCache.retireObjects()).thenReturn(() -> {
+ generationSwap.countDown();
+ Uninterruptibles.awaitUninterruptibly(completeSwap);
+ });
+ Field metaCacheField = ExternalDatabase.class.getDeclaredField("metaCache");
+ metaCacheField.setAccessible(true);
+ metaCacheField.set(db, tableCache);
+
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ Future<?> reset = executor.submit(() -> {
+ db.resetMetaToUninitialized();
+ });
+ Assertions.assertTrue(generationSwap.await(10, TimeUnit.SECONDS));
+ // The replacement cache has been published while routed invalidation and the final
+ // fence have not run yet. The opening fence must already be visible.
+ Mockito.verify(cacheMgr).invalidateRowCountCache(1L, 2L);
+ Mockito.verify(cacheMgr, Mockito.never()).invalidateDb(db);
+ completeSwap.countDown();
+ reset.get(10, TimeUnit.SECONDS);
+ } finally {
+ completeSwap.countDown();
+ executor.shutdownNow();
+ }
+
+ org.mockito.InOrder order = Mockito.inOrder(cacheMgr, tableCache);
+ order.verify(cacheMgr).invalidateRowCountCache(1L, 2L);
+ order.verify(tableCache).retireObjects();
+ order.verify(cacheMgr).invalidateDb(db);
+ order.verify(cacheMgr).invalidateRowCountCache(1L, 2L);
+ }
+
+ @Test
+ public void testUnresolvedTableFencesRowCountBeforeObjectGenerationSwap() throws Exception {
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(1L);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ new MockUp<Env>() {
+ @Mock
+ Env getCurrentEnv() {
+ return env;
+ }
+ };
+ ExternalDatabase<ExternalTable> db = new ExternalDatabase<ExternalTable>(
+ catalog, 2L, "db", "db", InitDatabaseLog.Type.TEST) {
+ @Override
+ protected ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
+ ExternalCatalog externalCatalog, ExternalDatabase externalDatabase) {
+ return null;
+ }
+ };
+ @SuppressWarnings("unchecked")
+ MetaCache<ExternalTable> tableCache = Mockito.mock(MetaCache.class);
+ CountDownLatch swapStarted = new CountDownLatch(1);
+ CountDownLatch releaseSwap = new CountDownLatch(1);
+ Mockito.doAnswer(invocation -> {
+ swapStarted.countDown();
+ Assertions.assertTrue(releaseSwap.await(10, TimeUnit.SECONDS));
+ return null;
+ }).when(tableCache).invalidateObjects();
+ Field metaCacheField = ExternalDatabase.class.getDeclaredField("metaCache");
+ metaCacheField.setAccessible(true);
+ metaCacheField.set(db, tableCache);
+
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ Future<?> retirement = executor.submit(() -> {
+ db.retireAllTableObjectsWithoutEngineInvalidation();
+ });
+ Assertions.assertTrue(swapStarted.await(10, TimeUnit.SECONDS));
+ Mockito.verify(cacheMgr).invalidateRowCountCache(1L, 2L);
+ releaseSwap.countDown();
+ retirement.get(10, TimeUnit.SECONDS);
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(1L, 2L);
+ } finally {
+ releaseSwap.countDown();
+ executor.shutdownNow();
+ }
+ }
+
@Test
public void testRowCountKeyUsesTableIdAsCacheIdentity() {
ExternalRowCountCache.RowCountKey key1 = new ExternalRowCountCache.RowCountKey(1, 2, 3);
@@ -84,20 +267,365 @@ public class ExternalRowCountCacheTest {
Mockito.verify(table).fetchRowCountWithMetaCache(false);
}
+ @Test
+ public void testInvalidationScopes() {
+ ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() {
+ @Override
+ protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey rowCountKey) {
+ return Optional.of(rowCountKey.getTableId());
+ }
+ };
+
+ ExternalRowCountCache cache = new ExternalRowCountCache(
+ MoreExecutors.newDirectExecutorService(), null, loader);
+ Assertions.assertEquals(100L, cache.getCachedRowCount(1, 10, 100, false));
+ Assertions.assertEquals(101L, cache.getCachedRowCount(1, 10, 101, false));
+ Assertions.assertEquals(102L, cache.getCachedRowCount(1, 11, 102, false));
+ Assertions.assertEquals(200L, cache.getCachedRowCount(2, 20, 200, false));
+
+ cache.invalidateTable(1, 10, 100);
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cache.getCachedRowCountIfPresent(1, 10, 100));
+ Assertions.assertEquals(101L, cache.getCachedRowCountIfPresent(1, 10, 101));
+
+ cache.invalidateDb(1, 10);
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cache.getCachedRowCountIfPresent(1, 10, 101));
+ Assertions.assertEquals(102L, cache.getCachedRowCountIfPresent(1, 11, 102));
+
+ cache.invalidateCatalog(1);
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cache.getCachedRowCountIfPresent(1, 11, 102));
+ Assertions.assertEquals(200L, cache.getCachedRowCountIfPresent(2, 20, 200));
+ }
+
+ @Test
+ public void testCatalogGenerationFenceDoesNotReuseInFlightOrCompletedCount() throws Exception {
+ AtomicInteger loadCount = new AtomicInteger();
+ CountDownLatch oldLoadStarted = new CountDownLatch(1);
+ CountDownLatch releaseOldLoad = new CountDownLatch(1);
+ ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() {
+ @Override
+ protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey key) {
+ int load = loadCount.incrementAndGet();
+ if (load == 1) {
+ oldLoadStarted.countDown();
+ Uninterruptibles.awaitUninterruptibly(releaseOldLoad);
+ }
+ return Optional.of(load * 100L);
+ }
+ };
+ ExecutorService cacheExecutor = Executors.newFixedThreadPool(2);
+ ExecutorService readerExecutor = Executors.newSingleThreadExecutor();
+ try {
+ ExternalRowCountCache cache = new ExternalRowCountCache(cacheExecutor, null, loader);
+ Future<Long> oldRead = readerExecutor.submit(() -> cache.getCachedRowCount(1, 10, 100, false));
+ Assertions.assertTrue(oldLoadStarted.await(10, TimeUnit.SECONDS));
+
+ cache.invalidateCatalog(1);
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cache.getCachedRowCountIfPresent(1, 10, 100));
+ Assertions.assertEquals(200L, cache.getCachedRowCount(1, 10, 100, false));
+
+ releaseOldLoad.countDown();
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, oldRead.get(10, TimeUnit.SECONDS));
+ Assertions.assertEquals(200L, cache.getCachedRowCountIfPresent(1, 10, 100));
+ } finally {
+ releaseOldLoad.countDown();
+ readerExecutor.shutdownNow();
+ cacheExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testDatabaseGenerationFenceKeepsOtherDatabasesReadable() throws Exception {
+ AtomicInteger loadCount = new AtomicInteger();
+ CountDownLatch oldLoadStarted = new CountDownLatch(1);
+ CountDownLatch releaseOldLoad = new CountDownLatch(1);
+ ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() {
+ @Override
+ protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey key) {
+ if (key.getDbId() == 10 && loadCount.incrementAndGet() == 1) {
+ oldLoadStarted.countDown();
+ Uninterruptibles.awaitUninterruptibly(releaseOldLoad);
+ return Optional.of(100L);
+ }
+ return Optional.of(key.getDbId() * 100L + loadCount.get());
+ }
+ };
+ ExecutorService cacheExecutor = Executors.newFixedThreadPool(2);
+ ExecutorService readerExecutor = Executors.newSingleThreadExecutor();
+ try {
+ ExternalRowCountCache cache = new ExternalRowCountCache(cacheExecutor, null, loader);
+ Assertions.assertEquals(2000L, cache.getCachedRowCount(1, 20, 100, false));
+ Future<Long> oldRead = readerExecutor.submit(() -> cache.getCachedRowCount(1, 10, 100, false));
+ Assertions.assertTrue(oldLoadStarted.await(10, TimeUnit.SECONDS));
+
+ cache.invalidateDb(1, 10);
+ Assertions.assertEquals(2000L, cache.getCachedRowCountIfPresent(1, 20, 100));
+ Assertions.assertEquals(1002L, cache.getCachedRowCount(1, 10, 100, false));
+ releaseOldLoad.countDown();
+
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, oldRead.get(10, TimeUnit.SECONDS));
+ Assertions.assertEquals(1002L, cache.getCachedRowCountIfPresent(1, 10, 100));
+ Assertions.assertEquals(2000L, cache.getCachedRowCountIfPresent(1, 20, 100));
+ } finally {
+ releaseOldLoad.countDown();
+ readerExecutor.shutdownNow();
+ cacheExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testInvalidatedNullLoadReturnsUnknownWithoutReusingIt() throws Exception {
+ AtomicInteger loadCount = new AtomicInteger();
+ CountDownLatch loadStarted = new CountDownLatch(1);
+ CountDownLatch releaseLoad = new CountDownLatch(1);
+ ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() {
+ @Override
+ protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey key) {
+ if (loadCount.incrementAndGet() == 1) {
+ loadStarted.countDown();
+ Uninterruptibles.awaitUninterruptibly(releaseLoad);
+ }
+ return Optional.of(100L);
+ }
+ };
+ ExecutorService cacheExecutor = Executors.newSingleThreadExecutor();
+ ExecutorService readerExecutor = Executors.newSingleThreadExecutor();
+ try {
+ ExternalRowCountCache cache = new ExternalRowCountCache(cacheExecutor, null, loader);
+ Future<Long> oldRead = readerExecutor.submit(() -> cache.getCachedRowCount(1, 10, 100, false));
+ Assertions.assertTrue(loadStarted.await(10, TimeUnit.SECONDS));
+
+ cache.invalidateTable(1, 10, 100);
+ releaseLoad.countDown();
+
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, oldRead.get(10, TimeUnit.SECONDS));
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cache.getCachedRowCountIfPresent(1, 10, 100));
+ Assertions.assertEquals(100L, cache.getCachedRowCount(1, 10, 100, false));
+ Assertions.assertEquals(2, loadCount.get());
+ } finally {
+ releaseLoad.countDown();
+ readerExecutor.shutdownNow();
+ cacheExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testPermanentCatalogRemovalReleasesGenerationWithoutRevivingOldReader() throws Exception {
+ CountDownLatch loadStarted = new CountDownLatch(1);
+ CountDownLatch releaseLoad = new CountDownLatch(1);
+ ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() {
+ @Override
+ protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey key) {
+ loadStarted.countDown();
+ Uninterruptibles.awaitUninterruptibly(releaseLoad);
+ return Optional.of(100L);
+ }
+ };
+ ExecutorService cacheExecutor = Executors.newSingleThreadExecutor();
+ ExecutorService readerExecutor = Executors.newSingleThreadExecutor();
+ try {
+ ExternalRowCountCache cache = new ExternalRowCountCache(cacheExecutor, null, loader);
+ cache.invalidateCatalog(2L);
+ Assertions.assertEquals(0, cache.getCatalogGenerationCountForTest());
+ Future<Long> oldRead = readerExecutor.submit(() -> cache.getCachedRowCount(1L, 10L, 100L, false));
+ Assertions.assertTrue(loadStarted.await(10, TimeUnit.SECONDS));
+ Assertions.assertEquals(1, cache.getCatalogGenerationCountForTest());
+
+ cache.invalidateCatalog(1L);
+ cache.releaseCatalog(1L);
+ Assertions.assertEquals(0, cache.getCatalogGenerationCountForTest());
+ releaseLoad.countDown();
+
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, oldRead.get(10, TimeUnit.SECONDS));
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cache.getCachedRowCountIfPresent(1L, 10L, 100L));
+ Assertions.assertEquals(0, cache.getCatalogGenerationCountForTest());
+ } finally {
+ releaseLoad.countDown();
+ readerExecutor.shutdownNow();
+ cacheExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testLateReadsAfterPermanentRemovalDoNotGrowCatalogGenerations() {
+ java.util.Set<Long> activeCatalogs = new java.util.HashSet<>();
+ ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() {
+ @Override
+ protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey key) {
+ return activeCatalogs.contains(key.getCatalogId()) ? Optional.of(100L) : null;
+ }
+ };
+ ExternalRowCountCache cache = new ExternalRowCountCache(
+ MoreExecutors.newDirectExecutorService(), null, loader, 2);
+ for (long catalogId = 1; catalogId <= 5; catalogId++) {
+ activeCatalogs.add(catalogId);
+ Assertions.assertEquals(100L, cache.getCachedRowCount(catalogId, 10, 100, false));
+ activeCatalogs.remove(catalogId);
+ cache.invalidateCatalog(catalogId);
+ cache.releaseCatalog(catalogId);
+
+ // A statement retaining an old table can still enter the cache after DROP cleanup.
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT,
+ cache.getCachedRowCount(catalogId, 10, 100, false));
+ Assertions.assertTrue(cache.getCatalogGenerationCountForTest() <= 2);
+ }
+ }
+
+ @Test
+ public void testInvalidateWhileRefreshIsRunningDoesNotRepublishStaleValue() throws Exception {
+ AtomicInteger loadCount = new AtomicInteger();
+ CountDownLatch refreshStarted = new CountDownLatch(1);
+ CountDownLatch allowRefreshToFinish = new CountDownLatch(1);
+ ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() {
+ @Override
+ protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey rowCountKey) {
+ int currentLoad = loadCount.incrementAndGet();
+ if (currentLoad == 2) {
+ refreshStarted.countDown();
+ Assertions.assertTrue(Uninterruptibles.awaitUninterruptibly(
+ allowRefreshToFinish, 30, TimeUnit.SECONDS));
+ }
+ return Optional.of(currentLoad * 100L);
+ }
+ };
+
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ ExternalRowCountCache cache = new ExternalRowCountCache(executor, null, loader);
+ Assertions.assertEquals(100L, cache.getCachedRowCount(1, 10, 100, false));
+
+ cache.refreshForTest(1, 10, 100);
+ Assertions.assertTrue(refreshStarted.await(30, TimeUnit.SECONDS));
+ Assertions.assertEquals(100L, cache.getCachedRowCount(1, 10, 100, false));
+
+ cache.invalidateTable(1, 10, 100);
+ allowRefreshToFinish.countDown();
+ executor.submit(() -> { }).get(30, TimeUnit.SECONDS);
+
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT,
+ cache.getCachedRowCountIfPresent(1, 10, 100));
+ Assertions.assertEquals(300L, cache.getCachedRowCount(1, 10, 100, false));
+ } finally {
+ allowRefreshToFinish.countDown();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testCatalogInvalidationUsesFullInFlightLoadIdentity() throws Exception {
+ CountDownLatch oldLoadStarted = new CountDownLatch(1);
+ CountDownLatch newLoadStarted = new CountDownLatch(1);
+ CountDownLatch allowLoadsToFinish = new CountDownLatch(1);
+ ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() {
+ @Override
+ protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey rowCountKey) {
+ if (rowCountKey.getCatalogId() == 1L) {
+ oldLoadStarted.countDown();
+ } else {
+ newLoadStarted.countDown();
+ }
+ Assertions.assertTrue(Uninterruptibles.awaitUninterruptibly(
+ allowLoadsToFinish, 30, TimeUnit.SECONDS));
+ return Optional.of(rowCountKey.getCatalogId() * 100L);
+ }
+ };
+
+ ExecutorService loaderExecutor = Executors.newFixedThreadPool(2);
+ ExecutorService callers = Executors.newFixedThreadPool(2);
+ try {
+ ExternalRowCountCache cache = new ExternalRowCountCache(loaderExecutor, null, loader);
+ Future<Long> oldLoad = callers.submit(() -> cache.getCachedRowCount(1, 10, 100, false));
+ Assertions.assertTrue(oldLoadStarted.await(30, TimeUnit.SECONDS));
+ cache.invalidateCatalog(1L);
+
+ Future<Long> newLoad = callers.submit(() -> cache.getCachedRowCount(2, 20, 100, false));
+ Assertions.assertTrue(newLoadStarted.await(30, TimeUnit.SECONDS));
+ cache.invalidateCatalog(2L);
+ allowLoadsToFinish.countDown();
+
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, oldLoad.get(30, TimeUnit.SECONDS));
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, newLoad.get(30, TimeUnit.SECONDS));
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT,
+ cache.getCachedRowCountIfPresent(2, 20, 100));
+ } finally {
+ allowLoadsToFinish.countDown();
+ callers.shutdownNow();
+ loaderExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testClosingFenceInvalidatesLoadAdmittedBeforeIt() throws Exception {
+ CountDownLatch loadAdmitted = new CountDownLatch(1);
+ CountDownLatch allowLoadToFinish = new CountDownLatch(1);
+ ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() {
+ @Override
+ protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey rowCountKey) {
+ loadAdmitted.countDown();
+ Assertions.assertTrue(Uninterruptibles.awaitUninterruptibly(
+ allowLoadToFinish, 30, TimeUnit.SECONDS));
+ return Optional.of(100L);
+ }
+ };
+
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ ExecutorService callers = Executors.newSingleThreadExecutor();
+ try {
+ ExternalRowCountCache cache = new ExternalRowCountCache(executor, null, loader);
+ // Opening fence, then a load admitted inside the admission window.
+ cache.invalidateTable(1, 10, 100);
+ Future<Long> staleLoad = callers.submit(() -> cache.getCachedRowCount(1, 10, 100, false));
+ Assertions.assertTrue(loadAdmitted.await(30, TimeUnit.SECONDS));
+
+ // The selective refresh finishes and closes the window before the load can publish.
+ cache.invalidateTable(1, 10, 100);
+ allowLoadToFinish.countDown();
+
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, staleLoad.get(30, TimeUnit.SECONDS));
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT,
+ cache.getCachedRowCountIfPresent(1, 10, 100));
+ } finally {
+ allowLoadToFinish.countDown();
+ callers.shutdownNow();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testRejectedSubmissionRemovesInFlightFence() {
+ ExecutorService rejectingExecutor = Mockito.mock(ExecutorService.class);
+ Mockito.doThrow(new RejectedExecutionException("rejected"))
+ .when(rejectingExecutor).execute(Mockito.any(Runnable.class));
+ ExternalRowCountCache cache = new ExternalRowCountCache(rejectingExecutor);
+
+ Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT,
+ cache.getCachedRowCount(1, 10, 100, false));
+ Assertions.assertEquals(0, cache.getInFlightLoadCountForTest());
+ }
+
@Test
public void testLoadWithException() throws Exception {
ThreadPoolExecutor executor = ThreadPoolManager.newDaemonFixedThreadPool(
1, Integer.MAX_VALUE, "TEST", true);
AtomicInteger counter = new AtomicInteger(0);
- new MockUp<ExternalRowCountCache.RowCountCacheLoader>() {
- @Mock
+ ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() {
+ @Override
protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey rowCountKey) {
- counter.incrementAndGet();
- return null;
+ int currentLoad = counter.incrementAndGet();
+ if (currentLoad == 1) {
+ return null;
+ }
+ if (currentLoad == 3) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ return Optional.of(100L);
}
};
- ExternalRowCountCache cache = new ExternalRowCountCache(executor);
+ ExternalRowCountCache cache = new ExternalRowCountCache(executor, null, loader);
long cachedRowCount = cache.getCachedRowCount(1, 1, 1, false);
Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cachedRowCount);
for (int i = 0; i < 60; i++) {
@@ -108,13 +636,6 @@ public class ExternalRowCountCacheTest {
}
Assertions.assertEquals(1, counter.get());
- new MockUp<ExternalRowCountCache.RowCountCacheLoader>() {
- @Mock
- protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey rowCountKey) {
- counter.incrementAndGet();
- return Optional.of(100L);
- }
- };
cache.getCachedRowCount(1, 1, 1, false);
for (int i = 0; i < 60; i++) {
cachedRowCount = cache.getCachedRowCount(1, 1, 1, false);
@@ -128,18 +649,6 @@ public class ExternalRowCountCacheTest {
Assertions.assertEquals(100, cachedRowCount);
Assertions.assertEquals(2, counter.get());
- new MockUp<ExternalRowCountCache.RowCountCacheLoader>() {
- @Mock
- protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey rowCountKey) {
- counter.incrementAndGet();
- try {
- Thread.sleep(2000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- return Optional.of(100L);
- }
- };
cachedRowCount = cache.getCachedRowCount(2, 2, 2, false);
Assertions.assertEquals(100, cachedRowCount);
Thread.sleep(1000);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetaCacheTest.java
index 8c5b29a2103..cf16443e9be 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetaCacheTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetaCacheTest.java
@@ -88,6 +88,15 @@ public class MetaCacheTest {
Assert.assertTrue(names.contains("local2"));
}
+ @Test
+ public void testCachedIdentityMatchingRespectsDatabaseCaseMode() {
+ metaCache.addObjForTest(1L, "foo", "foo");
+
+ Assert.assertTrue(metaCache.getCachedIdentitiesMatching("Foo", false).isEmpty());
+ Assert.assertEquals(1, metaCache.getCachedIdentitiesMatching("Foo", true).size());
+ Assert.assertEquals("foo", metaCache.getCachedIdentitiesMatching("foo", false).get(0).first);
+ }
+
@Test
public void testRefreshNamesReloadsCompleteSnapshot() {
ExecutorService refreshExecutor = Executors.newSingleThreadExecutor();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java
new file mode 100644
index 00000000000..a980cd66425
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java
@@ -0,0 +1,120 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.hive;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.datasource.ExternalDatabase;
+import org.apache.doris.datasource.ExternalMetaCacheMgr;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.Optional;
+
+public class HiveMetadataOpsTest {
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testAlternateCaseCreateResetsWarmDatabaseTableNames() {
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(catalog.getDbForReplay("mixeddb")).thenReturn((Optional) Optional.of(db));
+
+ new HiveMetadataOps(catalog, Mockito.mock(HMSCachedClient.class))
+ .afterCreateTable("mixeddb", "new_table");
+
+ Mockito.verify(db).resetMetaCacheNames();
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testColdTruncateReplayInvalidatesCanonicalDatabaseScope() {
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(1L);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(db.getId()).thenReturn(2L);
+ Mockito.when(db.getFullName()).thenReturn("MixedDb");
+ Mockito.when(db.getTableForReplay("mixedtbl")).thenReturn(Optional.empty());
+ Mockito.when(catalog.getDbForReplay("mixeddb")).thenReturn((Optional) Optional.of(db));
+
+ ExternalMetaCacheMgr metaCacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(metaCacheMgr);
+ new MockUp<Env>() {
+ @Mock
+ Env getCurrentEnv() {
+ return env;
+ }
+ };
+
+ HiveMetadataOps metadataOps = new HiveMetadataOps(catalog, Mockito.mock(HMSCachedClient.class));
+ metadataOps.afterTruncateTable("mixeddb", "mixedtbl", 100L);
+
+ // The event's lower-cased spelling must not be routed as a cache key directly.
+ Mockito.verify(metaCacheMgr).invalidateTableByNameOrWider(1L, "mixeddb", "mixedtbl");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testColdTruncateReplayWithColdDatabaseUsesWideningHelper() {
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(3L);
+ Mockito.when(catalog.getDbForReplay("db")).thenReturn(Optional.empty());
+
+ ExternalMetaCacheMgr metaCacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(metaCacheMgr);
+ new MockUp<Env>() {
+ @Mock
+ Env getCurrentEnv() {
+ return env;
+ }
+ };
+
+ HiveMetadataOps metadataOps = new HiveMetadataOps(catalog, Mockito.mock(HMSCachedClient.class));
+ metadataOps.afterTruncateTable("db", "tbl", 100L);
+
+ Mockito.verify(metaCacheMgr).invalidateTableByNameOrWider(3L, "db", "tbl");
+ }
+
+ @Test
+ public void testAfterDropTableWithoutResolvedDatabaseRetiresGeneration() {
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(4L);
+ Mockito.when(catalog.getName()).thenReturn("hive");
+ Mockito.when(catalog.getDbForReplay("db")).thenReturn(Optional.empty());
+
+ HiveMetadataOps metadataOps = new HiveMetadataOps(catalog, Mockito.mock(HMSCachedClient.class));
+ metadataOps.afterDropTable("db", "tbl");
+
+ // An empty lookup can hide a resident canonical database object after a lost mode-2 mapping.
+ Mockito.verify(catalog).invalidateColdDatabaseForReplay("db");
+ }
+
+ @Test
+ public void testNoOpDropDatabaseUsesOnlyCachedIdentityCleanup() {
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ HiveMetadataOps metadataOps = new HiveMetadataOps(catalog, Mockito.mock(HMSCachedClient.class));
+
+ metadataOps.afterDropDbNoOp("absent");
+
+ Mockito.verify(catalog).retireCachedDatabaseForNoOp("absent");
+ Mockito.verify(catalog, Mockito.never()).unregisterDatabase("absent");
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java
index a0eb338c4b6..9ac97941b5a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java
@@ -72,6 +72,16 @@ public class JdbcExternalCatalogTest {
jdbcExternalCatalog.getCatalogProperty().deleteProperty("lower_case_table_names");
}
+ @Test
+ public void testMalformedAlterMappingIsRejectedBeforePublication() {
+ Map<String, String> original = new HashMap<>(jdbcExternalCatalog.getProperties());
+ Map<String, String> update = new HashMap<>();
+ update.put(ExternalCatalog.META_NAMES_MAPPING, "not-json");
+ Assert.assertThrows(RuntimeException.class,
+ () -> jdbcExternalCatalog.validatePropertiesBeforeUpdate(original, update));
+ Assert.assertEquals(original, jdbcExternalCatalog.getProperties());
+ }
+
@Test
public void replayJdbcCatalogTest() throws DdlException {
jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.CONNECTION_POOL_MIN_SIZE, "1");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java
index 0b24a89139b..8b4dbd73660 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.RefreshManager;
import org.apache.doris.datasource.CatalogMgr;
import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.ExternalDatabase;
import org.apache.doris.datasource.ExternalMetaCacheMgr;
import org.apache.doris.datasource.ExternalObjectLog;
import org.apache.doris.datasource.ExternalTable;
@@ -217,8 +218,12 @@ public class LanceCatalogLifecycleTest {
// Leader REFRESH TABLE reaches only the typed table route, which must still retire the
// Lance access cache even though it bypasses the name-based invalidateTable() entry.
+ ExternalDatabase<?> lanceDb = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(lanceDb.getId()).thenReturn(70L);
ExternalTable lanceTable = Mockito.mock(ExternalTable.class);
Mockito.when(lanceTable.getCatalog()).thenReturn(catalog);
+ Mockito.doReturn(lanceDb).when(lanceTable).getDb();
+ Mockito.when(lanceTable.getId()).thenReturn(71L);
caches.invalidateTableCache(lanceTable);
Mockito.verify(client, Mockito.times(3)).invalidateTableAccessCache();
Mockito.verify(session, Mockito.never()).close();
@@ -228,6 +233,64 @@ public class LanceCatalogLifecycleTest {
}
}
+ @Test
+ public void testCatalogRefreshAndDatabaseDropFenceCountsBeforeRetiringLanceAccess() {
+ Map<String, String> properties = new HashMap<>();
+ properties.put("type", "lance");
+ properties.put(LanceExternalCatalog.WAREHOUSE, "/unused/lance-warehouse");
+ LanceExternalCatalog catalog = Mockito.spy(
+ new LanceExternalCatalog(903, "lance_fence", null, properties, ""));
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.doAnswer(invocation -> {
+ Mockito.verify(cacheMgr).invalidateRowCountCache(903L);
+ return null;
+ }).when(catalog).refreshSessionCache();
+
+ try (MockedStatic<Env> currentEnv = Mockito.mockStatic(Env.class)) {
+ currentEnv.when(Env::getCurrentEnv).thenReturn(env);
+ catalog.onRefreshCache(true);
+ Mockito.verify(cacheMgr).invalidateCatalog(903L);
+ Mockito.clearInvocations(cacheMgr);
+ Mockito.doAnswer(invocation -> {
+ Mockito.verify(cacheMgr).invalidateRowCountCache(903L);
+ return null;
+ }).when(catalog).invalidateTableAccessCache();
+ catalog.unregisterDatabase("db1");
+ }
+
+ Mockito.verify(cacheMgr).invalidateDb(903L, "db1");
+ }
+
+ @Test
+ public void testHeldTableInvalidationResetsLanceAccess() throws Exception {
+ Session session = Mockito.mock(Session.class);
+ LanceCatalogClient client = Mockito.spy(client(session));
+ LanceExternalCatalog catalog = catalog(client);
+ Env env = Mockito.mock(Env.class);
+ CatalogMgr catalogs = Mockito.mock(CatalogMgr.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogs);
+ long catalogId = catalog.getId();
+ Mockito.doReturn(catalog).when(catalogs).getCatalog(catalogId);
+ ExternalMetaCacheMgr caches = new ExternalMetaCacheMgr(true);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ ExternalTable table = Mockito.mock(ExternalTable.class);
+ Mockito.when(db.getId()).thenReturn(70L);
+ Mockito.doReturn(db).when(table).getDb();
+ Mockito.when(table.getId()).thenReturn(71L);
+ Mockito.when(table.getCatalog()).thenReturn(catalog);
+ Mockito.when(table.getDbName()).thenReturn("mapped_db");
+ Mockito.when(table.getName()).thenReturn("mapped_table");
+ try (MockedStatic<Env> currentEnv = Mockito.mockStatic(Env.class)) {
+ currentEnv.when(Env::getCurrentEnv).thenReturn(env);
+ caches.invalidateTableCache(table);
+ Mockito.verify(client).invalidateTableAccessCache();
+ } finally {
+ catalog.onClose();
+ }
+ }
+
@Test
public void testIndexJobLocatorBypassesQueryAccessCache() throws Exception {
LanceNamespace namespace = Mockito.mock(LanceNamespace.class);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java
index 48c0297dd61..4608013b5fb 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java
@@ -95,6 +95,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.Callable;
@@ -2217,6 +2218,7 @@ public class PaimonExternalMetaCacheTest {
Mockito.when(dorisDatabase.getCatalog()).thenReturn(dorisCatalog);
Mockito.when(dorisDatabase.getFullName()).thenReturn("db");
Mockito.when(dorisDatabase.getRemoteName()).thenReturn("db");
+ Mockito.when(dorisTable.getDb()).thenReturn(dorisDatabase);
// TVFs and table-existence checks can populate only Paimon's CachingCatalog. A
// REFRESH TABLE still has the resolved ExternalTable and must invalidate that direct
@@ -2569,6 +2571,62 @@ public class PaimonExternalMetaCacheTest {
}
}
+ @Test
+ public void testMode2ColdReplayRefreshInvalidatesReboundSdkOnlyPaimonHandle() throws Exception {
+ java.io.File warehouse = temporaryFolder.newFolder("mode2_rebound_refresh_sdk_only");
+ Map<String, String> properties = new HashMap<>();
+ properties.put("type", "paimon");
+ properties.put(PaimonExternalCatalog.PAIMON_CATALOG_TYPE,
+ PaimonExternalCatalog.PAIMON_FILESYSTEM);
+ properties.put("warehouse", warehouse.toURI().toString());
+ properties.put(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2");
+ long catalogId = 120L;
+ PaimonExternalCatalog dorisCatalog = new PaimonExternalCatalog(
+ catalogId, "mode2_rebound_refresh_test", null, properties, "");
+ dorisCatalog.makeSureInitialized();
+ dorisCatalog.catalog.createDatabase("Foo", false);
+ Identifier oldIdentifier = Identifier.create("Foo", "table");
+ Schema schema = Schema.newBuilder().column("id", DataTypes.INT()).build();
+ dorisCatalog.catalog.createTable(oldIdentifier, schema, false);
+
+ Map<String, String> externalProperties = new HashMap<>(properties);
+ externalProperties.put("paimon.cache-enabled", "false");
+ PaimonExternalCatalog externalCatalog = new PaimonExternalCatalog(
+ 121L, "external_paimon", null, externalProperties, "");
+ externalCatalog.makeSureInitialized();
+
+ CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+ Mockito.when(env.getAccessManager()).thenReturn(Mockito.mock(AccessControllerManager.class));
+ ExternalMetaCacheMgr cacheMgr = new ExternalMetaCacheMgr(true);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.doReturn(dorisCatalog).when(catalogMgr).getCatalog(catalogId);
+ Mockito.doReturn(dorisCatalog).when(catalogMgr)
+ .getCatalogOrException(Mockito.eq(catalogId), Mockito.any());
+ NameMapping oldMapping = new NameMapping(catalogId, "Foo", "table", "Foo", "table");
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ dorisCatalog.getPaimonTable(oldMapping);
+ externalCatalog.catalog.dropTable(oldIdentifier, false);
+ externalCatalog.catalog.dropDatabase("Foo", false, false);
+ externalCatalog.catalog.createDatabase("FOO", false);
+
+ Field mappingField = ExternalCatalog.class.getDeclaredField("lowerCaseToDatabaseName");
+ mappingField.setAccessible(true);
+ Map<String, String> reboundNames = new HashMap<>();
+ reboundNames.put("foo", "FOO");
+ mappingField.set(dorisCatalog, reboundNames);
+ Assert.assertFalse(dorisCatalog.getDbForReplay("Foo").isPresent());
+
+ new RefreshManager().replayRefreshDb(ExternalObjectLog.createForRefreshDb(catalogId, "Foo"));
+ assertPaimonCatalogTableMissing(dorisCatalog, oldMapping);
+ } finally {
+ dorisCatalog.catalog.close();
+ externalCatalog.catalog.close();
+ }
+ }
+
@Test
public void testReplayDropInvalidatesSdkOnlyPaimonCache() throws Exception {
java.io.File warehouse = temporaryFolder.newFolder("replay_drop_sdk_only");
@@ -2623,6 +2681,51 @@ public class PaimonExternalMetaCacheTest {
}
}
+ @Test
+ public void testModeOneSdkOnlyDatabaseInvalidationUsesRootLocale() throws Exception {
+ Locale previousLocale = Locale.getDefault();
+ Locale.setDefault(new Locale("tr", "TR"));
+ PaimonExternalCatalog dorisCatalog = null;
+ PaimonExternalCatalog externalCatalog = null;
+ try {
+ java.io.File warehouse = temporaryFolder.newFolder("mode_one_turkish_sdk_invalidation");
+ Map<String, String> properties = new HashMap<>();
+ properties.put("type", "paimon");
+ properties.put(PaimonExternalCatalog.PAIMON_CATALOG_TYPE,
+ PaimonExternalCatalog.PAIMON_FILESYSTEM);
+ properties.put("warehouse", warehouse.toURI().toString());
+ properties.put(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "1");
+ dorisCatalog = new PaimonExternalCatalog(121L, "mode_one_turkish_sdk_test", null, properties, "");
+ dorisCatalog.makeSureInitialized();
+ dorisCatalog.catalog.createDatabase("I", false);
+ Identifier identifier = Identifier.create("I", "table");
+ Schema schema = Schema.newBuilder().column("id", DataTypes.INT()).build();
+ dorisCatalog.catalog.createTable(identifier, schema, false);
+ dorisCatalog.catalog.getTable(identifier); // Populate only the SDK cache.
+
+ Map<String, String> uncachedProperties = new HashMap<>(properties);
+ uncachedProperties.put("paimon.cache-enabled", "false");
+ externalCatalog = new PaimonExternalCatalog(122L, "mode_one_turkish_external", null,
+ uncachedProperties, "");
+ externalCatalog.makeSureInitialized();
+ externalCatalog.catalog.dropTable(identifier, false);
+
+ // Doris mode 1 uses a locale-independent local name on every FE.
+ dorisCatalog.invalidatePaimonDatabaseByLocalName("i");
+ PaimonExternalCatalog cachedCatalog = dorisCatalog;
+ Assert.assertThrows(Catalog.TableNotExistException.class,
+ () -> cachedCatalog.catalog.getTable(identifier));
+ } finally {
+ if (dorisCatalog != null) {
+ dorisCatalog.catalog.close();
+ }
+ if (externalCatalog != null) {
+ externalCatalog.catalog.close();
+ }
+ Locale.setDefault(previousLocale);
+ }
+ }
+
@Test
public void testCatalogResetWaitsForInFlightSdkTableLoad() throws Exception {
java.io.File warehouse = temporaryFolder.newFolder("reset_fences_inflight_load");
@@ -3203,8 +3306,9 @@ public class PaimonExternalMetaCacheTest {
cache.invalidateDb(catalogId, "db1");
}
- Mockito.verify(dorisCatalog).invalidatePaimonTable(db1Table);
- Mockito.verify(dorisCatalog, Mockito.never()).invalidatePaimonTable(db2Table);
+ // Name-only invalidation must also cover SDK-only tables not present in Doris's
+ // table entry; the SDK scope is catalog-wide when the remote DB name is unknown.
+ Mockito.verify(dorisCatalog).invalidatePaimonDatabaseByLocalName("db1");
Assert.assertNull(tableEntry.getIfPresent(db1Table));
Assert.assertNotNull(tableEntry.getIfPresent(db2Table));
Assert.assertNull(schemaEntry.getIfPresent(db1Schema));
@@ -3280,7 +3384,7 @@ public class PaimonExternalMetaCacheTest {
}
Mockito.clearInvocations(cachingCatalog, delegate);
- dorisCatalog.invalidatePaimonDatabase("db1");
+ dorisCatalog.invalidatePaimonDatabaseByLocalName("db1");
// The batch path must not re-scan the table cache once per matched identifier.
Mockito.verify(cachingCatalog, Mockito.never()).invalidateTable(Mockito.any(Identifier.class));
@@ -3379,9 +3483,11 @@ public class PaimonExternalMetaCacheTest {
metadataOps.afterDropDb("db");
- // The cached database's synchronous removal listener performs the typed SDK
- // invalidation; the explicit duplicate must not run a second scan under the fence.
- Mockito.verify(cacheMgr, Mockito.times(1)).invalidateDb(Mockito.any(ExternalDatabase.class));
+ // The explicit DROP owns the one routed SDK invalidation, rather than duplicating
+ // the removal listener's work after the local object has been retired.
+ Mockito.verify(cacheMgr, Mockito.never()).invalidateDb(Mockito.any(ExternalDatabase.class));
+ Mockito.verify(cacheMgr, Mockito.times(1)).invalidateDb(
+ Mockito.eq(catalogId), Mockito.anyLong(), Mockito.eq("db"));
} finally {
dorisCatalog.catalog.close();
}
@@ -3567,8 +3673,8 @@ public class PaimonExternalMetaCacheTest {
// still retire it even though the drop is not journaled.
PaimonMetadataOps noOpMetadataOps = new PaimonMetadataOps(dorisCatalog, dorisCatalog.catalog) {
@Override
- public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) {
- return false;
+ public Optional<String> dropDbImplWithResolvedName(String dbName, boolean ifExists, boolean force) {
+ return Optional.empty();
}
};
Assert.assertFalse(noOpMetadataOps.dropDb("DB", true, false));
@@ -3765,11 +3871,3 @@ public class PaimonExternalMetaCacheTest {
}
}
}
-
-
-
-
-
-
-
-
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java
index fb43b5ccefa..ac8b33281c7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java
@@ -493,6 +493,22 @@ public class PaimonMetadataOpsTest {
}
}
+ @Test
+ public void testDropDatabaseUsesResolvedNameForLeaderCleanup() throws Exception {
+ ExternalCatalog dorisCatalog = Mockito.mock(ExternalCatalog.class);
+ ExternalDatabase<?> resolvedDb = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(resolvedDb.getFullName()).thenReturn("FOO");
+ Mockito.doReturn(resolvedDb).when(dorisCatalog).getDbNullable("Foo");
+ Catalog remoteCatalog = Mockito.mock(Catalog.class);
+ PaimonMetadataOps metadataOps = newMetadataOps(dorisCatalog, remoteCatalog);
+
+ Assert.assertEquals(Optional.of("FOO"), metadataOps.dropDbWithResolvedName("Foo", false, false));
+
+ Mockito.verify(remoteCatalog).dropDatabase("Foo", false, false);
+ Mockito.verify(dorisCatalog).unregisterDatabase("FOO");
+ Mockito.verify(dorisCatalog, Mockito.never()).unregisterDatabase("Foo");
+ }
+
@Test
public void testCreateDatabaseWithPropertiesForSupportedCatalogs() throws Exception {
List<String> supportedCatalogTypes = Arrays.asList(
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java
new file mode 100644
index 00000000000..f7b8affcfe3
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java
@@ -0,0 +1,343 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.insert;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.EnvFactory;
+import org.apache.doris.catalog.RefreshManager;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.ExternalDatabase;
+import org.apache.doris.datasource.ExternalMetaCacheMgr;
+import org.apache.doris.datasource.ExternalObjectLog;
+import org.apache.doris.datasource.ExternalTable;
+import org.apache.doris.datasource.hive.HMSExternalCatalog;
+import org.apache.doris.datasource.hive.HMSExternalTable;
+import org.apache.doris.datasource.hive.HiveExternalMetaCache;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.persist.EditLog;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.Coordinator;
+import org.apache.doris.thrift.THivePartitionUpdate;
+import org.apache.doris.transaction.TransactionManager;
+import org.apache.doris.transaction.TransactionType;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.Optional;
+
+class HiveInsertExecutorTest {
+
+ @AfterEach
+ void tearDown() {
+ ConnectContext.remove();
+ }
+
+ @Test
+ void testCommittedInsertPublishesRowCountFenceWhenPartitionRefreshFails() throws Exception {
+ ConnectContext ctx = new ConnectContext();
+ ctx.setThreadLocalInfo();
+
+ long catalogId = 61L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getName()).thenReturn("hms");
+ Mockito.when(catalog.getTransactionManager()).thenReturn(Mockito.mock(TransactionManager.class));
+ Mockito.when(db.getId()).thenReturn(62L);
+ Mockito.when(db.getFullName()).thenReturn("db1");
+ Mockito.when(table.getCatalog()).thenReturn(catalog);
+ Mockito.when(table.getDatabase()).thenReturn(db);
+ Mockito.when(table.getName()).thenReturn("tbl1");
+ Mockito.when(table.getNameWithFullQualifiers()).thenReturn("hms.db1.tbl1");
+ Mockito.when(table.isPartitionedTable()).thenReturn(true);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ HiveExternalMetaCache hiveCache = Mockito.mock(HiveExternalMetaCache.class);
+ EditLog editLog = Mockito.mock(EditLog.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(env.getEditLog()).thenReturn(editLog);
+ Mockito.when(cacheMgr.hive(catalogId)).thenReturn(hiveCache);
+ Mockito.doThrow(new IllegalStateException("partition cache failure"))
+ .when(hiveCache).refreshAffectedPartitions(
+ Mockito.eq(table), Mockito.anyList(), Mockito.anyList(), Mockito.anyList());
+
+ EnvFactory envFactory = Mockito.mock(EnvFactory.class);
+ Mockito.when(envFactory.createCoordinator(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong()))
+ .thenReturn(Mockito.mock(Coordinator.class));
+ try (MockedStatic<EnvFactory> mockedEnvFactory = Mockito.mockStatic(EnvFactory.class);
+ MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnvFactory.when(EnvFactory::getInstance).thenReturn(envFactory);
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+
+ TestingHiveInsertExecutor executor = new TestingHiveInsertExecutor(
+ ctx, table, Mockito.mock(NereidsPlanner.class));
+ executor.setPartitionUpdates(Collections.singletonList(new THivePartitionUpdate()));
+ Assertions.assertDoesNotThrow(executor::runAfterCommit);
+ }
+
+ InOrder order = Mockito.inOrder(cacheMgr, hiveCache);
+ order.verify(cacheMgr).invalidateRowCountCache(table);
+ order.verify(cacheMgr).hive(catalogId);
+ order.verify(hiveCache).refreshAffectedPartitions(
+ Mockito.eq(table), Mockito.anyList(), Mockito.anyList(), Mockito.anyList());
+ Mockito.verify(cacheMgr).invalidateTableCache(table);
+
+ ArgumentCaptor<ExternalObjectLog> logCaptor = ArgumentCaptor.forClass(ExternalObjectLog.class);
+ Mockito.verify(editLog).logRefreshExternalTable(logCaptor.capture());
+ ExternalObjectLog log = logCaptor.getValue();
+ Assertions.assertEquals(catalogId, log.getCatalogId());
+ Assertions.assertEquals("db1", log.getDbName());
+ Assertions.assertEquals("tbl1", log.getTableName());
+ Assertions.assertNull(log.getPartitionNames());
+ Assertions.assertNull(log.getNewPartitionNames());
+ }
+
+ @Test
+ void testDefaultPostCommitRefreshUsesHeldTableIdentity() throws Exception {
+ ConnectContext ctx = new ConnectContext();
+ ctx.setThreadLocalInfo();
+
+ ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ ExternalTable table = Mockito.mock(ExternalTable.class);
+ Mockito.when(catalog.getName()).thenReturn("iceberg");
+ Mockito.when(catalog.getTransactionManager()).thenReturn(Mockito.mock(TransactionManager.class));
+ Mockito.when(table.getCatalog()).thenReturn(catalog);
+ Mockito.when(table.getDatabase()).thenReturn(db);
+
+ Env env = Mockito.mock(Env.class);
+ RefreshManager refreshManager = Mockito.mock(RefreshManager.class);
+ Mockito.when(env.getRefreshManager()).thenReturn(refreshManager);
+ EnvFactory envFactory = Mockito.mock(EnvFactory.class);
+ Mockito.when(envFactory.createCoordinator(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong()))
+ .thenReturn(Mockito.mock(Coordinator.class));
+ try (MockedStatic<EnvFactory> mockedEnvFactory = Mockito.mockStatic(EnvFactory.class);
+ MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnvFactory.when(EnvFactory::getInstance).thenReturn(envFactory);
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+
+ TestingExternalInsertExecutor executor = new TestingExternalInsertExecutor(
+ ctx, table, Mockito.mock(NereidsPlanner.class));
+ executor.runAfterCommit();
+ }
+
+ Mockito.verify(refreshManager).refreshTableAfterCommit(table);
+ }
+
+ @Test
+ void testCommittedInsertFencesRowCountWhenPartitionedPredicateThrows() throws Exception {
+ ConnectContext ctx = new ConnectContext();
+ ctx.setThreadLocalInfo();
+
+ long catalogId = 63L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getName()).thenReturn("hms");
+ Mockito.when(catalog.getTransactionManager()).thenReturn(Mockito.mock(TransactionManager.class));
+ Mockito.when(db.getId()).thenReturn(64L);
+ Mockito.when(db.getFullName()).thenReturn("db1");
+ Mockito.when(table.getCatalog()).thenReturn(catalog);
+ Mockito.when(table.getDatabase()).thenReturn(db);
+ Mockito.when(table.getName()).thenReturn("tbl1");
+ Mockito.when(table.getNameWithFullQualifiers()).thenReturn("hms.db1.tbl1");
+ Mockito.when(table.isPartitionedTable()).thenThrow(new IllegalStateException("reinit failed"));
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ EditLog editLog = Mockito.mock(EditLog.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(env.getEditLog()).thenReturn(editLog);
+
+ EnvFactory envFactory = Mockito.mock(EnvFactory.class);
+ Mockito.when(envFactory.createCoordinator(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong()))
+ .thenReturn(Mockito.mock(Coordinator.class));
+ try (MockedStatic<EnvFactory> mockedEnvFactory = Mockito.mockStatic(EnvFactory.class);
+ MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnvFactory.when(EnvFactory::getInstance).thenReturn(envFactory);
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+
+ TestingHiveInsertExecutor executor = new TestingHiveInsertExecutor(
+ ctx, table, Mockito.mock(NereidsPlanner.class));
+ executor.setPartitionUpdates(Collections.singletonList(new THivePartitionUpdate()));
+ Assertions.assertDoesNotThrow(executor::runAfterCommit);
+ }
+
+ InOrder order = Mockito.inOrder(cacheMgr, editLog);
+ order.verify(cacheMgr).invalidateRowCountCache(table);
+ order.verify(cacheMgr).invalidateTableCache(table);
+ ArgumentCaptor<ExternalObjectLog> logCaptor = ArgumentCaptor.forClass(ExternalObjectLog.class);
+ order.verify(editLog).logRefreshExternalTable(logCaptor.capture());
+ Assertions.assertNull(logCaptor.getValue().getPartitionNames());
+ Assertions.assertNull(logCaptor.getValue().getNewPartitionNames());
+ }
+
+ @Test
+ void testSuccessfulSelectiveRefreshClosesRowCountAdmissionWindow() throws Exception {
+ ConnectContext ctx = new ConnectContext();
+ ctx.setThreadLocalInfo();
+
+ long catalogId = 67L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getName()).thenReturn("hms");
+ Mockito.when(catalog.getTransactionManager()).thenReturn(Mockito.mock(TransactionManager.class));
+ Mockito.when(db.getId()).thenReturn(68L);
+ Mockito.when(db.getFullName()).thenReturn("db1");
+ Mockito.when(table.getCatalog()).thenReturn(catalog);
+ Mockito.when(table.getDatabase()).thenReturn(db);
+ Mockito.when(table.getName()).thenReturn("tbl1");
+ Mockito.when(table.getNameWithFullQualifiers()).thenReturn("hms.db1.tbl1");
+ Mockito.when(table.isPartitionedTable()).thenReturn(true);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ HiveExternalMetaCache hiveCache = Mockito.mock(HiveExternalMetaCache.class);
+ EditLog editLog = Mockito.mock(EditLog.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(env.getEditLog()).thenReturn(editLog);
+ Mockito.when(cacheMgr.hive(catalogId)).thenReturn(hiveCache);
+
+ EnvFactory envFactory = Mockito.mock(EnvFactory.class);
+ Mockito.when(envFactory.createCoordinator(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong()))
+ .thenReturn(Mockito.mock(Coordinator.class));
+ try (MockedStatic<EnvFactory> mockedEnvFactory = Mockito.mockStatic(EnvFactory.class);
+ MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnvFactory.when(EnvFactory::getInstance).thenReturn(envFactory);
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+
+ TestingHiveInsertExecutor executor = new TestingHiveInsertExecutor(
+ ctx, table, Mockito.mock(NereidsPlanner.class));
+ executor.setPartitionUpdates(Collections.singletonList(new THivePartitionUpdate()));
+ executor.runAfterCommit();
+ }
+
+ // One fence before the selective refresh, one after it closes the admission window.
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(table);
+ InOrder order = Mockito.inOrder(hiveCache, table, cacheMgr);
+ order.verify(hiveCache).refreshAffectedPartitions(Mockito.eq(table), Mockito.anyList(),
+ Mockito.anyList(), Mockito.anyList());
+ order.verify(table).unsetObjectCreated();
+ order.verify(cacheMgr).invalidateRowCountCache(table);
+ }
+
+ @Test
+ void testCommittedFullInvalidationFailureStillLogsFullRefresh() throws Exception {
+ ConnectContext ctx = new ConnectContext();
+ ctx.setThreadLocalInfo();
+
+ long catalogId = 65L;
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ ExternalDatabase<?> db = Mockito.mock(ExternalDatabase.class);
+ HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
+ Mockito.when(catalog.getId()).thenReturn(catalogId);
+ Mockito.when(catalog.getName()).thenReturn("hms");
+ Mockito.when(catalog.getTransactionManager()).thenReturn(Mockito.mock(TransactionManager.class));
+ Mockito.when(db.getId()).thenReturn(66L);
+ Mockito.when(db.getFullName()).thenReturn("db1");
+ Mockito.when(table.getCatalog()).thenReturn(catalog);
+ Mockito.when(table.getDatabase()).thenReturn(db);
+ Mockito.when(table.getName()).thenReturn("tbl1");
+ Mockito.when(table.getNameWithFullQualifiers()).thenReturn("hms.db1.tbl1");
+ Mockito.when(table.isPartitionedTable()).thenReturn(false);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ EditLog editLog = Mockito.mock(EditLog.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+ Mockito.when(env.getEditLog()).thenReturn(editLog);
+ Mockito.doThrow(new IllegalStateException("table cache failure"))
+ .when(cacheMgr).invalidateTableCache(table);
+
+ EnvFactory envFactory = Mockito.mock(EnvFactory.class);
+ Mockito.when(envFactory.createCoordinator(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong()))
+ .thenReturn(Mockito.mock(Coordinator.class));
+ try (MockedStatic<EnvFactory> mockedEnvFactory = Mockito.mockStatic(EnvFactory.class);
+ MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnvFactory.when(EnvFactory::getInstance).thenReturn(envFactory);
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+
+ TestingHiveInsertExecutor executor = new TestingHiveInsertExecutor(
+ ctx, table, Mockito.mock(NereidsPlanner.class));
+ executor.setPartitionUpdates(Collections.emptyList());
+ Assertions.assertDoesNotThrow(executor::runAfterCommit);
+ }
+
+ Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(table);
+ Mockito.verify(table).unsetObjectCreated();
+ Mockito.verify(cacheMgr, Mockito.atLeastOnce()).invalidateTableCache(table);
+ ArgumentCaptor<ExternalObjectLog> logCaptor = ArgumentCaptor.forClass(ExternalObjectLog.class);
+ Mockito.verify(editLog).logRefreshExternalTable(logCaptor.capture());
+ Assertions.assertNull(logCaptor.getValue().getPartitionNames());
+ Assertions.assertNull(logCaptor.getValue().getNewPartitionNames());
+ }
+
+ private static class TestingHiveInsertExecutor extends HiveInsertExecutor {
+ TestingHiveInsertExecutor(ConnectContext ctx, HMSExternalTable table, NereidsPlanner planner) {
+ super(ctx, table, "label", planner, Optional.empty(), false, 0L);
+ }
+
+ void setPartitionUpdates(java.util.List<THivePartitionUpdate> updates) throws Exception {
+ Field field = HiveInsertExecutor.class.getDeclaredField("partitionUpdates");
+ field.setAccessible(true);
+ field.set(this, updates);
+ }
+
+ void runAfterCommit() throws DdlException {
+ doAfterCommit();
+ }
+ }
+
+ private static class TestingExternalInsertExecutor extends BaseExternalTableInsertExecutor {
+ TestingExternalInsertExecutor(ConnectContext ctx, ExternalTable table, NereidsPlanner planner) {
+ super(ctx, table, "label", planner, Optional.empty(), false, 0L);
+ }
+
+ @Override
+ protected void beforeExec() throws UserException {
+ }
+
+ @Override
+ protected void doBeforeCommit() throws UserException {
+ }
+
+ @Override
+ protected TransactionType transactionType() {
+ return TransactionType.ICEBERG;
+ }
+
+ void runAfterCommit() throws DdlException {
+ doAfterCommit();
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java
index 63afe375548..65dc072dead 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java
@@ -19,6 +19,7 @@ package org.apache.doris.persist;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.meta.MetaContext;
+import org.apache.doris.persist.gson.GsonUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -30,6 +31,18 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
public class DropAndRecoverInfoTest {
+ @Test
+ public void testExternalDropTargetSurvivesSerializationAndLegacyLogHasNoTarget() {
+ DropInfo current = new DropInfo("catalog", "Foo", "t", "FOO");
+ DropInfo decoded = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(current), DropInfo.class);
+ Assert.assertEquals("Foo", decoded.getDb());
+ Assert.assertEquals("FOO", decoded.getResolvedDb());
+
+ DropInfo legacy = GsonUtils.GSON.fromJson(
+ GsonUtils.GSON.toJson(new DropInfo("catalog", "Foo", "t")), DropInfo.class);
+ Assert.assertNull(legacy.getResolvedDb());
+ }
+
@Test
public void testDropInfoSerialization() throws Exception {
MetaContext metaContext = new MetaContext();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/DropDbInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/DropDbInfoTest.java
index aea8530afeb..e57dcbf2f25 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/persist/DropDbInfoTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/persist/DropDbInfoTest.java
@@ -19,6 +19,7 @@ package org.apache.doris.persist;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.meta.MetaContext;
+import org.apache.doris.persist.gson.GsonUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -29,6 +30,18 @@ import java.io.File;
import java.nio.file.Files;
public class DropDbInfoTest {
+ @Test
+ public void testExternalDropTargetSurvivesSerializationAndLegacyLogHasNoTarget() {
+ DropDbInfo current = new DropDbInfo("catalog", "Foo", "FOO");
+ DropDbInfo decoded = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(current), DropDbInfo.class);
+ Assert.assertEquals("Foo", decoded.getDbName());
+ Assert.assertEquals("FOO", decoded.getResolvedDbName());
+
+ DropDbInfo legacy = GsonUtils.GSON.fromJson(
+ GsonUtils.GSON.toJson(new DropDbInfo("catalog", "Foo")), DropDbInfo.class);
+ Assert.assertNull(legacy.getResolvedDbName());
+ }
+
@Test
public void testSerialization() throws Exception {
MetaContext metaContext = new MetaContext();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 22/35: [fix](flight-sql) Return analyzed query schemas and table type metadata (#68631)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ce1bc2c43bd543807afdf7756c8f3937e615489b
Author: Gabriel <li...@selectdb.com>
AuthorDate: Wed Sep 30 21:11:40 2026 +0800
[fix](flight-sql) Return analyzed query schemas and table type metadata (#68631)
### What problem does this PR solve?
Flight SQL preparation currently returns the same `ResultMeta: string`
schema for every SQL statement without analyzing it. Python ADBC
`adbc_execute_schema()` consumes that prepared dataset schema directly,
so valid queries report incorrect columns and invalid queries appear to
succeed. Direct `GetSchema` is also unimplemented.
Resolve output slots with the FE-only Nereids prepare planner and check
table/column privileges before converting them to Arrow fields. Set
query state before binding result columns, matching execution so
unaliased expressions retain their SQL labels instead of synthetic
command aliases. Both Prepare and GetSchema use this resolver, including
nested types, nullability, decimal precision and timestamp units.
Invalid queries fail before a prepared handle is registered; metadata
requests do not construct a query executor or dispatch BE fragments.
Preserve JDBC preparation of commands using their FE result metadata
(including the actual `StatusResult` protocol schema for commands
without rows), and resolve USE/SWITCH prefixes within a temporary
namespace scope. Session hints, statement resources and worker thread
context are restored on success and failure. Schema analysis and
execution share a per-session lock, so concurrent execution cannot
update temporary analysis variables. Dialect conversion and the
configured original-SQL parse fallback run before analysis, and
USE/SWITCH prefixes honor remembered catalog databases without changing
session history. Parameter binding, arbitrary multi-statement batches,
and commands without discoverable result metadata remain explicitly
unsupported by schema discovery.
Prepared handles are invalidated when used from a different catalog or
database. After a successful prepared USE/SWITCH, that handle is rebound
to its resulting namespace so it remains reusable; other handles retain
their original namespace checks. Before GetSchema or prepared execution,
reanalyze and compare against the schema advertised at Prepare;
result-affecting session changes such as `sql_mode` require preparing
again, while settings that preserve the schema keep the handle valid.
Check the actual execution schema again before returning FlightInfo to
cover concurrent DDL between analysis and replanning. For BE results,
compare the final planner's semantic column labels and positional Arrow
types: wire labels may contain expression text or a TVF prefix, and
constant folding may narrow nullability. Keep nested field names, type
metadata and dictionary encoding checks, and reject nullable widening.
Retain the final executor alongside FlightInfo before processor cleanup
clears the session context. On mismatch, expire the handle, cancel that
retained executor before finalizing deferred executors, and clear
buffered results/endpoints.
Resolve HELP headers from the local help catalog and SHOW SNAPSHOT
headers from validated predicates without reading snapshot data. Resolve
fixed headers for SHOW CONFIG, SHOW CREATE TABLE, procedure SHOW
commands, ordinary EXPLAIN, PLAN REPLAYER and WARM UP SELECT. Concrete
no-row DML and transaction commands advertise `StatusResult`; Lance
CREATE/DROP INDEX advertise `JobId` without admission or remote table
resolution. SHOW PROC exposes static headers for `/` and
`/current_queries` without opening nodes or fetching rows. Other PROC
nodes, EXPLAIN PLAN PROCESS (which lacks Flight result serialization),
DESCRIBE, SHOW DATA, SHOW PARTITIONS, SHOW QUERY STATS, SHOW PYTHON
PACKAGES, COPY INTO and WARM UP CLUSTER remain explicitly unsupported by
schema discovery. The string-serialized `group_concat_state` output maps
to Arrow UTF8; unmodeled aggregate-state serializations remain
unsupported.
Prepare and prepared execution now incur an additional FE analysis pass
to discover and validate the advertised schema. Physical planning and
query execution remain in the existing execution path.
The External Regression investigation also exposed an existing remote
Doris metadata timeout unit mismatch: the catalog supplies seconds, but
pooled Thrift sockets require milliseconds. Convert the configured value
with overflow checking so the default 10 seconds does not become 10
milliseconds.
Implement GetTableTypes data streaming for ADBC/JDBC metadata discovery.
GetFlightInfo previously returned the standard schema and ticket, but
DoGet always failed with UNIMPLEMENTED. Return sorted, distinct non-null
public names from the existing TableType.toMysqlType mapping (BASE
TABLE, SYSTEM VIEW, VIEW), using the standard non-null UTF8 field and
releasing vectors on success or failure. This does not enumerate
catalogs or execute queries; GetTables type-filter behavior is outside
this change.
### Release note
Fix Arrow Flight SQL schema discovery returning placeholder metadata and
implement table-type metadata streaming for ADBC/JDBC clients.
### Check List (For Author)
- Test
- [x] Unit Test
- [x] Regression Test
- [x] Manual test
- Behavior changed:
- [x] Yes. Prepare validates queries and reports their real output
schema; GetSchema is implemented without executing the query.
- Does this need documentation?
- [x] No.
Validation:
- FE Checkstyle passed. Recompiled changed FE classes and tests against
the existing FE test build; all 62 focused tests passed.
Execution-schema tests now run real
FlightSqlConnectProcessor.handleQuery(), covering the query-state
transition that direct planner tests missed. The updated tests failed
for SELECT 1, unhex and CAST before the fix and passed afterward.
Negative schema-drift and deferred-executor lifecycle tests remain
covered.
- Ran the actual regression framework against an isolated local FE + BE
cluster using the BE binary from this PR's CI compile and current FE
classes. With the previous FE classes, all five Flight suites from P0
build 1060727 reproduced schema-drift failures. With the fix,
test_flight_timestamp_range, test_flight_utf8_validation,
test_sql_cache_over_arrow_flight, test_select and test_auth_remote_ip
all passed, along with the expanded test_schema_only and new
test_table_types suites: 7 suites, 0 failures, 0 skipped.
- GetTableTypes tests cover the full GetFlightInfo-to-DoGet RPC,
standard schema, sorted unique values, repeated requests, JDBC metadata,
subsequent same-connection queries, and vector cleanup on stream
failure. Python ADBC also passed 10 consecutive adbc_get_table_types
calls followed by a query against the local cluster. Both new FE tests
failed before implementation and passed afterward.
- The new persistent regression checks unaliased literal/function/CAST
metadata and performs prepared execution, so metadata-only agreement
cannot hide a mismatch with execution.
- The separately muted mtmv_with_sql_cache failure also occurs on
unrelated PRs and is not changed here. Full P0/external CI reruns remain
separate from the focused local validation.
- Earlier validation covered prepared namespace reuse, HELP and snapshot
headers, session isolation, and the remote Doris timeout correction
using a real local Thrift server with a delayed response.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../doris/datasource/doris/FeServiceClient.java | 11 +-
.../nereids/trees/plans/commands/HelpCommand.java | 23 +
.../trees/plans/commands/ShowConfigCommand.java | 6 +
.../plans/commands/ShowCreateProcedureCommand.java | 6 +
.../plans/commands/ShowCreateTableCommand.java | 13 +
.../trees/plans/commands/ShowProcCommand.java | 24 +
.../plans/commands/ShowProcedureStatusCommand.java | 6 +
.../trees/plans/commands/ShowSnapshotCommand.java | 18 +-
.../plans/commands/insert/WarmupSelectCommand.java | 6 +
.../trees/plans/commands/use/UseCommand.java | 8 +
.../java/org/apache/doris/qe/ConnectContext.java | 48 +-
.../arrowflight/DorisFlightSqlProducer.java | 213 ++++--
.../service/arrowflight/FlightSqlQuerySchema.java | 413 +++++++++++
.../service/arrowflight/FlightSqlSchemaHelper.java | 4 +-
.../datasource/doris/FeServiceClientTest.java | 83 +++
.../commands/SchemaOnlyCommandMetadataTest.java | 123 ++++
.../arrowflight/DorisFlightSqlProducerTest.java | 114 ---
.../arrowflight/DorisFlightSqlSchemaTest.java | 775 +++++++++++++++++++++
.../arrowflight/FlightSqlExecutionSchemaTest.java | 178 +++++
.../FlightSqlSchemaHelperArrowTypeTest.java | 11 +
.../arrowflight/FlightSqlTableTypesTest.java | 91 +++
.../arrow_flight_sql_p0/test_schema_only.groovy | 288 ++++++++
.../arrow_flight_sql_p0/test_table_types.groovy | 73 ++
23 files changed, 2348 insertions(+), 187 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/FeServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/FeServiceClient.java
index d5b21a872e7..f76add8c29f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/FeServiceClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/FeServiceClient.java
@@ -56,16 +56,17 @@ public class FeServiceClient {
private final String user;
private final String password;
private final int retryCount;
- private final int timeout;
+ private final int timeoutMs;
public FeServiceClient(String name, List<TNetworkAddress> addresses, String user, String password,
- int retryCount, int timeout) {
+ int retryCount, int timeoutSeconds) {
this.name = name;
this.addresses = addresses;
this.user = user;
this.password = password;
this.retryCount = retryCount;
- this.timeout = timeout;
+ // Catalog metadata timeouts are in seconds, but pooled Thrift sockets require milliseconds.
+ this.timeoutMs = Math.multiplyExact(timeoutSeconds, 1000);
}
private List<TNetworkAddress> getAddresses() {
@@ -127,7 +128,7 @@ public class FeServiceClient {
return result.getBackends().stream()
.map(b -> Backend.fromThrift(b))
.collect(Collectors.toList());
- }, msg, timeout);
+ }, msg, timeoutMs);
}
public RemoteOlapTable getOlapTable(String dbName, String table, long tableId, List<Partition> partitions) {
@@ -174,7 +175,7 @@ public class FeServiceClient {
}
remoteOlapTable.rebuildPartitions(partitions, updatedPartitions, removedPartitions);
return remoteOlapTable;
- }, msg, timeout);
+ }, msg, timeoutMs);
}
private interface ThriftCall<T> {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HelpCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HelpCommand.java
index 32ea7d85a91..5569c1b7387 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HelpCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HelpCommand.java
@@ -125,6 +125,29 @@ public class HelpCommand extends ShowCommand {
return resultSet;
}
+ /** Resolve the HELP result shape without materializing topic text or category rows. */
+ public ShowResultSetMetaData getMetaData(ConnectContext ctx) throws AnalysisException {
+ if (Strings.isNullOrEmpty(mark)) {
+ throw new AnalysisException("Help empty info.");
+ }
+ HelpModule module = HelpModule.getInstance();
+ // HELP has three result shapes; a topic match takes precedence over keywords and categories.
+ HelpTopic topic = module.getTopic(mark);
+ if (topic == null) {
+ List<String> topics = module.listTopicByKeyword(mark);
+ if (topics.size() > 1) {
+ return KEYWORD_META_DATA;
+ }
+ if (topics.size() == 1) {
+ topic = module.getTopic(topics.get(0));
+ }
+ }
+ if (topic != null) {
+ return TOPIC_META_DATA;
+ }
+ return module.listCategoryByName(mark).size() > 1 ? CATEGORY_META_DATA : KEYWORD_META_DATA;
+ }
+
@Override
public ShowResultSetMetaData getMetaData() {
return ShowResultSetMetaData.builder().build();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConfigCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConfigCommand.java
index d106a63897f..5e3d10f085a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConfigCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConfigCommand.java
@@ -81,6 +81,12 @@ public class ShowConfigCommand extends Command implements NoForward {
this.isShowSingleBackend = true;
}
+ @Override
+ public ShowResultSetMetaData getResultSetMetaData() {
+ // Config commands produce rows even though their statement category is OTHER.
+ return getMetaData(nodeType == NodeType.FRONTEND ? FE_TITLE_NAMES : BE_TITLE_NAMES);
+ }
+
private ShowResultSetMetaData getMetaData(ImmutableList<String> metaNames) {
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
for (String title : metaNames) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateProcedureCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateProcedureCommand.java
index 1000100c459..df2f660f33a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateProcedureCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateProcedureCommand.java
@@ -54,6 +54,12 @@ public class ShowCreateProcedureCommand extends Command implements NoForward {
}
+ @Override
+ public ShowResultSetMetaData getResultSetMetaData() {
+ // These fixed headers are available before procedure lookup or execution.
+ return getMetaData();
+ }
+
public ShowResultSetMetaData getMetaData() {
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
for (String title : TITLE_NAMES) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateTableCommand.java
index 12412e2aa0c..636f9056bc4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateTableCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateTableCommand.java
@@ -128,6 +128,19 @@ public class ShowCreateTableCommand extends ShowCommand {
return ShowResultSetMetaData.builder().build();
}
+ /** Resolve the result header without rendering the table DDL. */
+ public ShowResultSetMetaData getMetaData(ConnectContext ctx) throws AnalysisException {
+ // The table kind determines the header; rendering DDL is unnecessary for schema discovery.
+ validate(ctx);
+ DatabaseIf db = ctx.getEnv().getCatalogMgr().getCatalogOrAnalysisException(tblNameInfo.getCtl())
+ .getDbOrAnalysisException(tblNameInfo.getDb());
+ TableIf table = resolveShowCreateTarget(db);
+ if (table instanceof View) {
+ return VIEW_META_DATA;
+ }
+ return table.getType() == Table.TableType.MATERIALIZED_VIEW ? MATERIALIZED_VIEW_META_DATA : META_DATA;
+ }
+
@Override
public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
validate(ctx);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcCommand.java
index 2c7d5e225b0..8672a9d90f4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcCommand.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.ScalarType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.proc.CurrentQueryStatisticsProcDir;
import org.apache.doris.common.proc.ProcNodeInterface;
import org.apache.doris.common.proc.ProcResult;
import org.apache.doris.common.proc.ProcService;
@@ -35,6 +36,7 @@ import org.apache.doris.qe.ShowResultSet;
import org.apache.doris.qe.ShowResultSetMetaData;
import org.apache.doris.qe.StmtExecutor;
+import java.util.Collections;
import java.util.List;
/**
@@ -74,6 +76,28 @@ public class ShowProcCommand extends ShowCommand {
return ShowResultSetMetaData.builder().build();
}
+ /** Resolve a static PROC header, or return null when discovery would require fetching rows. */
+ public ShowResultSetMetaData getMetaData(ConnectContext ctx) throws AnalysisException {
+ // PROC headers belong to the resolved node and require the same privilege as reading it.
+ if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.ADMIN_OR_NODE)) {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+ }
+ // Even opening nested nodes can perform remote lookups. Only use known static headers.
+ List<String> names;
+ if ("/".equals(path)) {
+ names = Collections.singletonList("name");
+ } else if ("/current_queries".equals(path)) {
+ names = CurrentQueryStatisticsProcDir.TITLE_NAMES;
+ } else {
+ return null;
+ }
+ ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
+ for (String name : names) {
+ builder.addColumn(new Column(name, ScalarType.createVarchar(30)));
+ }
+ return builder.build();
+ }
+
private ShowResultSet handleShowProc(ConnectContext ctx, StmtExecutor executor) throws Exception {
if (!Env.getCurrentEnv().getAccessManager()
.checkGlobalPriv(ctx, PrivPredicate.ADMIN_OR_NODE)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcedureStatusCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcedureStatusCommand.java
index c77c77c875e..1c0a25d4b3f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcedureStatusCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcedureStatusCommand.java
@@ -67,6 +67,12 @@ public class ShowProcedureStatusCommand extends Command implements NoForward {
this.whereExpr = Objects.requireNonNull(whereExpr, "whereExpr should not be null");
}
+ @Override
+ public ShowResultSetMetaData getResultSetMetaData() {
+ // These fixed headers are available before procedure lookup or execution.
+ return getMetaData();
+ }
+
public ShowResultSetMetaData getMetaData() {
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
for (String title : TITLE_NAMES) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowSnapshotCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowSnapshotCommand.java
index 3e5bd97ecad..f5886e08fcd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowSnapshotCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowSnapshotCommand.java
@@ -109,6 +109,16 @@ public class ShowSnapshotCommand extends ShowCommand {
return builder.build();
}
+ /** Resolve the snapshot header without accessing the remote repository. */
+ public ShowResultSetMetaData getMetaData(ConnectContext ctx) throws UserException {
+ // Both filters must be analyzed before choosing the detail header, even during Prepare.
+ if (!validate(ctx)) {
+ throw new AnalysisException("Where clause should looks like: SNAPSHOT = 'your_snapshot_name'"
+ + " [AND TIMESTAMP = '2018-04-18-19-19-10'] [AND SNAPSHOTTYPE = 'remote' | 'local']");
+ }
+ return getMetaData();
+ }
+
private boolean analyzeSubExpression(Expression expr) {
if (expr instanceof ComparisonPredicate) {
return analyzeComparisonPredicate((ComparisonPredicate) expr);
@@ -194,18 +204,14 @@ public class ShowSnapshotCommand extends ShowCommand {
* handle show backup
*/
private ShowResultSet handleShowSnapshot(ConnectContext ctx, StmtExecutor executor) throws Exception {
- boolean valid = validate(ctx);
- if (!valid) {
- throw new AnalysisException("Where clause should looks like: SNAPSHOT = 'your_snapshot_name'"
- + " [AND TIMESTAMP = '2018-04-18-19-19-10'] [AND SNAPSHOTTYPE = 'remote' | 'local']");
- }
+ ShowResultSetMetaData metadata = getMetaData(ctx);
Repository repo = Env.getCurrentEnv().getBackupHandler().getRepoMgr().getRepo(repoName);
if (repo == null) {
throw new AnalysisException("Repository " + repoName + " does not exist");
}
List<List<String>> snapshotInfos = repo.getSnapshotInfos(snapshotName, timestamp);
- return new ShowResultSet(getMetaData(), snapshotInfos);
+ return new ShowResultSet(metadata, snapshotInfos);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/WarmupSelectCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/WarmupSelectCommand.java
index e997f362b0e..77d45477898 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/WarmupSelectCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/WarmupSelectCommand.java
@@ -216,6 +216,12 @@ public class WarmupSelectCommand extends InsertIntoTableCommand {
executor.sendResultSet(resultSet);
}
+ @Override
+ public ShowResultSetMetaData getResultSetMetaData() {
+ // Unlike ordinary INSERT, warmup returns per-backend statistics rather than OK.
+ return metaData;
+ }
+
/**
* Skip auth check for warmup select command.
* The blackhole table is a system table and doesn't require LOAD privilege.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/use/UseCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/use/UseCommand.java
index 9223e7d5ad6..e5b810276b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/use/UseCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/use/UseCommand.java
@@ -54,6 +54,14 @@ public class UseCommand extends Command implements NoForward {
this.databaseName = databaseName;
}
+ public String getCatalogName() {
+ return catalogName;
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
@Override
public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
validate(ctx);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
index 0f3e092130c..674c2a99548 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
@@ -91,6 +91,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import lombok.Getter;
import lombok.Setter;
+import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -106,6 +107,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
@@ -153,7 +155,22 @@ public class ConnectContext {
protected volatile long loginTime;
// for arrow flight
protected volatile String peerIdentity;
- private final Map<String, String> preparedQuerys = new HashMap<>();
+ private final Map<String, PreparedQuery> preparedQuerys = new HashMap<>();
+
+ private static class PreparedQuery {
+ private final String sql;
+ private final String catalog;
+ private final String database;
+ private final Schema schema;
+
+ private PreparedQuery(String sql, String catalog, String database, Schema schema) {
+ this.sql = sql;
+ this.catalog = catalog;
+ this.database = database;
+ this.schema = schema;
+ }
+ }
+
private String runningQuery;
private final List<FlightSqlEndpointsLocation> flightSqlEndpointsLocations = Lists.newArrayList();
private boolean returnResultFromLocal = true;
@@ -907,15 +924,34 @@ public class ConnectContext {
this.loginTime = System.currentTimeMillis();
}
- public void addPreparedQuery(String preparedStatementId, String preparedQuery) {
- preparedQuerys.put(preparedStatementId, preparedQuery);
+ public synchronized void addPreparedQuery(String preparedStatementId, String preparedQuery) {
+ addPreparedQuery(preparedStatementId, preparedQuery, null);
+ }
+
+ public synchronized void addPreparedQuery(String preparedStatementId, String preparedQuery, Schema schema) {
+ preparedQuerys.put(preparedStatementId,
+ new PreparedQuery(preparedQuery, getDefaultCatalog(), getDatabase(), schema));
}
- public String getPreparedQuery(String preparedStatementId) {
- return preparedQuerys.get(preparedStatementId);
+ public synchronized Schema getPreparedQuerySchema(String preparedStatementId) {
+ PreparedQuery query = preparedQuerys.get(preparedStatementId);
+ return query == null ? null : query.schema;
+ }
+
+ public synchronized String getPreparedQuery(String preparedStatementId) {
+ PreparedQuery query = preparedQuerys.get(preparedStatementId);
+ if (query == null) {
+ return null;
+ }
+ // A handle must not execute unqualified SQL in a different namespace than its advertised schema.
+ if (!Objects.equals(query.catalog, getDefaultCatalog()) || !Objects.equals(query.database, getDatabase())) {
+ preparedQuerys.remove(preparedStatementId);
+ return null;
+ }
+ return query.sql;
}
- public void removePreparedQuery(String preparedStatementId) {
+ public synchronized void removePreparedQuery(String preparedStatementId) {
preparedQuerys.remove(preparedStatementId);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java
index e64690a54cb..153701dbd2d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java
@@ -20,11 +20,15 @@
package org.apache.doris.service.arrowflight;
+import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.common.Status;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.Util;
import org.apache.doris.mysql.MysqlCommand;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.QueryState.MysqlStateType;
+import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.service.arrowflight.results.FlightSqlEndpointsLocation;
import org.apache.doris.service.arrowflight.results.FlightSqlResultCacheEntry;
import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager;
@@ -42,6 +46,7 @@ import org.apache.arrow.flight.Criteria;
import org.apache.arrow.flight.FlightDescriptor;
import org.apache.arrow.flight.FlightEndpoint;
import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightRuntimeException;
import org.apache.arrow.flight.FlightStream;
import org.apache.arrow.flight.Location;
import org.apache.arrow.flight.PutResult;
@@ -74,10 +79,12 @@ import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.util.AutoCloseables;
+import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.WriteChannel;
import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -86,11 +93,11 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
import java.util.Base64;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
+import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -186,6 +193,17 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext connectContext, String query,
final FlightDescriptor descriptor) {
+ // Schema discovery temporarily installs session state; execution must not mutate that clone.
+ synchronized (connectContext) {
+ return executeQueryStatementLocked(peerIdentity, connectContext, query, descriptor).getLeft();
+ }
+ }
+
+ // Processor.close() clears the context's executor before returning. Retain the final executor
+ // with its result so prepared-schema validation and cancellation use the query that actually ran.
+ private Pair<FlightInfo, StmtExecutor> executeQueryStatementLocked(String peerIdentity,
+ ConnectContext connectContext, String query,
+ final FlightDescriptor descriptor) {
try {
Preconditions.checkState(null != connectContext);
Preconditions.checkState(!query.isEmpty());
@@ -212,9 +230,9 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
final ByteString handle = ByteString.copyFromUtf8(peerIdentity + ":" + queryId);
TicketStatementQuery ticketStatement = TicketStatementQuery.newBuilder()
.setStatementHandle(handle).build();
- return getFlightInfoForSchema(ticketStatement, descriptor,
+ return Pair.of(getFlightInfoForSchema(ticketStatement, descriptor,
connectContext.getFlightSqlChannel().getResult(queryId).getVectorSchemaRoot()
- .getSchema());
+ .getSchema()), connectContext.getExecutor());
} else {
// A Flight Sql request can only contain one statement that returns result,
// otherwise expected thrown exception during execution.
@@ -228,9 +246,10 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
peerIdentity + ":" + DebugUtil.printId(connectContext.queryId()));
TicketStatementQuery ticketStatement = TicketStatementQuery.newBuilder()
.setStatementHandle(handle).build();
- return getFlightInfoForSchema(ticketStatement, descriptor, connectContext.getFlightSqlChannel()
- .getResult(DebugUtil.printId(connectContext.queryId())).getVectorSchemaRoot()
- .getSchema());
+ return Pair.of(getFlightInfoForSchema(ticketStatement, descriptor,
+ connectContext.getFlightSqlChannel()
+ .getResult(DebugUtil.printId(connectContext.queryId())).getVectorSchemaRoot()
+ .getSchema()), connectContext.getExecutor());
}
} else {
// Now only query stmt will pull results from BE.
@@ -280,7 +299,8 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
endpoints.add(new FlightEndpoint(ticket, location));
}
// TODO Set in BE callback after query end, Client will not callback.
- return new FlightInfo(flightSQLConnectProcessor.getArrowSchema(), descriptor, endpoints, -1, -1);
+ return Pair.of(new FlightInfo(flightSQLConnectProcessor.getArrowSchema(), descriptor,
+ endpoints, -1, -1), connectContext.getExecutor());
}
}
} catch (Throwable e) {
@@ -318,18 +338,95 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
@Override
public FlightInfo getFlightInfoPreparedStatement(final CommandPreparedStatementQuery command,
final CallContext context, final FlightDescriptor descriptor) {
- String[] handleParts = command.getPreparedStatementHandle().toStringUtf8().split(":");
- String executedPeerIdentity = handleParts[0];
- String preparedStatementId = handleParts[1];
- ConnectContext connectContext = flightSessionsManager.getConnectContext(executedPeerIdentity);
- return executeQueryStatement(executedPeerIdentity, connectContext,
- connectContext.getPreparedQuery(preparedStatementId), descriptor);
+ ConnectContext connection = flightSessionsManager.getConnectContext(context.peerIdentity());
+ synchronized (connection) {
+ Pair<String, Schema> prepared = preparedQuery(connection, context, command);
+ Pair<FlightInfo, StmtExecutor> result = executeQueryStatementLocked(
+ context.peerIdentity(), connection, prepared.getLeft(), descriptor);
+ FlightInfo info = result.getLeft();
+ String id = command.getPreparedStatementHandle().toStringUtf8()
+ .substring(context.peerIdentity().length() + 1);
+ // Another session's DDL can change the result after reanalysis but before execution
+ // acquires table locks. Reject the actual schema before publishing a DoGet ticket.
+ StmtExecutor executor = result.getRight();
+ boolean matches = !connection.isReturnResultFromLocal() && executor != null
+ && executor.getParsedStmt() instanceof LogicalPlanAdapter
+ ? FlightSqlQuerySchema.matchesExecutionSchema(prepared.getRight(), info.getSchema(),
+ ((LogicalPlanAdapter) executor.getParsedStmt()).getColLabels())
+ : prepared.getRight().equals(info.getSchema());
+ if (!matches) {
+ connection.removePreparedQuery(id);
+ try {
+ if (executor != null) {
+ executor.cancel(Status.CANCELLED);
+ }
+ } catch (Exception e) {
+ LOG.warn("Failed to cancel Flight query after prepared schema changed", e);
+ } finally {
+ connection.closeFlightSqlDeferredExecutors();
+ connection.getFlightSqlChannel().reset();
+ connection.clearFlightSqlEndpointsLocations();
+ }
+ throw CallStatus.NOT_FOUND.withDescription("Prepared statement schema changed; prepare again")
+ .toRuntimeException();
+ }
+ // A successful USE/SWITCH in this statement may change its own namespace. Rebind
+ // only this handle; unrelated handles must still reject an external namespace change.
+ connection.addPreparedQuery(id, prepared.getLeft(), prepared.getRight());
+ return info;
+ }
}
@Override
public SchemaResult getSchemaStatement(final CommandStatementQuery command, final CallContext context,
final FlightDescriptor descriptor) {
- throw CallStatus.UNIMPLEMENTED.withDescription("getSchemaStatement unimplemented").toRuntimeException();
+ return new SchemaResult(analyzeQuerySchema(
+ flightSessionsManager.getConnectContext(context.peerIdentity()), command.getQuery()));
+ }
+
+ @Override
+ public SchemaResult getSchemaPreparedStatement(final CommandPreparedStatementQuery command,
+ final CallContext context, final FlightDescriptor descriptor) {
+ ConnectContext connection = flightSessionsManager.getConnectContext(context.peerIdentity());
+ synchronized (connection) {
+ return new SchemaResult(preparedQuery(connection, context, command).getRight());
+ }
+ }
+
+ private Pair<String, Schema> preparedQuery(ConnectContext connection, CallContext context,
+ CommandPreparedStatementQuery command) {
+ String prefix = context.peerIdentity() + ":";
+ String handle = command.getPreparedStatementHandle().toStringUtf8();
+ if (!handle.startsWith(prefix)) {
+ throw CallStatus.INVALID_ARGUMENT.withDescription("Invalid prepared statement handle").toRuntimeException();
+ }
+ String id = handle.substring(prefix.length());
+ String query = connection.getPreparedQuery(id);
+ if (query == null) {
+ throw CallStatus.NOT_FOUND
+ .withDescription("Prepared statement expired; prepare again in the current namespace")
+ .toRuntimeException();
+ }
+ Schema schema = analyzeQuerySchema(connection, query);
+ // Execution reparses SQL using the current session. Never silently replace the schema
+ // advertised by Prepare when settings such as sql_mode or time_zone change its result.
+ if (!schema.equals(connection.getPreparedQuerySchema(id))) {
+ connection.removePreparedQuery(id);
+ throw CallStatus.NOT_FOUND.withDescription("Prepared statement schema changed; prepare again")
+ .toRuntimeException();
+ }
+ return Pair.of(query, schema);
+ }
+
+ private Schema analyzeQuerySchema(ConnectContext context, String query) {
+ try {
+ return FlightSqlQuerySchema.analyze(context, query);
+ } catch (FlightRuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw CallStatus.INVALID_ARGUMENT.withDescription("Cannot determine query schema: " + e.getMessage())
+ .withCause(e).toRuntimeException();
+ }
}
@Override
@@ -355,51 +452,32 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
@Override
public void createPreparedStatement(final ActionCreatePreparedStatementRequest request, final CallContext context,
final StreamListener<Result> listener) {
- // TODO can only execute complete SQL, not support SQL parameters.
- // For Python: the Python code will try to create a prepared statement (this is to fit DBAPI, IIRC) and
- // if the server raises any error except for NotImplemented it will fail. (If it gets NotImplemented,
- // it will ignore and execute without a prepared statement.) see: https://github.com/apache/arrow/issues/38786
executorService.submit(() -> {
- ConnectContext connectContext = flightSessionsManager.getConnectContext(context.peerIdentity());
+ ConnectContext connectContext = null;
+ String preparedStatementId = null;
try {
- connectContext.setCommand(MysqlCommand.COM_QUERY);
- final String query = request.getQuery();
- String preparedStatementId = UUID.randomUUID().toString();
- final ByteString handle = ByteString.copyFromUtf8(context.peerIdentity() + ":" + preparedStatementId);
- connectContext.addPreparedQuery(preparedStatementId, query);
-
- // Close the temporary VectorSchemaRoot after extracting its Schema, otherwise the
- // off-heap buffers backing its vectors are leaked on every prepare (FE direct memory leak).
- final Schema parameterSchema;
- try (VectorSchemaRoot emptyVectorSchemaRoot =
- new VectorSchemaRoot(new ArrayList<>(), new ArrayList<>())) {
- parameterSchema = emptyVectorSchemaRoot.getSchema();
+ connectContext = flightSessionsManager.getConnectContext(context.peerIdentity());
+ synchronized (connectContext) {
+ String query = request.getQuery();
+ // ADBC ExecuteSchema reads this dataset schema directly without calling GetSchema.
+ // Analyze before registering a handle so failed preparation does not retain a query.
+ Schema schema = analyzeQuerySchema(connectContext, query);
+ preparedStatementId = UUID.randomUUID().toString();
+ ByteString handle = ByteString.copyFromUtf8(context.peerIdentity() + ":" + preparedStatementId);
+ Result result = new Result(Any.pack(buildCreatePreparedStatementResult(handle,
+ new Schema(Collections.emptyList()), schema)).toByteArray());
+ connectContext.addPreparedQuery(preparedStatementId, query, schema);
+ listener.onNext(result);
+ listener.onCompleted();
}
- // TODO FE does not have the ability to convert root fragment output expr into arrow schema.
- // However, the metaData schema returned by createPreparedStatement is usually not used by the client,
- // but it cannot be empty, otherwise it will be mistaken by the client as an updata statement.
- // see: https://github.com/apache/arrow/issues/38911
- final Schema metaData;
- try (VectorSchemaRoot metaSchemaRoot = connectContext.getFlightSqlChannel()
- .createOneOneSchemaRoot("ResultMeta", "UNIMPLEMENTED")) {
- metaData = metaSchemaRoot.getSchema();
+ } catch (Throwable e) {
+ if (connectContext != null && preparedStatementId != null) {
+ connectContext.removePreparedQuery(preparedStatementId);
}
- listener.onNext(new Result(
- Any.pack(buildCreatePreparedStatementResult(handle, parameterSchema, metaData)).toByteArray()));
- } catch (Exception e) {
- String errMsg = "create prepared statement failed, " + e.getMessage() + ", " + Util.getRootCauseMessage(
- e) + ", error code: " + connectContext.getState().getErrorCode() + ", error msg: "
- + connectContext.getState().getErrorMessage();
- LOG.error(errMsg, e);
- listener.onError(CallStatus.INTERNAL.withDescription(errMsg).withCause(e).toRuntimeException());
- return;
- } catch (final Throwable t) {
- listener.onError(CallStatus.INTERNAL.withDescription("Unknown error: " + t).toRuntimeException());
- return;
- } finally {
- connectContext.setCommand(MysqlCommand.COM_SLEEP);
+ listener.onError(e instanceof FlightRuntimeException ? e
+ : CallStatus.INTERNAL.withDescription("Create prepared statement failed: " + e.getMessage())
+ .withCause(e).toRuntimeException());
}
- listener.onCompleted();
});
}
@@ -564,7 +642,32 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
@Override
public void getStreamTableTypes(final CallContext context, final ServerStreamListener listener) {
- throw CallStatus.UNIMPLEMENTED.withDescription("getStreamTableTypes unimplemented").toRuntimeException();
+ try {
+ flightSessionsManager.getConnectContext(context.peerIdentity());
+ // Match GetTables' public type names without scanning catalogs: supported types
+ // must remain available even when this session cannot see any user tables.
+ TreeSet<String> tableTypes = new TreeSet<>();
+ for (TableType type : TableType.values()) {
+ String mysqlType = type.toMysqlType();
+ if (mysqlType != null) {
+ tableTypes.add(mysqlType);
+ }
+ }
+ try (VectorSchemaRoot root = VectorSchemaRoot.create(Schemas.GET_TABLE_TYPES_SCHEMA, rootAllocator)) {
+ root.allocateNew();
+ VarCharVector vector = (VarCharVector) root.getVector("table_type");
+ int row = 0;
+ for (String type : tableTypes) {
+ vector.setSafe(row++, type.getBytes(StandardCharsets.UTF_8));
+ }
+ root.setRowCount(row);
+ listener.start(root);
+ listener.putNext();
+ listener.completed();
+ }
+ } catch (Throwable e) {
+ handleStreamException(e, "get table types failed", listener);
+ }
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java
new file mode 100644
index 00000000000..e0f89217690
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java
@@ -0,0 +1,413 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service.arrowflight;
+
+import org.apache.doris.analysis.StatementBase;
+import org.apache.doris.catalog.AggStateType;
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.StructField;
+import org.apache.doris.catalog.StructType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.es.EsExternalCatalog;
+import org.apache.doris.datasource.lance.LanceExternalCatalog;
+import org.apache.doris.mysql.MysqlCommand;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.parser.SqlDialectHelper;
+import org.apache.doris.nereids.rules.rewrite.CheckPrivileges;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PrepareCommandPlanner;
+import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.nereids.trees.plans.commands.DeleteFromCommand;
+import org.apache.doris.nereids.trees.plans.commands.DescribeCommand;
+import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
+import org.apache.doris.nereids.trees.plans.commands.HelpCommand;
+import org.apache.doris.nereids.trees.plans.commands.KillCommand;
+import org.apache.doris.nereids.trees.plans.commands.ReplayCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowDataCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowPartitionsCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowProcCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowPythonPackagesCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowQueryStatsCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowSnapshotCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.TransactionCommand;
+import org.apache.doris.nereids.trees.plans.commands.UpdateCommand;
+import org.apache.doris.nereids.trees.plans.commands.info.CreateIndexOp;
+import org.apache.doris.nereids.trees.plans.commands.info.DropIndexOp;
+import org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTVFCommand;
+import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.merge.MergeIntoCommand;
+import org.apache.doris.nereids.trees.plans.commands.use.SwitchCommand;
+import org.apache.doris.nereids.trees.plans.commands.use.UseCommand;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.QueryState;
+import org.apache.doris.qe.ResultSetMetaData;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.qe.VariableMgr;
+
+import org.apache.arrow.flight.CallStatus;
+import org.apache.arrow.util.AutoCloseables;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/** Resolves result metadata without scheduling fragments or evaluating query expressions. */
+final class FlightSqlQuerySchema {
+ private FlightSqlQuerySchema() {
+ }
+
+ static Schema analyze(ConnectContext context, String query) throws Exception {
+ synchronized (context) {
+ ConnectContext previousThreadContext = ConnectContext.get();
+ StatementContext previousStatement = context.getStatementContext();
+ SessionVariable previousSession = context.getSessionVariable();
+ QueryState previousState = context.getState();
+ StmtExecutor previousExecutor = context.getExecutor();
+ String previousCatalog = context.getDefaultCatalog();
+ String previousDatabase = context.getDatabase();
+ List<StatementBase> statements = Collections.emptyList();
+ try {
+ context.setThreadLocalInfo();
+ context.setCommand(MysqlCommand.COM_QUERY);
+ // Parsing SET_VAR hints already mutates session variables. Isolate them even when parsing fails.
+ context.setSessionVariable(VariableMgr.cloneSessionVariable(previousSession));
+ context.setState(new QueryState());
+ context.setExecutor(null);
+ context.setStatementContext(null);
+ // Match execution's HTTP/plugin conversion before the dialect parser sees the SQL.
+ String converted = SqlDialectHelper.convertSqlByDialect(query, context.getSessionVariable());
+ try {
+ statements = new NereidsParser().parseSQL(converted, context.getSessionVariable());
+ } catch (Exception convertedError) {
+ if (!context.getSessionVariable().isRetryOriginSqlOnConvertFail() || converted.equals(query)) {
+ throw convertedError;
+ }
+ // Match execution's parse fallback while discarding any failed parser context.
+ StatementContext failed = context.getStatementContext();
+ if (failed != null) {
+ failed.close();
+ context.setStatementContext(null);
+ }
+ statements = new NereidsParser().parseSQL(query, context.getSessionVariable());
+ }
+ Map<String, String> scopedDatabases = new HashMap<>();
+ if (statements.isEmpty()) {
+ throw CallStatus.UNIMPLEMENTED.withDescription(
+ "Schema discovery requires a statement").toRuntimeException();
+ }
+ // JDBC clients commonly prefix their query with USE. Resolve that namespace only within this scope.
+ for (int i = 0; i < statements.size() - 1; ++i) {
+ Plan prefix = ((LogicalPlanAdapter) statements.get(i)).getLogicalPlan();
+ if (!(prefix instanceof UseCommand) && !(prefix instanceof SwitchCommand)) {
+ throw CallStatus.UNIMPLEMENTED.withDescription(
+ "Schema discovery only supports USE or SWITCH before the result statement")
+ .toRuntimeException();
+ }
+ resolveNamespace(context, prefix, scopedDatabases);
+ }
+ LogicalPlanAdapter statement = (LogicalPlanAdapter) statements.get(statements.size() - 1);
+ StatementContext statementContext = statement.getStatementContext();
+ context.setStatementContext(statementContext);
+ statementContext.setParsedStatement(statement);
+ if (!statementContext.getPlaceholders().isEmpty()) {
+ throw CallStatus.UNIMPLEMENTED.withDescription(
+ "Flight SQL parameter binding is not supported").toRuntimeException();
+ }
+ List<Field> fields = new ArrayList<>();
+ Plan plan = statement.getLogicalPlan();
+ if (plan instanceof Command) {
+ resolveNamespace(context, plan, scopedDatabases);
+ ResultSetMetaData metadata = commandMetadata(context, (Command) plan);
+ if (metadata == null) {
+ throw CallStatus.UNIMPLEMENTED.withDescription("Command result metadata is unavailable")
+ .toRuntimeException();
+ }
+ // FE-local result sets are serialized as nullable strings by FlightSqlChannel.
+ for (Column column : metadata.getColumns()) {
+ fields.add(Field.nullable(column.getName(), new ArrowType.Utf8()));
+ }
+ if (fields.isEmpty()) {
+ switch (((Command) plan).stmtType()) {
+ case SET:
+ case USE:
+ case SWITCH:
+ case CREATE:
+ case ALTER:
+ case DROP:
+ case TRUNCATE:
+ // Only known no-row command categories have the protocol OK schema.
+ fields.add(Field.nullable("StatusResult", new ArrowType.Utf8()));
+ break;
+ default:
+ throw CallStatus.UNIMPLEMENTED.withDescription(
+ "Result metadata is unavailable without executing this command")
+ .toRuntimeException();
+ }
+ }
+ } else {
+ // BindResultSink preserves SQL column labels only in query mode, just as
+ // StmtExecutor does; command mode would infer synthetic aliases instead.
+ context.getState().setIsQuery(true);
+ PrepareCommandPlanner planner = new PrepareCommandPlanner(statementContext);
+ planner.plan(statement, context.getSessionVariable().toThrift());
+ CascadesContext cascades = planner.getCascadesContext();
+ Plan analyzed = cascades.getRewritePlan();
+ // PrepareCommandPlanner stops before the rewrite phase that normally checks privileges.
+ new CheckPrivileges().rewriteRoot(analyzed, cascades.getCurrentJobContext());
+ for (Slot slot : analyzed.getOutput()) {
+ fields.add(field(slot.getName(), slot.getDataType().toCatalogDataType(), slot.nullable(),
+ true, context.getSessionVariable().getTimeZone()));
+ }
+ }
+ return new Schema(fields);
+ } finally {
+ try {
+ List<AutoCloseable> resources = new ArrayList<>();
+ for (StatementBase statement : statements) {
+ if (statement instanceof LogicalPlanAdapter) {
+ resources.add(((LogicalPlanAdapter) statement).getStatementContext());
+ }
+ }
+ // A parser failure can leave a context that was never added to the returned list.
+ StatementContext current = context.getStatementContext();
+ if (current != null && current != previousStatement && !resources.contains(current)) {
+ resources.add(current);
+ }
+ AutoCloseables.close(resources);
+ } finally {
+ try {
+ context.setStatementContext(previousStatement);
+ context.setSessionVariable(previousSession);
+ context.setState(previousState);
+ context.setExecutor(previousExecutor);
+ if (!previousCatalog.equals(context.getDefaultCatalog())
+ || !previousDatabase.equals(context.getDatabase())) {
+ context.changeDefaultCatalog(previousCatalog);
+ context.setDatabase(previousDatabase);
+ }
+ } finally {
+ context.setCommand(MysqlCommand.COM_SLEEP);
+ if (previousThreadContext == null) {
+ ConnectContext.remove();
+ } else {
+ previousThreadContext.setThreadLocalInfo();
+ }
+ }
+ }
+ }
+ }
+ }
+
+ static boolean matchesExecutionSchema(Schema prepared, Schema actual, List<String> columnLabels) {
+ List<Field> expectedFields = prepared.getFields();
+ List<Field> actualFields = actual.getFields();
+ if (columnLabels == null || expectedFields.size() != actualFields.size()
+ || expectedFields.size() != columnLabels.size()
+ || !prepared.getCustomMetadata().equals(actual.getCustomMetadata())) {
+ return false;
+ }
+ for (int i = 0; i < expectedFields.size(); ++i) {
+ Field expected = expectedFields.get(i);
+ // Compare the final planner's semantic labels: BE labels may instead contain SQL
+ // text or type_name_index. Ignoring all names would hide a concurrent column rename.
+ if (!expected.getName().equals(columnLabels.get(i))
+ || !matchesExecutionField(expected, actualFields.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean matchesExecutionField(Field expected, Field actual) {
+ // Rewrites can prove an expression non-null (e.g. a folded CAST). Such narrowing is
+ // compatible with Prepare's nullable field; the reverse violates its advertised contract.
+ if ((!expected.isNullable() && actual.isNullable()) || !expected.getType().equals(actual.getType())
+ || !expected.getMetadata().equals(actual.getMetadata())
+ || !Objects.equals(expected.getFieldType().getDictionary(), actual.getFieldType().getDictionary())
+ || expected.getChildren().size() != actual.getChildren().size()) {
+ return false;
+ }
+ for (int i = 0; i < expected.getChildren().size(); ++i) {
+ Field child = expected.getChildren().get(i);
+ Field actualChild = actual.getChildren().get(i);
+ if (!child.getName().equals(actualChild.getName()) || !matchesExecutionField(child, actualChild)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static ResultSetMetaData commandMetadata(ConnectContext context, Command command) throws Exception {
+ // These getters depend on execution-time state or remote responses. Do not advertise a
+ // guessed schema, or run the command merely to discover it.
+ if (command instanceof ShowPythonPackagesCommand || command instanceof DescribeCommand
+ || command instanceof ShowDataCommand || command instanceof ShowPartitionsCommand
+ || command instanceof ShowQueryStatsCommand) {
+ throw CallStatus.UNIMPLEMENTED.withDescription("Command schema requires execution-time metadata")
+ .toRuntimeException();
+ }
+ if (command instanceof HelpCommand) {
+ return ((HelpCommand) command).getMetaData(context);
+ } else if (command instanceof ShowSnapshotCommand) {
+ return ((ShowSnapshotCommand) command).getMetaData(context);
+ } else if (command instanceof ShowTableCommand) {
+ ((ShowTableCommand) command).validate(context);
+ } else if (command instanceof ShowCreateTableCommand) {
+ return ((ShowCreateTableCommand) command).getMetaData(context);
+ } else if (command instanceof ShowProcCommand) {
+ return ((ShowProcCommand) command).getMetaData(context);
+ }
+ if (command instanceof ExplainCommand) {
+ // PLAN PROCESS has no Flight serialization path in StmtExecutor.
+ if (((ExplainCommand) command).showPlanProcess()) {
+ throw CallStatus.UNIMPLEMENTED.withDescription("EXPLAIN PLAN PROCESS is not supported over Flight SQL")
+ .toRuntimeException();
+ }
+ return stringMetadata("Explain String(Nereids Planner)");
+ } else if (command instanceof ReplayCommand) {
+ return stringMetadata("Plan Replayer dump url");
+ } else if (command instanceof AlterTableCommand) {
+ AlterTableCommand alter = (AlterTableCommand) command;
+ String catalog = alter.getTbl().getCtl();
+ // Lance index admission returns a JobId header even for an IF no-op. Do not run
+ // validation/admission here: those paths can resolve remote tables or allocate IDs.
+ if (context.getCatalog(catalog == null ? context.getDefaultCatalog() : catalog)
+ instanceof LanceExternalCatalog && alter.getNereidsOps().stream().anyMatch(op ->
+ (op instanceof CreateIndexOp && !((CreateIndexOp) op).isAlter())
+ || (op instanceof DropIndexOp && !((DropIndexOp) op).isAlter()))) {
+ return stringMetadata("JobId");
+ }
+ }
+ ResultSetMetaData metadata = command.getResultSetMetaData();
+ // Concrete DML commands return OK, but subclasses such as WARM UP SELECT supply rows.
+ if (metadata != null && metadata.getColumnCount() == 0 && (command instanceof InsertIntoTableCommand
+ || command instanceof InsertOverwriteTableCommand || command instanceof BatchInsertIntoTableCommand
+ || command instanceof InsertIntoTVFCommand || command instanceof UpdateCommand
+ || command instanceof DeleteFromCommand || command instanceof MergeIntoCommand
+ || command instanceof KillCommand || command instanceof TransactionCommand)) {
+ return stringMetadata("StatusResult");
+ }
+ return metadata;
+ }
+
+ private static ResultSetMetaData stringMetadata(String name) {
+ return ShowResultSetMetaData.builder().addColumn(new Column(name, Type.STRING)).build();
+ }
+
+ private static void resolveNamespace(ConnectContext context, Plan plan, Map<String, String> scopedDatabases)
+ throws Exception {
+ if (plan instanceof UseCommand) {
+ UseCommand use = (UseCommand) plan;
+ String catalog = use.getCatalogName() == null ? context.getDefaultCatalog() : use.getCatalogName();
+ CatalogIf catalogObject = context.getCatalog(catalog);
+ if (catalogObject == null || !context.getEnv().getAccessManager()
+ .checkDbPriv(context, catalog, use.getDatabaseName(), PrivPredicate.SHOW)) {
+ throw CallStatus.UNAUTHORIZED.withDescription("Database access denied").toRuntimeException();
+ }
+ catalogObject.getDbOrAnalysisException(use.getDatabaseName());
+ if (use.getCatalogName() != null && !context.getDatabase().isEmpty()) {
+ scopedDatabases.put(context.getDefaultCatalog(), context.getDatabase());
+ }
+ context.changeDefaultCatalog(catalog);
+ context.setDatabase(use.getDatabaseName());
+ } else if (plan instanceof SwitchCommand) {
+ String catalog = ((SwitchCommand) plan).getCatalogName();
+ if (context.getCatalog(catalog) == null || !context.getEnv().getAccessManager()
+ .checkCtlPriv(context, catalog, PrivPredicate.SHOW)) {
+ throw CallStatus.UNAUTHORIZED.withDescription("Catalog access denied").toRuntimeException();
+ }
+ // Mirror Env.changeCatalog, keeping remembered databases local to this analysis.
+ if (!context.getDatabase().isEmpty()) {
+ scopedDatabases.put(context.getDefaultCatalog(), context.getDatabase());
+ }
+ String database = scopedDatabases.getOrDefault(catalog, context.getLastDBOfCatalog(catalog));
+ context.changeDefaultCatalog(catalog);
+ if (database != null && !database.isEmpty()) {
+ context.setDatabase(database);
+ }
+ if (context.getCatalog(catalog) instanceof EsExternalCatalog) {
+ context.setDatabase(EsExternalCatalog.DEFAULT_DB);
+ }
+ }
+ }
+
+ private static Field field(String name, Type type, boolean nullable, boolean topLevel, String timezone) {
+ // group_concat uses IAggregateFunction's string serialization, unlike fixed-size states
+ // such as sum/count. Match that BE wire type instead of treating every AGG_STATE as Null.
+ if (type instanceof AggStateType && "group_concat".equals(((AggStateType) type).getFunctionName())) {
+ type = Type.STRING;
+ }
+ PrimitiveType primitive = type.getPrimitiveType();
+ int precision = type instanceof ScalarType ? ((ScalarType) type).getScalarPrecision() : 0;
+ int scale = type instanceof ScalarType ? ((ScalarType) type).getScalarScale() : 0;
+ ArrowType arrowType = FlightSqlSchemaHelper.getArrowType(primitive, precision, scale);
+ if (primitive == PrimitiveType.TIMESTAMPTZ) {
+ arrowType = new ArrowType.Timestamp(((ArrowType.Timestamp) arrowType).getUnit(),
+ "Z".equals(timezone) ? "UTC" : timezone);
+ }
+ if (arrowType instanceof ArrowType.Null && primitive != PrimitiveType.NULL_TYPE) {
+ throw CallStatus.UNIMPLEMENTED.withDescription("Unsupported Arrow result type: " + type)
+ .toRuntimeException();
+ }
+ List<Field> children = new ArrayList<>();
+ if (type instanceof ArrayType) {
+ // BE constructs ListType and MapType from data types, so item/value fields are nullable.
+ children.add(field("item", ((ArrayType) type).getItemType(), true, false, timezone));
+ } else if (type instanceof MapType) {
+ MapType map = (MapType) type;
+ children.add(new Field("entries", FieldType.notNullable(new ArrowType.Struct()), Arrays.asList(
+ field("key", map.getKeyType(), false, false, timezone),
+ field("value", map.getValueType(), true, false, timezone))));
+ } else if (type instanceof StructType) {
+ for (StructField child : ((StructType) type).getFields()) {
+ children.add(field(child.getName(), child.getType(), child.getContainsNull(), false, timezone));
+ }
+ }
+ Map<String, String> metadata = null;
+ if (topLevel && (primitive == PrimitiveType.LARGEINT || primitive == PrimitiveType.IPV4
+ || primitive == PrimitiveType.IPV6)) {
+ metadata = Collections.singletonMap("doris_type", primitive.toString());
+ }
+ return new Field(name, new FieldType(nullable, arrowType, null, metadata), children);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java
index a66d2e9648a..9d91153ebd6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java
@@ -91,7 +91,7 @@ public class FlightSqlSchemaHelper {
* Ref: `convert_to_arrow_type` in be/src/util/arrow/row_batch.cpp.
* which is consistent with the type of Arrow data returned by Doris Arrow Flight Sql query.
*/
- private static ArrowType getArrowType(PrimitiveType primitiveType, Integer precision, Integer scale) {
+ static ArrowType getArrowType(PrimitiveType primitiveType, Integer precision, Integer scale) {
switch (primitiveType) {
case BOOLEAN:
return new ArrowType.Bool();
@@ -107,6 +107,7 @@ public class FlightSqlSchemaHelper {
case FLOAT:
return new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE);
case DOUBLE:
+ case TIMEV2:
return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
case LARGEINT:
case VARCHAR:
@@ -148,6 +149,7 @@ public class FlightSqlSchemaHelper {
return new ArrowType.Decimal(precision, scale, 256);
case DECIMALV2:
return new ArrowType.Decimal(27, 9, 128);
+ case VARBINARY:
case HLL:
case BITMAP:
case QUANTILE_STATE:
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/doris/FeServiceClientTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/doris/FeServiceClientTest.java
new file mode 100644
index 00000000000..8a0eba96d4c
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/doris/FeServiceClientTest.java
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.doris;
+
+import org.apache.doris.common.ClientPool;
+import org.apache.doris.thrift.FrontendService;
+import org.apache.doris.thrift.TGetBackendMetaResult;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TStatus;
+import org.apache.doris.thrift.TStatusCode;
+
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.transport.TSocket;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.util.Collections;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+class FeServiceClientTest {
+ @Test
+ void rejectsTimeoutThatCannotFitInSocketMilliseconds() {
+ Assertions.assertThrows(ArithmeticException.class, () -> new FeServiceClient("remote_catalog",
+ Collections.emptyList(), "test_user", "", 1, Integer.MAX_VALUE));
+ }
+
+ @Test
+ void metadataReadTimeoutUsesSeconds() throws Exception {
+ FrontendService.Iface handler = Mockito.mock(FrontendService.Iface.class);
+ Mockito.when(handler.getBackendMeta(Mockito.any())).thenAnswer(invocation -> {
+ // Ordinary metadata latency must remain within the configured seconds-based timeout.
+ Thread.sleep(100);
+ return new TGetBackendMetaResult().setStatus(new TStatus(TStatusCode.OK))
+ .setBackends(Collections.emptyList());
+ });
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) {
+ server.setSoTimeout(5000);
+ Future<?> response = executor.submit(() -> {
+ try (TSocket socket = new TSocket(server.accept())) {
+ TBinaryProtocol protocol = new TBinaryProtocol(socket);
+ new FrontendService.Processor<>(handler).process(protocol, protocol);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ TNetworkAddress address = new TNetworkAddress(server.getInetAddress().getHostAddress(),
+ server.getLocalPort());
+ try {
+ FeServiceClient client = new FeServiceClient("remote_catalog", Collections.singletonList(address),
+ "test_user", "", 1, 10);
+ Assertions.assertTrue(client.listBackends().isEmpty());
+ response.get(5, TimeUnit.SECONDS);
+ } finally {
+ ClientPool.frontendPool.clearPool(address);
+ }
+ } finally {
+ executor.shutdownNow();
+ Assertions.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/SchemaOnlyCommandMetadataTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/SchemaOnlyCommandMetadataTest.java
new file mode 100644
index 00000000000..e02e5f5e283
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/SchemaOnlyCommandMetadataTest.java
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ResultSetMetaData;
+import org.apache.doris.qe.help.HelpModule;
+import org.apache.doris.qe.help.HelpTopic;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+class SchemaOnlyCommandMetadataTest {
+ private static List<String> names(ResultSetMetaData metadata) {
+ return metadata.getColumns().stream().map(column -> column.getName()).collect(Collectors.toList());
+ }
+
+ @Test
+ void helpTopicDoesNotReadDescriptionOrExample() throws Exception {
+ HelpModule module = Mockito.mock(HelpModule.class);
+ HelpTopic topic = Mockito.mock(HelpTopic.class);
+ Mockito.when(module.getTopic("topic")).thenReturn(topic);
+ try (MockedStatic<HelpModule> ignored = Mockito.mockStatic(HelpModule.class)) {
+ ignored.when(HelpModule::getInstance).thenReturn(module);
+ HelpCommand command = new HelpCommand("topic");
+ Assertions.assertEquals(Arrays.asList("name", "description", "example"), names(command.getMetaData(null)));
+ Mockito.verifyNoInteractions(topic);
+ Assertions.assertEquals(names(command.doRun(null, null).getMetaData()), names(command.getMetaData(null)));
+ }
+ }
+
+ @Test
+ void helpKeywordWithOneTopicUsesTopicHeader() throws Exception {
+ HelpModule module = Mockito.mock(HelpModule.class);
+ Mockito.when(module.listTopicByKeyword("keyword")).thenReturn(Collections.singletonList("topic"));
+ Mockito.when(module.getTopic("topic")).thenReturn(Mockito.mock(HelpTopic.class));
+ try (MockedStatic<HelpModule> ignored = Mockito.mockStatic(HelpModule.class)) {
+ ignored.when(HelpModule::getInstance).thenReturn(module);
+ HelpCommand command = new HelpCommand("keyword");
+ Assertions.assertEquals(Arrays.asList("name", "description", "example"), names(command.getMetaData(null)));
+ Assertions.assertEquals(names(command.doRun(null, null).getMetaData()), names(command.getMetaData(null)));
+ }
+ }
+
+ @Test
+ void helpKeywordAndCategoryHeadersMatchExecution() throws Exception {
+ HelpModule module = Mockito.mock(HelpModule.class);
+ Mockito.when(module.listTopicByKeyword("keyword")).thenReturn(Arrays.asList("first", "second"));
+ Mockito.when(module.listCategoryByName("categories")).thenReturn(Arrays.asList("first", "second"));
+ Mockito.when(module.listCategoryByName("category")).thenReturn(Collections.singletonList("first"));
+ try (MockedStatic<HelpModule> ignored = Mockito.mockStatic(HelpModule.class)) {
+ ignored.when(HelpModule::getInstance).thenReturn(module);
+ for (String mark : Arrays.asList("keyword", "categories", "category", "missing")) {
+ HelpCommand command = new HelpCommand(mark);
+ List<String> expected = mark.equals("categories")
+ ? Arrays.asList("source_category_name", "name", "is_it_category")
+ : Arrays.asList("name", "is_it_category");
+ Assertions.assertEquals(expected, names(command.getMetaData(null)));
+ Assertions.assertEquals(names(command.doRun(null, null).getMetaData()), names(command.getMetaData(null)));
+ }
+ }
+ }
+
+ @Test
+ void invalidHelpAndSnapshotFiltersAreRejected() throws Exception {
+ Assertions.assertThrows(AnalysisException.class, () -> new HelpCommand("").getMetaData(null));
+ Env env = Mockito.mock(Env.class, Mockito.RETURNS_DEEP_STUBS);
+ Mockito.when(env.getAccessManager().checkGlobalPriv(Mockito.nullable(ConnectContext.class),
+ Mockito.eq(PrivPredicate.ADMIN))).thenReturn(true);
+ try (MockedStatic<Env> ignored = Mockito.mockStatic(Env.class)) {
+ ignored.when(Env::getCurrentEnv).thenReturn(env);
+ ShowSnapshotCommand command = (ShowSnapshotCommand) new NereidsParser()
+ .parseSingle("SHOW SNAPSHOT ON example_repo WHERE TIMESTAMP = '2026-01-01-00-00-00'");
+ Assertions.assertThrows(AnalysisException.class, () -> command.getMetaData(null));
+ Mockito.verify(env, Mockito.never()).getBackupHandler();
+ }
+ }
+
+ @Test
+ void filteredSnapshotsResolveWithoutRepositoryAccess() throws Exception {
+ Env env = Mockito.mock(Env.class, Mockito.RETURNS_DEEP_STUBS);
+ Mockito.when(env.getAccessManager().checkGlobalPriv(Mockito.nullable(ConnectContext.class),
+ Mockito.eq(PrivPredicate.ADMIN))).thenReturn(true);
+ try (MockedStatic<Env> ignored = Mockito.mockStatic(Env.class)) {
+ ignored.when(Env::getCurrentEnv).thenReturn(env);
+ for (String filter : Arrays.asList("", " WHERE SNAPSHOT = 'sample'",
+ " WHERE SNAPSHOT = 'sample' AND TIMESTAMP = '2026-01-01-00-00-00'")) {
+ ShowSnapshotCommand command = (ShowSnapshotCommand) new NereidsParser()
+ .parseSingle("SHOW SNAPSHOT ON example_repo" + filter);
+ Assertions.assertEquals(filter.contains("TIMESTAMP")
+ ? ShowSnapshotCommand.SNAPSHOT_DETAIL : ShowSnapshotCommand.SNAPSHOT_ALL,
+ names(command.getMetaData(null)));
+ }
+ Mockito.verify(env, Mockito.never()).getBackupHandler();
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java
index eede1268851..8b66ccf3d54 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java
@@ -17,136 +17,22 @@
package org.apache.doris.service.arrowflight;
-import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.service.arrowflight.results.FlightSqlChannel;
import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager;
-import org.apache.doris.service.arrowflight.sessions.FlightSqlConnectContext;
import org.apache.arrow.flight.FlightDescriptor;
import org.apache.arrow.flight.FlightProducer.CallContext;
-import org.apache.arrow.flight.FlightProducer.StreamListener;
import org.apache.arrow.flight.Location;
-import org.apache.arrow.flight.Result;
-import org.apache.arrow.flight.sql.impl.FlightSql.ActionCreatePreparedStatementRequest;
import org.apache.arrow.flight.sql.impl.FlightSql.CommandStatementQuery;
-import org.junit.After;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.Test;
import org.mockito.MockedConstruction;
import org.mockito.Mockito;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
public class DorisFlightSqlProducerTest {
- private boolean prevRunningUnitTest;
-
- @Before
- public void setUp() {
- // FlightSqlConnectContext.init() only reaches Env when this is false; keep it true so the
- // context can be built without a running FE.
- prevRunningUnitTest = FeConstants.runningUnitTest;
- FeConstants.runningUnitTest = true;
- }
-
- @After
- public void tearDown() {
- FeConstants.runningUnitTest = prevRunningUnitTest;
- }
-
- /**
- * Regression test for the FE direct-memory leak in {@code createPreparedStatement}
- * (issue apache/doris#65305, fixed by PR #65311).
- *
- * <p>Before the fix, each prepare allocated a {@link org.apache.arrow.vector.VectorSchemaRoot}
- * via {@code FlightSqlChannel.createOneOneSchemaRoot("ResultMeta", ...)} and read only its
- * {@code Schema}, never closing the root, leaking one off-heap {@code VarCharVector} buffer per
- * prepare. Because Arrow Flight has no parameter binding, every client query triggers a fresh
- * prepare, so the leak grows monotonically until {@code MaxDirectMemorySize} is exhausted.
- *
- * <p>This test drives {@code createPreparedStatement} against a real {@link FlightSqlChannel} many
- * times and asserts the channel's Arrow allocator holds zero bytes afterwards. It goes red if the
- * try-with-resources that closes the temporary roots is removed.
- */
- @Test
- public void createPreparedStatementDoesNotLeakChannelAllocator() throws Exception {
- // A real flight session context owns a real FlightSqlChannel (and thus a real Arrow allocator),
- // so allocator bookkeeping is exercised for real instead of mocked away.
- FlightSqlConnectContext connectContext = new FlightSqlConnectContext("test-peer-identity");
- FlightSqlChannel channel = connectContext.getFlightSqlChannel();
- Assert.assertEquals("channel allocator should start empty", 0L, channel.getAllocatedMemory());
-
- FlightSessionsManager sessionsManager = new FlightSessionsManager() {
- @Override
- public ConnectContext getConnectContext(String peerIdentity) {
- return connectContext;
- }
-
- @Override
- public ConnectContext createConnectContext(String peerIdentity) {
- return connectContext;
- }
-
- @Override
- public void closeConnectContext(String peerIdentity) {
- // not exercised by this test
- }
- };
- DorisFlightSqlProducer producer =
- new DorisFlightSqlProducer(Location.forGrpcInsecure("127.0.0.1", 9090), sessionsManager);
-
- CallContext callContext = Mockito.mock(CallContext.class);
- Mockito.when(callContext.peerIdentity()).thenReturn("test-peer-identity");
-
- final int rounds = 100;
- AtomicInteger errors = new AtomicInteger(0);
- try {
- // createPreparedStatement runs asynchronously and mutates a shared, non-thread-safe
- // ConnectContext, so drive it serially: each prepare completes before the next is issued.
- // The leak, if any, still accumulates on the single channel allocator across rounds.
- for (int i = 0; i < rounds; i++) {
- CountDownLatch finished = new CountDownLatch(1);
- StreamListener<Result> listener = new StreamListener<Result>() {
- @Override
- public void onNext(Result val) {
- // discard the placeholder prepared-statement result
- }
-
- @Override
- public void onError(Throwable t) {
- errors.incrementAndGet();
- finished.countDown();
- }
-
- @Override
- public void onCompleted() {
- finished.countDown();
- }
- };
- ActionCreatePreparedStatementRequest request = ActionCreatePreparedStatementRequest.newBuilder()
- .setQuery("select * from t where id = " + i).build();
- producer.createPreparedStatement(request, callContext, listener);
- Assert.assertTrue("createPreparedStatement #" + i + " did not finish in time",
- finished.await(30, TimeUnit.SECONDS));
- }
-
- // Guard against a false pass: if a prepare failed before reaching the allocation, no buffer
- // would be leaked and the memory assertion below could not detect a regression.
- Assert.assertEquals("no createPreparedStatement call should fail", 0, errors.get());
- // Every temporary VectorSchemaRoot must have been closed, so the channel's Arrow allocator
- // is back to zero. Reverting the fix leaves `rounds` ResultMeta buffers allocated here.
- Assert.assertEquals("createPreparedStatement leaked off-heap memory in the channel allocator",
- 0L, channel.getAllocatedMemory());
- } finally {
- producer.close();
- }
- }
-
// Arrow Flight SQL keeps a query's coordinator alive across GetFlightInfo -> DoGet (see #62259):
// executeAndSendResult() registers it as a deferred executor on the ConnectContext right after
// submitting it to the BE. GetFlightInfo then still has to fetch the Arrow schema from the BE.
diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlSchemaTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlSchemaTest.java
new file mode 100644
index 00000000000..70fb507e502
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlSchemaTest.java
@@ -0,0 +1,775 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service.arrowflight;
+
+import org.apache.doris.analysis.UserIdentity;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.Status;
+import org.apache.doris.common.proc.ProcNodeInterface;
+import org.apache.doris.common.proc.ProcService;
+import org.apache.doris.datasource.lance.LanceExternalCatalog;
+import org.apache.doris.datasource.test.TestExternalCatalog;
+import org.apache.doris.mysql.MysqlCommand;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.parser.SqlDialectHelper;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.qe.SqlModeHelper;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.service.arrowflight.results.FlightSqlChannel;
+import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager;
+import org.apache.doris.utframe.TestWithFeService;
+
+import com.google.protobuf.Any;
+import org.apache.arrow.flight.FlightDescriptor;
+import org.apache.arrow.flight.FlightProducer.CallContext;
+import org.apache.arrow.flight.FlightProducer.StreamListener;
+import org.apache.arrow.flight.FlightRuntimeException;
+import org.apache.arrow.flight.FlightStatusCode;
+import org.apache.arrow.flight.Location;
+import org.apache.arrow.flight.Result;
+import org.apache.arrow.flight.sql.impl.FlightSql.ActionCreatePreparedStatementRequest;
+import org.apache.arrow.flight.sql.impl.FlightSql.ActionCreatePreparedStatementResult;
+import org.apache.arrow.flight.sql.impl.FlightSql.CommandPreparedStatementQuery;
+import org.apache.arrow.flight.sql.impl.FlightSql.CommandStatementQuery;
+import org.apache.arrow.vector.ipc.ReadChannel;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
+import org.apache.arrow.vector.types.TimeUnit;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.InOrder;
+import org.mockito.MockedConstruction;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.io.ByteArrayInputStream;
+import java.nio.channels.Channels;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+public class DorisFlightSqlSchemaTest extends TestWithFeService {
+ private DorisFlightSqlProducer producer;
+ private CallContext callContext;
+ private boolean previousUnitTest;
+
+ public static class SchemaCatalogProvider implements TestExternalCatalog.TestCatalogProvider {
+ @Override
+ public Map<String, Map<String, List<Column>>> getMetadata() {
+ return Collections.singletonMap("db1", Collections.emptyMap());
+ }
+ }
+
+ @Override
+ protected void runBeforeAll() throws Exception {
+ previousUnitTest = FeConstants.runningUnitTest;
+ FeConstants.runningUnitTest = true;
+ createDatabase("schema_test");
+ connectContext.setDatabase("schema_test");
+ createTable("CREATE TABLE schema_input (id BIGINT NOT NULL, name VARCHAR(20)) "
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES ('replication_num' = '1')");
+ createCatalog("CREATE CATALOG schema_catalog PROPERTIES ('type'='test', "
+ + "'catalog_provider.class'='" + SchemaCatalogProvider.class.getName() + "')");
+ connectContext = Mockito.spy(connectContext);
+ Mockito.doReturn(new FlightSqlChannel()).when(connectContext).getFlightSqlChannel();
+ FlightSessionsManager sessions = Mockito.mock(FlightSessionsManager.class);
+ Mockito.when(sessions.getConnectContext("schema-peer")).thenReturn(connectContext);
+ callContext = Mockito.mock(CallContext.class);
+ Mockito.when(callContext.peerIdentity()).thenReturn("schema-peer");
+ producer = new DorisFlightSqlProducer(Location.forGrpcInsecure("127.0.0.1", 0), sessions);
+ }
+
+ @Override
+ protected void runAfterAll() throws Exception {
+ FeConstants.runningUnitTest = previousUnitTest;
+ producer.close();
+ connectContext.getFlightSqlChannel().close();
+ }
+
+ private Schema schema(String sql) {
+ return producer.getSchemaStatement(CommandStatementQuery.newBuilder().setQuery(sql).build(),
+ callContext, FlightDescriptor.command(new byte[0])).getSchema();
+ }
+
+ private ActionCreatePreparedStatementResult prepare(String sql) throws Exception {
+ CompletableFuture<ActionCreatePreparedStatementResult> result = new CompletableFuture<>();
+ producer.createPreparedStatement(ActionCreatePreparedStatementRequest.newBuilder().setQuery(sql).build(),
+ callContext, new StreamListener<Result>() {
+ private ActionCreatePreparedStatementResult response;
+
+ @Override
+ public void onNext(Result value) {
+ try {
+ response = Any.parseFrom(value.getBody()).unpack(ActionCreatePreparedStatementResult.class);
+ } catch (Exception e) {
+ result.completeExceptionally(e);
+ }
+ }
+
+ @Override
+ public void onError(Throwable error) {
+ result.completeExceptionally(error);
+ }
+
+ @Override
+ public void onCompleted() {
+ result.complete(response);
+ }
+ });
+ return result.get(30, java.util.concurrent.TimeUnit.SECONDS);
+ }
+
+ private Schema preparedSchema(String sql) throws Exception {
+ ActionCreatePreparedStatementResult result = prepare(sql);
+ String handle = result.getPreparedStatementHandle().toStringUtf8();
+ connectContext.removePreparedQuery(handle.substring(handle.indexOf(':') + 1));
+ byte[] bytes = result.getDatasetSchema().toByteArray();
+ return MessageSerializer.deserializeSchema(new ReadChannel(
+ Channels.newChannel(new ByteArrayInputStream(bytes))));
+ }
+
+ @Test
+ void prepareReturnsAnalyzedOutput() throws Exception {
+ Schema schema = preparedSchema("SELECT id AS key_alias, name FROM schema_input");
+ Assertions.assertEquals(Arrays.asList("key_alias", "name"), schema.getFields().stream()
+ .map(Field::getName).collect(Collectors.toList()));
+ Assertions.assertEquals(new ArrowType.Int(64, true), schema.getFields().get(0).getType());
+ Assertions.assertFalse(schema.getFields().get(0).isNullable());
+ Assertions.assertEquals(new ArrowType.Utf8(), schema.getFields().get(1).getType());
+ Assertions.assertTrue(schema.getFields().get(1).isNullable());
+ }
+
+ @Test
+ void getSchemaAndPrepareAgree() throws Exception {
+ String sql = "SELECT CAST(7 AS BIGINT) AS id, CAST('2025-01-02 03:04:05.123456' "
+ + "AS DATETIME(6)) AS event_time, ARRAY('first', 'second') AS tags";
+ Schema schema = schema(sql);
+ Assertions.assertEquals(schema, preparedSchema(sql));
+ Assertions.assertEquals(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null),
+ schema.getFields().get(1).getType());
+ Assertions.assertEquals(new ArrowType.List(), schema.getFields().get(2).getType());
+ Assertions.assertEquals(new ArrowType.Utf8(), schema.getFields().get(2).getChildren().get(0).getType());
+ }
+
+ @Test
+ void invalidSqlIsRejectedAndConnectionRecovers() throws Exception {
+ for (String sql : Arrays.asList("SELEC 1", "SELECT missing_column FROM schema_input")) {
+ Assertions.assertThrows(FlightRuntimeException.class, () -> schema(sql));
+ Mockito.clearInvocations(connectContext);
+ Assertions.assertThrows(java.util.concurrent.ExecutionException.class, () -> prepare(sql));
+ Mockito.verify(connectContext, Mockito.never()).addPreparedQuery(Mockito.anyString(), Mockito.anyString(), Mockito.any());
+ Assertions.assertEquals("id", preparedSchema("SELECT id FROM schema_input")
+ .getFields().get(0).getName());
+ }
+ }
+
+ @Test
+ void showVariablesUsesFeResultTypes() throws Exception {
+ String sql = "SHOW VARIABLES LIKE 'query_timeout'";
+ Schema schema = schema(sql);
+ Assertions.assertEquals(schema, preparedSchema(sql));
+ Assertions.assertEquals(Arrays.asList("Variable_name", "Value", "Default_Value", "Changed"),
+ schema.getFields().stream().map(Field::getName).collect(Collectors.toList()));
+ Assertions.assertTrue(schema.getFields().stream().allMatch(f -> f.getType().equals(new ArrowType.Utf8())));
+ }
+
+ @Test
+ void metadataDoesNotExecuteOrChangeSession() {
+ SessionVariable session = connectContext.getSessionVariable();
+ StatementContext statement = connectContext.getStatementContext();
+ ConnectContext threadContext = ConnectContext.get();
+ int timeout = session.getQueryTimeoutS();
+ schema("SELECT /*+ SET_VAR(query_timeout=17) */ id FROM schema_input");
+ Assertions.assertSame(session, connectContext.getSessionVariable());
+ Assertions.assertEquals(timeout, session.getQueryTimeoutS());
+ Assertions.assertSame(statement, connectContext.getStatementContext());
+ Assertions.assertSame(threadContext, ConnectContext.get());
+ Assertions.assertEquals("StatusResult", schema("SET query_timeout=17").getFields().get(0).getName());
+ Assertions.assertEquals(timeout, session.getQueryTimeoutS());
+ Assertions.assertThrows(FlightRuntimeException.class, () -> schema("SELECT 1; SELECT 2"));
+ Assertions.assertEquals(MysqlCommand.COM_SLEEP, connectContext.getCommand());
+ }
+
+ @Test
+ void createPreparedStatementDoesNotLeakChannelAllocator() throws Exception {
+ // Each prepare used to allocate an off-heap placeholder vector. Repeated prepares must stay bounded.
+ for (int i = 0; i < 100; ++i) {
+ preparedSchema("SELECT " + i + " AS id");
+ Assertions.assertEquals(0, connectContext.getFlightSqlChannel().getAllocatedMemory());
+ Assertions.assertEquals(0, connectContext.getFlightSqlChannel().resultNum());
+ }
+ }
+
+ @Test
+ void getSchemaPreparedStatementValidatesHandle() throws Exception {
+ ActionCreatePreparedStatementResult result = prepare("SELECT id FROM schema_input");
+ CommandPreparedStatementQuery command = CommandPreparedStatementQuery.newBuilder()
+ .setPreparedStatementHandle(result.getPreparedStatementHandle()).build();
+ Assertions.assertEquals(schema("SELECT id FROM schema_input"), producer.getSchemaPreparedStatement(
+ command, callContext, FlightDescriptor.command(new byte[0])).getSchema());
+ String handle = result.getPreparedStatementHandle().toStringUtf8();
+ connectContext.removePreparedQuery(handle.substring(handle.indexOf(':') + 1));
+ FlightRuntimeException failure = Assertions.assertThrows(FlightRuntimeException.class,
+ () -> producer.getSchemaPreparedStatement(command, callContext, FlightDescriptor.command(new byte[0])));
+ Assertions.assertEquals(FlightStatusCode.NOT_FOUND, failure.status().code());
+ }
+
+ @Test
+ void schemaDiscoveryChecksTablePrivileges() {
+ UserIdentity originalUser = connectContext.getCurrentUserIdentity();
+ connectContext.setCurrentUserIdentity(UserIdentity.createAnalyzedUserIdentWithIp("schema_reader", "%"));
+ try {
+ for (String sql : Arrays.asList("SELECT id FROM schema_input",
+ "WITH source AS (SELECT id FROM schema_input) SELECT * FROM source",
+ "SELECT (SELECT MAX(id) FROM schema_input) AS value",
+ "SELECT SUM(id) FROM schema_input")) {
+ FlightRuntimeException failure = Assertions.assertThrows(FlightRuntimeException.class,
+ () -> schema(sql), sql);
+ Assertions.assertTrue(failure.getMessage().contains("denied"), failure.getMessage());
+ }
+ } finally {
+ connectContext.setCurrentUserIdentity(originalUser);
+ }
+ }
+
+ @Test
+ void prepareKeepsCommandAndUsePrefixCompatibility() throws Exception {
+ Assertions.assertEquals("StatusResult", preparedSchema("CREATE DATABASE schema_not_created")
+ .getFields().get(0).getName());
+ Assertions.assertFalse(connectContext.getCurrentCatalog().getDb("schema_not_created").isPresent());
+ String previousDatabase = connectContext.getDatabase();
+ connectContext.setDatabase("information_schema");
+ try {
+ Assertions.assertEquals("id", preparedSchema("USE schema_test; SELECT id FROM schema_input")
+ .getFields().get(0).getName());
+ Assertions.assertEquals("information_schema", connectContext.getDatabase());
+ } finally {
+ connectContext.setDatabase(previousDatabase);
+ }
+ }
+
+ @Test
+ void nestedFieldsAndLogicalTypeMetadata() {
+ Schema result = schema("SELECT MAP('key', CAST(1 AS LARGEINT)) AS properties, "
+ + "NAMED_STRUCT('value', CAST(1 AS DECIMAL(18,4))) AS record, "
+ + "CAST(1 AS LARGEINT) AS large_value");
+ Field map = result.getFields().get(0);
+ Assertions.assertEquals(new ArrowType.Map(false), map.getType());
+ Field entries = map.getChildren().get(0);
+ Assertions.assertFalse(entries.isNullable());
+ Assertions.assertFalse(entries.getChildren().get(0).isNullable());
+ Assertions.assertEquals(new ArrowType.Utf8(), entries.getChildren().get(1).getType());
+ Assertions.assertEquals(new ArrowType.Decimal(18, 4, 128),
+ result.getFields().get(1).getChildren().get(0).getType());
+ Assertions.assertEquals("LARGEINT", result.getFields().get(2).getMetadata().get("doris_type"));
+ }
+
+ @Test
+ void informationSchemaAndUnevaluatedExpression() {
+ Schema catalog = schema("SELECT TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME "
+ + "FROM information_schema.tables WHERE 1=0");
+ Assertions.assertEquals(Arrays.asList("TABLE_CATALOG", "TABLE_SCHEMA", "TABLE_NAME"),
+ catalog.getFields().stream().map(Field::getName).collect(Collectors.toList()));
+ // A schema-only request must never construct the executor that could evaluate SLEEP or launch fragments.
+ try (MockedConstruction<StmtExecutor> executors =
+ Mockito.mockConstruction(StmtExecutor.class)) {
+ Assertions.assertEquals(new ArrowType.Bool(), schema("SELECT sleep(30) AS sleeping")
+ .getFields().get(0).getType());
+ Assertions.assertTrue(executors.constructed().isEmpty());
+ }
+ }
+
+ @Test
+ void aggregateAndCteSchemas() {
+ Assertions.assertEquals(Arrays.asList("total", "count_value"),
+ schema("SELECT SUM(id) AS total, COUNT(*) AS count_value FROM schema_input")
+ .getFields().stream().map(Field::getName).collect(Collectors.toList()));
+ Assertions.assertEquals("id", schema("WITH source AS (SELECT id FROM schema_input) SELECT * FROM source")
+ .getFields().get(0).getName());
+ }
+
+ @Test
+ void showTableLabelsUseResolvedDatabase() {
+ Assertions.assertEquals("Tables_in_schema_test", schema("SHOW TABLES").getFields().get(0).getName());
+ }
+
+ @Test
+ void schemaConvertsDialectBeforeParsing() {
+ String query = "dialect_only_query";
+ try (MockedStatic<SqlDialectHelper> converter = Mockito.mockStatic(SqlDialectHelper.class)) {
+ converter.when(() -> SqlDialectHelper.convertSqlByDialect(Mockito.eq(query), Mockito.any()))
+ .thenReturn("SELECT id FROM schema_input");
+ Assertions.assertEquals("id", schema(query).getFields().get(0).getName());
+ converter.verify(() -> SqlDialectHelper.convertSqlByDialect(Mockito.eq(query), Mockito.any()));
+ }
+ }
+
+ @Test
+ void switchUsesRememberedDatabaseWithoutChangingSession() throws Exception {
+ String database = connectContext.getDatabase();
+ String remembered = connectContext.getLastDBOfCatalog("internal");
+ try {
+ connectContext.addLastDBOfCatalog("internal", "information_schema");
+ Assertions.assertEquals("id", preparedSchema(
+ "USE schema_catalog.db1; SWITCH internal; SELECT id FROM schema_input")
+ .getFields().get(0).getName());
+ connectContext.addLastDBOfCatalog("internal", "schema_test");
+ connectContext.changeDefaultCatalog("schema_catalog");
+ connectContext.setDatabase("db1");
+ Assertions.assertEquals("id", preparedSchema("SWITCH internal; SELECT id FROM schema_input")
+ .getFields().get(0).getName());
+ Assertions.assertEquals("schema_catalog", connectContext.getDefaultCatalog());
+ Assertions.assertEquals("db1", connectContext.getDatabase());
+ Assertions.assertEquals("schema_test", connectContext.getLastDBOfCatalog("internal"));
+ } finally {
+ connectContext.changeDefaultCatalog("internal");
+ connectContext.setDatabase(database);
+ connectContext.addLastDBOfCatalog("internal", remembered == null ? "" : remembered);
+ }
+ }
+
+ @Test
+ void commandsWithUnknownOrDynamicMetadataAreNotOkResults() {
+ for (String query : Arrays.asList("SHOW PYTHON PACKAGES IN '3.11'", "DESC schema_input ALL",
+ "SHOW DATA FROM schema_input", "SHOW PARTITIONS FROM schema_input", "SHOW QUERY STATS",
+ "COPY INTO schema_input FROM @~", "WARM UP CLUSTER target WITH CLUSTER source")) {
+ FlightRuntimeException error = Assertions.assertThrows(FlightRuntimeException.class,
+ () -> schema(query), query);
+ Assertions.assertEquals(FlightStatusCode.UNIMPLEMENTED, error.status().code(), query);
+ java.util.concurrent.ExecutionException prepareError = Assertions.assertThrows(
+ java.util.concurrent.ExecutionException.class, () -> prepare(query), query);
+ Assertions.assertEquals(FlightStatusCode.UNIMPLEMENTED,
+ ((FlightRuntimeException) prepareError.getCause()).status().code(), query);
+ }
+ Assertions.assertEquals(6, schema("SHOW FRONTEND CONFIG").getFields().size());
+ }
+
+ @Test
+ void dynamicShowCreateAndProcCanBePrepared() throws Exception {
+ Assertions.assertEquals(Arrays.asList("Table", "Create Table"),
+ preparedSchema("SHOW CREATE TABLE schema_input").getFields().stream()
+ .map(Field::getName).collect(Collectors.toList()));
+ Assertions.assertEquals(schema("SHOW PROC '/'"), preparedSchema("SHOW PROC '/'"));
+ Assertions.assertFalse(schema("SHOW PROC '/'").getFields().isEmpty());
+ }
+
+ @Test
+ void aggregateStateUsesSerializedStringType() throws Exception {
+ String query = "SELECT group_concat_state('x') AS s";
+ Assertions.assertEquals(new ArrowType.Utf8(), schema(query).getFields().get(0).getType());
+ Assertions.assertEquals(schema(query), preparedSchema(query));
+ }
+
+ @Test
+ void preparedHandleCannotChangeNamespace() throws Exception {
+ ActionCreatePreparedStatementResult result = prepare("SELECT id FROM schema_input");
+ CommandPreparedStatementQuery command = CommandPreparedStatementQuery.newBuilder()
+ .setPreparedStatementHandle(result.getPreparedStatementHandle()).build();
+ String database = connectContext.getDatabase();
+ connectContext.setDatabase("information_schema");
+ try {
+ FlightRuntimeException error = Assertions.assertThrows(FlightRuntimeException.class,
+ () -> producer.getSchemaPreparedStatement(command, callContext, FlightDescriptor.command(new byte[0])));
+ Assertions.assertEquals(FlightStatusCode.NOT_FOUND, error.status().code());
+ error = Assertions.assertThrows(FlightRuntimeException.class,
+ () -> producer.getFlightInfoPreparedStatement(command, callContext,
+ FlightDescriptor.command(new byte[0])));
+ Assertions.assertEquals(FlightStatusCode.NOT_FOUND, error.status().code());
+ } finally {
+ connectContext.setDatabase(database);
+ String handle = result.getPreparedStatementHandle().toStringUtf8();
+ connectContext.removePreparedQuery(handle.substring(handle.indexOf(':') + 1));
+ }
+ }
+
+ @Test
+ void executionCannotMutateSchemaAnalysisSessionClone() throws Exception {
+ SessionVariable original = connectContext.getSessionVariable();
+ int timeout = original.getQueryTimeoutS();
+ CountDownLatch cloned = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ CountDownLatch executing = new CountDownLatch(1);
+ ExecutorService workers = Executors.newFixedThreadPool(2);
+ Mockito.doAnswer(invocation -> {
+ invocation.callRealMethod();
+ if (invocation.getArgument(0) != original) {
+ cloned.countDown();
+ Assertions.assertTrue(release.await(10, java.util.concurrent.TimeUnit.SECONDS));
+ }
+ return null;
+ }).when(connectContext).setSessionVariable(Mockito.any());
+ try {
+ Future<?> analysis = workers.submit(() -> schema("SELECT 1"));
+ Assertions.assertTrue(cloned.await(10, java.util.concurrent.TimeUnit.SECONDS));
+ Future<?> execution = workers.submit(() -> {
+ try (MockedConstruction<FlightSqlConnectProcessor> processors = Mockito.mockConstruction(
+ FlightSqlConnectProcessor.class, (processor, context) -> {
+ Mockito.doAnswer(invocation -> {
+ connectContext.getSessionVariable().setQueryTimeoutS(17);
+ connectContext.setReturnResultFromLocal(true);
+ return null;
+ }).when(processor).handleQuery(Mockito.anyString());
+ })) {
+ executing.countDown();
+ producer.getFlightInfoStatement(CommandStatementQuery.newBuilder()
+ .setQuery("SET query_timeout=17").build(), callContext,
+ FlightDescriptor.command(new byte[0]));
+ }
+ });
+ Assertions.assertTrue(executing.await(10, java.util.concurrent.TimeUnit.SECONDS));
+ // Execution must wait until schema analysis restores the real session variables.
+ Assertions.assertThrows(TimeoutException.class,
+ () -> execution.get(200, java.util.concurrent.TimeUnit.MILLISECONDS));
+ release.countDown();
+ analysis.get(10, java.util.concurrent.TimeUnit.SECONDS);
+ execution.get(10, java.util.concurrent.TimeUnit.SECONDS);
+ Assertions.assertSame(original, connectContext.getSessionVariable());
+ Assertions.assertEquals(17, original.getQueryTimeoutS());
+ } finally {
+ release.countDown();
+ workers.shutdown();
+ Assertions.assertTrue(workers.awaitTermination(10, java.util.concurrent.TimeUnit.SECONDS));
+ Mockito.doCallRealMethod().when(connectContext).setSessionVariable(Mockito.any());
+ original.setQueryTimeoutS(timeout);
+ connectContext.getFlightSqlChannel().reset();
+ }
+ }
+
+ @Test
+ void preparedSchemaCannotDriftWithSqlMode() throws Exception {
+ long mode = connectContext.getSessionVariable().getSqlMode();
+ try {
+ connectContext.getSessionVariable().setSqlMode(0);
+ for (boolean fetchSchema : Arrays.asList(true, false)) {
+ ActionCreatePreparedStatementResult result = prepare("SELECT 1 || 2 AS x");
+ CommandPreparedStatementQuery command = CommandPreparedStatementQuery.newBuilder()
+ .setPreparedStatementHandle(result.getPreparedStatementHandle()).build();
+ connectContext.getSessionVariable().setSqlMode(SqlModeHelper.MODE_PIPES_AS_CONCAT);
+ Assertions.assertEquals(new ArrowType.Utf8(), schema("SELECT 1 || 2 AS x")
+ .getFields().get(0).getType());
+ FlightRuntimeException error = Assertions.assertThrows(FlightRuntimeException.class, () -> {
+ if (fetchSchema) {
+ producer.getSchemaPreparedStatement(command, callContext, FlightDescriptor.command(new byte[0]));
+ } else {
+ producer.getFlightInfoPreparedStatement(command, callContext,
+ FlightDescriptor.command(new byte[0]));
+ }
+ });
+ Assertions.assertEquals(FlightStatusCode.NOT_FOUND, error.status().code());
+ String handle = result.getPreparedStatementHandle().toStringUtf8();
+ Assertions.assertNull(connectContext.getPreparedQuery(handle.substring(handle.indexOf(':') + 1)));
+ connectContext.getSessionVariable().setSqlMode(0);
+ }
+ } finally {
+ connectContext.getSessionVariable().setSqlMode(mode);
+ }
+ }
+
+ @Test
+ void preparedHandleSurvivesSettingsThatDoNotChangeSchema() throws Exception {
+ int timeout = connectContext.getSessionVariable().getQueryTimeoutS();
+ ActionCreatePreparedStatementResult result = prepare("SELECT 1 AS x");
+ CommandPreparedStatementQuery command = CommandPreparedStatementQuery.newBuilder()
+ .setPreparedStatementHandle(result.getPreparedStatementHandle()).build();
+ try {
+ connectContext.getSessionVariable().setQueryTimeoutS(timeout + 1);
+ Assertions.assertEquals(schema("SELECT 1 AS x"), producer.getSchemaPreparedStatement(
+ command, callContext, FlightDescriptor.command(new byte[0])).getSchema());
+ } finally {
+ connectContext.getSessionVariable().setQueryTimeoutS(timeout);
+ String handle = result.getPreparedStatementHandle().toStringUtf8();
+ connectContext.removePreparedQuery(handle.substring(handle.indexOf(':') + 1));
+ }
+ }
+
+ @Test
+ void dialectParseFailureRetriesOriginalOnlyWhenEnabled() {
+ SessionVariable session = connectContext.getSessionVariable();
+ boolean retry = session.retryOriginSqlOnConvertFail;
+ String query = "SELECT id FROM schema_input";
+ try (MockedStatic<SqlDialectHelper> converter = Mockito.mockStatic(SqlDialectHelper.class)) {
+ converter.when(() -> SqlDialectHelper.convertSqlByDialect(Mockito.eq(query), Mockito.any()))
+ .thenReturn("SELECT FROM");
+ session.retryOriginSqlOnConvertFail = true;
+ Assertions.assertEquals("id", schema(query).getFields().get(0).getName());
+ Assertions.assertSame(session, connectContext.getSessionVariable());
+ session.retryOriginSqlOnConvertFail = false;
+ Assertions.assertEquals(FlightStatusCode.INVALID_ARGUMENT,
+ Assertions.assertThrows(FlightRuntimeException.class, () -> schema(query)).status().code());
+ } finally {
+ session.retryOriginSqlOnConvertFail = retry;
+ }
+ }
+
+ @Test
+ void procSchemaDoesNotFetchRowsOrTraverseRemoteNodes() throws Exception {
+ ProcService service = Mockito.mock(ProcService.class);
+ ProcNodeInterface node = Mockito.mock(ProcNodeInterface.class);
+ Mockito.when(service.open(Mockito.anyString())).thenReturn(node);
+ Mockito.when(node.fetchResult()).thenThrow(new AssertionError("Schema discovery fetched PROC rows"));
+ try (MockedStatic<ProcService> services = Mockito.mockStatic(ProcService.class)) {
+ services.when(ProcService::getInstance).thenReturn(service);
+ Assertions.assertEquals("name", schema("SHOW PROC '/'").getFields().get(0).getName());
+ Assertions.assertEquals("QueryId", schema("SHOW PROC '/current_queries'")
+ .getFields().get(0).getName());
+ Assertions.assertEquals(FlightStatusCode.UNIMPLEMENTED, Assertions.assertThrows(
+ FlightRuntimeException.class, () -> schema("SHOW PROC '/dbs/1/2/index_schema/3'"))
+ .status().code());
+ Mockito.verifyNoInteractions(service, node);
+ }
+ }
+
+ @Test
+ void lanceIndexSchemaIsJobIdWithoutAdmission() throws Exception {
+ LanceExternalCatalog catalog = Mockito.mock(LanceExternalCatalog.class);
+ Mockito.doReturn(catalog).when(connectContext).getCatalog("schema_lance");
+ for (String query : Arrays.asList(
+ "CREATE INDEX idx ON schema_lance.db1.source (v) USING ANN",
+ "CREATE INDEX IF NOT EXISTS idx ON schema_lance.db1.source (v) USING ANN",
+ "DROP INDEX IF EXISTS idx ON schema_lance.db1.source")) {
+ Assertions.assertEquals("JobId", preparedSchema(query).getFields().get(0).getName(), query);
+ Assertions.assertEquals(schema(query), preparedSchema(query));
+ }
+ Mockito.verifyNoInteractions(catalog);
+ Assertions.assertEquals("StatusResult", schema("CREATE INDEX idx ON schema_input (name) USING INVERTED")
+ .getFields().get(0).getName());
+ }
+
+ @Test
+ void procedureCommandsExposeFixedHeaders() throws Exception {
+ Assertions.assertEquals(Arrays.asList("Procedure", "Create Procedure"),
+ preparedSchema("SHOW CREATE PROCEDURE schema_proc").getFields().stream()
+ .map(Field::getName).collect(Collectors.toList()));
+ Assertions.assertEquals(Arrays.asList("ProcedureName", "CatalogId", "DbId", "DbName", "PackageName",
+ "OwnerName", "CreateTime", "ModifyTime"), preparedSchema("SHOW PROCEDURE STATUS")
+ .getFields().stream().map(Field::getName).collect(Collectors.toList()));
+ }
+
+ @Test
+ void noRowCommandsRemainPreparableWithoutExecution() throws Exception {
+ try (MockedConstruction<StmtExecutor> executors = Mockito.mockConstruction(StmtExecutor.class)) {
+ for (String query : Arrays.asList("INSERT INTO schema_input VALUES (1, 'x')",
+ "INSERT OVERWRITE TABLE schema_input SELECT * FROM schema_input",
+ "UPDATE schema_input SET name='x' WHERE id=1", "DELETE FROM schema_input WHERE id=1",
+ "MERGE INTO schema_input t USING schema_input s ON t.id=s.id WHEN MATCHED THEN DELETE",
+ "KILL QUERY 123", "BEGIN", "COMMIT", "ROLLBACK")) {
+ Assertions.assertEquals("StatusResult", preparedSchema(query).getFields().get(0).getName(), query);
+ Assertions.assertEquals(schema(query), preparedSchema(query));
+ }
+ Assertions.assertTrue(executors.constructed().isEmpty());
+ }
+ }
+
+ @Test
+ void warmupSelectHasStatisticsInsteadOfOkHeader() throws Exception {
+ String query = "WARM UP SELECT * FROM schema_input";
+ Assertions.assertEquals(Arrays.asList("BackendId", "ScanRows", "ScanBytes", "ScanBytesFromLocalStorage",
+ "ScanBytesFromRemoteStorage", "BytesWriteIntoCache"), preparedSchema(query).getFields().stream()
+ .map(Field::getName).collect(Collectors.toList()));
+ Assertions.assertEquals(schema(query), preparedSchema(query));
+ }
+
+ @Test
+ void explainAndReplayExposeHeadersWithoutRunningPlanner() throws Exception {
+ Assertions.assertEquals("Explain String(Nereids Planner)",
+ preparedSchema("EXPLAIN SELECT 1").getFields().get(0).getName());
+ Assertions.assertEquals("Plan Replayer dump url",
+ preparedSchema("PLAN REPLAYER DUMP SELECT 1").getFields().get(0).getName());
+ // PLAN PROCESS currently has no Flight result serialization, unlike ordinary EXPLAIN.
+ Assertions.assertEquals(FlightStatusCode.UNIMPLEMENTED, Assertions.assertThrows(
+ FlightRuntimeException.class, () -> schema("EXPLAIN PLAN PROCESS SELECT 1")).status().code());
+ }
+
+ @Test
+ void preparedNamespaceTransitionIsReusable() throws Exception {
+ String database = connectContext.getDatabase();
+ try {
+ for (String query : Arrays.asList("USE information_schema",
+ "USE information_schema; SELECT TABLE_NAME FROM tables WHERE 1=0")) {
+ connectContext.setDatabase(database);
+ Schema expected = schema(query);
+ ActionCreatePreparedStatementResult result = prepare(query);
+ CommandPreparedStatementQuery command = CommandPreparedStatementQuery.newBuilder()
+ .setPreparedStatementHandle(result.getPreparedStatementHandle()).build();
+ try (MockedConstruction<FlightSqlConnectProcessor> processors = Mockito.mockConstruction(
+ FlightSqlConnectProcessor.class, (processor, context) -> {
+ Mockito.doAnswer(invocation -> {
+ connectContext.setDatabase("information_schema");
+ connectContext.setReturnResultFromLocal(false);
+ return null;
+ }).when(processor).handleQuery(Mockito.anyString());
+ Mockito.when(processor.getArrowSchema()).thenReturn(expected);
+ })) {
+ for (int i = 0; i < 2; ++i) {
+ Assertions.assertEquals(expected, producer.getFlightInfoPreparedStatement(command,
+ callContext, FlightDescriptor.command(new byte[0])).getSchema());
+ Assertions.assertEquals(expected, producer.getSchemaPreparedStatement(command,
+ callContext, FlightDescriptor.command(new byte[0])).getSchema());
+ }
+ } finally {
+ String handle = result.getPreparedStatementHandle().toStringUtf8();
+ connectContext.removePreparedQuery(handle.substring(handle.indexOf(':') + 1));
+ }
+ }
+ } finally {
+ connectContext.setDatabase(database);
+ }
+ }
+
+ @Test
+ void executionSchemaDriftExpiresHandleAndCleansResults() throws Exception {
+ Schema changed = new Schema(Arrays.asList(Field.nullable("id", new ArrowType.Int(64, true)),
+ Field.nullable("new_column", new ArrowType.Utf8())));
+ StmtExecutor previous = connectContext.getExecutor();
+ try {
+ for (boolean remote : Arrays.asList(true, false)) {
+ ActionCreatePreparedStatementResult result = prepare("SELECT id FROM schema_input");
+ CommandPreparedStatementQuery command = CommandPreparedStatementQuery.newBuilder()
+ .setPreparedStatementHandle(result.getPreparedStatementHandle()).build();
+ StmtExecutor executor = Mockito.mock(StmtExecutor.class);
+ FlightSqlConnectProcessor cleanup = new FlightSqlConnectProcessor(connectContext);
+ try (MockedConstruction<FlightSqlConnectProcessor> processors = Mockito.mockConstruction(
+ FlightSqlConnectProcessor.class, (processor, context) -> {
+ // Simulate replanning after another session changes the table, after schema reanalysis.
+ Mockito.doAnswer(invocation -> {
+ connectContext.setReturnResultFromLocal(!remote);
+ connectContext.setExecutor(executor);
+ connectContext.addFlightSqlDeferredExecutor(executor);
+ return null;
+ }).when(processor).handleQuery(Mockito.anyString());
+ Mockito.when(processor.getArrowSchema()).thenReturn(changed);
+ Mockito.doAnswer(invocation -> {
+ cleanup.close();
+ return null;
+ }).when(processor).close();
+ })) {
+ FlightRuntimeException error = Assertions.assertThrows(FlightRuntimeException.class,
+ () -> producer.getFlightInfoPreparedStatement(command, callContext,
+ FlightDescriptor.command(new byte[0])));
+ Assertions.assertEquals(FlightStatusCode.NOT_FOUND, error.status().code());
+ Assertions.assertNull(connectContext.getExecutor());
+ Mockito.verify(processors.constructed().get(0)).close();
+ InOrder cleanupOrder = Mockito.inOrder(executor);
+ cleanupOrder.verify(executor).cancel(Mockito.any(Status.class));
+ cleanupOrder.verify(executor).finalizeArrowFlightQuery();
+ Assertions.assertEquals(0, connectContext.getFlightSqlChannel().resultNum());
+ Assertions.assertTrue(connectContext.getFlightSqlEndpointsLocations().isEmpty());
+ String handle = result.getPreparedStatementHandle().toStringUtf8();
+ Assertions.assertNull(connectContext.getPreparedQuery(handle.substring(handle.indexOf(':') + 1)));
+ connectContext.closeFlightSqlDeferredExecutors();
+ Mockito.verify(executor, Mockito.times(1)).finalizeArrowFlightQuery();
+ } finally {
+ connectContext.closeFlightSqlDeferredExecutors();
+ connectContext.getFlightSqlChannel().reset();
+ String handle = result.getPreparedStatementHandle().toStringUtf8();
+ connectContext.removePreparedQuery(handle.substring(handle.indexOf(':') + 1));
+ }
+ }
+ } finally {
+ connectContext.setExecutor(previous);
+ }
+ }
+
+ @Test
+ void backendWireLabelsAndNarrowedNullabilityDoNotExpirePreparedQueries() throws Exception {
+ Schema actual = new Schema(Arrays.asList(
+ Field.notNullable("Int64_0", new ArrowType.Int(64, true)),
+ Field.notNullable("String_1", new ArrowType.Utf8())));
+ assertPreparedExecutionSchema("SELECT id, name FROM schema_input WHERE name IS NOT NULL",
+ Arrays.asList("id", "name"), actual, true);
+ }
+
+ @Test
+ void actualSemanticRenameAndNullableWideningStillExpirePreparedQueries() throws Exception {
+ Schema renamed = new Schema(Collections.singletonList(
+ Field.notNullable("Int64_0", new ArrowType.Int(64, true))));
+ assertPreparedExecutionSchema("SELECT id FROM schema_input", Collections.singletonList("renamed"),
+ renamed, false);
+ Schema widened = new Schema(Collections.singletonList(
+ Field.nullable("id", new ArrowType.Int(64, true))));
+ assertPreparedExecutionSchema("SELECT id FROM schema_input", Collections.singletonList("id"),
+ widened, false);
+ }
+
+ private void assertPreparedExecutionSchema(String query, List<String> labels, Schema actual, boolean compatible)
+ throws Exception {
+ StmtExecutor previous = connectContext.getExecutor();
+ ActionCreatePreparedStatementResult result = prepare(query);
+ CommandPreparedStatementQuery command = CommandPreparedStatementQuery.newBuilder()
+ .setPreparedStatementHandle(result.getPreparedStatementHandle()).build();
+ StmtExecutor executor = Mockito.mock(StmtExecutor.class);
+ LogicalPlanAdapter statement = Mockito.mock(LogicalPlanAdapter.class);
+ Mockito.when(statement.getColLabels()).thenReturn(new ArrayList<>(labels));
+ Mockito.when(executor.getParsedStmt()).thenReturn(statement);
+ FlightSqlConnectProcessor cleanup = new FlightSqlConnectProcessor(connectContext);
+ try (MockedConstruction<FlightSqlConnectProcessor> processors = Mockito.mockConstruction(
+ FlightSqlConnectProcessor.class, (processor, context) -> {
+ Mockito.doAnswer(invocation -> {
+ connectContext.setExecutor(executor);
+ connectContext.addFlightSqlDeferredExecutor(executor);
+ connectContext.setReturnResultFromLocal(false);
+ return null;
+ }).when(processor).handleQuery(Mockito.anyString());
+ Mockito.when(processor.getArrowSchema()).thenReturn(actual);
+ // Exercise real processor cleanup: it clears the context's executor before validation.
+ Mockito.doAnswer(invocation -> {
+ cleanup.close();
+ return null;
+ }).when(processor).close();
+ })) {
+ if (compatible) {
+ Assertions.assertEquals(actual, producer.getFlightInfoPreparedStatement(command,
+ callContext, FlightDescriptor.command(new byte[0])).getSchema());
+ Assertions.assertEquals(schema(query), producer.getSchemaPreparedStatement(command,
+ callContext, FlightDescriptor.command(new byte[0])).getSchema());
+ Mockito.verify(executor, Mockito.never()).cancel(Mockito.any(Status.class));
+ Mockito.verify(executor, Mockito.never()).finalizeArrowFlightQuery();
+ } else {
+ FlightRuntimeException error = Assertions.assertThrows(FlightRuntimeException.class,
+ () -> producer.getFlightInfoPreparedStatement(command, callContext,
+ FlightDescriptor.command(new byte[0])));
+ Assertions.assertEquals(FlightStatusCode.NOT_FOUND, error.status().code());
+ InOrder cleanupOrder = Mockito.inOrder(executor);
+ cleanupOrder.verify(executor).cancel(Mockito.any(Status.class));
+ cleanupOrder.verify(executor).finalizeArrowFlightQuery();
+ }
+ Assertions.assertNull(connectContext.getExecutor());
+ Mockito.verify(processors.constructed().get(0)).close();
+ } finally {
+ connectContext.closeFlightSqlDeferredExecutors();
+ connectContext.setExecutor(previous);
+ String handle = result.getPreparedStatementHandle().toStringUtf8();
+ connectContext.removePreparedQuery(handle.substring(handle.indexOf(':') + 1));
+ }
+ }
+
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlExecutionSchemaTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlExecutionSchemaTest.java
new file mode 100644
index 00000000000..bebf43f8189
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlExecutionSchemaTest.java
@@ -0,0 +1,178 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service.arrowflight;
+
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.FeConstants;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.qe.QueryState.MysqlStateType;
+import org.apache.doris.service.arrowflight.sessions.FlightSqlConnectContext;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
+import org.apache.doris.utframe.TestWithFeService;
+
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+class FlightSqlExecutionSchemaTest extends TestWithFeService {
+ private FlightSqlConnectContext flightContext;
+ private boolean previousUnitTest;
+
+ @Override
+ protected void runBeforeAll() throws Exception {
+ previousUnitTest = FeConstants.runningUnitTest;
+ FeConstants.runningUnitTest = true;
+ createDatabase("execution_schema_test");
+ connectContext.setDatabase("execution_schema_test");
+ createTable("CREATE TABLE schema_input (id INT NOT NULL, encoded STRING NULL) "
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES ('replication_num' = '1')");
+ flightContext = new FlightSqlConnectContext("schema-peer");
+ flightContext.setEnv(connectContext.getEnv());
+ flightContext.setCurrentUserIdentity(connectContext.getCurrentUserIdentity());
+ flightContext.setSessionVariable(connectContext.getSessionVariable());
+ flightContext.setDatabase("execution_schema_test");
+ }
+
+ @Override
+ protected void runAfterAll() throws Exception {
+ FeConstants.runningUnitTest = previousUnitTest;
+ flightContext.getFlightSqlChannel().close();
+ }
+
+ @Test
+ void unaliasedLiteralRetainsExecutionLabel() throws Exception {
+ assertExecutionSchema("SELECT 1", false, false);
+ assertExecutionSchema("SELECT 1 AS id", false, false);
+ }
+
+ @Test
+ void unaliasedFunctionRetainsExecutionLabel() throws Exception {
+ assertExecutionSchema("SELECT unhex(encoded) FROM schema_input WHERE id <= 4 ORDER BY id", false, false);
+ }
+
+ @Test
+ void constantFoldingNarrowsNullableCast() throws Exception {
+ assertExecutionSchema("SELECT CAST(CAST('2026-01-01' AS DATETIME(6)) AS STRING)", false, true);
+ }
+
+ @Test
+ void nullableColumnRemainsCompatibleWithFilter() throws Exception {
+ assertExecutionSchema("SELECT encoded FROM schema_input WHERE encoded IS NOT NULL", false, false);
+ }
+
+ @Test
+ void tableValuedFunctionUsesQualifiedWireLabel() throws Exception {
+ Schema actual = assertExecutionSchema("SELECT * FROM numbers('number' = '1')", true, false);
+ Assertions.assertTrue(actual.getFields().get(0).getName().startsWith("_tvf_numbers."));
+ }
+
+ private Schema assertExecutionSchema(String sql, boolean differentLabel, boolean narrowedNullability)
+ throws Exception {
+ flightContext.setThreadLocalInfo();
+ try (FlightSqlConnectProcessor processor = new FlightSqlConnectProcessor(flightContext)) {
+ Schema prepared = FlightSqlQuerySchema.analyze(flightContext, sql);
+ // Execution marks the query state before binding output labels. A bare planner call
+ // misses that transition and can reproduce Prepare's incorrect command-style aliases.
+ processor.handleQuery(sql);
+ Assertions.assertNotEquals(MysqlStateType.ERR, flightContext.getState().getStateType(),
+ flightContext.getState().getErrorMessage());
+ Assertions.assertFalse(flightContext.isReturnResultFromLocal());
+ Assertions.assertNotNull(flightContext.getExecutor());
+ LogicalPlanAdapter statement = (LogicalPlanAdapter) flightContext.getExecutor().getParsedStmt();
+ List<Field> wireFields = new ArrayList<>();
+ for (Expr expression : statement.getResultExprs()) {
+ TExprNode node = expression.treeToThrift().getNodes().get(0);
+ Assertions.assertEquals(TExprNodeType.SLOT_REF, node.getNodeType());
+ ScalarType type = (ScalarType) expression.getType();
+ ArrowType arrowType = FlightSqlSchemaHelper.getArrowType(type.getPrimitiveType(),
+ type.getScalarPrecision(), type.getScalarScale());
+ // BE derives these fields from the serialized output expressions, not analyzed Slot names.
+ wireFields.add(new Field(node.getLabel(),
+ new FieldType(node.isIsNullable(), arrowType, null), Collections.emptyList()));
+ }
+ Schema actual = new Schema(wireFields);
+ Assertions.assertEquals(differentLabel,
+ !prepared.getFields().get(0).getName().equals(actual.getFields().get(0).getName()));
+ Assertions.assertEquals(narrowedNullability,
+ prepared.getFields().get(0).isNullable() && !actual.getFields().get(0).isNullable());
+ Assertions.assertTrue(FlightSqlQuerySchema.matchesExecutionSchema(
+ prepared, actual, statement.getColLabels()));
+ return actual;
+ } finally {
+ flightContext.closeFlightSqlDeferredExecutors();
+ flightContext.clearFlightSqlEndpointsLocations();
+ flightContext.getFlightSqlChannel().reset();
+ connectContext.setThreadLocalInfo();
+ }
+ }
+
+ @Test
+ void sameArityRenameIsRejectedDespiteIdenticalWireTypes() {
+ Schema schema = schema(Field.nullable("old_name", new ArrowType.Utf8()));
+ Assertions.assertFalse(FlightSqlQuerySchema.matchesExecutionSchema(schema, schema,
+ Collections.singletonList("new_name")));
+ }
+
+ @Test
+ void differentTypesAreRejected() {
+ Schema prepared = schema(Field.nullable("value", new ArrowType.Int(32, true)));
+ Schema actual = schema(Field.nullable("value", new ArrowType.Int(64, true)));
+ Assertions.assertFalse(FlightSqlQuerySchema.matchesExecutionSchema(prepared, actual,
+ Collections.singletonList("value")));
+ }
+
+ @Test
+ void nestedFieldNamesAreNotTransportLabels() {
+ Schema prepared = schema(new Field("value", FieldType.nullable(new ArrowType.Struct()),
+ Collections.singletonList(Field.nullable("old_name", new ArrowType.Utf8()))));
+ Schema actual = schema(new Field("value", FieldType.nullable(new ArrowType.Struct()),
+ Collections.singletonList(Field.nullable("new_name", new ArrowType.Utf8()))));
+ Assertions.assertFalse(FlightSqlQuerySchema.matchesExecutionSchema(prepared, actual,
+ Collections.singletonList("value")));
+ }
+
+ @Test
+ void typeMetadataChangesAreRejected() {
+ Schema prepared = schema(new Field("value", new FieldType(true, new ArrowType.Utf8(), null,
+ Collections.singletonMap("doris_type", "LARGEINT")), Collections.emptyList()));
+ Schema actual = schema(Field.nullable("value", new ArrowType.Utf8()));
+ Assertions.assertFalse(FlightSqlQuerySchema.matchesExecutionSchema(prepared, actual,
+ Collections.singletonList("value")));
+ }
+
+ @Test
+ void nullableWideningIsRejected() {
+ Schema prepared = schema(Field.notNullable("value", new ArrowType.Utf8()));
+ Schema actual = schema(Field.nullable("value", new ArrowType.Utf8()));
+ Assertions.assertFalse(FlightSqlQuerySchema.matchesExecutionSchema(prepared, actual,
+ Collections.singletonList("value")));
+ }
+
+ private static Schema schema(Field field) {
+ return new Schema(Collections.singletonList(field));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java
index 147a52aa55e..94e182bc8d5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.service.arrowflight;
+import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.thrift.TColumnDesc;
import org.apache.doris.thrift.TPrimitiveType;
@@ -26,6 +27,7 @@ import org.apache.arrow.vector.complex.MapVector;
import org.apache.arrow.vector.ipc.ReadChannel;
import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.apache.arrow.vector.types.DateUnit;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
@@ -206,4 +208,13 @@ public class FlightSqlSchemaHelperArrowTypeTest {
Assertions.assertEquals(ArrowType.ArrowTypeID.List, array.getType().getTypeID());
Assertions.assertEquals(new ArrowType.Int(32, true), array.getChildren().get(0).getType());
}
+
+ @Test
+ public void timeAndBinaryResultTypes() {
+ Assertions.assertEquals(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE),
+ FlightSqlSchemaHelper.getArrowType(PrimitiveType.TIMEV2, 0, 6));
+ Assertions.assertEquals(new ArrowType.Binary(),
+ FlightSqlSchemaHelper.getArrowType(PrimitiveType.VARBINARY, 0, 0));
+ }
+
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlTableTypesTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlTableTypesTest.java
new file mode 100644
index 00000000000..b504ff9c33f
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlTableTypesTest.java
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service.arrowflight;
+
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager;
+
+import org.apache.arrow.flight.FlightClient;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightProducer.CallContext;
+import org.apache.arrow.flight.FlightProducer.ServerStreamListener;
+import org.apache.arrow.flight.FlightRuntimeException;
+import org.apache.arrow.flight.FlightServer;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.Location;
+import org.apache.arrow.flight.sql.FlightSqlClient;
+import org.apache.arrow.flight.sql.FlightSqlProducer.Schemas;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+class FlightSqlTableTypesTest {
+ @Test
+ void tableTypesAreSortedAndStableThroughFlightRpc() throws Exception {
+ FlightSessionsManager sessions = Mockito.mock(FlightSessionsManager.class);
+ ConnectContext context = Mockito.mock(ConnectContext.class);
+ Mockito.when(sessions.getConnectContext(Mockito.anyString())).thenReturn(context);
+ Location location = Location.forGrpcInsecure("127.0.0.1", 0);
+ try (DorisFlightSqlProducer producer = new DorisFlightSqlProducer(location, sessions);
+ RootAllocator allocator = new RootAllocator();
+ FlightServer server = FlightServer.builder(allocator, location, producer).build().start();
+ FlightClient client = FlightClient.builder(allocator,
+ Location.forGrpcInsecure("127.0.0.1", server.getPort())).build()) {
+ FlightSqlClient sql = new FlightSqlClient(client);
+ for (int iteration = 0; iteration < 3; ++iteration) {
+ FlightInfo info = sql.getTableTypes();
+ Assertions.assertEquals(Schemas.GET_TABLE_TYPES_SCHEMA, info.getSchema());
+ Assertions.assertEquals(1, info.getEndpoints().size());
+ List<String> types = new ArrayList<>();
+ try (FlightStream stream = client.getStream(info.getEndpoints().get(0).getTicket())) {
+ Assertions.assertEquals(info.getSchema(), stream.getSchema());
+ while (stream.next()) {
+ VectorSchemaRoot root = stream.getRoot();
+ for (int row = 0; row < root.getRowCount(); ++row) {
+ types.add(root.getVector("table_type").getObject(row).toString());
+ }
+ }
+ }
+ Assertions.assertEquals(Arrays.asList("BASE TABLE", "SYSTEM VIEW", "VIEW"), types);
+ }
+ // Metadata must not plan queries or enumerate catalogs, including on empty sessions.
+ Mockito.verify(sessions, Mockito.times(3)).getConnectContext(Mockito.anyString());
+ Mockito.verifyNoInteractions(context);
+ }
+ }
+
+ @Test
+ void streamFailureReleasesAllocatedVectors() throws Exception {
+ FlightSessionsManager sessions = Mockito.mock(FlightSessionsManager.class);
+ CallContext context = Mockito.mock(CallContext.class);
+ ServerStreamListener listener = Mockito.mock(ServerStreamListener.class);
+ Mockito.doThrow(new IllegalStateException("stream failed")).when(listener).putNext();
+ try (DorisFlightSqlProducer producer = new DorisFlightSqlProducer(
+ Location.forGrpcInsecure("127.0.0.1", 0), sessions)) {
+ Assertions.assertThrows(FlightRuntimeException.class, () -> producer.getStreamTableTypes(context, listener));
+ Mockito.verify(listener).error(Mockito.any(FlightRuntimeException.class));
+ Mockito.verify(listener, Mockito.never()).completed();
+ }
+ }
+}
diff --git a/regression-test/suites/arrow_flight_sql_p0/test_schema_only.groovy b/regression-test/suites/arrow_flight_sql_p0/test_schema_only.groovy
new file mode 100644
index 00000000000..26bd75c2d94
--- /dev/null
+++ b/regression-test/suites/arrow_flight_sql_p0/test_schema_only.groovy
@@ -0,0 +1,288 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CallOption
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CallOptions
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CloseSessionRequest
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightClient
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightRuntimeException
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightStatusCode
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.Location
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.sql.FlightSqlClient
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.memory.RootAllocator
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.vector.types.TimeUnit
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.vector.types.pojo.ArrowType
+
+suite("test_schema_only", "arrow_flight_sql") {
+ def config = context.config.otherConfigs
+ def location = Location.forGrpcInsecure(config.get("extArrowFlightSqlHost"),
+ Integer.parseInt(config.get("extArrowFlightSqlPort")))
+ new RootAllocator().withCloseable { allocator ->
+ FlightClient.builder(allocator, location).build().withCloseable { flight ->
+ def token = flight.authenticateBasicToken(config.get("extArrowFlightSqlUser"),
+ config.get("extArrowFlightSqlPassword")).get()
+ CallOption[] options = [token, CallOptions.timeout(30, java.util.concurrent.TimeUnit.SECONDS)]
+ def client = new FlightSqlClient(flight)
+ String showTimeout = "SHOW VARIABLES LIKE 'query_timeout'"
+ def readVariable = { String name ->
+ def info = client.execute("SHOW VARIABLES LIKE '${name}'", options)
+ assertEquals(1, info.getEndpoints().size())
+ def values = []
+ flight.getStream(info.getEndpoints().get(0).getTicket(), options).withCloseable { stream ->
+ while (stream.next()) {
+ def root = stream.getRoot()
+ for (int i = 0; i < root.getRowCount(); ++i) {
+ values.add(root.getVector("Value").getObject(i).toString())
+ }
+ }
+ }
+ assertEquals(1, values.size())
+ return values[0]
+ }
+ def readTimeout = { readVariable("query_timeout") }
+ def prepareSchema = { String query ->
+ def prepared = client.prepare(query, options)
+ try {
+ // ADBC ExecuteSchema consumes this schema before executing or fetching FlightInfo.
+ def schema = prepared.getResultSetSchema()
+ assertTrue(schema.getFields()*.getName() != ["ResultMeta"])
+ assertEquals(schema, prepared.fetchSchema(options).getSchema())
+ return schema
+ } finally {
+ prepared.close(options)
+ }
+ }
+ try {
+ client.execute("USE information_schema", options)
+ // Prepare must bind unaliased expressions in query mode, as execution does.
+ // Comparing metadata-only calls misses synthetic aliases that execution rejects.
+ ["1", "unhex('616263')", "CAST(CAST('2026-01-01' AS DATETIME(6)) AS STRING)"].each { expression ->
+ def prepared = client.prepare("SELECT ${expression}", options)
+ try {
+ def advertised = prepared.getResultSetSchema()
+ assertEquals([expression], advertised.getFields()*.getName())
+ def executed = prepared.execute(options).getSchema()
+ assertEquals(advertised.getFields()*.getName(), executed.getFields()*.getName())
+ assertEquals(advertised.getFields()*.getType(), executed.getFields()*.getType())
+ assertEquals(advertised, prepared.fetchSchema(options).getSchema())
+ } finally {
+ prepared.close(options)
+ }
+ }
+ def cases = [
+ ["SELECT CAST(1 AS BIGINT) AS id, CAST(NULL AS VARCHAR(20)) AS name",
+ ["id", "name"], [new ArrowType.Int(64, true), new ArrowType.Utf8()]],
+ ["""SELECT CAST('2025-01-02 03:04:05.123456' AS DATETIME(6)) AS event_time,
+ ARRAY('first', 'second') AS tags,
+ MAP('key', CAST(1 AS BIGINT)) AS properties,
+ NAMED_STRUCT('amount', CAST(1 AS DECIMAL(18,4))) AS record""",
+ ["event_time", "tags", "properties", "record"],
+ [new ArrowType.Timestamp(TimeUnit.MICROSECOND, null), new ArrowType.List(),
+ new ArrowType.Map(false), new ArrowType.Struct()]],
+ ["SELECT TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME FROM information_schema.tables WHERE 1=0",
+ ["TABLE_CATALOG", "TABLE_SCHEMA", "TABLE_NAME"],
+ [new ArrowType.Utf8(), new ArrowType.Utf8(), new ArrowType.Utf8()]],
+ ["SELECT group_concat_state('x') AS state", ["state"], [new ArrowType.Utf8()]],
+ [showTimeout, ["Variable_name", "Value", "Default_Value", "Changed"],
+ [new ArrowType.Utf8(), new ArrowType.Utf8(), new ArrowType.Utf8(), new ArrowType.Utf8()]]
+ ]
+ [
+ ["SHOW PROCEDURE STATUS", ["ProcedureName", "CatalogId", "DbId", "DbName", "PackageName",
+ "OwnerName", "CreateTime", "ModifyTime"]],
+ ["SHOW CREATE PROCEDURE schema_proc", ["Procedure", "Create Procedure"]],
+ ["EXPLAIN SELECT 1", ["Explain String(Nereids Planner)"]],
+ ["PLAN REPLAYER DUMP SELECT 1", ["Plan Replayer dump url"]],
+ ["WARM UP SELECT * FROM schema_source", ["BackendId", "ScanRows", "ScanBytes",
+ "ScanBytesFromLocalStorage", "ScanBytesFromRemoteStorage", "BytesWriteIntoCache"]],
+ ["INSERT INTO schema_source VALUES (1)", ["StatusResult"]],
+ ["UPDATE schema_source SET id=1", ["StatusResult"]],
+ ["DELETE FROM schema_source WHERE id=1", ["StatusResult"]],
+ ["MERGE INTO schema_target t USING schema_source s ON t.id=s.id WHEN MATCHED THEN DELETE",
+ ["StatusResult"]],
+ ["KILL QUERY 123", ["StatusResult"]],
+ ["BEGIN", ["StatusResult"]], ["COMMIT", ["StatusResult"]], ["ROLLBACK", ["StatusResult"]]
+ ].each { entry ->
+ def schema = prepareSchema(entry[0])
+ assertEquals(entry[1], schema.getFields()*.getName())
+ assertEquals(schema, client.getExecuteSchema(entry[0], options).getSchema())
+ assertTrue(schema.getFields().every { it.getType() == new ArrowType.Utf8() })
+ }
+ // Re-parsing a prepared statement must not silently change its advertised result type.
+ String previousMode = readVariable("sql_mode")
+ try {
+ [true, false].each { fetchSchema ->
+ client.execute("SET sql_mode=''", options)
+ def prepared = client.prepare("SELECT 1 || 2 AS x", options)
+ try {
+ assertEquals(new ArrowType.Bool(), prepared.getResultSetSchema().getFields()[0].getType())
+ client.execute("SET sql_mode='PIPES_AS_CONCAT'", options)
+ try {
+ if (fetchSchema) {
+ prepared.fetchSchema(options)
+ } else {
+ prepared.execute(options)
+ }
+ assertTrue(false, "A changed result schema must expire the prepared handle")
+ } catch (FlightRuntimeException error) {
+ assertEquals(FlightStatusCode.NOT_FOUND, error.status().code())
+ }
+ } finally {
+ prepared.close(options)
+ }
+ }
+ } finally {
+ client.execute("SET sql_mode='${previousMode}'", options)
+ }
+ cases.eachWithIndex { entry, index ->
+ def schema = prepareSchema(entry[0])
+ assertEquals(entry[1], schema.getFields()*.getName())
+ assertEquals(entry[2], schema.getFields()*.getType())
+ assertEquals(schema, client.getExecuteSchema(entry[0], options).getSchema())
+ if (index == 0) {
+ assertTrue(!schema.getFields()[0].isNullable())
+ assertTrue(schema.getFields()[1].isNullable())
+ } else if (index == 1) {
+ assertEquals(new ArrowType.Utf8(), schema.getFields()[1].getChildren()[0].getType())
+ def entries = schema.getFields()[2].getChildren()[0]
+ assertTrue(!entries.isNullable())
+ assertTrue(!entries.getChildren()[0].isNullable())
+ assertEquals(new ArrowType.Int(64, true), entries.getChildren()[1].getType())
+ assertEquals(new ArrowType.Decimal(18, 4, 128),
+ schema.getFields()[3].getChildren()[0].getType())
+ }
+ }
+
+ // Resolving command metadata must not execute SET or leak query-scoped SET_VAR hints.
+ def originalTimeout = readTimeout()
+ def changedTimeout = originalTimeout == "17" ? "19" : "17"
+ def setQuery = "SET query_timeout=${changedTimeout}".toString()
+ assertEquals(["StatusResult"], prepareSchema(setQuery).getFields()*.getName())
+ client.getExecuteSchema(setQuery, options)
+ assertEquals(originalTimeout, readTimeout())
+ def hintedQuery = "SELECT /*+ SET_VAR(query_timeout=${changedTimeout}) */ 1 AS id".toString()
+ prepareSchema(hintedQuery)
+ client.getExecuteSchema(hintedQuery, options)
+ assertEquals(originalTimeout, readTimeout())
+
+ ["SELEC 1", "SELECT definitely_missing_schema_column"].each { query ->
+ [true, false].each { preparedRoute ->
+ FlightRuntimeException failure = null
+ try {
+ if (preparedRoute) {
+ prepareSchema(query)
+ } else {
+ client.getExecuteSchema(query, options)
+ }
+ } catch (FlightRuntimeException e) {
+ failure = e
+ }
+ assertTrue(failure != null, "Schema discovery accepted invalid SQL: ${query}")
+ assertEquals(FlightStatusCode.INVALID_ARGUMENT, failure.status().code())
+ assertEquals(["id", "name"], prepareSchema(cases[0][0]).getFields()*.getName())
+ assertEquals(originalTimeout, readTimeout())
+ }
+ }
+
+ assertEquals(6, prepareSchema("SHOW FRONTEND CONFIG").getFields().size())
+ assertEquals(prepareSchema("SHOW PROC '/'"), client.getExecuteSchema("SHOW PROC '/'", options).getSchema())
+ ["SHOW PYTHON PACKAGES IN '3.11'", "SHOW QUERY STATS"].each { query ->
+ [true, false].each { preparedRoute ->
+ FlightRuntimeException failure = null
+ try {
+ if (preparedRoute) {
+ prepareSchema(query)
+ } else {
+ client.getExecuteSchema(query, options)
+ }
+ } catch (FlightRuntimeException e) {
+ failure = e
+ }
+ assertTrue(failure != null)
+ assertEquals(FlightStatusCode.UNIMPLEMENTED, failure.status().code())
+ }
+ }
+
+ // Executing an explicit namespace change must not invalidate that same handle.
+ ["USE mysql", "USE mysql; SHOW VARIABLES LIKE 'query_timeout'"].each { query ->
+ client.execute("USE information_schema", options)
+ def changing = client.prepare(query, options)
+ try {
+ def expected = changing.getResultSetSchema()
+ 2.times {
+ def info = changing.execute(options)
+ assertEquals(expected, info.getSchema())
+ assertEquals(expected, changing.fetchSchema(options).getSchema())
+ flight.getStream(info.getEndpoints()[0].getTicket(), options).withCloseable { stream ->
+ while (stream.next()) {
+ assertEquals(expected, stream.getRoot().getSchema())
+ }
+ }
+ }
+ } finally {
+ changing.close(options)
+ }
+ }
+ def help = client.prepare("HELP 'schema_only_unknown_help_topic'", options)
+ try {
+ def expected = help.getResultSetSchema()
+ assertEquals(["name", "is_it_category"], expected.getFields()*.getName())
+ assertEquals(expected, help.fetchSchema(options).getSchema())
+ def info = help.execute(options)
+ assertEquals(expected, info.getSchema())
+ flight.getStream(info.getEndpoints()[0].getTicket(), options).withCloseable { stream ->
+ while (stream.next()) {
+ assertEquals(expected, stream.getRoot().getSchema())
+ }
+ }
+ } finally {
+ help.close(options)
+ }
+ assertEquals(["Snapshot", "Timestamp", "Status"],
+ prepareSchema("SHOW SNAPSHOT ON schema_repository").getFields()*.getName())
+ assertEquals(["Snapshot", "Timestamp", "Database", "Details", "Status"],
+ prepareSchema("SHOW SNAPSHOT ON schema_repository WHERE SNAPSHOT='s' AND TIMESTAMP='t'")
+ .getFields()*.getName())
+
+ client.execute("USE information_schema", options)
+ def prepared = client.prepare("SELECT TABLE_NAME FROM tables WHERE 1=0", options)
+ try {
+ client.execute("USE mysql", options)
+ // A prepared header must not silently describe a different database's table.
+ [true, false].each { schemaRoute ->
+ FlightRuntimeException failure = null
+ try {
+ if (schemaRoute) {
+ prepared.fetchSchema(options)
+ } else {
+ prepared.execute(options)
+ }
+ } catch (FlightRuntimeException e) {
+ failure = e
+ }
+ assertTrue(failure != null)
+ assertEquals(FlightStatusCode.NOT_FOUND, failure.status().code())
+ }
+ } finally {
+ prepared.close(options)
+ }
+
+ } finally {
+ client.closeSession(new CloseSessionRequest(), options)
+ }
+ }
+ }
+}
diff --git a/regression-test/suites/arrow_flight_sql_p0/test_table_types.groovy b/regression-test/suites/arrow_flight_sql_p0/test_table_types.groovy
new file mode 100644
index 00000000000..3fb61f37bb5
--- /dev/null
+++ b/regression-test/suites/arrow_flight_sql_p0/test_table_types.groovy
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CallOption
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CallOptions
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightClient
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.Location
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.sql.FlightSqlClient
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.memory.RootAllocator
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.vector.types.pojo.ArrowType
+
+suite("test_table_types", "arrow_flight_sql") {
+ def expected = ["BASE TABLE", "SYSTEM VIEW", "VIEW"]
+ def config = context.config.otherConfigs
+ def location = Location.forGrpcInsecure(config.get("extArrowFlightSqlHost"),
+ Integer.parseInt(config.get("extArrowFlightSqlPort")))
+ new RootAllocator().withCloseable { allocator ->
+ FlightClient.builder(allocator, location).build().withCloseable { flight ->
+ def token = flight.authenticateBasicToken(config.get("extArrowFlightSqlUser"),
+ config.get("extArrowFlightSqlPassword")).get()
+ CallOption[] options = [token, CallOptions.timeout(30, java.util.concurrent.TimeUnit.SECONDS)]
+ def client = new FlightSqlClient(flight)
+ 3.times {
+ def info = client.getTableTypes(options)
+ assertEquals(["table_type"], info.getSchema().getFields()*.getName())
+ assertEquals(new ArrowType.Utf8(), info.getSchema().getFields()[0].getType())
+ assertFalse(info.getSchema().getFields()[0].isNullable())
+ assertEquals(1, info.getEndpoints().size())
+ def types = []
+ // GetFlightInfo already succeeded before this fix; consume DoGet to detect missing data support.
+ flight.getStream(info.getEndpoints()[0].getTicket(), options).withCloseable { stream ->
+ assertEquals(info.getSchema(), stream.getSchema())
+ while (stream.next()) {
+ def root = stream.getRoot()
+ for (int row = 0; row < root.getRowCount(); ++row) {
+ types.add(root.getVector("table_type").getObject(row).toString())
+ }
+ }
+ }
+ assertEquals(expected, types)
+ }
+ def info = client.execute("SHOW VARIABLES LIKE 'query_timeout'", options)
+ int rows = 0
+ flight.getStream(info.getEndpoints()[0].getTicket(), options).withCloseable { stream ->
+ while (stream.next()) {
+ rows += stream.getRoot().getRowCount()
+ }
+ }
+ assertEquals(1, rows)
+ }
+ }
+ def jdbcTypes = []
+ context.getArrowFlightSqlConnection().getMetaData().getTableTypes().withCloseable { rows ->
+ while (rows.next()) {
+ jdbcTypes.add(rows.getString(1))
+ }
+ }
+ assertEquals(expected, jdbcTypes)
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 35/35: [fix](arrow) Correct year-zero DATE conversion on branch-4.1 (#68649)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0e741f9162bd03cdca1822ee3f9c1e3bc31aa91f
Author: Gabriel <li...@selectdb.com>
AuthorDate: Thu Oct 8 10:48:32 2026 +0800
[fix](arrow) Correct year-zero DATE conversion on branch-4.1 (#68649)
### What problem does this PR solve?
Related issue: #67366
Related PR: #67449
On branch-4.1, Arrow Flight SQL returns `0000-01-02` for a stored DATE
of `0000-01-01`. Doris follows the MySQL calendar, which excludes year
zero's leap day, while Arrow DATE uses the proleptic Gregorian calendar.
Subtracting the Unix epoch day number directly shifts dates in
`0000-01-01` through `0000-02-28` by one day, including elements of
`ARRAY<DATE>`.
Adapt the DATE portion of #67449 to branch-4.1, with credit to its
author. Convert between the calendars at Arrow and ORC write boundaries
and Arrow/default Parquet/ORC read boundaries. Use the same conversion
for ORC statistics so pushed-down MIN/MAX agrees with row decoding.
Reject unrepresentable external dates and ignore undefined payloads in
null Arrow slots.
The two DATE regression suites and their expected results are copied
unchanged from #67449. Add its DATE unit coverage plus an explicit
null-payload test and an exhaustive comparison against the civil
calendar for all 3,652,424 representable dates.
Also align Iceberg DATE day/bucket partition transforms with file
ordinals. For ORC/Parquet, metadata is usable only when its raw DATE
range proves that every ordinal is representable, including the interior
of the range. Unsafe ranges fall back to decoding: strict reads retain
their errors, non-strict Parquet reads retain conversion-generated
NULLs, and COUNT uses materialized values when physical null counts are
insufficient. ORC uses a metadata-only SARG to check projected DATE
leaves per stripe: safe excluded stripes retain SARG pruning, while
unsafe stripes remain readable with SDK row-group pruning disabled. ORC
COUNT validates every DATE leaf of its argument in each selected stripe,
and strict Parquet COUNT validates projected DATE descendants before
reading only levels.
Scope: DATE, including its Iceberg partition transforms. This does not
include DATETIME changes or the independent legacy file scanner
conversions used when `enable_file_scanner_v2=false`. V1 reader changes
and mixed-version/rolling-upgrade compatibility gates are explicitly
outside this PR's scope.
### Validation
- Baseline: 9 of the original 10 DATE unit tests fail; the unchanged
Flight regression reproduces the one-day shift.
- Fixed: 250 unit tests pass under ASAN, including the complete ORC
reader suite, DATE SerDe exhaustive calendar check, Iceberg partition
transforms, Parquet statistics, DATE scans and COUNT tests. These
compile the modified branch-4.1 sources against cached test
dependencies.
- The unchanged `test_date_year_zero` Flight regression passes locally,
including nullable and array values.
- `test_outfile_date_year_zero` passes for Parquet and ORC using the
original rows and golden assertions, with S3 transport replaced by local
outfile/TVF only for the local run. The committed suite retains S3
transport for CI.
- Local integration runs use the same fix rebuilt into an existing
ASAN/UBSAN BE build at `5bcb996a5ff`, with an existing FE. A full
branch-4.1 build and the original S3 run remain for CI.
- clang-format 16 passes for all affected C++ files. Source whitespace
checks pass; generated golden files retain their conventional trailing
blank line from #67449.
- Review coverage: new tests reproduce partition mismatches,
invalid-interior MIN/MAX bypass, ORC SARG/Parquet footer/page pruning
inconsistencies, and Parquet IS NULL/COUNT discrepancies. All five
reproducing tests fail before the review fixes.
- Added a DATE-only Spark/Iceberg partitioned write/read regression,
including selective day/bucket predicates and partition metadata
comparison. Its Groovy compilation passes; execution requires the
external CI environment.
- Follow-up review: scalar/nested ORC COUNT and strict nested Parquet
COUNT reproductions fail before the guards and pass afterward. ARRAY,
STRUCT, MAP, all-null values, safe ranges, non-strict Parquet, and
unprojected DATE siblings are covered.
- External ORC regression: retain the original LZ4 file as a BE fixture
and verify that ordinal `-719530` raises a data-quality error. Replace
the obsolete `1900-01-01` golden result with an explicit error
assertion, restoring the scanner setting afterward. The fixture test and
Groovy compilation pass locally. The complete group6 suite is also
validated locally with HDFS transport replaced by LOCAL TVF; the
committed suite retains HDFS for CI.
- Pruning review: both new tests fail before the fixes and pass
afterward. Partial ORC STRUCT projection retains SARG when the invalid
DATE sibling is not read; selecting that DATE still preserves its error.
Multi-page Parquet tests retain the unsafe DATE page while pruning safe
pages for IS NULL and range predicates, through both single-slot and
multi-slot OR evaluators.
- Mixed-stripe ORC coverage: the original file-wide guard prunes zero
safe stripes; the fix prunes both safe excluded stripes and still
reports the invalid DATE in the retained stripe. The active SDK options
outlive their reader, and the cached SARG evaluator is discarded before
opening an unsafe range.
- External group6 coverage: the original `non_vec_orc_scanner.orc`
fixture encodes `20191111` as an invalid epoch-day ordinal. Replace its
obsolete `1900-01-01` golden result with an explicit data-quality error
assertion and keep the original bytes as a small BE fixture; projecting
other columns still succeeds.
- Final local integration after the stripe fix: all three suites pass
with no failures or skips (complete group6: 41 fixtures, 39
golden-result assertions and two expected errors; Flight year zero;
Parquet/ORC outfile year zero). These use the incremental ASAN/UBSAN
build and local transport substitutions described above.
- Metadata safety review: the malformed pruned-stripe index and
truncated nested DATE statistics reproduce SDK bounds assertions before
the fix. Both regressions pass afterward; the DATE preflight now adds no
file read calls or bytes compared with the same excluded stripe without
DATE projection.
- Historical-file migration: an explicit offline tool converts only
confirmed legacy Doris DATE ordinals, including nested values. It
publishes a separate output atomically and refuses overwrites. Tests
cover year-zero boundaries, NULLs, hidden null-container payloads,
nested ORC/Parquet round trips, schema metadata, timestamp precision,
and failed-write cleanup.
- Final follow-up validation: all 250 ASAN tests and 10 migration-tool
tests pass. The final incremental production BE passes the three
existing local suites plus six legacy migration/read/export/read checks,
covering both ORC and Parquet. Original HDFS/S3 transport and the full
branch build remain for CI as described above.
### Release note
Fix year-zero DATE values being returned one day late through Arrow
Flight SQL, align DATE encoding and decoding in the default Parquet/ORC
paths, and keep Iceberg DATE partition values consistent with file
contents.
Existing files written with the former year-zero encoding cannot be
distinguished automatically from correctly encoded files. Before
switching their consumers to the corrected reader, use the explicit
[legacy DATE migration tool and
procedure](https://github.com/apache/doris/blob/082429b00f2392f8e8eabcee6e6a5a94a403e7aa/tools/legacy_date_migration/README.md)
on files confirmed to use the old Doris encoding, or re-export from
authoritative original data. The tool writes new ORC/Parquet files
without modifying the source. Existing Iceberg files must be migrated
through a new table or a supported table-format transaction that
recomputes partition metadata; do not replace files in existing
snapshots. The proleptic-only `0000-02-29` remains unrepresentable as a
Doris DATE.
### Check List (For Author)
- Test
- [x] Regression test
- [x] Unit Test
- Behavior changed:
- [x] Yes, external DATE ordinals use the correct calendar; internal
DATE arithmetic is unchanged.
- Does this need documentation?
- [x] No.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../data_type_serde/data_type_datev2_serde.cpp | 73 ++-
be/src/core/value/vdatetime_value.h | 53 +-
.../sink/writer/iceberg/partition_transformers.h | 8 +-
be/src/format_v2/orc/orc_reader.cpp | 164 +++++-
be/src/format_v2/parquet/parquet_reader.cpp | 43 ++
be/src/format_v2/parquet/parquet_statistics.cpp | 10 +
.../data_type_datev2_serde_calendar_test.cpp | 358 +++++++++++++
.../writer/iceberg/partition_transformers_test.cpp | 27 +
.../orc_scanner/date_count_truncated_stats.orc | Bin 0 -> 230 bytes
.../test_data/orc_scanner/date_large_ordinal.orc | Bin 0 -> 981 bytes
.../orc_scanner/date_metadata_fixtures.md | 39 ++
.../test_data/orc_scanner/date_out_of_range.orc | Bin 0 -> 619 bytes
.../exec/test_data/orc_scanner/date_preflight.orc | Bin 0 -> 390 bytes
.../orc_scanner/date_preflight_bad_index.orc | Bin 0 -> 390 bytes
be/test/format_v2/orc/orc_reader_test.cpp | 559 ++++++++++++++++++++-
be/test/format_v2/parquet/parquet_reader_test.cpp | 248 +++++++++
.../format_v2/parquet/parquet_statistics_test.cpp | 174 +++++++
.../arrow_flight_sql_p0/test_date_year_zero.out | 11 +
.../outfile/test_outfile_date_year_zero.out | 37 ++
.../tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out | 34 --
.../arrow_flight_sql_p0/test_date_year_zero.groovy | 95 ++++
.../outfile/test_outfile_date_year_zero.groovy | 85 ++++
.../write/test_iceberg_write_date_year_zero.groovy | 79 +++
.../orc_tvf/test_hdfs_orc_group6_orc_files.groovy | 38 +-
tools/legacy_date_migration/README.md | 117 +++++
.../legacy_date_migration/migrate_legacy_dates.py | 162 ++++++
.../test_migrate_legacy_dates.py | 179 +++++++
27 files changed, 2505 insertions(+), 88 deletions(-)
diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.cpp b/be/src/core/data_type_serde/data_type_datev2_serde.cpp
index a4cc34f49fc..3b6f5f7fb17 100644
--- a/be/src/core/data_type_serde/data_type_datev2_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_datev2_serde.cpp
@@ -22,6 +22,7 @@
#include <fmt/core.h>
#include <cstdint>
+#include <limits>
#include <vector>
#include "common/config.h"
@@ -42,13 +43,9 @@
namespace doris {
#include "common/compile_check_begin.h"
-// This number represents the number of days from 0000-01-01 to 1970-01-01.
-static constexpr int32_t date_threshold = 719528;
namespace {
-constexpr int32_t DORIS_DATE_EPOCH_DAYNR = 719528;
-
Status decode_date_orc_values(const DataTypeSerDe& serde, IColumn& column,
const OrcDecodedColumnView& orc_view) {
const auto* orc_batch = dynamic_cast<const ::orc::LongVectorBatch*>(orc_view.batch);
@@ -65,22 +62,44 @@ Status decode_date_orc_values(const DataTypeSerDe& serde, IColumn& column,
orc_serde_utils::orc_decode_row_count(orc_view.rows, orc_view.selected_rows);
std::vector<int32_t> date_values;
date_values.resize(output_rows);
- auto& date_dict = date_day_offset_dict::get();
for (size_t row = 0; row < output_rows; ++row) {
const auto source_row = orc_serde_utils::orc_source_row_at(row, orc_view.selected_rows);
- const auto date = date_dict[cast_set<int>(orc_batch->data[source_row])];
- date_values[row] = cast_set<int32_t>(date.daynr() - DORIS_DATE_EPOCH_DAYNR);
+ // The payload of a null slot is undefined, so it is zeroed rather than range-checked;
+ // `read_column_from_decoded_values` never decodes it.
+ if (view.null_map != nullptr && view.null_map[row] != 0) {
+ date_values[row] = 0;
+ continue;
+ }
+ // ORC DATE is days since 1970-01-01 in the proleptic Gregorian calendar, the same encoding
+ // `decode_epoch_days()` expects, but ORC hands it over in an int64 batch. Reject a value
+ // that does not fit the int32 view here, while the real file value is still available for
+ // the message, instead of narrowing it blindly.
+ const int64_t file_days = orc_batch->data[source_row];
+ if (file_days < std::numeric_limits<int32_t>::min() ||
+ file_days > std::numeric_limits<int32_t>::max()) {
+ return Status::DataQualityError(
+ "DATE value {} is outside the Doris DATE range (0000-01-01..9999-12-31, "
+ "0000-02-29 excluded)",
+ file_days);
+ }
+ date_values[row] = static_cast<int32_t>(file_days);
}
view.values = reinterpret_cast<const uint8_t*>(date_values.data());
RETURN_IF_ERROR(orc_serde_utils::read_decoded_values(serde, column, &view));
return Status::OK();
}
-Status decode_parquet_date(int32_t encoded_date, DateV2Value<DateV2ValueType>* value) {
+// Shared by every "days since 1970-01-01" source: Parquet, ORC and Arrow date32/date64. The
+// message must therefore not name one format, and it carries the value so a broken file can be
+// identified without re-reading it.
+Status decode_epoch_days(int64_t encoded_date, DateV2Value<DateV2ValueType>* value) {
DORIS_CHECK(value != nullptr);
- const int64_t day_number = static_cast<int64_t>(encoded_date) + date_threshold;
- if (day_number < 0 || !value->get_date_from_daynr(static_cast<uint64_t>(day_number))) {
- return Status::DataQualityError("Parquet DATE value is out of range");
+ const int64_t day_number = epoch_days_to_daynr(encoded_date);
+ if (day_number == 0 || !value->get_date_from_daynr(static_cast<uint64_t>(day_number))) {
+ return Status::DataQualityError(
+ "DATE value {} is outside the Doris DATE range (0000-01-01..9999-12-31, "
+ "0000-02-29 excluded)",
+ encoded_date);
}
return Status::OK();
}
@@ -100,7 +119,7 @@ public:
_data.resize(old_size + num_values);
for (size_t row = 0; row < num_values; ++row) {
DateV2Value<DateV2ValueType> value;
- const auto status = decode_parquet_date(
+ const auto status = decode_epoch_days(
unaligned_load<int32_t>(values + row * sizeof(int32_t)), &value);
if (!status.ok()) {
if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) {
@@ -217,13 +236,12 @@ Status DataTypeDateV2SerDe::write_column_to_arrow(const IColumn& column, const N
const auto& col_data = static_cast<const ColumnDateV2&>(column).get_data();
auto& date32_builder = assert_cast<arrow::Date32Builder&>(*array_builder);
for (size_t i = start; i < end; ++i) {
- auto daynr = col_data[i].daynr() - date_threshold;
if (null_map && (*null_map)[i]) {
RETURN_IF_ERROR(checkArrowStatus(date32_builder.AppendNull(), column, *array_builder));
} else {
- RETURN_IF_ERROR(
- checkArrowStatus(date32_builder.Append(cast_set<int, int64_t, false>(daynr)),
- column, *array_builder));
+ RETURN_IF_ERROR(checkArrowStatus(
+ date32_builder.Append(daynr_to_epoch_days(col_data[i].daynr())), column,
+ *array_builder));
}
}
return Status::OK();
@@ -258,10 +276,9 @@ Status DataTypeDateV2SerDe::read_column_from_arrow(IColumn& column, const arrow:
"Arrow Date64 value must contain whole days: row={}, milliseconds={}",
value_i, milliseconds);
}
- const int64_t daynr = milliseconds / MILLISECONDS_PER_DAY + date_threshold;
DateV2Value<DateV2ValueType> value;
- if (daynr <= 0 || daynr > DATE_MAX_DAYNR ||
- !value.get_date_from_daynr(static_cast<uint64_t>(daynr))) {
+ if (const auto status = decode_epoch_days(milliseconds / MILLISECONDS_PER_DAY, &value);
+ !status.ok()) {
return Status::InvalidArgument(
"Arrow Date64 value is outside the Doris DATE range: "
"row={}, milliseconds={}",
@@ -278,12 +295,20 @@ Status DataTypeDateV2SerDe::read_column_from_arrow(IColumn& column, const arrow:
const auto* base_ptr = reinterpret_cast<const uint8_t*>(concrete_array->raw_values());
const size_t element_size = sizeof(int32_t);
for (auto value_i = start; value_i < end; ++value_i) {
- int32_t date_value = 0;
+ // A null slot has no calendar value and its payload may be outside the DATE range.
+ if (concrete_array->IsNull(value_i)) {
+ col_data.emplace_back(DateV2Value<DateV2ValueType>());
+ continue;
+ }
const uint8_t* raw_byte_ptr = base_ptr + value_i * element_size;
- memcpy(&date_value, raw_byte_ptr, element_size);
+ auto date_value = unaligned_load<int32_t>(raw_byte_ptr);
DateV2Value<DateV2ValueType> v;
- v.get_date_from_daynr(date_value + date_threshold);
+ if (const auto status = decode_epoch_days(date_value, &v); !status.ok()) {
+ return Status::InvalidArgument(
+ "Arrow Date32 value is outside the Doris DATE range: row={}, days={}",
+ value_i, date_value);
+ }
col_data.emplace_back(v);
}
} else {
@@ -311,7 +336,7 @@ Status DataTypeDateV2SerDe::read_column_from_decoded_values(IColumn& column,
continue;
}
DateV2Value<DateV2ValueType> date_v2;
- const auto status = decode_parquet_date(values[row], &date_v2);
+ const auto status = decode_epoch_days(values[row], &date_v2);
if (!status.ok()) {
// Decoded values back both metadata conversion and native materialization. Preserve
// strict errors while allowing nullable non-strict scans to mark only the bad row.
@@ -403,7 +428,7 @@ Status DataTypeDateV2SerDe::write_column_to_orc(const std::string& timezone, con
if (cur_batch->notNull[row_id] == 0) {
continue;
}
- cur_batch->data[row_id] = col_data[row_id].daynr() - date_threshold;
+ cur_batch->data[row_id] = daynr_to_epoch_days(col_data[row_id].daynr());
}
cur_batch->numElements = end - start;
return Status::OK();
diff --git a/be/src/core/value/vdatetime_value.h b/be/src/core/value/vdatetime_value.h
index 5b5f768c0e2..02fefdf3abf 100644
--- a/be/src/core/value/vdatetime_value.h
+++ b/be/src/core/value/vdatetime_value.h
@@ -1697,8 +1697,6 @@ private:
static constexpr int START_YEAR = 1900; // 1900-01-01
static constexpr int END_YEAR = 2039; // 2039-10-24
- static constexpr int DAY_OFFSET_CAL_START_POINT_DAYNR =
- 719528; // 1970-01-01 (start from 0000-01-01, 0000-01-01 is day 1, returns 1)
static std::array<DateV2Value<DateV2ValueType>, DICT_DAYS> DATE_DAY_OFFSET_ITEMS;
static std::array<std::array<std::array<int, 31>, 12>, 140> DATE_DAY_OFFSET_DICT;
@@ -1713,6 +1711,9 @@ private:
date_day_offset_dict& operator=(const date_day_offset_dict&) = default;
public:
+ static constexpr int DAY_OFFSET_CAL_START_POINT_DAYNR =
+ 719528; // 1970-01-01 (start from 0000-01-01, 0000-01-01 is day 1, returns 1)
+
static bool can_speed_up_calc_daynr(int year) { return year >= START_YEAR && year <= END_YEAR; }
static int get_offset_by_daynr(int daynr) { return daynr - DAY_OFFSET_CAL_START_POINT_DAYNR; }
@@ -1773,6 +1774,54 @@ inline uint32_t calc_daynr(uint16_t year, uint8_t month, uint8_t day) {
return delsum + y / 4 - y / 100 + y / 400;
}
+// Doris follows MySQL's calendar, in which year 0 is NOT a leap year (see `is_leap()`): 0000-02-29
+// does not exist and `calc_daynr()` therefore numbers 0000-01-01 .. 0000-02-28 one day ahead of the
+// proleptic Gregorian calendar. Arrow `date32`, Parquet/ORC `DATE` and the Iceberg spec all define
+// their day ordinal in the proleptic Gregorian calendar, where year 0 IS a leap year. The two
+// numberings coincide from 0000-03-01 (daynr 60) onwards, so the whole difference is the missing
+// 0000-02-29. Every conversion between a Doris DATE and an external "days since 1970-01-01" value
+// must go through the two helpers below instead of adding/subtracting the epoch daynr directly.
+inline constexpr int64_t DAYNR_OF_UNIX_EPOCH =
+ date_day_offset_dict::DAY_OFFSET_CAL_START_POINT_DAYNR;
+inline constexpr int64_t DAYNR_OF_0000_03_01 = 60; // first daynr shared by both calendars
+
+// Doris daynr -> days since 1970-01-01 in the proleptic Gregorian calendar.
+inline constexpr int32_t daynr_to_epoch_days(int64_t daynr) {
+ return static_cast<int32_t>(daynr - DAYNR_OF_UNIX_EPOCH -
+ (daynr < DAYNR_OF_0000_03_01 ? 1 : 0));
+}
+
+// The three boundaries of the external ordinal domain, derived from the daynr domain rather than
+// typed out, so a change to either calendar constant cannot leave them behind.
+inline constexpr int64_t EPOCH_DAYS_MIN = daynr_to_epoch_days(1); // 0000-01-01, smallest Doris DATE
+inline constexpr int64_t EPOCH_DAYS_0000_02_29 =
+ daynr_to_epoch_days(DAYNR_OF_0000_03_01) - 1; // exists in proleptic Gregorian only
+inline constexpr int64_t EPOCH_DAYS_MAX =
+ daynr_to_epoch_days(DATE_MAX_DAYNR); // 9999-12-31, the largest Doris DATE
+
+static_assert(DAYNR_OF_UNIX_EPOCH == 719528, "calc_daynr(1970, 1, 1)");
+static_assert(EPOCH_DAYS_MIN == -719528);
+static_assert(EPOCH_DAYS_0000_02_29 == -719469);
+static_assert(EPOCH_DAYS_MAX == 2932896);
+
+// Inverse of `daynr_to_epoch_days()`. Returns 0, which is never a valid daynr, when the value has
+// no Doris DATE representation: before 0000-01-01, after 9999-12-31, or the proleptic-only
+// 0000-02-29. Callers must reject 0 instead of feeding it to `get_date_from_daynr()`.
+inline constexpr int64_t epoch_days_to_daynr(int64_t epoch_days) {
+ if (epoch_days < EPOCH_DAYS_MIN || epoch_days > EPOCH_DAYS_MAX ||
+ epoch_days == EPOCH_DAYS_0000_02_29) {
+ return 0;
+ }
+ return epoch_days + DAYNR_OF_UNIX_EPOCH + (epoch_days < EPOCH_DAYS_0000_02_29 ? 1 : 0);
+}
+
+// Valid endpoints alone do not prove a DATE range is decodable: it may enclose the
+// Gregorian-only leap day. Metadata shortcuts require every ordinal in the range to be valid.
+inline constexpr bool epoch_days_range_is_representable(int64_t min, int64_t max) {
+ return min <= max && epoch_days_to_daynr(min) != 0 && epoch_days_to_daynr(max) != 0 &&
+ (max < EPOCH_DAYS_0000_02_29 || min > EPOCH_DAYS_0000_02_29);
+}
+
// DAY / WEEK fast path. Real workloads hold dates that cluster in a narrow range, so the two
// dictionary lookups below (`daynr(y, m, d)` and `daynr -> date`) are L1-resident; measured about
// 3x faster than the generic `date_add_interval<DAY>` (no TimeInterval, no second-level
diff --git a/be/src/exec/sink/writer/iceberg/partition_transformers.h b/be/src/exec/sink/writer/iceberg/partition_transformers.h
index 46db5ed2263..da63554ff03 100644
--- a/be/src/exec/sink/writer/iceberg/partition_transformers.h
+++ b/be/src/exec/sink/writer/iceberg/partition_transformers.h
@@ -658,7 +658,8 @@ public:
DateV2Value<DateV2ValueType> value =
binary_cast<uint32_t, DateV2Value<DateV2ValueType>>(*(UInt32*)p_in);
- int64_t days_from_unix_epoch = value.daynr() - 719528;
+ // Partition hashes must use the same Gregorian ordinal as the DATE file payload.
+ int64_t days_from_unix_epoch = daynr_to_epoch_days(value.daynr());
uint32_t hash_value = HashUtil::murmur_hash3_32(&days_from_unix_epoch,
sizeof(days_from_unix_epoch), 0);
@@ -1149,9 +1150,8 @@ public:
while (p_in < end_in) {
DateV2Value<DateV2ValueType> value =
binary_cast<uint32_t, DateV2Value<DateV2ValueType>>(*(UInt32*)p_in);
- // datetime_diff<DAY> actually returns int
- *p_out = cast_set<int, int64_t, false>(
- datetime_diff<DAY>(PartitionColumnTransformUtils::epoch_date(), value));
+ // Iceberg day partitions describe external DATE ordinals, not MySQL day numbers.
+ *p_out = daynr_to_epoch_days(value.daynr());
++p_in;
++p_out;
}
diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp
index 856f38f0eee..d87bad294c7 100644
--- a/be/src/format_v2/orc/orc_reader.cpp
+++ b/be/src/format_v2/orc/orc_reader.cpp
@@ -487,10 +487,30 @@ bool set_date_zone_map(const ::orc::ColumnStatistics& statistics, segment_v2::Zo
!date_statistics->hasMaximum()) {
return false;
}
- auto& date_dict = date_day_offset_dict::get();
- return set_validated_zone_map(
- Field::create_field<TYPE_DATEV2>(date_dict[date_statistics->getMinimum()]),
- Field::create_field<TYPE_DATEV2>(date_dict[date_statistics->getMaximum()]), zone_map);
+ if (!epoch_days_range_is_representable(date_statistics->getMinimum(),
+ date_statistics->getMaximum())) {
+ return false;
+ }
+ // ORC DATE statistics are proleptic-Gregorian day ordinals, the same domain the row decoder
+ // (DataTypeDateV2SerDe::read_column_from_orc) interprets. Converting them through
+ // `date_day_offset_dict` instead would put the year-zero window one day off the rows and let a
+ // pushed-down MIN/MAX report a value no row holds. A bound with no Doris DATE disables the
+ // statistics, so MIN/MAX falls back to scanning rows.
+ const auto to_date = [](int64_t epoch_days) -> std::optional<DateV2Value<DateV2ValueType>> {
+ const int64_t daynr = epoch_days_to_daynr(epoch_days);
+ DateV2Value<DateV2ValueType> value;
+ if (daynr == 0 || !value.get_date_from_daynr(static_cast<uint64_t>(daynr))) {
+ return std::nullopt;
+ }
+ return value;
+ };
+ const auto min_value = to_date(date_statistics->getMinimum());
+ const auto max_value = to_date(date_statistics->getMaximum());
+ if (!min_value.has_value() || !max_value.has_value()) {
+ return false;
+ }
+ return set_validated_zone_map(Field::create_field<TYPE_DATEV2>(*min_value),
+ Field::create_field<TYPE_DATEV2>(*max_value), zone_map);
}
std::optional<DateV2Value<DateTimeV2ValueType>> datetime_v2_from_orc_millis(
@@ -740,6 +760,7 @@ struct OrcReaderScanState {
uint64_t last_stripe = 0;
uint64_t offset = 0;
uint64_t length = 0;
+ bool disable_sarg = false;
};
std::unique_ptr<::orc::Reader> reader;
@@ -747,6 +768,7 @@ struct OrcReaderScanState {
const ::orc::Type* root_type = nullptr;
::orc::ReaderMetrics reader_metrics;
::orc::RowReaderOptions row_reader_options; // projection + filter + SARG + stripe range
+ ::orc::RowReaderOptions active_row_reader_options;
std::string timezone = TimezoneUtils::default_time_zone;
cctz::time_zone timezone_obj;
std::unique_ptr<::orc::RowReader> row_reader;
@@ -771,6 +793,7 @@ struct OrcReaderScanState {
bool orc_lazy_selection_valid = false;
OrcSargBuildOptions sarg_build_options;
+ std::vector<uint64_t> decoded_date_column_ids;
std::vector<StripeRange> selected_stripe_ranges;
size_t current_stripe_range = 0;
bool stripe_pruning_applied = false;
@@ -1502,6 +1525,35 @@ Status OrcReader::_init_search_argument_from_local_filters() {
}
try {
+ const auto collect_dates = [&](const auto& self, const ::orc::Type& type,
+ const std::set<uint64_t>* projected_ids) -> void {
+ if (projected_ids != nullptr && !projected_ids->contains(type.getColumnId())) {
+ return;
+ }
+ if (type.getKind() == ::orc::TypeKind::DATE) {
+ _state->decoded_date_column_ids.push_back(type.getColumnId());
+ }
+ for (uint64_t child = 0; child < type.getSubtypeCount(); ++child) {
+ self(self, *type.getSubtype(child), projected_ids);
+ }
+ };
+ for (const auto column_id : _state->read_columns) {
+ if (is_virtual_column(column_id)) {
+ continue;
+ }
+ const auto& type =
+ *_state->root_type->getSubtype(static_cast<uint64_t>(column_id.value()));
+ const auto* projection = find_request_projection(*_request, column_id);
+ DORIS_CHECK(projection != nullptr);
+ std::set<uint64_t> projected_ids;
+ const bool partial = has_pruned_projection(*projection);
+ if (partial) {
+ // Match includeTypes exactly: an unread DATE sibling cannot raise a conversion
+ // error and must not disable pruning for the projected columns.
+ RETURN_IF_ERROR(collect_projected_type_ids(type, *projection, &projected_ids));
+ }
+ collect_dates(collect_dates, type, partial ? &projected_ids : nullptr);
+ }
auto builder = ::orc::SearchArgumentFactory::newBuilder();
bool has_pushdown = false;
builder->startAnd();
@@ -1670,8 +1722,45 @@ Status OrcReader::_select_stripe_ranges_by_statistics() {
}
std::vector<int> sarg_needed_stripes;
+ std::set<uint64_t> unsafe_date_stripes;
try {
+ if (!_state->decoded_date_column_ids.empty()) {
+ // getStripeStatistics also reads every ROW_INDEX stream. A metadata-only SARG
+ // proves which stripes cannot contain an invalid DATE without touching skipped data
+ // or exposing malformed, unprojected row-index streams to the SDK.
+ constexpr auto date_type = ::orc::PredicateDataType::DATE;
+ auto builder = ::orc::SearchArgumentFactory::newBuilder();
+ builder->startOr();
+ for (const auto column_id : _state->decoded_date_column_ids) {
+ builder->lessThan(column_id, date_type, ::orc::Literal(date_type, EPOCH_DAYS_MIN));
+ builder->startNot();
+ builder->lessThanEquals(column_id, date_type,
+ ::orc::Literal(date_type, EPOCH_DAYS_MAX));
+ builder->end();
+ builder->equals(column_id, date_type,
+ ::orc::Literal(date_type, EPOCH_DAYS_0000_02_29));
+ }
+ builder->end();
+ // The SDK non-const copy constructor transfers ownership instead of copying.
+ ::orc::RowReaderOptions validation_options(std::as_const(_state->row_reader_options));
+ validation_options.searchArgument(builder->build());
+ const auto needs_validation = _state->reader->getNeedReadStripes(validation_options);
+ // The SDK caches a borrowed evaluator for its next row reader. Consume it while
+ // validation_options is alive, before installing the query's actual SARG.
+ _state->reader->createRowReader(validation_options).reset();
+ for (const auto stripe_index : split_stripes) {
+ if (stripe_index >= needs_validation.size() ||
+ needs_validation[stripe_index] != 0) {
+ unsafe_date_stripes.insert(stripe_index);
+ }
+ }
+ }
sarg_needed_stripes = _state->reader->getNeedReadStripes(_state->row_reader_options);
+ if (!unsafe_date_stripes.empty()) {
+ // getNeedReadStripes caches its SARG evaluator for the first row reader, even when
+ // that reader has no SARG. Consume the cache before opening an unsafe range.
+ _state->reader->createRowReader(_state->row_reader_options).reset();
+ }
} catch (const Exception& e) {
if (is_orc_stop(_io_ctx.get(), e)) {
return Status::EndOfFile("stop");
@@ -1695,7 +1784,8 @@ Status OrcReader::_select_stripe_ranges_by_statistics() {
int64_t filtered_bytes = 0;
for (const auto stripe_index : split_stripes) {
bool drop = false;
- if (stripe_index < sarg_needed_stripes.size() && sarg_needed_stripes[stripe_index] == 0) {
+ if (!unsafe_date_stripes.contains(stripe_index) &&
+ stripe_index < sarg_needed_stripes.size() && sarg_needed_stripes[stripe_index] == 0) {
drop = true;
}
if (!drop) {
@@ -1712,7 +1802,7 @@ Status OrcReader::_select_stripe_ranges_by_statistics() {
}
}
- if (filtered_stripes == 0) {
+ if (filtered_stripes == 0 && unsafe_date_stripes.empty()) {
return Status::OK();
}
@@ -1739,6 +1829,7 @@ Status OrcReader::_select_stripe_ranges_by_statistics() {
.last_stripe = last_stripe,
.offset = offset,
.length = end_offset - offset,
+ .disable_sarg = unsafe_date_stripes.contains(first_stripe),
});
} catch (const std::exception& e) {
return Status::InternalError("Failed to build ORC stripe read range: {}", e.what());
@@ -1750,7 +1841,8 @@ Status OrcReader::_select_stripe_ranges_by_statistics() {
uint64_t previous = range_first;
for (size_t idx = 1; idx < selected_stripes.size(); ++idx) {
const auto stripe_index = selected_stripes[idx];
- if (stripe_index == previous + 1) {
+ if (stripe_index == previous + 1 &&
+ unsafe_date_stripes.contains(stripe_index) == unsafe_date_stripes.contains(previous)) {
previous = stripe_index;
continue;
}
@@ -1790,9 +1882,18 @@ Status OrcReader::_create_row_reader() {
if (_state->orc_lazy_read_enabled && _orc_filter == nullptr) {
_orc_filter = std::make_unique<OrcFilterImpl>(this);
}
+ // The SDK retains its SARG by reference, so the active options must outlive the reader.
+ _state->row_reader.reset();
+ auto& options = _state->active_row_reader_options;
+ options = _state->row_reader_options;
+ if (_state->stripe_pruning_applied &&
+ _state->selected_stripe_ranges[_state->current_stripe_range].disable_sarg) {
+ // The SDK also prunes row groups; clearing only a copy lets later safe ranges keep
+ // their SARG while the retained unsafe range reaches DATE decoding.
+ options.searchArgument(nullptr);
+ }
_state->row_reader = _state->reader->createRowReader(
- _state->row_reader_options,
- _state->orc_lazy_read_enabled ? _orc_filter.get() : nullptr);
+ options, _state->orc_lazy_read_enabled ? _orc_filter.get() : nullptr);
_state->selected_type = &_state->row_reader->getSelectedType();
DORIS_CHECK(_state->selected_type->getKind() == ::orc::TypeKind::STRUCT);
// Row-id fetch seeks before every read; a one-row batch preserves exact selection instead
@@ -2275,6 +2376,17 @@ Status OrcReader::get_aggregate_result(const format::FileAggregateRequest& reque
_state->root_type->getSubtype(static_cast<uint64_t>(count_projection.local_id()));
DORIS_CHECK(count_type != nullptr);
+ std::vector<uint32_t> date_column_ids;
+ const auto collect_dates = [&](auto&& self, const ::orc::Type& type) -> void {
+ if (type.getKind() == ::orc::TypeKind::DATE) {
+ date_column_ids.push_back(cast_set<uint32_t>(type.getColumnId()));
+ }
+ for (uint64_t i = 0; i < type.getSubtypeCount(); ++i) {
+ self(self, *type.getSubtype(i));
+ }
+ };
+ collect_dates(collect_dates, *count_type);
+
result->count = 0;
const auto stripe_statistics_count = _state->reader->getNumberOfStripeStatistics();
for (const auto stripe_index : selected_stripes) {
@@ -2294,6 +2406,26 @@ Status OrcReader::get_aggregate_result(const format::FileAggregateRequest& reque
return Status::NotSupported("Missing ORC stripe statistics for stripe {}",
stripe_index);
}
+ // COUNT skips decoding even for complex arguments. Every DATE leaf must be safe,
+ // including an unrepresentable day inside otherwise valid bounds, to preserve errors.
+ for (const auto column_id : date_column_ids) {
+ // Schema IDs can outlive truncated stripe statistics; the SDK lookup is unchecked.
+ if (column_id >= stripe_statistics->getNumberOfColumns()) {
+ return Status::NotSupported("Missing ORC DATE statistics for COUNT");
+ }
+ const auto* date_statistics = dynamic_cast<const ::orc::DateColumnStatistics*>(
+ stripe_statistics->getColumnStatistics(column_id));
+ if (date_statistics == nullptr ||
+ (date_statistics->getNumberOfValues() != 0 &&
+ (!date_statistics->hasMinimum() || !date_statistics->hasMaximum() ||
+ !epoch_days_range_is_representable(date_statistics->getMinimum(),
+ date_statistics->getMaximum())))) {
+ return Status::NotSupported("ORC DATE COUNT requires value validation");
+ }
+ }
+ if (count_type->getColumnId() >= stripe_statistics->getNumberOfColumns()) {
+ return Status::NotSupported("Missing ORC COUNT column statistics");
+ }
const auto* column_statistics = stripe_statistics->getColumnStatistics(
cast_set<uint32_t>(count_type->getColumnId()));
if (column_statistics == nullptr) {
@@ -2353,6 +2485,9 @@ Status OrcReader::get_aggregate_result(const format::FileAggregateRequest& reque
return Status::NotSupported("Missing ORC stripe statistics for stripe {}",
stripe_index);
}
+ if (leaf_type->getColumnId() >= stripe_statistics->getNumberOfColumns()) {
+ return Status::NotSupported("Missing ORC min/max column statistics");
+ }
const auto* column_statistics = stripe_statistics->getColumnStatistics(
cast_set<uint32_t>(leaf_type->getColumnId()));
if (column_statistics == nullptr) {
@@ -2409,8 +2544,15 @@ Status OrcReader::_decode_column_into_block(const ::orc::StructVectorBatch& stru
const auto* selected_type = _state->selected_type->getSubtype(selected_batch_idx);
DORIS_CHECK(selected_type != nullptr);
auto column = file_block->get_by_position(block_position.value()).column->assert_mutable();
- RETURN_IF_ERROR(_decode_column(*type, *selected_type, *struct_batch.fields[selected_batch_idx],
- column, rows, selected_rows));
+ auto status = _decode_column(*type, *selected_type, *struct_batch.fields[selected_batch_idx],
+ column, rows, selected_rows);
+ if (!status.ok()) {
+ // The decoders work on a bare value buffer and cannot name the column themselves; without
+ // this the user only learns that some DATE/TIMESTAMP in some file is unrepresentable.
+ return status.prepend(fmt::format(
+ "Failed to decode ORC column '{}': ",
+ _state->root_type->getFieldName(static_cast<uint64_t>(file_column_id.value()))));
+ }
file_block->replace_by_position(block_position.value(), std::move(column));
return Status::OK();
}
diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp
index ff493bc071f..917b17083a8 100644
--- a/be/src/format_v2/parquet/parquet_reader.cpp
+++ b/be/src/format_v2/parquet/parquet_reader.cpp
@@ -1175,6 +1175,49 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r
}
const auto& count_projection = request.columns[0].projection;
const auto& root_schema = projected_root_schema(_state->file_schema, count_projection);
+ std::vector<const ParquetColumnSchema*> date_leaves;
+ if (_state->enable_strict_mode ||
+ remove_nullable(root_schema.type)->get_primitive_type() == TYPE_DATEV2) {
+ const auto collect_dates = [&](auto&& self, const ParquetColumnSchema& schema,
+ const format::LocalColumnIndex* projection) -> void {
+ if (schema.kind == ParquetColumnSchemaKind::PRIMITIVE) {
+ if (remove_nullable(schema.type)->get_primitive_type() == TYPE_DATEV2) {
+ date_leaves.push_back(&schema);
+ }
+ return;
+ }
+ for (const auto& child : schema.children) {
+ if (format::is_child_projected(projection, child->local_id)) {
+ self(self, *child,
+ format::find_child_projection(projection, child->local_id));
+ }
+ }
+ };
+ collect_dates(collect_dates, root_schema, &count_projection);
+ }
+ for (const auto* date_leaf : date_leaves) {
+ // Definition levels count physical NULLs only. DATE conversion can add logical NULLs
+ // or raise an error, including for nested children in strict mode. Validate every
+ // projected DATE leaf before allowing a levels-only count to bypass conversion.
+ for (const auto& row_group_plan : _state->scan_plan->row_groups) {
+ const auto& group = _state->file_context.native_metadata->to_thrift()
+ .row_groups[row_group_plan.row_group_id];
+ const auto& chunk = group.columns[date_leaf->leaf_column_id];
+ if (!chunk.__isset.meta_data || !chunk.meta_data.__isset.statistics) {
+ return Status::NotSupported("Parquet DATE COUNT requires value validation");
+ }
+ const auto stats = detail::sanitize_native_footer_statistics(
+ date_leaf->type_descriptor, chunk.meta_data.statistics,
+ detail::has_supported_type_defined_order(
+ _state->file_context.native_metadata->to_thrift(),
+ date_leaf->leaf_column_id));
+ const auto decoded = ParquetStatisticsUtils::TransformColumnStatistics(
+ *date_leaf, &stats, chunk.meta_data.num_values, _state->timezone);
+ if (!decoded.has_min_max && !(decoded.has_null_count && !decoded.has_not_null)) {
+ return Status::NotSupported("Parquet DATE COUNT requires value validation");
+ }
+ }
+ }
// A required primitive COUNT(col) still carries its projection so the unsupported-type
// validation above cannot be bypassed. Once validated, its definition level proves that
// every selected row is non-NULL, so preserve the already-computed footer row count and
diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp
index 6ea6fed0797..e4928e65a7c 100644
--- a/be/src/format_v2/parquet/parquet_statistics.cpp
+++ b/be/src/format_v2/parquet/parquet_statistics.cpp
@@ -1661,6 +1661,16 @@ bool set_native_page_scalar_min_max(const tparquet::ColumnIndex& column_index,
}
const auto min_value = unaligned_load<ValueType>(column_index.min_values[page_idx].data());
const auto max_value = unaligned_load<ValueType>(column_index.max_values[page_idx].data());
+ if constexpr (std::is_same_v<ValueType, int32_t>) {
+ if (remove_nullable(column_schema.type)->get_primitive_type() == TYPE_DATEV2 &&
+ !epoch_days_range_is_representable(min_value, max_value)) {
+ // An invalid interior DATE errors in strict mode and becomes NULL otherwise.
+ // Neither min/max nor physical null counts prove the materialized result here.
+ // Retain this page without discarding usable statistics on the other pages.
+ *page_statistics = {};
+ return true;
+ }
+ }
if constexpr (std::is_integral_v<ValueType>) {
if (remove_nullable(column_schema.type)->get_primitive_type() == TYPE_TIMEV2) {
int64_t units_per_day = 0;
diff --git a/be/test/core/data_type_serde/data_type_datev2_serde_calendar_test.cpp b/be/test/core/data_type_serde/data_type_datev2_serde_calendar_test.cpp
new file mode 100644
index 00000000000..a03784222bc
--- /dev/null
+++ b/be/test/core/data_type_serde/data_type_datev2_serde_calendar_test.cpp
@@ -0,0 +1,358 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Doris DATE values are numbered in MySQL's calendar, where year 0 is not a leap year. Arrow
+// `date32`, Parquet `DATE` and ORC `DATE` are all days since 1970-01-01 in the proleptic
+// Gregorian calendar, where year 0 IS a leap year. These tests pin the boundary so the two
+// numberings cannot drift apart again: 0000-01-01 must leave Doris as -719528, not -719527.
+
+#include <arrow/api.h>
+#include <cctz/time_zone.h>
+#include <gtest/gtest.h>
+
+#include <limits>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "core/assert_cast.h"
+#include "core/column/column_array.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_vector.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_date_or_datetime_v2.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type_serde/data_type_datev2_serde.h"
+#include "core/data_type_serde/decoded_column_view.h"
+#include "core/value/vdatetime_value.h"
+
+namespace doris {
+
+namespace {
+
+struct DateCase {
+ int year;
+ int month;
+ int day;
+ int32_t epoch_days;
+};
+
+// 0000-01-01 .. 0000-02-28 are the only dates where Doris's daynr and the proleptic Gregorian
+// ordinal disagree, so the set brackets that window on both sides.
+const std::vector<DateCase>& boundary_cases() {
+ static const std::vector<DateCase> cases = {
+ {0, 1, 1, -719528}, {0, 2, 28, -719470}, {0, 3, 1, -719468}, {1, 1, 1, -719162},
+ {1969, 12, 31, -1}, {1970, 1, 1, 0}, {2024, 1, 1, 19723}, {9999, 12, 31, 2932896}};
+ return cases;
+}
+
+DateV2Value<DateV2ValueType> make_date(int year, int month, int day) {
+ DateV2Value<DateV2ValueType> value;
+ value.unchecked_set_time(year, month, day, 0, 0, 0, 0);
+ return value;
+}
+
+ColumnDateV2::MutablePtr boundary_column() {
+ auto column = ColumnDateV2::create();
+ auto& data = column->get_data();
+ for (const auto& c : boundary_cases()) {
+ auto value = make_date(c.year, c.month, c.day);
+ data.push_back(*reinterpret_cast<UInt32*>(&value));
+ }
+ return column;
+}
+
+std::shared_ptr<arrow::Array> build_date32(const std::vector<int32_t>& days) {
+ arrow::Date32Builder builder;
+ for (int32_t d : days) {
+ EXPECT_TRUE(builder.Append(d).ok());
+ }
+ std::shared_ptr<arrow::Array> array;
+ EXPECT_TRUE(builder.Finish(&array).ok());
+ return array;
+}
+
+} // namespace
+
+class DataTypeDateV2SerDeCalendarTest : public ::testing::Test {
+protected:
+ DataTypeDateV2SerDe serde;
+ cctz::time_zone tz = cctz::utc_time_zone();
+};
+
+TEST_F(DataTypeDateV2SerDeCalendarTest, WriteArrowUsesProlepticGregorian) {
+ auto column = boundary_column();
+ arrow::Date32Builder builder;
+ ASSERT_TRUE(serde.write_column_to_arrow(*column, nullptr, &builder, 0,
+ static_cast<int64_t>(column->size()), tz)
+ .ok());
+ std::shared_ptr<arrow::Array> array;
+ ASSERT_TRUE(builder.Finish(&array).ok());
+
+ const auto& date32 = assert_cast<const arrow::Date32Array&>(*array);
+ ASSERT_EQ(boundary_cases().size(), static_cast<size_t>(date32.length()));
+ for (size_t i = 0; i < boundary_cases().size(); ++i) {
+ EXPECT_EQ(boundary_cases()[i].epoch_days, date32.Value(static_cast<int64_t>(i)))
+ << "row " << i;
+ }
+}
+
+TEST_F(DataTypeDateV2SerDeCalendarTest, WriteArrowHonoursNullMap) {
+ auto column = boundary_column();
+ NullMap null_map;
+ null_map.resize_fill(column->size(), 0);
+ null_map[0] = 1; // 0000-01-01
+ null_map[2] = 1; // 0000-03-01
+
+ arrow::Date32Builder builder;
+ ASSERT_TRUE(serde.write_column_to_arrow(*column, &null_map, &builder, 0,
+ static_cast<int64_t>(column->size()), tz)
+ .ok());
+ std::shared_ptr<arrow::Array> array;
+ ASSERT_TRUE(builder.Finish(&array).ok());
+
+ const auto& date32 = assert_cast<const arrow::Date32Array&>(*array);
+ for (size_t i = 0; i < boundary_cases().size(); ++i) {
+ if (null_map[i]) {
+ EXPECT_TRUE(date32.IsNull(static_cast<int64_t>(i))) << "row " << i;
+ } else {
+ EXPECT_EQ(boundary_cases()[i].epoch_days, date32.Value(static_cast<int64_t>(i)))
+ << "row " << i;
+ }
+ }
+}
+
+TEST_F(DataTypeDateV2SerDeCalendarTest, ReadDate32RestoresTheSameCalendarDay) {
+ std::vector<int32_t> days;
+ for (const auto& c : boundary_cases()) {
+ days.push_back(c.epoch_days);
+ }
+ auto array = build_date32(days);
+
+ auto column = ColumnDateV2::create();
+ ASSERT_TRUE(serde.read_column_from_arrow(*column, array.get(), 0, array->length(), tz).ok());
+
+ ASSERT_EQ(boundary_cases().size(), column->size());
+ const auto& values = column->get_data();
+ for (size_t i = 0; i < boundary_cases().size(); ++i) {
+ const auto& c = boundary_cases()[i];
+ EXPECT_EQ(c.year, values[i].year()) << "row " << i;
+ EXPECT_EQ(c.month, values[i].month()) << "row " << i;
+ EXPECT_EQ(c.day, values[i].day()) << "row " << i;
+ }
+}
+
+TEST_F(DataTypeDateV2SerDeCalendarTest, ArrowRoundTripIsLossless) {
+ auto column = boundary_column();
+ arrow::Date32Builder builder;
+ ASSERT_TRUE(serde.write_column_to_arrow(*column, nullptr, &builder, 0,
+ static_cast<int64_t>(column->size()), tz)
+ .ok());
+ std::shared_ptr<arrow::Array> array;
+ ASSERT_TRUE(builder.Finish(&array).ok());
+
+ auto restored = ColumnDateV2::create();
+ ASSERT_TRUE(serde.read_column_from_arrow(*restored, array.get(), 0, array->length(), tz).ok());
+ ASSERT_EQ(column->size(), restored->size());
+ for (size_t i = 0; i < column->size(); ++i) {
+ EXPECT_EQ(column->get_data()[i], restored->get_data()[i]) << "row " << i;
+ }
+}
+
+TEST_F(DataTypeDateV2SerDeCalendarTest, ReadDate32RejectsUnrepresentableDays) {
+ const auto expect_rejected = [&](int32_t days, const char* why) {
+ auto array = build_date32({days});
+ auto column = ColumnDateV2::create();
+ const auto status = serde.read_column_from_arrow(*column, array.get(), 0, 1, tz);
+ EXPECT_FALSE(status.ok()) << why << " (days=" << days << ")";
+ EXPECT_NE(std::string::npos, status.to_string().find("outside the Doris DATE range"))
+ << status.to_string();
+ };
+
+ expect_rejected(-719529, "one day before 0000-01-01");
+ // 0000-02-29 exists in the proleptic Gregorian calendar but Doris has no such date. The old
+ // `encoded + 719528` mapping decoded it as 0000-02-28; it must be rejected instead.
+ expect_rejected(-719469, "proleptic-only leap day 0000-02-29");
+ expect_rejected(2932897, "one day after 9999-12-31");
+}
+
+TEST_F(DataTypeDateV2SerDeCalendarTest, ReadDate64UsesTheSameCalendar) {
+ constexpr int64_t millis_per_day = 24LL * 60 * 60 * 1000;
+ arrow::Date64Builder builder;
+ for (const auto& c : boundary_cases()) {
+ ASSERT_TRUE(builder.Append(static_cast<int64_t>(c.epoch_days) * millis_per_day).ok());
+ }
+ std::shared_ptr<arrow::Array> array;
+ ASSERT_TRUE(builder.Finish(&array).ok());
+
+ auto column = ColumnDateV2::create();
+ ASSERT_TRUE(serde.read_column_from_arrow(*column, array.get(), 0, array->length(), tz).ok());
+
+ ASSERT_EQ(boundary_cases().size(), column->size());
+ const auto& values = column->get_data();
+ for (size_t i = 0; i < boundary_cases().size(); ++i) {
+ const auto& c = boundary_cases()[i];
+ EXPECT_EQ(c.year, values[i].year()) << "row " << i;
+ EXPECT_EQ(c.month, values[i].month()) << "row " << i;
+ EXPECT_EQ(c.day, values[i].day()) << "row " << i;
+ }
+}
+
+TEST_F(DataTypeDateV2SerDeCalendarTest, NestedArrayOfDateUsesTheSameCalendar) {
+ // ARRAY<DATE> has no encoding of its own; it delegates every element to the DATE SerDe. The
+ // reported bug showed up through this path too, so keep it covered.
+ // DataTypeArray stores its element type as DataTypeNullablePtr, so the nested column has to be
+ // a ColumnNullable for the element SerDe to match.
+ auto nested = boundary_column();
+ const auto element_count = nested->size();
+ auto element_null_map = ColumnUInt8::create();
+ element_null_map->get_data().resize_fill(element_count, 0);
+ auto nullable_nested = ColumnNullable::create(std::move(nested), std::move(element_null_map));
+ auto offsets = ColumnArray::ColumnOffsets::create();
+ offsets->get_data().push_back(static_cast<ColumnArray::Offset64>(element_count));
+ auto array_column = ColumnArray::create(std::move(nullable_nested), std::move(offsets));
+
+ auto array_type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeDateV2>());
+ auto array_serde = array_type->get_serde();
+
+ auto value_builder = std::make_shared<arrow::Date32Builder>();
+ arrow::ListBuilder list_builder(arrow::default_memory_pool(), value_builder);
+ ASSERT_TRUE(array_serde->write_column_to_arrow(*array_column, nullptr, &list_builder, 0, 1, tz)
+ .ok());
+ std::shared_ptr<arrow::Array> arrow_array;
+ ASSERT_TRUE(list_builder.Finish(&arrow_array).ok());
+
+ const auto& list = assert_cast<const arrow::ListArray&>(*arrow_array);
+ const auto& values = assert_cast<const arrow::Date32Array&>(*list.values());
+ ASSERT_EQ(boundary_cases().size(), static_cast<size_t>(values.length()));
+ for (size_t i = 0; i < boundary_cases().size(); ++i) {
+ EXPECT_EQ(boundary_cases()[i].epoch_days, values.Value(static_cast<int64_t>(i)))
+ << "element " << i;
+ }
+}
+
+// The Parquet and ORC readers do not go through Arrow: rows, dictionary entries and column
+// statistics all land in `read_column_from_decoded_values()`. Cover that entry point with the same
+// boundary set, in both the strict and the null-on-failure mode the file scanners use.
+TEST_F(DataTypeDateV2SerDeCalendarTest, ReadDecodedValuesUsesTheSameCalendar) {
+ std::vector<int32_t> values;
+ for (const auto& c : boundary_cases()) {
+ values.push_back(c.epoch_days);
+ }
+ values.push_back(0); // payload of the null row below, never decoded
+ std::vector<uint8_t> null_map(values.size(), 0);
+ null_map.back() = 1;
+
+ DecodedColumnView view;
+ view.value_kind = DecodedValueKind::INT32;
+ view.row_count = static_cast<int64_t>(values.size());
+ view.values = reinterpret_cast<const uint8_t*>(values.data());
+ view.null_map = null_map.data();
+
+ auto column = ColumnDateV2::create();
+ ASSERT_TRUE(serde.read_column_from_decoded_values(*column, view).ok());
+ ASSERT_EQ(values.size(), column->size());
+ const auto& data = column->get_data();
+ for (size_t i = 0; i < boundary_cases().size(); ++i) {
+ const auto& c = boundary_cases()[i];
+ EXPECT_EQ(c.year, data[i].year()) << "row " << i;
+ EXPECT_EQ(c.month, data[i].month()) << "row " << i;
+ EXPECT_EQ(c.day, data[i].day()) << "row " << i;
+ }
+}
+
+TEST_F(DataTypeDateV2SerDeCalendarTest, ReadDecodedValuesRejectsUnrepresentableDaysWhenStrict) {
+ // -719469 is the proleptic-only 0000-02-29: the only value inside the file-format range that
+ // Doris cannot represent, and the one the old dictionary fallback decoded as 0000-02-28.
+ const std::vector<int32_t> values = {-719528, -719470, -719469, -719468, 19723};
+ DecodedColumnView view;
+ view.value_kind = DecodedValueKind::INT32;
+ view.row_count = static_cast<int64_t>(values.size());
+ view.values = reinterpret_cast<const uint8_t*>(values.data());
+ view.enable_strict_mode = true;
+
+ auto column = ColumnDateV2::create();
+ const auto status = serde.read_column_from_decoded_values(*column, view);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(std::string::npos, status.to_string().find("outside the Doris DATE range"))
+ << status.to_string();
+ EXPECT_NE(std::string::npos, status.to_string().find("-719469")) << status.to_string();
+ // A failed batch must not leave a half-written column behind.
+ EXPECT_EQ(0, column->size());
+}
+
+TEST_F(DataTypeDateV2SerDeCalendarTest, ReadDecodedValuesNullsOnlyTheUnrepresentableRow) {
+ const std::vector<int32_t> values = {-719528, -719470, -719469, -719468, 19723};
+ NullMap conversion_failures;
+ conversion_failures.resize_fill(values.size(), 0);
+
+ DecodedColumnView view;
+ view.value_kind = DecodedValueKind::INT32;
+ view.row_count = static_cast<int64_t>(values.size());
+ view.values = reinterpret_cast<const uint8_t*>(values.data());
+ view.conversion_failure_null_map = &conversion_failures;
+
+ auto column = ColumnDateV2::create();
+ ASSERT_TRUE(serde.read_column_from_decoded_values(*column, view).ok());
+ ASSERT_EQ(values.size(), column->size());
+ const std::vector<uint8_t> expected_failures = {0, 0, 1, 0, 0};
+ for (size_t i = 0; i < expected_failures.size(); ++i) {
+ EXPECT_EQ(expected_failures[i], conversion_failures[i]) << "row " << i;
+ }
+ const auto& data = column->get_data();
+ EXPECT_EQ(0, data[0].year());
+ EXPECT_EQ(1, data[0].month());
+ EXPECT_EQ(1, data[0].day());
+ EXPECT_EQ(0, data[1].year());
+ EXPECT_EQ(2, data[1].month());
+ EXPECT_EQ(28, data[1].day());
+ EXPECT_EQ(0, data[3].year());
+ EXPECT_EQ(3, data[3].month());
+ EXPECT_EQ(1, data[3].day());
+ EXPECT_EQ(2024, data[4].year());
+}
+
+TEST_F(DataTypeDateV2SerDeCalendarTest, NullDate32PayloadIsNotDecoded) {
+ auto array = build_date32({std::numeric_limits<int32_t>::min(), -719528});
+ auto data = array->data()->Copy();
+ data->buffers[0] = arrow::Buffer::FromVector(std::vector<uint8_t> {0x02});
+ data->SetNullCount(1);
+ auto nullable_array = arrow::MakeArray(data);
+ auto column = ColumnDateV2::create();
+ ASSERT_TRUE(serde.read_column_from_arrow(*column, nullable_array.get(), 0, 2, tz).ok());
+ ASSERT_EQ(column->size(), 2);
+ EXPECT_EQ(column->get_data()[1], make_date(0, 1, 1));
+}
+
+TEST_F(DataTypeDateV2SerDeCalendarTest, EveryRepresentableDateMatchesCivilCalendar) {
+ const cctz::civil_day epoch(1970, 1, 1);
+ for (auto civil = cctz::civil_day(0, 1, 1); civil < cctz::civil_day(10000, 1, 1); ++civil) {
+ // The external calendar has this leap day, while Doris intentionally follows MySQL.
+ if (civil.year() == 0 && civil.month() == 2 && civil.day() == 29) {
+ continue;
+ }
+ const auto value = make_date(civil.year(), civil.month(), civil.day());
+ const auto days = civil - epoch;
+ ASSERT_EQ(days, daynr_to_epoch_days(value.daynr())) << value.to_string();
+ ASSERT_EQ(value.daynr(), epoch_days_to_daynr(days)) << value.to_string();
+ }
+ EXPECT_EQ(epoch_days_to_daynr(-719469), 0);
+ EXPECT_EQ(epoch_days_to_daynr(std::numeric_limits<int64_t>::min()), 0);
+ EXPECT_EQ(epoch_days_to_daynr(std::numeric_limits<int64_t>::max()), 0);
+}
+
+} // namespace doris
diff --git a/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp b/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp
index 8595b19e029..f6909877350 100644
--- a/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp
+++ b/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp
@@ -20,6 +20,7 @@
#include <gtest/gtest.h>
#include <limits>
+#include <tuple>
#include "core/data_type/data_type_date_or_datetime_v2.h"
#include "core/data_type/data_type_timestamptz.h"
@@ -721,4 +722,30 @@ TEST_F(PartitionTransformersTest, test_nullable_column_string_truncate_transform
EXPECT_EQ("db", result_strings->get_data_at(2).to_string());
}
+TEST_F(PartitionTransformersTest, DatePartitionsMatchYearZeroFileOrdinals) {
+ auto column = ColumnDateV2::create();
+ const std::vector<std::tuple<int, int, int, int64_t>> cases = {
+ {0, 1, 1, -719528}, {0, 2, 28, -719470}, {0, 3, 1, -719468},
+ {1969, 12, 31, -1}, {1970, 1, 1, 0}, {2024, 1, 1, 19723}};
+ for (const auto& [year, month, day, ordinal] : cases) {
+ DateV2Value<DateV2ValueType> value;
+ value.unchecked_set_time(year, month, day, 0, 0, 0, 0);
+ column->get_data().push_back(value);
+ }
+ auto type = std::make_shared<DataTypeDateV2>();
+ Block block({{std::move(column), type, "d"}});
+ DateDayPartitionColumnTransform day_transform(type);
+ DateBucketPartitionColumnTransform bucket_transform(type, 16);
+ const auto days = day_transform.apply(block, 0);
+ const auto buckets = bucket_transform.apply(block, 0);
+ for (size_t i = 0; i < cases.size(); ++i) {
+ // Iceberg hashes a signed, eight-byte epoch day, including for DATE's INT32 carrier.
+ const int64_t ordinal = std::get<3>(cases[i]);
+ const auto hash = HashUtil::murmur_hash3_32(&ordinal, sizeof(ordinal), 0);
+ EXPECT_EQ(ordinal, assert_cast<const ColumnInt32&>(*days.column).get_data()[i]);
+ EXPECT_EQ((hash & INT32_MAX) % 16,
+ assert_cast<const ColumnInt32&>(*buckets.column).get_data()[i]);
+ }
+}
+
} // namespace doris
diff --git a/be/test/exec/test_data/orc_scanner/date_count_truncated_stats.orc b/be/test/exec/test_data/orc_scanner/date_count_truncated_stats.orc
new file mode 100644
index 00000000000..7c942f141f3
Binary files /dev/null and b/be/test/exec/test_data/orc_scanner/date_count_truncated_stats.orc differ
diff --git a/be/test/exec/test_data/orc_scanner/date_large_ordinal.orc b/be/test/exec/test_data/orc_scanner/date_large_ordinal.orc
new file mode 100644
index 00000000000..b0c21072394
Binary files /dev/null and b/be/test/exec/test_data/orc_scanner/date_large_ordinal.orc differ
diff --git a/be/test/exec/test_data/orc_scanner/date_metadata_fixtures.md b/be/test/exec/test_data/orc_scanner/date_metadata_fixtures.md
new file mode 100644
index 00000000000..47bbff013f7
--- /dev/null
+++ b/be/test/exec/test_data/orc_scanner/date_metadata_fixtures.md
@@ -0,0 +1,39 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# ORC DATE metadata fixtures
+
+These small, uncompressed files exercise metadata validation independently of
+DATE value decoding. They were generated with PyArrow 24.0.0's ORC writer.
+
+- `date_preflight.orc`: schema `struct<id:int,d:date,unused:string>`, two rows
+ `(0, DATE ordinal 0, 'a')` and `(0, DATE ordinal 1, 'b')`, one stripe, and
+ row-index stride 1000. A predicate `id > 7` excludes the stripe. Projecting
+ `d` must not add stripe-footer or row-index reads.
+- `date_preflight_bad_index.orc`: identical bytes except that the unused
+ column's `ROW_INDEX` stream column ID in the stripe footer is changed from
+ 3 to 100. The stream itself remains nonempty and valid. No data or index in
+ this excluded stripe should be read or parsed during DATE preflight.
+- `date_count_truncated_stats.orc`: schema `struct<id:int,s:struct<d:date>>`,
+ two rows with IDs 0 and 1 and DATE ordinals 0 and 1. The stripe metadata's
+ fourth `colStats` entry (nested DATE ID 3) is removed, while the schema and
+ file-level statistics remain complete. Row-index streams are removed, the
+ stripe index length and file row-index stride are zero, and footer lengths
+ are adjusted. `COUNT(s)` must decline metadata pushdown without an unchecked
+ lookup; ordinary reading still returns both intact struct values.
diff --git a/be/test/exec/test_data/orc_scanner/date_out_of_range.orc b/be/test/exec/test_data/orc_scanner/date_out_of_range.orc
new file mode 100644
index 00000000000..9a7acdd055b
Binary files /dev/null and b/be/test/exec/test_data/orc_scanner/date_out_of_range.orc differ
diff --git a/be/test/exec/test_data/orc_scanner/date_preflight.orc b/be/test/exec/test_data/orc_scanner/date_preflight.orc
new file mode 100644
index 00000000000..41350b1c924
Binary files /dev/null and b/be/test/exec/test_data/orc_scanner/date_preflight.orc differ
diff --git a/be/test/exec/test_data/orc_scanner/date_preflight_bad_index.orc b/be/test/exec/test_data/orc_scanner/date_preflight_bad_index.orc
new file mode 100644
index 00000000000..3b8e2e5f7be
Binary files /dev/null and b/be/test/exec/test_data/orc_scanner/date_preflight_bad_index.orc differ
diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp
index e3e6d686cea..1817793936e 100644
--- a/be/test/format_v2/orc/orc_reader_test.cpp
+++ b/be/test/format_v2/orc/orc_reader_test.cpp
@@ -150,8 +150,9 @@ DateV2Value<DateV2ValueType> make_date_v2(uint16_t year, uint8_t month, uint8_t
}
int64_t orc_date_offset(uint16_t year, uint8_t month, uint8_t day) {
- static constexpr int32_t DATE_THRESHOLD = 719528;
- return make_date_v2(year, month, day).daynr() - DATE_THRESHOLD;
+ // ORC DATE is days since 1970-01-01 in the proleptic Gregorian calendar, which is not the
+ // same as Doris's daynr minus the epoch daynr for year-zero dates.
+ return daynr_to_epoch_days(make_date_v2(year, month, day).daynr());
}
DateV2Value<DateTimeV2ValueType> make_datetime_v2(uint16_t year, uint8_t month, uint8_t day,
@@ -3339,7 +3340,6 @@ void write_complex_orc_file(const std::string& file_path) {
void write_map_decimal_date_orc_file(const std::string& file_path) {
constexpr size_t ROWS = 4;
- constexpr int64_t HIVE_012_1900_DAY_OFFSET = -719530;
constexpr int64_t YEAR_0000_12_29_DAY_OFFSET = -719165;
constexpr int64_t YEAR_1000_10_16_DAY_OFFSET = -353997;
@@ -3371,7 +3371,9 @@ void write_map_decimal_date_orc_file(const std::string& file_path) {
key_batch.values[1] = 9999999999L;
key_batch.values[2] = 0;
key_batch.values[3] = 1;
- value_batch.data[0] = HIVE_012_1900_DAY_OFFSET;
+ // The smallest DATE Doris can represent: -719528 in the proleptic Gregorian calendar the ORC
+ // spec defines DATE in, one day below what Doris's own MySQL-calendar daynr would suggest.
+ value_batch.data[0] = orc_date_offset(0, 1, 1);
value_batch.data[1] = orc_date_offset(9999, 12, 31);
value_batch.data[2] = YEAR_0000_12_29_DAY_OFFSET;
value_batch.data[3] = YEAR_1000_10_16_DAY_OFFSET;
@@ -3389,6 +3391,42 @@ void write_map_decimal_date_orc_file(const std::string& file_path) {
out.write(memory_stream.getData(), static_cast<std::streamsize>(memory_stream.getLength()));
}
+// A flat `struct<id:int,d:date>` file whose DATE ordinals are written verbatim, so a test can
+// place a value that no Doris DATE maps to (the proleptic-only 0000-02-29, or an ordinal outside
+// the type's range) next to representable ones.
+void write_date_orc_file(const std::string& file_path,
+ const std::vector<std::optional<int64_t>>& day_offsets) {
+ auto type =
+ std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString("struct<id:int,d:date>"));
+
+ MemoryOutputStream memory_stream(1024 * 1024);
+ ::orc::WriterOptions options;
+ options.setCompression(::orc::CompressionKind_NONE);
+ options.setMemoryPool(::orc::getDefaultPool());
+ auto writer = ::orc::createWriter(*type, &memory_stream, options);
+ auto batch = writer->createRowBatch(day_offsets.size());
+ auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch);
+ auto& id_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]);
+ auto& date_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[1]);
+
+ date_batch.hasNulls = true;
+ for (size_t row = 0; row < day_offsets.size(); ++row) {
+ id_batch.data[row] = static_cast<int64_t>(row) + 1;
+ date_batch.notNull[row] = day_offsets[row].has_value() ? 1 : 0;
+ // Deliberately garbage under a null slot: a decoder must never look at it.
+ date_batch.data[row] = day_offsets[row].value_or(std::numeric_limits<int64_t>::min());
+ }
+
+ struct_batch.numElements = day_offsets.size();
+ id_batch.numElements = day_offsets.size();
+ date_batch.numElements = day_offsets.size();
+ writer->add(*batch);
+ writer->close();
+
+ std::ofstream out(file_path, std::ios::binary);
+ out.write(memory_stream.getData(), static_cast<std::streamsize>(memory_stream.getLength()));
+}
+
void write_two_stripe_orc_array_map_file(const std::string& file_path) {
auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString(
"struct<array_col:array<int>,map_col:map<string,int>,payload:string>"));
@@ -11096,12 +11134,140 @@ TEST_F(NewOrcReaderTest, ReadMapDecimalDateWithCenturyBoundary) {
const auto& map_column = assert_cast<const ColumnMap&>(map_nullable.get_nested_column());
ASSERT_EQ(map_column.get_offsets().size(), 4);
ASSERT_EQ(map_column.get_values().size(), 4);
- EXPECT_EQ(schema[1].children[1].type->to_string(map_column.get_values(), 0), "1900-01-01");
+ EXPECT_EQ(schema[1].children[1].type->to_string(map_column.get_values(), 0), "0000-01-01");
EXPECT_EQ(schema[1].children[1].type->to_string(map_column.get_values(), 1), "9999-12-31");
EXPECT_EQ(schema[1].children[1].type->to_string(map_column.get_values(), 2), "0000-12-29");
EXPECT_EQ(schema[1].children[1].type->to_string(map_column.get_values(), 3), "1000-10-16");
}
+// ORC DATE is a proleptic Gregorian day ordinal, so year zero is the window where Doris's own
+// MySQL-calendar day number disagrees with the file. Pin the decode there, including a null row
+// whose payload must never be looked at.
+TEST_F(NewOrcReaderTest, ReadDateYearZeroWindow) {
+ const auto file_path = (_test_dir / "date_year_zero.orc").string();
+ write_date_orc_file(file_path, {orc_date_offset(0, 1, 1), orc_date_offset(0, 2, 28),
+ orc_date_offset(0, 3, 1), std::nullopt,
+ orc_date_offset(1970, 1, 1), orc_date_offset(2024, 1, 1)});
+ auto reader = create_reader_for_path(file_path);
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ ASSERT_EQ(schema.size(), 2);
+
+ Block block = build_file_block(schema);
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {field_projection(0), field_projection(1)};
+ ASSERT_TRUE(reader->open(request).ok());
+
+ size_t rows = 0;
+ bool eof = false;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ ASSERT_EQ(rows, 6);
+
+ const auto& nullable = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ const std::vector<std::string> expected = {"0000-01-01", "0000-02-28", "0000-03-01",
+ "", "1970-01-01", "2024-01-01"};
+ for (size_t row = 0; row < expected.size(); ++row) {
+ if (expected[row].empty()) {
+ EXPECT_TRUE(nullable.is_null_at(row)) << "row " << row;
+ continue;
+ }
+ ASSERT_FALSE(nullable.is_null_at(row)) << "row " << row;
+ EXPECT_EQ(expected[row], schema[1].type->to_string(nullable, row)) << "row " << row;
+ }
+}
+
+TEST_F(NewOrcReaderTest, ReadDateRejectsUnrepresentableOrdinals) {
+ // -719469 is 0000-02-29, which exists in the proleptic Gregorian calendar but not in Doris;
+ // -719530 is below 0000-01-01. Both used to decode as 1900-01-01 through the day dictionary.
+ for (const int64_t bad_offset : {int64_t {-719469}, int64_t {-719530}, int64_t {2932897}}) {
+ const auto file_path = (_test_dir / fmt::format("date_bad_{}.orc", bad_offset)).string();
+ write_date_orc_file(file_path, {orc_date_offset(2024, 1, 1), bad_offset});
+ auto reader = create_reader_for_path(file_path);
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ Block block = build_file_block(schema);
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {field_projection(0), field_projection(1)};
+ ASSERT_TRUE(reader->open(request).ok());
+
+ size_t rows = 0;
+ bool eof = false;
+ const auto status = reader->get_block(&block, &rows, &eof);
+ EXPECT_FALSE(status.ok()) << "offset " << bad_offset;
+ // The message must name the column and the offending value, and must not claim a format
+ // it did not come from: the helper is shared with Parquet.
+ EXPECT_NE(std::string::npos, status.to_string().find("outside the Doris DATE range"))
+ << status.to_string();
+ EXPECT_NE(std::string::npos, status.to_string().find(std::to_string(bad_offset)))
+ << status.to_string();
+ EXPECT_NE(std::string::npos, status.to_string().find("'d'")) << status.to_string();
+ EXPECT_EQ(std::string::npos, status.to_string().find("Parquet")) << status.to_string();
+ }
+}
+
+// A pushed-down MIN/MAX is answered from stripe statistics without reading a row, so the
+// statistics have to be decoded with exactly the same calendar as the rows. Decoding them through
+// the day dictionary used to report 1900-01-01 as the minimum of a file whose smallest row is
+// 0000-01-01 -- a value present in no row at all.
+TEST_F(NewOrcReaderTest, AggregatePushdownDateMinMaxAgreesWithRowDecode) {
+ const auto file_path = (_test_dir / "date_year_zero_minmax.orc").string();
+ write_date_orc_file(file_path, {orc_date_offset(0, 1, 1), orc_date_offset(0, 2, 1),
+ std::nullopt, orc_date_offset(0, 2, 28)});
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+
+ auto reader = create_reader_for_path(file_path);
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {field_projection(1)};
+ ASSERT_TRUE(reader->open(request).ok());
+
+ format::FileAggregateRequest aggregate_request;
+ aggregate_request.agg_type = TPushAggOp::type::MINMAX;
+ aggregate_request.columns.push_back(
+ {.projection = format::LocalColumnIndex::top_level(format::LocalColumnId(1))});
+ format::FileAggregateResult aggregate_result;
+ const auto status = reader->get_aggregate_result(aggregate_request, &aggregate_result);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_EQ(aggregate_result.columns.size(), 1);
+ EXPECT_EQ(aggregate_result.count, 4);
+ ASSERT_TRUE(aggregate_result.columns[0].has_min);
+ ASSERT_TRUE(aggregate_result.columns[0].has_max);
+ EXPECT_EQ(aggregate_result.columns[0].min_value.get<TYPE_DATEV2>(), make_date_v2(0, 1, 1));
+ EXPECT_EQ(aggregate_result.columns[0].max_value.get<TYPE_DATEV2>(), make_date_v2(0, 2, 28));
+}
+
+TEST_F(NewOrcReaderTest, AggregatePushdownDateMinMaxFallsBackForUnrepresentableBound) {
+ const auto file_path = (_test_dir / "date_minmax_fallback.orc").string();
+ // The minimum is the proleptic-only 0000-02-29: no Doris DATE bounds this file, so the
+ // statistics must be refused rather than silently rounded onto a neighbouring day.
+ write_date_orc_file(file_path, {std::optional<int64_t> {-719469}, orc_date_offset(2024, 1, 1)});
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+
+ auto reader = create_reader_for_path(file_path);
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {field_projection(1)};
+ ASSERT_TRUE(reader->open(request).ok());
+
+ format::FileAggregateRequest aggregate_request;
+ aggregate_request.agg_type = TPushAggOp::type::MINMAX;
+ aggregate_request.columns.push_back(
+ {.projection = format::LocalColumnIndex::top_level(format::LocalColumnId(1))});
+ format::FileAggregateResult aggregate_result;
+ const auto status = reader->get_aggregate_result(aggregate_request, &aggregate_result);
+ EXPECT_TRUE(status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) << status;
+}
+
TEST_F(NewOrcReaderTest, ReadDeepNestedComplexTypes) {
const auto complex_file_path = (_test_dir / "deep_complex.orc").string();
write_deep_nested_complex_orc_file(complex_file_path);
@@ -11567,5 +11733,388 @@ TEST_F(NewOrcReaderTest, ReadProjectedComplexChildrenWithNulls) {
EXPECT_EQ(value_a.get_element(1), 202);
}
+TEST_F(NewOrcReaderTest, DateInteriorLeapDayDisablesAggregateAndSarg) {
+ const auto path = (_test_dir / "date_interior_leap_day.orc").string();
+ write_date_orc_file(path, {-719528, -719469, -719468});
+ for (bool pruning : {false, true}) {
+ auto reader = create_reader_for_path(path);
+ TQueryOptions options;
+ options.__set_enable_orc_filter_by_min_max(pruning);
+ RuntimeState state {options, TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->predicate_columns = {field_projection(1)};
+ request->conjuncts = {
+ VExprContext::create_shared(std::make_shared<NullableGreaterThanExpr<TYPE_DATEV2>>(
+ 0, remove_nullable(schema[1].type),
+ Field::create_field<TYPE_DATEV2>(make_date_v2(1, 1, 1)), "d"))};
+ ASSERT_TRUE(reader->open(request).ok());
+ Block block = build_file_block({schema[1]});
+ size_t rows = 0;
+ bool eof = false;
+ const auto status = reader->get_block(&block, &rows, &eof);
+ EXPECT_FALSE(status.ok()) << "pruning=" << pruning;
+ EXPECT_NE(status.to_string().find("-719469"), std::string::npos);
+ }
+ auto reader = create_reader_for_path(path);
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {field_projection(1)};
+ ASSERT_TRUE(reader->open(request).ok());
+ format::FileAggregateRequest aggregate;
+ aggregate.agg_type = TPushAggOp::type::MINMAX;
+ aggregate.columns.push_back({.projection = field_projection(1)});
+ format::FileAggregateResult result;
+ EXPECT_TRUE(reader->get_aggregate_result(aggregate, &result)
+ .is<ErrorCode::NOT_IMPLEMENTED_ERROR>());
+}
+
+TEST_F(NewOrcReaderTest, DateCountValidatesScalarAndNestedValues) {
+ const std::vector<std::vector<std::optional<int64_t>>> cases = {
+ {-719528, -719469, -719468},
+ {-719530, 0, 1},
+ {0, 1, 2932897},
+ {-719528, -719470, -719470},
+ {-719468, 0, 1},
+ {std::nullopt, std::nullopt, std::nullopt}};
+ for (size_t case_id = 0; case_id < cases.size(); ++case_id) {
+ SCOPED_TRACE(case_id);
+ const auto& days = cases[case_id];
+ const auto path = (_test_dir / "count_nested_dates.orc").string();
+ auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString(
+ "struct<d:date,a:array<date>,s:struct<d:date>,m:map<int,date>>"));
+ MemoryOutputStream stream(1024 * 1024);
+ auto writer = ::orc::createWriter(*type, &stream, ::orc::WriterOptions());
+ auto batch = writer->createRowBatch(days.size());
+ auto& root = dynamic_cast<::orc::StructVectorBatch&>(*batch);
+ auto fill = [&](auto&& self, ::orc::ColumnVectorBatch& column) -> void {
+ column.numElements = days.size();
+ if (auto* record = dynamic_cast<::orc::StructVectorBatch*>(&column)) {
+ for (auto* field : record->fields) {
+ self(self, *field);
+ }
+ } else if (auto* list = dynamic_cast<::orc::ListVectorBatch*>(&column)) {
+ for (size_t i = 0; i <= days.size(); ++i) {
+ list->offsets[i] = i;
+ }
+ self(self, *list->elements);
+ } else if (auto* map = dynamic_cast<::orc::MapVectorBatch*>(&column)) {
+ for (size_t i = 0; i <= days.size(); ++i) {
+ map->offsets[i] = i;
+ }
+ auto& keys = dynamic_cast<::orc::LongVectorBatch&>(*map->keys);
+ keys.numElements = days.size();
+ for (size_t i = 0; i < days.size(); ++i) {
+ keys.data[i] = i;
+ }
+ self(self, *map->elements);
+ } else {
+ auto& dates = dynamic_cast<::orc::LongVectorBatch&>(column);
+ dates.hasNulls = true;
+ for (size_t i = 0; i < days.size(); ++i) {
+ dates.notNull[i] = days[i].has_value();
+ dates.data[i] = days[i].value_or(0);
+ }
+ }
+ };
+ fill(fill, root);
+ writer->add(*batch);
+ writer->close();
+ std::ofstream out(path, std::ios::binary);
+ out.write(stream.getData(), static_cast<std::streamsize>(stream.getLength()));
+ out.close();
+
+ for (int column_id = 0; column_id < 4; ++column_id) {
+ SCOPED_TRACE(column_id);
+ auto reader = create_reader_for_path(path);
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {field_projection(column_id)};
+ ASSERT_TRUE(reader->open(request).ok());
+ format::FileAggregateRequest aggregate;
+ aggregate.agg_type = TPushAggOp::type::COUNT;
+ format::FileAggregateResult result;
+ ASSERT_TRUE(reader->get_aggregate_result(aggregate, &result).ok());
+ EXPECT_EQ(result.count, 3);
+ aggregate.columns.push_back({.projection = field_projection(column_id)});
+ const auto status = reader->get_aggregate_result(aggregate, &result);
+ const bool invalid = case_id < 3;
+ if (invalid) {
+ EXPECT_TRUE(status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) << status;
+ } else {
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(result.count, column_id == 0 && case_id == 5 ? 0 : 3);
+ }
+ auto block = build_file_block({schema[column_id]});
+ size_t rows = 0;
+ bool eof = false;
+ const auto scan_status = reader->get_block(&block, &rows, &eof);
+ if (invalid) {
+ EXPECT_TRUE(scan_status.is<ErrorCode::DATA_QUALITY_ERROR>()) << scan_status;
+ EXPECT_NE(scan_status.to_string().find("outside the Doris DATE range"),
+ std::string::npos);
+ } else {
+ ASSERT_TRUE(scan_status.ok()) << scan_status;
+ EXPECT_EQ(rows, 3);
+ }
+ }
+ }
+}
+
+TEST_F(NewOrcReaderTest, DateSargGuardIgnoresUnprojectedStructChild) {
+ const auto path = (_test_dir / "partial_struct_date.orc").string();
+ auto type = std::unique_ptr<::orc::Type>(
+ ::orc::Type::buildTypeFromString("struct<id:int,s:struct<x:int,d:date>>"));
+ MemoryOutputStream stream(1024 * 1024);
+ auto writer = ::orc::createWriter(*type, &stream, ::orc::WriterOptions());
+ auto batch = writer->createRowBatch(3);
+ auto& root = dynamic_cast<::orc::StructVectorBatch&>(*batch);
+ auto& ids = dynamic_cast<::orc::LongVectorBatch&>(*root.fields[0]);
+ auto& record = dynamic_cast<::orc::StructVectorBatch&>(*root.fields[1]);
+ auto& values = dynamic_cast<::orc::LongVectorBatch&>(*record.fields[0]);
+ auto& dates = dynamic_cast<::orc::LongVectorBatch&>(*record.fields[1]);
+ root.numElements = ids.numElements = record.numElements = values.numElements =
+ dates.numElements = 3;
+ for (int row = 0; row < 3; ++row) {
+ ids.data[row] = values.data[row] = row;
+ dates.data[row] = -719469;
+ }
+ writer->add(*batch);
+ writer->close();
+ std::ofstream out(path, std::ios::binary);
+ out.write(stream.getData(), static_cast<std::streamsize>(stream.getLength()));
+ out.close();
+
+ for (bool project_date : {false, true}) {
+ SCOPED_TRACE(project_date);
+ auto reader = create_reader_for_path(path);
+ TQueryOptions options;
+ options.__set_enable_orc_lazy_mat(false);
+ RuntimeState state {options, TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->predicate_columns = {field_projection(0)};
+ request->non_predicate_columns = {project_date ? field_projection(1)
+ : struct_child_projection(1, 0)};
+ request->conjuncts = {
+ VExprContext::create_shared(std::make_shared<NullableInt32GreaterThanExpr>(0, 7))};
+ ASSERT_TRUE(reader->open(request).ok());
+ // Only a decoded DATE can require disabling SARG to preserve a conversion error.
+ EXPECT_EQ(reader->get_total_rows(), project_date ? 3 : 0);
+ EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, project_date ? 0 : 1);
+ if (project_date) {
+ auto block = build_file_block(schema);
+ size_t rows = 0;
+ bool eof = false;
+ const auto status = reader->get_block(&block, &rows, &eof);
+ EXPECT_TRUE(status.is<ErrorCode::DATA_QUALITY_ERROR>()) << status;
+ }
+ }
+}
+
+TEST_F(NewOrcReaderTest, DateSargRetainsUnsafeStripeWithoutDisablingSafeStripePruning) {
+ const auto path = (_test_dir / "mixed_date_stripes.orc").string();
+ auto type = std::unique_ptr<::orc::Type>(
+ ::orc::Type::buildTypeFromString("struct<id:int,d:date,payload:string>"));
+ MemoryOutputStream stream(4 * 1024 * 1024);
+ ::orc::WriterOptions writer_options;
+ writer_options.setStripeSize(1);
+ writer_options.setCompression(::orc::CompressionKind_NONE);
+ writer_options.setDictionaryKeySizeThreshold(0);
+ auto writer = ::orc::createWriter(*type, &stream, writer_options);
+ // The safe stripes cannot match id > 7, but the invalid middle stripe must be decoded.
+ for (const int64_t day : {0, -719469, 1}) {
+ auto batch = writer->createRowBatch(200);
+ auto& root = dynamic_cast<::orc::StructVectorBatch&>(*batch);
+ auto& ids = dynamic_cast<::orc::LongVectorBatch&>(*root.fields[0]);
+ auto& dates = dynamic_cast<::orc::LongVectorBatch&>(*root.fields[1]);
+ auto& payload = dynamic_cast<::orc::StringVectorBatch&>(*root.fields[2]);
+ std::vector<std::string> values;
+ values.reserve(200);
+ root.numElements = ids.numElements = dates.numElements = payload.numElements = 200;
+ for (int row = 0; row < 200; ++row) {
+ ids.data[row] = 0;
+ dates.data[row] = day;
+ values.emplace_back(2048, static_cast<char>('a' + row % 26));
+ set_string_value(payload, row, values.back());
+ }
+ writer->add(*batch);
+ }
+ writer->close();
+ std::ofstream out(path, std::ios::binary);
+ out.write(stream.getData(), static_cast<std::streamsize>(stream.getLength()));
+ out.close();
+ ASSERT_EQ(get_orc_stripe_count(path), 3);
+
+ for (bool pruning : {false, true}) {
+ SCOPED_TRACE(pruning);
+ auto reader = create_reader_for_path(path);
+ TQueryOptions options;
+ options.__set_enable_orc_filter_by_min_max(pruning);
+ options.__set_enable_orc_lazy_mat(false);
+ RuntimeState state {options, TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->predicate_columns = {field_projection(0)};
+ request->non_predicate_columns = {field_projection(1)};
+ request->conjuncts = {
+ VExprContext::create_shared(std::make_shared<NullableInt32GreaterThanExpr>(0, 7))};
+ ASSERT_TRUE(reader->open(request).ok());
+ EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, pruning ? 2 : 0);
+ auto block = build_file_block({schema[0], schema[1]});
+ bool eof = false;
+ Status status;
+ while (!eof && status.ok()) {
+ block.clear_column_data();
+ size_t rows = 0;
+ status = reader->get_block(&block, &rows, &eof);
+ }
+ EXPECT_TRUE(status.is<ErrorCode::DATA_QUALITY_ERROR>()) << status;
+ EXPECT_NE(status.to_string().find("-719469"), std::string::npos);
+ }
+}
+
+TEST_F(NewOrcReaderTest, ReadExternalDateRejectsLargeOrdinal) {
+ // This external file encoded YYYYMMDD as an ORC day ordinal; the old dictionary hid it
+ // by substituting 1900-01-01. Reading other columns must remain possible.
+ const auto path = find_repo_file("be/test/exec/test_data/orc_scanner/date_large_ordinal.orc");
+ ASSERT_TRUE(std::filesystem::exists(path));
+ for (bool project_date : {false, true}) {
+ SCOPED_TRACE(project_date);
+ auto reader = create_reader_for_path(path.string());
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ ASSERT_EQ(schema.size(), 9);
+ ASSERT_EQ(remove_nullable(schema[3].type)->get_primitive_type(), TYPE_DATEV2);
+ const int column_id = project_date ? 3 : 0;
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {field_projection(column_id)};
+ ASSERT_TRUE(reader->open(request).ok());
+ auto block = build_file_block({schema[column_id]});
+ size_t rows = 0;
+ bool eof = false;
+ const auto status = reader->get_block(&block, &rows, &eof);
+ if (project_date) {
+ EXPECT_TRUE(status.is<ErrorCode::DATA_QUALITY_ERROR>()) << status;
+ EXPECT_NE(
+ status.to_string().find("DATE value 20191111 is outside the Doris DATE range"),
+ std::string::npos);
+ } else {
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(rows, 10);
+ }
+ }
+}
+
+TEST_F(NewOrcReaderTest, ReadExternalMapDateRejectsOutOfRangeOrdinal) {
+ // Keep the external LZ4 fixture: the old offset dictionary silently replaced -719530
+ // with 1900-01-01, hiding an unrepresentable DATE inside the map values.
+ const auto path = find_repo_file("be/test/exec/test_data/orc_scanner/date_out_of_range.orc");
+ ASSERT_TRUE(std::filesystem::exists(path));
+ auto reader = create_reader_for_path(path.string());
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ ASSERT_EQ(schema.size(), 2);
+ ASSERT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_MAP);
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {field_projection(0), field_projection(1)};
+ ASSERT_TRUE(reader->open(request).ok());
+ auto block = build_file_block(schema);
+ size_t rows = 0;
+ bool eof = false;
+ const auto status = reader->get_block(&block, &rows, &eof);
+ EXPECT_TRUE(status.is<ErrorCode::DATA_QUALITY_ERROR>()) << status;
+ EXPECT_NE(status.to_string().find("DATE value -719530 is outside the Doris DATE range"),
+ std::string::npos);
+}
+
+TEST_F(NewOrcReaderTest, DatePreflightDoesNotReadPrunedStripeIndexes) {
+ std::array<size_t, 2> io_calls {};
+ std::array<size_t, 2> io_bytes {};
+ for (bool project_date : {false, true}) {
+ RuntimeProfile profile("date_preflight");
+ const auto path = find_repo_file("be/test/exec/test_data/orc_scanner/date_preflight.orc");
+ io::FileReaderStats stats;
+ auto io_ctx = std::make_shared<io::IOContext>();
+ io_ctx->file_reader_stats = &stats;
+ auto reader = create_reader_for_path(path.string(), &profile, io_ctx);
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->predicate_columns = {field_projection(0)};
+ if (project_date) {
+ request->non_predicate_columns = {field_projection(1)};
+ }
+ request->conjuncts = {
+ VExprContext::create_shared(std::make_shared<NullableInt32GreaterThanExpr>(0, 7))};
+ ASSERT_TRUE(reader->open(request).ok());
+ EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1);
+ ASSERT_TRUE(reader->close().ok());
+ io_calls[project_date] = stats.read_calls;
+ io_bytes[project_date] = stats.read_bytes;
+ }
+ // Projecting DATE must only inspect the already loaded file-level stripe metadata.
+ EXPECT_GT(io_calls[0], 0);
+ EXPECT_EQ(io_calls[1], io_calls[0]);
+ EXPECT_EQ(io_bytes[1], io_bytes[0]);
+}
+
+TEST_F(NewOrcReaderTest, DatePreflightSkipsMalformedIndexInPrunedStripe) {
+ // Only the unused column's ROW_INDEX stream ID is changed from 3 to 100.
+ const auto path =
+ find_repo_file("be/test/exec/test_data/orc_scanner/date_preflight_bad_index.orc");
+ auto reader = create_reader_for_path(path.string());
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->predicate_columns = {field_projection(0)};
+ request->non_predicate_columns = {field_projection(1)};
+ request->conjuncts = {
+ VExprContext::create_shared(std::make_shared<NullableInt32GreaterThanExpr>(0, 7))};
+ ASSERT_TRUE(reader->open(request).ok());
+ EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 1);
+}
+
+TEST_F(NewOrcReaderTest, DateCountRejectsTruncatedNestedStatistics) {
+ // The schema includes s.d (ID 3), but stripe statistics stop at its parent (ID 2).
+ const auto path =
+ find_repo_file("be/test/exec/test_data/orc_scanner/date_count_truncated_stats.orc");
+ auto reader = create_reader_for_path(path.string());
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {field_projection(1)};
+ ASSERT_TRUE(reader->open(request).ok());
+ format::FileAggregateRequest aggregate_request;
+ aggregate_request.agg_type = TPushAggOp::type::COUNT;
+ aggregate_request.columns.push_back(
+ {.projection = format::LocalColumnIndex::top_level(format::LocalColumnId(1))});
+ format::FileAggregateResult result;
+ const auto status = reader->get_aggregate_result(aggregate_request, &result);
+ EXPECT_TRUE(status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) << status;
+ // Falling back still reads both struct values from the intact data streams.
+ auto block = build_file_block({schema[1]});
+ size_t rows = 0;
+ bool eof = false;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ EXPECT_EQ(rows, 2);
+}
+
} // namespace
} // namespace doris
diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp
index a5f45839428..170800d6a95 100644
--- a/be/test/format_v2/parquet/parquet_reader_test.cpp
+++ b/be/test/format_v2/parquet/parquet_reader_test.cpp
@@ -5590,5 +5590,253 @@ TEST_F(NewParquetReaderTest, RowPositionReaderUsesFileLocalPositionsForScanRange
}
}
+class YearZeroDatePredicate final : public VExpr {
+public:
+ explicit YearZeroDatePredicate(bool is_null)
+ : VExpr(std::make_shared<DataTypeUInt8>(), false), _is_null(is_null) {}
+ const std::string& expr_name() const override { return _name; }
+ void collect_slot_column_ids(std::set<int>& ids) const override { ids.insert(0); }
+ bool can_evaluate_zonemap_filter() const override { return true; }
+ ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override {
+ auto map = ctx.zone_map(0);
+ if (map == nullptr) {
+ return ZoneMapFilterResult::kMayMatch;
+ }
+ if (_is_null) {
+ return map->has_null ? ZoneMapFilterResult::kMayMatch : ZoneMapFilterResult::kNoMatch;
+ }
+ DateV2Value<DateV2ValueType> bound;
+ bound.unchecked_set_time(1, 1, 1, 0, 0, 0, 0);
+ return map->has_not_null && map->max_value >= Field::create_field<TYPE_DATEV2>(bound)
+ ? ZoneMapFilterResult::kMayMatch
+ : ZoneMapFilterResult::kNoMatch;
+ }
+ Status execute_column_impl(VExprContext*, const Block* block, const Selector* selector,
+ size_t count, ColumnPtr& result) const override {
+ const auto& column = assert_cast<const ColumnNullable&>(*block->get_by_position(0).column);
+ const auto& dates = assert_cast<const ColumnDateV2&>(column.get_nested_column());
+ auto out = ColumnUInt8::create();
+ for (size_t i = 0; i < count; ++i) {
+ const auto row = selector == nullptr ? i : (*selector)[i];
+ out->insert_value(_is_null ? column.is_null_at(row)
+ : !column.is_null_at(row) &&
+ dates.get_data()[row].year() >= 1);
+ }
+ result = std::move(out);
+ return Status::OK();
+ }
+
+private:
+ bool _is_null;
+ const std::string _name = "YearZeroDatePredicate";
+};
+
+TEST_F(NewParquetReaderTest, DateInteriorLeapDayRequiresMaterializedAggregates) {
+ arrow::Date32Builder dates;
+ ASSERT_TRUE(dates.AppendValues({-719528, -719469, -719468}).ok());
+ std::shared_ptr<arrow::Array> array;
+ ASSERT_TRUE(dates.Finish(&array).ok());
+ const auto table =
+ arrow::Table::Make(arrow::schema({arrow::field("d", arrow::date32())}), {array});
+ auto output = arrow::io::FileOutputStream::Open(_file_path).ValueOrDie();
+ ::parquet::WriterProperties::Builder properties;
+ properties.disable_dictionary();
+ ASSERT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, 3,
+ properties.build())
+ .ok());
+ ASSERT_TRUE(output->Close().ok());
+
+ for (bool strict : {false, true}) {
+ for (bool pruning : {false, true}) {
+ auto filtered_reader = create_reader();
+ TQueryOptions options;
+ options.__set_enable_strict_cast(strict);
+ RuntimeState filtered_state {options, TQueryGlobals()};
+ ASSERT_TRUE(filtered_reader->init(&filtered_state).ok());
+ std::vector<format::ColumnDefinition> filtered_schema;
+ ASSERT_TRUE(filtered_reader->get_schema(&filtered_schema).ok());
+ auto filtered_request = std::make_shared<format::FileScanRequest>();
+ filtered_request->predicate_columns = {
+ format::LocalColumnIndex::top_level(format::LocalColumnId(0))};
+ filtered_request->conjuncts = {
+ VExprContext::create_shared(std::make_shared<YearZeroDatePredicate>(!strict))};
+ filtered_request->metadata_pruning_safe_conjunct_count = pruning ? 1 : 0;
+ ASSERT_TRUE(filtered_reader->open(filtered_request).ok());
+ auto filtered_block = build_file_block(filtered_schema);
+ size_t filtered_rows = 0;
+ bool filtered_eof = false;
+ const auto status =
+ filtered_reader->get_block(&filtered_block, &filtered_rows, &filtered_eof);
+ if (strict) {
+ EXPECT_FALSE(status.ok()) << "pruning=" << pruning;
+ EXPECT_NE(status.to_string().find("-719469"), std::string::npos);
+ } else {
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(filtered_rows, 1) << "pruning=" << pruning;
+ if (filtered_rows == 1) {
+ EXPECT_TRUE(filtered_block.get_by_position(0).column->is_null_at(0));
+ }
+ }
+ }
+ }
+ auto reader = create_reader();
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {
+ format::LocalColumnIndex::top_level(format::LocalColumnId(0))};
+ ASSERT_TRUE(reader->open(request).ok());
+ for (auto operation : {TPushAggOp::type::COUNT, TPushAggOp::type::MINMAX}) {
+ format::FileAggregateRequest aggregate;
+ aggregate.agg_type = operation;
+ aggregate.columns.push_back({.projection = request->non_predicate_columns[0]});
+ format::FileAggregateResult result;
+ EXPECT_TRUE(reader->get_aggregate_result(aggregate, &result)
+ .is<ErrorCode::NOT_IMPLEMENTED_ERROR>());
+ }
+ auto block = build_file_block(schema);
+ size_t rows = 0;
+ bool eof = false;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ ASSERT_EQ(rows, 3);
+ const auto& values = assert_cast<const ColumnNullable&>(*block.get_by_position(0).column);
+ EXPECT_FALSE(values.is_null_at(0));
+ EXPECT_TRUE(values.is_null_at(1));
+ EXPECT_FALSE(values.is_null_at(2));
+}
+
+TEST_F(NewParquetReaderTest, DateCountKeepsSafeMetadataShortcut) {
+ const std::vector<std::vector<std::optional<int32_t>>> cases = {
+ {-719528, -719470}, {-719468, 19723}, {std::nullopt, std::nullopt}};
+ for (const auto& values : cases) {
+ for (bool nullable : {false, true}) {
+ if (!nullable && !values[0].has_value()) {
+ continue;
+ }
+ arrow::Date32Builder dates;
+ for (auto value : values) {
+ ASSERT_TRUE((value.has_value() ? dates.Append(*value) : dates.AppendNull()).ok());
+ }
+ std::shared_ptr<arrow::Array> array;
+ ASSERT_TRUE(dates.Finish(&array).ok());
+ auto table = arrow::Table::Make(
+ arrow::schema({arrow::field("d", arrow::date32(), nullable)}), {array});
+ auto output = arrow::io::FileOutputStream::Open(_file_path).ValueOrDie();
+ ASSERT_TRUE(
+ ::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, 2)
+ .ok());
+ ASSERT_TRUE(output->Close().ok());
+ auto reader = create_reader();
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ ASSERT_TRUE(reader->open(std::make_shared<format::FileScanRequest>()).ok());
+ format::FileAggregateRequest request;
+ request.agg_type = TPushAggOp::type::COUNT;
+ request.columns.push_back(
+ {.projection = format::LocalColumnIndex::top_level(format::LocalColumnId(0))});
+ format::FileAggregateResult result;
+ ASSERT_TRUE(reader->get_aggregate_result(request, &result).ok());
+ EXPECT_EQ(result.count, values[0].has_value() ? 2 : 0);
+ }
+ }
+}
+
+TEST_F(NewParquetReaderTest, NestedDateCountPreservesStrictConversionErrors) {
+ for (bool invalid : {false, true}) {
+ for (bool all_null : {false, true}) {
+ if (invalid && all_null) {
+ continue;
+ }
+ auto dates = std::make_shared<arrow::Date32Builder>();
+ arrow::ListBuilder lists(arrow::default_memory_pool(), dates);
+ for (int32_t day : {-719528, invalid ? -719469 : -719470, -719470}) {
+ ASSERT_TRUE(lists.Append().ok());
+ ASSERT_TRUE((all_null ? dates->AppendNull() : dates->Append(day)).ok());
+ }
+ const auto list = finish_array(&lists);
+ const auto values = std::static_pointer_cast<arrow::ListArray>(list)->values();
+ const auto record = arrow::StructArray::Make({values, build_int32_array({0, 1, 2})},
+ {arrow::field("d", arrow::date32()),
+ arrow::field("n", arrow::int32())})
+ .ValueOrDie();
+ const auto offsets = build_int32_array({0, 1, 2, 3});
+ const auto map =
+ arrow::MapArray::FromArrays(offsets, build_int32_array({0, 1, 2}), values)
+ .ValueOrDie();
+ const auto table = arrow::Table::Make(arrow::schema({arrow::field("a", list->type()),
+ arrow::field("s", record->type()),
+ arrow::field("m", map->type())}),
+ {list, record, map});
+ auto output = arrow::io::FileOutputStream::Open(_file_path).ValueOrDie();
+ ::parquet::WriterProperties::Builder properties;
+ properties.disable_dictionary();
+ ASSERT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output,
+ 3, properties.build())
+ .ok());
+ ASSERT_TRUE(output->Close().ok());
+ for (bool strict : {false, true}) {
+ for (int column_id = 0; column_id < 3; ++column_id) {
+ SCOPED_TRACE(testing::Message()
+ << "invalid=" << invalid << " all_null=" << all_null
+ << " strict=" << strict << " column=" << column_id);
+ auto reader = create_reader();
+ TQueryOptions options;
+ options.__set_enable_strict_cast(strict);
+ RuntimeState state {options, TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ const auto projection =
+ format::LocalColumnIndex::top_level(format::LocalColumnId(column_id));
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {projection};
+ ASSERT_TRUE(reader->open(request).ok());
+ format::FileAggregateRequest aggregate;
+ aggregate.agg_type = TPushAggOp::type::COUNT;
+ aggregate.columns.push_back({.projection = projection});
+ format::FileAggregateResult result;
+ const auto status = reader->get_aggregate_result(aggregate, &result);
+ if (invalid && strict) {
+ EXPECT_TRUE(status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) << status;
+ } else {
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(result.count, 3);
+ }
+ auto block = build_file_block(schema);
+ size_t rows = 0;
+ bool eof = false;
+ const auto scan_status = reader->get_block(&block, &rows, &eof);
+ if (invalid && strict) {
+ EXPECT_TRUE(scan_status.is<ErrorCode::DATA_QUALITY_ERROR>()) << scan_status;
+ EXPECT_NE(scan_status.to_string().find("-719469"), std::string::npos);
+ } else {
+ ASSERT_TRUE(scan_status.ok()) << scan_status;
+ EXPECT_EQ(rows, 3);
+ }
+ }
+ }
+ // An unprojected DATE sibling must not disable a safe shape-only COUNT.
+ auto reader = create_reader();
+ TQueryOptions options;
+ options.__set_enable_strict_cast(true);
+ RuntimeState state {options, TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ auto projection = format::LocalColumnIndex::partial_local(1);
+ projection.children.push_back(format::LocalColumnIndex::local(1));
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {projection};
+ ASSERT_TRUE(reader->open(request).ok());
+ format::FileAggregateRequest aggregate;
+ aggregate.agg_type = TPushAggOp::type::COUNT;
+ aggregate.columns.push_back({.projection = projection});
+ format::FileAggregateResult result;
+ ASSERT_TRUE(reader->get_aggregate_result(aggregate, &result).ok());
+ EXPECT_EQ(result.count, 3);
+ }
+ }
+}
+
} // namespace
} // namespace doris
diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp
index 6f51499bc33..b06a72f0977 100644
--- a/be/test/format_v2/parquet/parquet_statistics_test.cpp
+++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp
@@ -28,6 +28,7 @@
#include <optional>
#include <set>
#include <string>
+#include <tuple>
#include <unordered_map>
#include <utility>
#include <vector>
@@ -2212,5 +2213,178 @@ TEST(NativeParquetStatisticsTest, ShreddedVariantTypedValueDrivesPageFiltering)
EXPECT_EQ(selected_ranges[0].length, 100);
}
+TEST(NativeParquetStatisticsTest, DateInteriorLeapDayInvalidatesPhysicalNullStatistics) {
+ format::parquet::ParquetColumnSchema schema;
+ schema.type = make_nullable(std::make_shared<DataTypeDateV2>());
+ schema.type_descriptor.doris_type = schema.type;
+ schema.type_descriptor.physical_type = tparquet::Type::INT32;
+ auto encode = [](int32_t value) {
+ return std::string(reinterpret_cast<const char*>(&value), sizeof(value));
+ };
+ for (const auto& [min, max, safe] :
+ std::vector<std::tuple<int32_t, int32_t, bool>> {{-719528, -719468, false},
+ {-719469, -719469, false},
+ {-719528, -719470, true},
+ {-719468, 19723, true}}) {
+ tparquet::Statistics stats;
+ stats.__set_null_count(0);
+ stats.__set_min_value(encode(min));
+ stats.__set_max_value(encode(max));
+ const auto decoded = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(
+ schema, &stats, 3, nullptr);
+ EXPECT_EQ(safe, decoded.has_min_max);
+ EXPECT_EQ(safe, decoded.has_null_count);
+ }
+}
+
+class DateStatisticsPredicate final : public VExpr {
+public:
+ explicit DateStatisticsPredicate(bool is_null, int slot = 0)
+ : VExpr(std::make_shared<DataTypeUInt8>(), false), _is_null(is_null), _slot(slot) {}
+ const std::string& expr_name() const override { return _name; }
+ Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t,
+ ColumnPtr&) const override {
+ return Status::InternalError("Metadata-only predicate");
+ }
+ bool can_evaluate_zonemap_filter() const override { return true; }
+ void collect_slot_column_ids(std::set<int>& ids) const override { ids.insert(_slot); }
+ ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override {
+ const auto map = ctx.zone_map(_slot);
+ if (map == nullptr) {
+ return ZoneMapFilterResult::kMayMatch;
+ }
+ if (_is_null) {
+ return map->has_null ? ZoneMapFilterResult::kMayMatch : ZoneMapFilterResult::kNoMatch;
+ }
+ DateV2Value<DateV2ValueType> bound;
+ bound.unchecked_set_time(1, 1, 1, 0, 0, 0, 0);
+ return map->has_not_null && map->max_value >= Field::create_field<TYPE_DATEV2>(bound)
+ ? ZoneMapFilterResult::kMayMatch
+ : ZoneMapFilterResult::kNoMatch;
+ }
+
+private:
+ bool _is_null;
+ int _slot;
+ const std::string _name = "DateStatisticsPredicate";
+};
+
+TEST(NativeParquetStatisticsTest, UnsafeDatePageDoesNotDisableSafePagePruning) {
+ std::vector<std::unique_ptr<format::parquet::ParquetColumnSchema>> schema;
+ format::FileScanRequest request;
+ for (int slot = 0; slot < 2; ++slot) {
+ auto column = std::make_unique<format::parquet::ParquetColumnSchema>();
+ column->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE;
+ column->local_id = column->leaf_column_id = slot;
+ column->type = make_nullable(std::make_shared<DataTypeDateV2>());
+ column->type_descriptor.doris_type = column->type;
+ column->type_descriptor.physical_type = tparquet::Type::INT32;
+ schema.push_back(std::move(column));
+ request.local_positions.emplace(format::LocalColumnId(slot), format::LocalIndex(slot));
+ request.predicate_columns.push_back(format::LocalColumnIndex::local(slot));
+ }
+ auto encode = [](int32_t value) {
+ return std::string(reinterpret_cast<const char*>(&value), sizeof(value));
+ };
+ format::parquet::NativeParquetPageIndex index;
+ index.column_index.__set_min_values({encode(-719528), encode(-719528), encode(19723)});
+ index.column_index.__set_max_values({encode(-719470), encode(-719468), encode(19724)});
+ index.column_index.__set_null_pages({false, false, false});
+ index.column_index.__set_null_counts({0, 0, 0});
+ for (int page = 0; page < 3; ++page) {
+ tparquet::PageLocation location;
+ location.__set_offset(page * 10);
+ location.__set_compressed_page_size(10);
+ location.__set_first_row_index(page * 3);
+ index.offset_index.page_locations.push_back(location);
+ }
+ std::unordered_map<int, format::parquet::NativeParquetPageIndex> indexes;
+ indexes.emplace(0, index);
+ indexes.emplace(1, index);
+ tparquet::ColumnOrder order;
+ order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder());
+ tparquet::FileMetaData metadata;
+ metadata.__set_column_orders({order, order});
+ tparquet::RowGroup group;
+ group.__set_num_rows(9);
+ for (bool is_null : {false, true}) {
+ for (bool compound : {false, true}) {
+ SCOPED_TRACE(testing::Message() << "is_null=" << is_null << " compound=" << compound);
+ VExprSPtr predicate = std::make_shared<DateStatisticsPredicate>(is_null);
+ if (compound) {
+ TExprNode node;
+ node.__set_node_type(TExprNodeType::COMPOUND_PRED);
+ node.__set_opcode(TExprOpcode::COMPOUND_OR);
+ node.__set_type(std::make_shared<DataTypeUInt8>()->to_thrift());
+ node.__set_num_children(2);
+ node.__set_is_nullable(false);
+ auto either = VCompoundPred::create_shared(node);
+ either->add_child(predicate);
+ either->add_child(std::make_shared<DateStatisticsPredicate>(is_null, 1));
+ predicate = either;
+ }
+ request.conjuncts = {VExprContext::create_shared(predicate)};
+ std::vector<format::parquet::RowRange> ranges;
+ std::map<int, format::parquet::ParquetPageSkipPlan> plans;
+ ASSERT_TRUE(
+ format::parquet::select_row_group_ranges_by_native_page_index(
+ metadata, group, indexes, schema, request, 9, &ranges, &plans, nullptr)
+ .ok());
+ ASSERT_EQ(ranges.size(), 1);
+ EXPECT_EQ(ranges[0].start, 3);
+ EXPECT_EQ(ranges[0].length, is_null ? 3 : 6);
+ }
+ }
+}
+
+TEST(NativeParquetStatisticsTest, DateInteriorLeapDayKeepsPageCandidates) {
+ auto column = std::make_unique<format::parquet::ParquetColumnSchema>();
+ column->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE;
+ column->local_id = 0;
+ column->leaf_column_id = 0;
+ column->type = make_nullable(std::make_shared<DataTypeDateV2>());
+ column->type_descriptor.doris_type = column->type;
+ column->type_descriptor.physical_type = tparquet::Type::INT32;
+ std::vector<std::unique_ptr<format::parquet::ParquetColumnSchema>> schema;
+ schema.push_back(std::move(column));
+ auto encode = [](int32_t value) {
+ return std::string(reinterpret_cast<const char*>(&value), sizeof(value));
+ };
+ format::parquet::NativeParquetPageIndex index;
+ index.column_index.__set_min_values({encode(-719528)});
+ index.column_index.__set_max_values({encode(-719468)});
+ index.column_index.__set_null_pages({false});
+ index.column_index.__set_null_counts({0});
+ tparquet::PageLocation location;
+ location.__set_offset(0);
+ location.__set_compressed_page_size(10);
+ location.__set_first_row_index(0);
+ index.offset_index.__set_page_locations({location});
+ std::unordered_map<int, format::parquet::NativeParquetPageIndex> indexes;
+ indexes.emplace(0, std::move(index));
+ tparquet::ColumnOrder order;
+ order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder());
+ tparquet::FileMetaData metadata;
+ metadata.__set_column_orders({order});
+ tparquet::RowGroup group;
+ group.__set_num_rows(3);
+ for (bool is_null : {false, true}) {
+ format::FileScanRequest request;
+ request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0));
+ request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))};
+ request.conjuncts = {
+ VExprContext::create_shared(std::make_shared<DateStatisticsPredicate>(is_null))};
+ std::vector<format::parquet::RowRange> ranges;
+ std::map<int, format::parquet::ParquetPageSkipPlan> plans;
+ ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index(
+ metadata, group, indexes, schema, request, 3, &ranges, &plans, nullptr)
+ .ok());
+ EXPECT_EQ(ranges.size(), 1);
+ if (ranges.size() == 1) {
+ EXPECT_EQ(ranges[0].length, 3);
+ }
+ }
+}
+
} // namespace
} // namespace doris
diff --git a/regression-test/data/arrow_flight_sql_p0/test_date_year_zero.out b/regression-test/data/arrow_flight_sql_p0/test_date_year_zero.out
new file mode 100644
index 00000000000..f338898c9cb
--- /dev/null
+++ b/regression-test/data/arrow_flight_sql_p0/test_date_year_zero.out
@@ -0,0 +1,11 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !source --
+1 0000-01-01 0000-01-01
+2 0000-02-28 \N
+3 0000-03-01 0000-03-01
+4 0001-01-01 0001-01-01
+5 1969-12-31 1969-12-31
+6 1970-01-01 1970-01-01
+7 2024-01-01 2024-01-01
+8 9999-12-31 9999-12-31
+
diff --git a/regression-test/data/export_p0/outfile/test_outfile_date_year_zero.out b/regression-test/data/export_p0/outfile/test_outfile_date_year_zero.out
new file mode 100644
index 00000000000..6c1d9003d9d
--- /dev/null
+++ b/regression-test/data/export_p0/outfile/test_outfile_date_year_zero.out
@@ -0,0 +1,37 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !source --
+1 0000-01-01
+10 \N
+2 0000-01-31
+3 0000-02-28
+4 0000-03-01
+5 0001-01-01
+6 1969-12-31
+7 1970-01-01
+8 2024-01-01
+9 9999-12-31
+
+-- !readback_parquet --
+1 0000-01-01
+10 \N
+2 0000-01-31
+3 0000-02-28
+4 0000-03-01
+5 0001-01-01
+6 1969-12-31
+7 1970-01-01
+8 2024-01-01
+9 9999-12-31
+
+-- !readback_orc --
+1 0000-01-01
+10 \N
+2 0000-01-31
+3 0000-02-28
+4 0000-03-01
+5 0001-01-01
+6 1969-12-31
+7 1970-01-01
+8 2024-01-01
+9 9999-12-31
+
diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out
index dcf3cdb63ef..f31f2a50d2f 100644
--- a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out
+++ b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out
@@ -209,28 +209,6 @@
-- !test_15 --
1 0 0 1 apple 3.5
--- !test_16 --
-1 {-99999.99999:"1900-01-01"}
-1 {-99999.99999:"1900-01-01"}
-10 {null:"1000-10-16"}
-10 {null:"1000-10-16"}
-2 {99999.99999:"9999-12-31"}
-2 {99999.99999:"9999-12-31"}
-3 {0.00000:null}
-3 {0.00000:null}
-4 {null:"0000-12-29"}
-4 {null:"0000-12-29"}
-5 {null:null}
-5 {null:null}
-6 {-101.00000:null}
-6 {-101.00000:null}
-7 {null:null}
-7 {null:null}
-8 {0.00001:"2022-03-02"}
-8 {0.00001:"2022-03-02"}
-9 {null:null}
-9 {null:null}
-
-- !test_17 --
10 hello 10 10.01 \N
20 world 20 20.01 \N
@@ -247,18 +225,6 @@
-- !test_20 --
2022-04-09T07:13 2022-04-09
--- !test_21 --
-0 \N doris \N 1.567 1.567 12345 1 doris
-1 true doris 1900-01-01 1.567 1.567 12345 1 doris
-2 \N doris 1900-01-01 1.567 1.567 12345 1 doris
-3 true doris 1900-01-01 1.567 1.567 12345 1 doris
-4 \N doris \N 1.567 1.567 12345 1 doris
-5 true doris 1900-01-01 1.567 1.567 12345 1 doris
-6 \N doris 1900-01-01 1.567 1.567 12345 1 doris
-7 true doris 1900-01-01 1.567 1.567 12345 1 doris
-8 \N doris \N 1.567 1.567 12345 1 doris
-9 true doris 1900-01-01 1.567 1.567 12345 1 doris
-
-- !test_22 --
-128
diff --git a/regression-test/suites/arrow_flight_sql_p0/test_date_year_zero.groovy b/regression-test/suites/arrow_flight_sql_p0/test_date_year_zero.groovy
new file mode 100644
index 00000000000..c480a29e24b
--- /dev/null
+++ b/regression-test/suites/arrow_flight_sql_p0/test_date_year_zero.groovy
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Doris numbers DATE values in MySQL's calendar, where year 0 is not a leap year. Arrow `date32`
+// is days since 1970-01-01 in the proleptic Gregorian calendar, where year 0 IS a leap year, so
+// the two disagree for 0000-01-01 .. 0000-02-28. The MySQL protocol ships the year/month/day
+// fields directly and therefore never depends on a calendar; it is the control path here. Any
+// difference between the two protocols means the Arrow day ordinal is wrong.
+// See https://github.com/apache/doris/issues/67366
+suite("test_date_year_zero", "arrow_flight_sql") {
+ sql "DROP TABLE IF EXISTS test_date_year_zero"
+ sql """
+ CREATE TABLE test_date_year_zero (
+ id INT,
+ d DATE,
+ d_null DATE NULL,
+ arr ARRAY<DATE>
+ ) DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES ("replication_num" = "1");
+ """
+ sql """
+ INSERT INTO test_date_year_zero VALUES
+ (1, '0000-01-01', '0000-01-01', ['0000-01-01', '0000-02-28']),
+ (2, '0000-02-28', NULL, ['0000-03-01']),
+ (3, '0000-03-01', '0000-03-01', ['0001-01-01']),
+ (4, '0001-01-01', '0001-01-01', ['1969-12-31', '1970-01-01']),
+ (5, '1969-12-31', '1969-12-31', ['2024-01-01']),
+ (6, '1970-01-01', '1970-01-01', ['9999-12-31']),
+ (7, '2024-01-01', '2024-01-01', ['0000-01-01']),
+ (8, '9999-12-31', '9999-12-31', NULL);
+ """
+
+ // The absolute anchor. The MySQL protocol carries year/month/day verbatim, so these strings
+ // are what the Arrow path below must reproduce. Cast to string: java.sql.Date runs year-zero
+ // values through the Julian/Gregorian hybrid calendar.
+ order_qt_source """
+ SELECT CAST(id AS STRING), CAST(d AS STRING), CAST(d_null AS STRING)
+ FROM test_date_year_zero
+ """
+
+ // Read through the driver's own rendering rather than getObject(), for the same reason.
+ def fetchStrings = { java.sql.Connection conn, String stmtSql ->
+ def rows = []
+ conn.prepareStatement(stmtSql).withCloseable { st ->
+ st.executeQuery().withCloseable { rs ->
+ def columnCount = rs.metaData.columnCount
+ while (rs.next()) {
+ def row = []
+ for (int i = 1; i <= columnCount; ++i) {
+ row.add(rs.getString(i))
+ }
+ rows.add(row)
+ }
+ }
+ }
+ return rows
+ }
+
+ def query = "SELECT id, d, d_null FROM test_date_year_zero ORDER BY id"
+ def viaMysql = fetchStrings(context.getConnection(), query)
+ def viaFlight = fetchStrings(context.getArrowFlightSqlConnection(),
+ "USE ${context.dbName};" + query)
+ assertEquals(8, viaMysql.size())
+ // Before the fix, Arrow reported 0000-01-02 for row 1 and 0000-02-29 for row 2.
+ assertEquals(viaMysql, viaFlight,
+ "the Arrow and MySQL protocols disagree on the DATE values")
+
+ // ARRAY<DATE> has no encoding of its own, it delegates each element to the DATE SerDe. The
+ // Arrow Flight JDBC driver renders a list<date32> as its raw day counts rather than as dates,
+ // which makes this the strictest check available here: it pins the exact wire values.
+ // 0000-01-01 is -719528 and 0000-02-28 is -719470 in the proleptic Gregorian calendar; the
+ // pre-fix encoding produced -719527 and -719469.
+ def arrayQuery = "SELECT arr FROM test_date_year_zero WHERE id = 1"
+ def arrayViaFlight =
+ fetchStrings(context.getArrowFlightSqlConnection(),
+ "USE ${context.dbName};" + arrayQuery)[0][0].toString()
+ def arrayElements = arrayViaFlight.replaceAll(/[\[\]\s"]/, "").split(",") as List
+ assertEquals(["-719528", "-719470"], arrayElements,
+ "ARRAY<DATE> elements are not proleptic Gregorian day counts, got: ${arrayViaFlight}")
+}
diff --git a/regression-test/suites/export_p0/outfile/test_outfile_date_year_zero.groovy b/regression-test/suites/export_p0/outfile/test_outfile_date_year_zero.groovy
new file mode 100644
index 00000000000..7d4cfaa1a13
--- /dev/null
+++ b/regression-test/suites/export_p0/outfile/test_outfile_date_year_zero.groovy
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Parquet and ORC both store DATE as days since 1970-01-01 in the proleptic Gregorian calendar,
+// which differs from Doris's MySQL-calendar day number for 0000-01-01 .. 0000-02-28. The writer
+// (which shares the Arrow date32 encoder) and the reader must therefore agree, otherwise a table
+// exported and re-read by Doris comes back shifted by a day.
+// See https://github.com/apache/doris/issues/67366
+suite("test_outfile_date_year_zero", "p0") {
+ String ak = getS3AK()
+ String sk = getS3SK()
+ String s3_endpoint = getS3Endpoint()
+ String region = getS3Region()
+ String bucket = context.config.otherConfigs.get("s3BucketName")
+
+ def outFilePath = "${bucket}/outfile/date_year_zero/exp_"
+
+ sql """ DROP TABLE IF EXISTS test_outfile_date_year_zero_table """
+ sql """
+ CREATE TABLE test_outfile_date_year_zero_table (
+ `id` INT NOT NULL,
+ `d` DATE NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES ("replication_num" = "1");
+ """
+ // 0000-01-01 .. 0000-02-28 are the window where the two calendars disagree; 0000-03-01 onwards
+ // they coincide, so keep dates on both sides of the boundary plus the two range limits.
+ sql """
+ INSERT INTO test_outfile_date_year_zero_table VALUES
+ (1, '0000-01-01'), (2, '0000-01-31'), (3, '0000-02-28'), (4, '0000-03-01'),
+ (5, '0001-01-01'), (6, '1969-12-31'), (7, '1970-01-01'), (8, '2024-01-01'),
+ (9, '9999-12-31'), (10, NULL);
+ """
+
+ // Cast to string: the MySQL JDBC driver cannot materialise a year-zero DATE as java.sql.Date.
+ order_qt_source """
+ SELECT CAST(id AS STRING), CAST(d AS STRING) FROM test_outfile_date_year_zero_table
+ """
+
+ def outfile_to_S3 = { format ->
+ def res = sql """
+ SELECT * FROM test_outfile_date_year_zero_table t
+ INTO OUTFILE "s3://${outFilePath}"
+ FORMAT AS ${format}
+ PROPERTIES (
+ "s3.endpoint" = "${s3_endpoint}",
+ "s3.region" = "${region}",
+ "s3.secret_key"="${sk}",
+ "s3.access_key" = "${ak}"
+ );
+ """
+ def outfile_url = res[0][3]
+ def uri = "http://${bucket}.${s3_endpoint}" +
+ outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1) + "0." + format
+ return """
+ SELECT CAST(id AS STRING), CAST(d AS STRING) FROM S3 (
+ "uri" = "${uri}",
+ "ACCESS_KEY" = "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "${format}",
+ "region" = "${region}"
+ )
+ """
+ }
+
+ // Both read-backs must return exactly the source rows: same values, same NULL.
+ order_qt_readback_parquet(outfile_to_S3("parquet"))
+ order_qt_readback_orc(outfile_to_S3("orc"))
+}
diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_date_year_zero.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_date_year_zero.groovy
new file mode 100644
index 00000000000..829353ae7f3
--- /dev/null
+++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_date_year_zero.groovy
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Partition metadata and Parquet DATE payloads must use the same Gregorian epoch days.
+suite("test_iceberg_write_date_year_zero",
+ "p0,external,iceberg,external_docker,external_docker_iceberg") {
+ if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableIcebergTest"))) {
+ logger.info("disable iceberg test")
+ return
+ }
+ def env = context.config.otherConfigs
+ def catalog = "test_iceberg_write_date_year_zero"
+ def database = "iceberg_write_date_year_zero_db"
+ sql "drop catalog if exists ${catalog}"
+ sql """create catalog ${catalog} properties (
+ "type" = "iceberg", "iceberg.catalog.type" = "rest",
+ "uri" = "http://${env.get('externalEnvIp')}:${env.get('iceberg_rest_uri_port')}",
+ "s3.access_key" = "admin", "s3.secret_key" = "password",
+ "s3.endpoint" = "http://${env.get('externalEnvIp')}:${env.get('iceberg_minio_port')}",
+ "s3.region" = "us-east-1", "meta.cache.iceberg.table.ttl-second" = "0",
+ "meta.cache.iceberg.schema.ttl-second" = "0")"""
+ sql "switch ${catalog}"
+ sql "drop database if exists ${database} force"
+ sql "create database ${database}"
+ sql "use ${database}"
+ try {
+ def dates = ['0000-01-01', '0000-02-28', '0000-03-01', '1969-12-31', '1970-01-01', '2024-01-01']
+ def rows = dates.withIndex().collect { d, i -> "(${i}, date '${d}', date '${d}')" }.join(', ')
+ rows += ', (6, NULL, NULL)'
+ for (def name : ['doris_dates', 'spark_dates']) {
+ sql """create table ${name} (id int, d_day date, d_bucket date)
+ partition by list (day(d_day), bucket(16, d_bucket)) ()
+ properties ("format-version" = "2", "write.format.default" = "parquet")"""
+ }
+ sql "insert into doris_dates values ${rows}"
+ spark_iceberg "insert into demo.${database}.spark_dates values ${rows}"
+ for (def name : ['doris_dates', 'spark_dates']) {
+ sql "refresh table ${name}"
+ spark_iceberg "refresh table demo.${database}.${name}"
+ def projection = 'cast(id as string), cast(d_day as string), cast(d_bucket as string)'
+ assertSparkDorisResultEquals(
+ spark_iceberg("select ${projection} from demo.${database}.${name} order by id"),
+ sql("select ${projection} from ${name} order by id"))
+ dates.eachWithIndex { d, i ->
+ // An unfiltered scan alone cannot detect partition metadata contradicting the file.
+ for (def field : ['d_day', 'd_bucket']) {
+ def predicate = "${field} = date '${d}'"
+ def actual = spark_iceberg("select id from demo.${database}.${name} where ${predicate}")
+ assertEquals([[i]].toString(), actual.toString())
+ assertSparkDorisResultEquals(actual, sql("select id from ${name} where ${predicate}"))
+ }
+ }
+ }
+ def partitionQuery = { name ->
+ spark_iceberg("""select cast(partition.d_day_day as string),
+ cast(partition.d_bucket_bucket as string)
+ from demo.${database}.${name}.partitions order by 1, 2""")
+ }
+ assertEquals(partitionQuery('spark_dates').toString(), partitionQuery('doris_dates').toString())
+ } finally {
+ sql "drop database if exists ${database} force"
+ sql "switch internal"
+ sql "drop catalog if exists ${catalog}"
+ }
+}
diff --git a/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.groovy b/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.groovy
index 0afe32746da..d4162027410 100644
--- a/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.groovy
+++ b/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.groovy
@@ -139,10 +139,21 @@ suite("test_hdfs_orc_group6_orc_files","external,hive,tvf,external_docker") {
uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_orc/group6/map_decimal_date.lz4.orc"
- order_qt_test_16 """ select * from HDFS(
- "uri" = "${uri}",
- "hadoop.username" = "${hdfsUserName}",
- "format" = "orc"); """
+ def scannerV2 = sql "show variables like 'enable_file_scanner_v2'"
+ try {
+ sql "set enable_file_scanner_v2 = true"
+ // The map contains ordinal -719530, below 0000-01-01. Returning 1900-01-01
+ // was an unchecked conversion failure, not a valid interpretation of the file.
+ test {
+ sql """ select * from HDFS(
+ "uri" = "${uri}",
+ "hadoop.username" = "${hdfsUserName}",
+ "format" = "orc"); """
+ exception "DATE value -719530 is outside the Doris DATE range"
+ }
+ } finally {
+ sql "set enable_file_scanner_v2 = ${scannerV2[0][1]}"
+ }
uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_orc/group6/string-2-double.orc"
@@ -174,10 +185,21 @@ suite("test_hdfs_orc_group6_orc_files","external,hive,tvf,external_docker") {
uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_orc/group6/non_vec_orc_scanner.orc"
- order_qt_test_21 """ select * from HDFS(
- "uri" = "${uri}",
- "hadoop.username" = "${hdfsUserName}",
- "format" = "orc"); """
+ scannerV2 = sql "show variables like 'enable_file_scanner_v2'"
+ try {
+ sql "set enable_file_scanner_v2 = true"
+ // ORC DATE stores epoch days, not YYYYMMDD. The raw ordinal 20191111 is
+ // out of range; the old 1900-01-01 result silently hid that invalid value.
+ test {
+ sql """ select * from HDFS(
+ "uri" = "${uri}",
+ "hadoop.username" = "${hdfsUserName}",
+ "format" = "orc"); """
+ exception "DATE value 20191111 is outside the Doris DATE range"
+ }
+ } finally {
+ sql "set enable_file_scanner_v2 = ${scannerV2[0][1]}"
+ }
uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_orc/group6/tinyint.orc"
diff --git a/tools/legacy_date_migration/README.md b/tools/legacy_date_migration/README.md
new file mode 100644
index 00000000000..7420e05d9f9
--- /dev/null
+++ b/tools/legacy_date_migration/README.md
@@ -0,0 +1,117 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Migrating files with the former Doris DATE encoding
+
+The corrected DATE encoder writes proleptic Gregorian epoch days to ORC,
+Parquet and Arrow. The former Doris encoder used MySQL calendar day numbers
+minus 719528. For dates from `0000-01-01` through `0000-02-28`, the former value
+is one day too large. For example, those two endpoints were written as
+`-719527` and `-719469`, respectively; their correct values are `-719528` and
+`-719470`. Dates from `0000-03-01` onward are unchanged.
+
+A corrected reader cannot infer the intended date from an old ordinal:
+`-719527` also encodes the legitimate Gregorian date `0000-01-02`.
+Consequently, old files do not become compatible merely by upgrading the
+reader. Identify files using writer provenance, not their date values.
+
+## Explicit offline migration
+
+This tool requires Python 3.8 or later and PyArrow with ORC and Parquet support. It
+accepts only files **confirmed to have been produced with the former Doris
+DATE encoding for every DATE field**. Do not run it on files from other
+writers, mixed-origin files, or already migrated outputs. Running the
+conversion twice can silently shift dates again; the confirmation flag is
+an operator assertion, not automatic format detection. The lower-bound
+check catches some repeated conversions but cannot identify all of them.
+
+```bash
+python tools/legacy_date_migration/migrate_legacy_dates.py \
+ --input legacy.orc --output migrated.orc --format orc \
+ --confirm-legacy-doris-date-encoding
+
+python tools/legacy_date_migration/migrate_legacy_dates.py \
+ --input legacy.parquet --output migrated.parquet --format parquet \
+ --confirm-legacy-doris-date-encoding
+```
+
+The tool reads raw DATE32 integers and subtracts one only within the legacy
+window `[-719527, -719469]`. It preserves NULLs, nested ARRAY/STRUCT/MAP fields
+and other column values. It rejects DATE ordinals outside the legacy Doris
+range. Conversion uses Parquet batches of at most 65536 rows or one ORC
+stripe at a time; a large stripe or large nested values can still require
+substantial memory. It never modifies its input or overwrites an existing
+output. Output is published only after writing finishes, using an exclusive
+hard link on the destination filesystem; filesystems without hard-link
+support will fail without publishing output.
+
+Arrow field/schema metadata and nullability are retained during conversion.
+Parquet preserves this Arrow schema metadata. ORC has different metadata and
+type conventions and cannot round-trip arbitrary Arrow metadata or field
+nullability; the tool rewrites ORC data and schema supported by PyArrow,
+not an exact copy of all writer-specific metadata. Compression, row groups,
+stripes, statistics and indexes are regenerated. Unsupported types or
+writer failures abort without publishing a partially converted file.
+
+Keep the original files and record which files were converted. Query the
+new files with `enable_file_scanner_v2=true`, comparing row counts, NULLs,
+modern dates, and the year-zero boundary dates against authoritative source
+values. Only switch consumers after these checks. The converted files must
+be read by the corrected reader; enabling an old reader again is not a
+rollback strategy for the new encoding.
+
+## Optional legacy ORC read path
+
+For confirmed legacy ORC files, the legacy scanner can recover the original
+Doris dates before rewriting them through the corrected writer:
+
+```sql
+SET enable_file_scanner_v2 = false;
+-- Read only the confirmed legacy ORC source and write to a NEW output location.
+SELECT * FROM legacy_orc_source
+INTO OUTFILE 's3://example-bucket/migrated/date_export_'
+FORMAT AS ORC
+PROPERTIES (...);
+SET enable_file_scanner_v2 = true;
+-- Validate the newly written files against the original source values.
+```
+
+The storage properties and source relation above are placeholders. Scope
+the setting to a dedicated migration session. This is not a general
+compatibility switch: legacy Parquet reading does not reliably restore
+these dates, and a source containing both corrected and legacy files must
+not be switched wholesale. Prefer the explicit offline tool when provenance
+is known and the source format is Parquet.
+
+## Iceberg and other table formats
+
+Do not replace data files in place or edit their paths in existing metadata.
+Existing snapshots, partition values and statistics still describe the old
+files. Use migrated files as an independent source, then use the table
+format's writer to populate a new table (or a supported transactional rewrite)
+so that partition transforms and metadata are recomputed consistently.
+Validate the new table before moving consumers and retain the old table and
+snapshots for rollback. This utility migrates individual files; it does not
+perform an Iceberg commit or change existing snapshots.
+
+## Tests
+
+```bash
+python -m unittest discover -s tools/legacy_date_migration -v
+```
diff --git a/tools/legacy_date_migration/migrate_legacy_dates.py b/tools/legacy_date_migration/migrate_legacy_dates.py
new file mode 100644
index 00000000000..8554972c0ae
--- /dev/null
+++ b/tools/legacy_date_migration/migrate_legacy_dates.py
@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Rewrite confirmed legacy Doris DATE32 files into proleptic Gregorian encoding."""
+
+import argparse
+import os
+import tempfile
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.compute as pc
+import pyarrow.orc as orc
+import pyarrow.parquet as parquet
+
+
+def convert_array(array, visible=None):
+ """Preserve Arrow structure and nulls while correcting legacy DATE32 ordinals."""
+ data_type = array.type
+ visible = array.is_valid() if visible is None else pc.and_(visible, array.is_valid())
+ if pa.types.is_date32(data_type):
+ # Children beneath a null container have no date value, regardless of their payload.
+ days = pc.if_else(visible, array.cast(pa.int32()), pa.scalar(None, pa.int32()))
+ bounds = pc.min_max(days).as_py()
+ if bounds['min'] is not None and (
+ bounds['min'] < -719527 or bounds['max'] > 2932896):
+ raise ValueError('DATE value outside the legacy Doris range')
+ # The old MySQL-calendar encoder is one day ahead only before March of year zero.
+ early = pc.and_(pc.greater_equal(days, -719527), pc.less_equal(days, -719469))
+ return pc.if_else(early, pc.subtract(days, pa.scalar(1, pa.int32())), days).cast(pa.date32())
+ if pa.types.is_struct(data_type):
+ return pa.StructArray.from_arrays(
+ [convert_array(array.field(i), visible) for i in range(data_type.num_fields)],
+ fields=list(data_type), mask=array.is_null())
+ if pa.types.is_list(data_type) or pa.types.is_large_list(data_type):
+ start, end = array.offsets[0].as_py(), array.offsets[-1].as_py()
+ raw_values = array.values.slice(start, end - start)
+ offsets = pc.subtract(array.offsets, start)
+ parents = pc.list_parent_indices(pa.LargeListArray.from_arrays(offsets, raw_values))
+ values = convert_array(raw_values, pc.take(visible, parents))
+ constructor = pa.LargeListArray if pa.types.is_large_list(data_type) else pa.ListArray
+ return constructor.from_arrays(offsets, values, type=data_type, mask=array.is_null())
+ if pa.types.is_fixed_size_list(data_type):
+ size = data_type.list_size
+ raw_values = array.values.slice(array.offset * size, len(array) * size)
+ parents = pc.list_parent_indices(pa.FixedSizeListArray.from_arrays(raw_values, size))
+ values = convert_array(raw_values, pc.take(visible, parents))
+ return pa.FixedSizeListArray.from_arrays(values, type=data_type, mask=array.is_null())
+ if pa.types.is_map(data_type):
+ start, end = array.offsets[0].as_py(), array.offsets[-1].as_py()
+ offsets = pc.subtract(array.offsets, start)
+ keys = array.keys.slice(start, end - start)
+ parents = pc.list_parent_indices(pa.LargeListArray.from_arrays(offsets, keys))
+ child_visible = pc.take(visible, parents)
+ converted_keys = convert_array(keys, child_visible)
+ if pa.types.is_date32(keys.type):
+ # Arrow requires non-null map keys even in an unreachable null map slot.
+ converted_keys = pc.fill_null(converted_keys, pa.scalar(0, pa.date32()))
+ return pa.MapArray.from_arrays(
+ offsets,
+ converted_keys,
+ convert_array(array.items.slice(start, end - start), child_visible),
+ type=data_type, mask=array.is_null())
+ if pa.types.is_dictionary(data_type):
+ used = pc.unique(pc.filter(array.indices, visible)).cast(pa.int64())
+ dictionary_visible = pc.is_in(pa.array(range(len(array.dictionary)), type=pa.int64()),
+ value_set=used)
+ return pa.DictionaryArray.from_arrays(
+ array.indices, convert_array(array.dictionary, dictionary_visible),
+ ordered=data_type.ordered)
+ if pa.types.is_union(data_type) or isinstance(data_type, pa.ExtensionType):
+ raise ValueError('Unsupported Arrow type; refusing a potentially partial migration: '
+ + str(data_type))
+ return array
+
+
+def convert_table(table):
+ """Convert all DATE32 fields, including nested fields and all chunks."""
+ columns = [pa.chunked_array([convert_array(chunk) for chunk in column.chunks],
+ type=column.type) for column in table.columns]
+ return pa.Table.from_arrays(columns, schema=table.schema)
+
+
+def migrate(source, destination, file_format, confirmed_legacy=False):
+ """Write a new file; source provenance must be confirmed by the caller."""
+ if not confirmed_legacy:
+ raise ValueError('Explicit confirmation of legacy Doris DATE encoding is required')
+ source, destination = Path(source), Path(destination)
+ if source.resolve() == destination.resolve():
+ raise ValueError('Source and destination must be different files')
+ if file_format not in ('orc', 'parquet'):
+ raise ValueError('Format must be orc or parquet')
+ # Convert one stripe/batch at a time to avoid loading the whole data file into memory.
+ reader = orc.ORCFile(source) if file_format == 'orc' else parquet.ParquetFile(source)
+ chunks = (reader.read_stripe(i) for i in range(reader.nstripes)) if file_format == 'orc' else (
+ reader.iter_batches(batch_size=65536))
+ schema = reader.schema if file_format == 'orc' else reader.schema_arrow
+ if destination.exists():
+ raise FileExistsError(destination)
+ # Publish with an exclusive hard link only after the writer succeeds, including its footer.
+ # An existence check alone races with another writer and could overwrite its output.
+ temporary = tempfile.NamedTemporaryFile(prefix='.legacy-date-', dir=destination.parent,
+ delete=False)
+ temporary_path = Path(temporary.name)
+ try:
+ with temporary:
+ writer = (orc.ORCWriter(temporary) if file_format == 'orc'
+ else parquet.ParquetWriter(temporary, schema))
+ try:
+ written = False
+ for chunk in chunks:
+ table = (pa.Table.from_batches([chunk])
+ if isinstance(chunk, pa.RecordBatch) else chunk)
+ converted = convert_table(table)
+ if file_format == 'orc':
+ writer.write(converted)
+ else:
+ writer.write_table(converted)
+ written = True
+ if not written:
+ empty = pa.Table.from_batches([], schema=schema)
+ if file_format == 'orc':
+ writer.write(empty)
+ else:
+ writer.write_table(empty)
+ finally:
+ writer.close()
+ temporary.flush()
+ os.fsync(temporary.fileno())
+ os.link(temporary_path, destination)
+ finally:
+ temporary_path.unlink(missing_ok=True)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--input', required=True, type=Path)
+ parser.add_argument('--output', required=True, type=Path)
+ parser.add_argument('--format', required=True, choices=('orc', 'parquet'))
+ parser.add_argument('--confirm-legacy-doris-date-encoding', action='store_true', required=True,
+ help='Confirm every DATE field in this file uses the old Doris encoding')
+ args = parser.parse_args()
+ migrate(args.input, args.output, args.format, args.confirm_legacy_doris_date_encoding)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/legacy_date_migration/test_migrate_legacy_dates.py b/tools/legacy_date_migration/test_migrate_legacy_dates.py
new file mode 100644
index 00000000000..69a5d5c1651
--- /dev/null
+++ b/tools/legacy_date_migration/test_migrate_legacy_dates.py
@@ -0,0 +1,179 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+import pyarrow as pa
+import pyarrow.orc as orc
+import pyarrow.parquet as parquet
+
+from migrate_legacy_dates import convert_array, convert_table, migrate
+
+
+class LegacyDateMigrationTest(unittest.TestCase):
+ def test_boundary_ordinals_and_nulls(self):
+ raw = [-719527, -719526, -719469, -719468, -719162, -1, 0, 19723, 2932896, None]
+ expected = [-719528, -719527, -719470] + raw[3:]
+ result = convert_array(pa.array(raw, pa.date32()))
+ self.assertEqual(expected, result.cast(pa.int32()).to_pylist())
+
+ def test_nested_values_and_slices(self):
+ date = pa.date32()
+ types_and_values = [
+ (pa.list_(date), [[-719527], None, [-719469, None], []]),
+ (pa.large_list(date), [[-719527], None, [-719469, None], []]),
+ (pa.list_(date, 2), [[-719527, 0], None, [-719469, None], [0, 1]]),
+ (pa.struct([('d', date), ('s', pa.string())]),
+ [{'d': -719527, 's': 'a'}, None, {'d': -719469, 's': 'b'}, {'d': None}]),
+ (pa.map_(pa.string(), pa.list_(date)),
+ [[('a', [-719527])], None, [('b', [-719469, None])], []]),
+ (pa.map_(date, pa.string()), [[(-719527, 'a')], None, [(-719469, 'b')], []]),
+ ]
+ for data_type, values in types_and_values:
+ with self.subTest(data_type=data_type):
+ original = pa.array(values, data_type)
+ converted = convert_array(original)
+ self.assertEqual(original.type, converted.type)
+ self.assertEqual(original.is_null(), converted.is_null())
+ self.assertEqual(converted.slice(1, 2), convert_array(original.slice(1, 2)))
+ array = convert_array(pa.array([[-719527, -719469, None]], pa.list_(date)))
+ self.assertEqual([-719528, -719470, None], array.values.cast(pa.int32()).to_pylist())
+
+ def test_null_parent_hides_invalid_date_payload(self):
+ hidden = pa.array([2147483647, -719527], pa.date32())
+ mask = pa.array([True, False])
+ arrays = [
+ pa.StructArray.from_arrays([hidden], names=['d'], mask=mask),
+ pa.ListArray.from_arrays([0, 1, 2], hidden, mask=mask),
+ pa.FixedSizeListArray.from_arrays(hidden, 1, mask=mask),
+ pa.MapArray.from_arrays([0, 1, 2], pa.array(['a', 'b']), hidden, mask=mask),
+ pa.MapArray.from_arrays([0, 1, 2], hidden, pa.array(['a', 'b']), mask=mask),
+ ]
+ for array in arrays:
+ with self.subTest(data_type=array.type):
+ result = convert_array(array)
+ self.assertTrue(result[0].as_py() is None)
+ # The visible second row is identical to converting it independently.
+ self.assertEqual(result.slice(1), convert_array(array.slice(1)))
+
+ def test_unused_dictionary_values_are_not_validated(self):
+ dictionary = pa.array([-719527] + [2147483647] * 199, pa.date32())
+ array = pa.DictionaryArray.from_arrays(pa.array([0, None], pa.int8()), dictionary)
+ result = convert_array(array)
+ self.assertEqual(array.indices, result.indices)
+ self.assertEqual([-719528, None], result.dictionary_decode().cast(pa.int32()).to_pylist())
+ invalid = pa.DictionaryArray.from_arrays(pa.array([1], pa.int8()), dictionary)
+ with self.assertRaises(ValueError):
+ convert_array(invalid)
+
+ def test_metadata_chunks_and_other_types(self):
+ schema = pa.schema([pa.field('d', pa.date32(), metadata={b'field': b'value'}),
+ pa.field('i', pa.int64(), nullable=False)], metadata={b'table': b'value'})
+ table = pa.Table.from_arrays([
+ pa.chunked_array([pa.array([-719527], pa.date32()), pa.array([None], pa.date32())]),
+ pa.array([1, 2], pa.int64())], schema=schema)
+ result = convert_table(table)
+ self.assertTrue(result.schema.equals(schema, check_metadata=True))
+ self.assertEqual(table['i'], result['i'])
+ self.assertEqual(2, result['d'].num_chunks)
+ self.assertEqual([-719528, None], result['d'].cast(pa.int32()).to_pylist())
+
+ def test_round_trip_files_and_no_overwrite(self):
+ with tempfile.TemporaryDirectory() as directory:
+ for fmt in ['orc', 'parquet']:
+ source = Path(directory) / ('source.' + fmt)
+ output = Path(directory) / ('output.' + fmt)
+ table = pa.table({'d': pa.array([-719527, -719469, -719468, 0, 2932896, None],
+ pa.date32())})
+ module = orc if fmt == 'orc' else parquet
+ module.write_table(table, source)
+ before = source.read_bytes()
+ with self.assertRaises(ValueError):
+ migrate(source, output, fmt)
+ migrate(source, output, fmt, True)
+ self.assertEqual([-719528, -719470, -719468, 0, 2932896, None],
+ module.read_table(output)['d'].cast(pa.int32()).to_pylist())
+ with self.assertRaises(FileExistsError):
+ migrate(source, output, fmt, True)
+ with self.assertRaises(ValueError):
+ migrate(source, source, fmt, True)
+ # A corrected lower-bound value is outside the legacy encoding's domain.
+ with self.assertRaises(ValueError):
+ migrate(output, Path(directory) / ('twice.' + fmt), fmt, True)
+ self.assertEqual(before, source.read_bytes())
+ self.assertFalse(list(Path(directory).glob('.legacy-date-*')))
+
+ def test_failure_never_publishes_output(self):
+ with tempfile.TemporaryDirectory() as directory:
+ source, output = Path(directory) / 'source.parquet', Path(directory) / 'output.parquet'
+ parquet.write_table(pa.table({'d': pa.array([-719528], pa.date32())}), source)
+ with self.assertRaises(ValueError):
+ migrate(source, output, 'parquet', True)
+ self.assertFalse(output.exists())
+ self.assertFalse(list(Path(directory).glob('.legacy-date-*')))
+
+ def test_concurrent_output_is_preserved(self):
+ with tempfile.TemporaryDirectory() as directory:
+ source, output = Path(directory) / 'source.parquet', Path(directory) / 'output.parquet'
+ parquet.write_table(pa.table({'d': pa.array([0], pa.date32())}), source)
+ def racing_writer(*args):
+ output.write_bytes(b'other writer')
+ raise FileExistsError(output)
+ with patch('migrate_legacy_dates.os.link', side_effect=racing_writer):
+ with self.assertRaises(FileExistsError):
+ migrate(source, output, 'parquet', True)
+ self.assertEqual(b'other writer', output.read_bytes())
+ self.assertFalse(list(Path(directory).glob('.legacy-date-*')))
+
+ def test_nested_file_values_and_timestamp_precision(self):
+ schema = pa.schema([
+ pa.field('nested', pa.struct([
+ pa.field('dates', pa.list_(pa.date32()), metadata={b'list': b'metadata'}),
+ pa.field('mapping', pa.map_(pa.string(), pa.date32()))])),
+ pa.field('ts', pa.timestamp('us'), nullable=False)], metadata={b'table': b'metadata'})
+ table = pa.Table.from_arrays([
+ pa.array([{'dates': [-719527, None], 'mapping': [('a', -719469)]}, None],
+ schema.field('nested').type),
+ pa.array([123456789, 987654321], pa.timestamp('us'))], schema=schema)
+ expected = convert_table(table)
+ with tempfile.TemporaryDirectory() as directory:
+ for fmt, module in [('orc', orc), ('parquet', parquet)]:
+ source, output = Path(directory) / ('s.' + fmt), Path(directory) / ('o.' + fmt)
+ module.write_table(table, source)
+ migrate(source, output, fmt, True)
+ actual = module.read_table(output)
+ self.assertEqual(expected['nested'].combine_chunks(),
+ actual['nested'].combine_chunks().cast(schema.field('nested').type))
+ self.assertEqual(expected['ts'], actual['ts'].cast(pa.timestamp('us')))
+ if fmt == 'parquet':
+ self.assertTrue(actual.schema.equals(module.read_table(source).schema,
+ check_metadata=True))
+
+ def test_empty_files(self):
+ with tempfile.TemporaryDirectory() as directory:
+ for fmt, module in [('orc', orc), ('parquet', parquet)]:
+ source, output = Path(directory) / ('s.' + fmt), Path(directory) / ('o.' + fmt)
+ module.write_table(pa.table({'d': pa.array([], pa.date32())}), source)
+ migrate(source, output, fmt, True)
+ self.assertEqual(pa.date32(), module.read_table(output).schema.field('d').type)
+
+
+if __name__ == '__main__':
+ unittest.main()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 27/35: [improvement](lance) Add ANN prefilter optimization and profiles (branch-4.1) (#68615)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 710f324af721f7ba05f8817b77548ff259c76cc4
Author: Gabriel <li...@selectdb.com>
AuthorDate: Wed Sep 30 22:15:36 2026 +0800
[improvement](lance) Add ANN prefilter optimization and profiles (branch-4.1) (#68615)
Unfiltered ANN searches scoped to complete index segments can
materialize redundant row-ID allowlists, and the existing scan profile
leaves the remaining search work unexplained. Update the upstream
dependency and expose ANN stage timings on `branch-4.1`.
This includes the dependency integration from #68613: adopt the merged
lance-format/lance#9599 and lance-format/lance#9602 through
lance-format/lance-c#89, retain the merged lance-format/lance#9537 PQ
scoring fix, remove superseded local lance-c patches, and retain only
the Foyer cache integration.
Add BE profile counters for index opening, partition
loading/preparation, prefilter readiness, CPU queue wait, partition
search, query lookup-table preparation, fused distance/TopK work, and
result materialization. Export operator baselines for ANN, sort/merge,
take, and vector-distance work. Extend the existing indexed multivector
regression to check that stage timers reach the Doris profile, alongside
existing result and prefilter assertions.
Timings accumulate across concurrent work and overlap parent stages.
Distance/TopK includes candidate filtering in fused paths; these
counters must not be summed to reconstruct wall time.
`docs/lance-ann-profile.md` documents the boundaries, supported paths,
and relationship to the existing scanner and prefilter timers. The BE
changes add observability; the prefilter optimization itself remains in
the upstream dependency.
Pin lance-c `9bd730add2ac70316c1d642b8459011e2dd92022`, using the merged
Lance #9602 revision `68c12dfd7efe02f90ad2d7f3a239a7eb64884e57`. This
includes parallel-path partition-preparation timing. The upstream
callback regression checks timer presence and nanosecond units while
accepting valid zero durations.
The Foyer wrapper caches immutable ObjectMeta and Attributes
independently of HTTP response extensions, avoiding repeated HEAD
requests on warm range reads. It does not replay transport extensions
from cached metadata. The HTTP regression checks HEAD/GET counts,
returned bytes/ranges/metadata, fresh dataset scopes, and explicit HEAD
bypass.
Generate the retained Foyer patch from source commit
`24c7ca4bcb9422c113b0d3e07e4efe1173b0bc9f` relative to the pinned
lance-c baseline. The header records both revisions and the regeneration
command. The source is submitted as
https://github.com/zhangstar333/lance-c/pull/3 against the branch behind
lance-format/lance-c#73, on top of merged source-alignment PR #2. Doris
consumes the exact reviewed source revision while that supplementary PR
awaits merge.
Scope metadata, size, and block keys to a random namespace for each live
underlying object-store instance. The wrapping API omits a complete
stable backend identity; identical bucket/path names alone cannot
distinguish S3-compatible endpoints. Weak identity records preserve
sharing for the same live store without retaining it, while new stores
and process restarts start cold. A fresh Dataset open that creates a new
store consequently needs to warm its own cache. This deliberately
reduces reuse across instances to prevent returning another origin's
data; it does not claim unchanged cache-hit rates or production latency.
Avoid inserting an unchanged size entry into the WriteOnInsertion hybrid
cache: look up both tiers first and populate only absent or invalid size
records. Regression tests cover real HTTP endpoints sharing
bucket/path/ETag, batched data and NotFound isolation, replaced origins
after disk recovery, weak-reference lifetime, and actual disk-write
bytes. Five warm range reads wrote 40 KB before the fix and zero bytes
after it in the regression.
Fingerprint the patch in the downloader so existing source trees refresh
after updates, including legacy empty markers. Identical patches reuse
cached sources. Check platform definitions under nounset with simulated
Darwin x86_64/arm64, and make the optional ADBC source guard safe when
unset on master. Downloader lifecycle and platform handling remain
downstream in Doris.
Validation: 461 regular Rust tests passed on the final source; Rust
formatting and diff checks passed. GNU patch and git apply checks
passed, and all 87 tracked source files match the recorded source
commit. Downloader tests passed on master, branch-4.1, and the
downstream hotfix branch for fresh extraction, idempotence,
re-extraction, generic markers, legacy/mismatched Foyer markers, and
patch failure. Shell syntax and simulated macOS initialization passed.
All three opt-in native consumer tests passed: C calls, C++ calls, and
static OSS HTTP transport. Full Doris builds and BE integration
execution remain in PR CI.
---
be/src/format_v2/table/lance_reader.cpp | 49 +
be/test/format_v2/table/lance_reader_test.cpp | 9 +
docs/lance-ann-profile.md | 67 +
thirdparty/download-thirdparty.sh | 35 +-
thirdparty/patches/lance-c-0.1.9-pr-74.patch | 1159 ----------
thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch | 2441 --------------------
thirdparty/patches/lance-c-0.1.9-pr-77.patch | 1863 ---------------
thirdparty/patches/lance-c-0.1.9-pr-79.patch | 1782 --------------
thirdparty/patches/lance-c-0.1.9-pr-80.patch | 225 --
thirdparty/patches/lance-c-0.1.9-pr-83.patch | 1914 ---------------
.../patches/lance-c-0.1.9-prefilter-fts.patch | 148 --
thirdparty/patches/lance-c-0.1.9-prefilter.patch | 147 --
...nce-c-0.1.9-pr-73.patch => lance-c-foyer.patch} | 883 ++++---
thirdparty/test/lance-prefilter-patch-test.sh | 124 +
thirdparty/vars.sh | 9 +-
15 files changed, 834 insertions(+), 10021 deletions(-)
diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp
index 724495bf990..52cf45533e5 100644
--- a/be/src/format_v2/table/lance_reader.cpp
+++ b/be/src/format_v2/table/lance_reader.cpp
@@ -697,6 +697,55 @@ void LanceTableReader::_init_scanner_profile() {
TUnit::UNIT, LANCE_READER_PROFILE, 1)},
};
_lance_time_metrics = {
+ // Partition stages accumulate across concurrent work and overlap their parent timers.
+ // DistanceTopK includes fused candidate filtering, scoring, and heap updates.
+ {"index_open_time", ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceIndexOpenTime",
+ LANCE_READER_PROFILE, 1)},
+ {"index_partition_load_time",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceIndexPartitionLoadTime",
+ LANCE_READER_PROFILE, 1)},
+ {"index_partition_prepare_time",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceIndexPartitionPrepareTime",
+ LANCE_READER_PROFILE, 1)},
+ {"index_prefilter_wait_time",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceIndexPrefilterWaitTime",
+ LANCE_READER_PROFILE, 1)},
+ {"index_cpu_queue_wait_time",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceIndexCpuQueueWaitTime",
+ LANCE_READER_PROFILE, 1)},
+ {"index_search_time",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceIndexSearchTime",
+ LANCE_READER_PROFILE, 1)},
+ {"index_query_prepare_time",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceIndexQueryPrepareTime",
+ LANCE_READER_PROFILE, 1)},
+ {"index_distance_topk_time",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceIndexDistanceTopKTime",
+ LANCE_READER_PROFILE, 1)},
+ {"index_result_materialize_time",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceIndexResultMaterializeTime",
+ LANCE_READER_PROFILE, 1)},
+ {"ANNIVFPartitionExec_elapsed_compute",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceANNPartitionExecTime",
+ LANCE_READER_PROFILE, 1)},
+ {"ANNSubIndexExec_elapsed_compute",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceANNSubIndexExecTime",
+ LANCE_READER_PROFILE, 1)},
+ {"ANNIvfBatchExec_elapsed_compute",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceANNBatchExecTime",
+ LANCE_READER_PROFILE, 1)},
+ {"SortExec_elapsed_compute",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceSortComputeTime",
+ LANCE_READER_PROFILE, 1)},
+ {"SortPreservingMergeExec_elapsed_compute",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceSortMergeComputeTime",
+ LANCE_READER_PROFILE, 1)},
+ {"TakeExec_elapsed_compute",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceTakeExecTime", LANCE_READER_PROFILE,
+ 1)},
+ {"KNNVectorDistanceExec_elapsed_compute",
+ ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceVectorDistanceComputeTime",
+ LANCE_READER_PROFILE, 1)},
// These are wall times in the ANN row-id loader. LoadTime includes input polling
// and set construction; it must not be added to its component timers.
{"prefilter_load_time",
diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp
index a751e3aa80e..93531688634 100644
--- a/be/test/format_v2/table/lance_reader_test.cpp
+++ b/be/test/format_v2/table/lance_reader_test.cpp
@@ -912,6 +912,15 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorScoresFiltersOffsetsAndIndexed
}
EXPECT_TRUE(reader.close().ok());
if (indexed) {
+ // Warm searches still perform scoring even when every index partition is cached.
+ for (const char* name :
+ {"LanceIndexPartitionLoadTime", "LanceIndexCpuQueueWaitTime",
+ "LanceIndexSearchTime", "LanceIndexQueryPrepareTime",
+ "LanceIndexDistanceTopKTime", "LanceIndexResultMaterializeTime"}) {
+ auto* counter = profile.get_counter(name);
+ ASSERT_NE(nullptr, counter) << name;
+ EXPECT_GT(counter->value(), 0) << name;
+ }
// Read metrics after close: lance-c publishes its final execution summary
// when the stream is released, including for an early top-k stop.
for (const char* name : {"LancePrefilterLoads", "LancePrefilterInputRows",
diff --git a/docs/lance-ann-profile.md b/docs/lance-ann-profile.md
new file mode 100644
index 00000000000..51fdc15d095
--- /dev/null
+++ b/docs/lance-ann-profile.md
@@ -0,0 +1,67 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Lance ANN profile timings
+
+`FileScannerV2` accumulates scanner initialization, open, block-read, and close
+wall time. Range acquisition and split preparation are nested within those
+calls, so their counters are not additional time. Scanner worker scheduling
+wait is reported separately. `LanceScannerReadTime` measures time spent calling
+the Lance scanner, including Rust execution and waits. Doris scanner CPU time
+does not include work done on Lance's CPU pool.
+
+The following counters expose the work within an indexed vector search:
+
+| Counter | Scope |
+| --- | --- |
+| `LanceIndexOpenTime` | Index-handle lookup/open, including metadata reads on a miss. |
+| `LanceIVFPartitionRankingTime` | Partition ranking, including its CPU dispatch wait. |
+| `LanceIndexPartitionLoadTime` | Partition cache lookup, coalesced-load wait, and read/decode on a miss. Also measured on cache hits. |
+| `LanceIndexPartitionPrepareTime` | Partition load and per-partition filter preparation. On the streaming path, shared-filter waiting overlaps loading. |
+| `LanceIndexPrefilterWaitTime` | Waiting for the shared prefilter to become ready. This is distinct from building the filter. |
+| `LanceIndexCpuQueueWaitTime` | Delay before a dispatched search or result-materialization CPU task starts. |
+| `LanceIndexSearchTime` | Search of prepared partitions on the CPU pool, including query preparation and any per-partition result construction. |
+| `LanceIndexQueryPrepareTime` | Distance-calculator / lookup-table construction in the IVF flat sub-index (including quantized storage). |
+| `LanceIndexDistanceTopKTime` | Candidate filtering, distance evaluation, and heap updates in that sub-index. These operations are fused in fast-scan paths. |
+| `LanceIndexResultMaterializeTime` | Converting result heaps into Arrow arrays and batches, excluding final global sorting. |
+| `LanceANNPartitionExecTime`, `LanceANNSubIndexExecTime`, `LanceANNBatchExecTime` | Baseline elapsed times reported by the corresponding Lance ANN operators. These include asynchronous waits. |
+| `LanceSortComputeTime`, `LanceSortMergeComputeTime` | DataFusion sort / sort-preserving merge operator compute times. |
+| `LanceTakeExecTime` | Baseline time reported by Lance's take operator within the scan plan. Doris second-phase row-ID fetch has separate counters. |
+| `LanceVectorDistanceComputeTime` | Baseline reported by the vector-distance operator, e.g. refinement or an unindexed tail. |
+
+Timings accumulate across partitions, tasks, and index segments. They are
+**nested and may overlap**, so summing them does not reconstruct query wall time.
+In particular, partition preparation contains loading; search contains query
+preparation and distance/TopK work; ANN operator baselines contain downstream
+search stages and waits. A zero counter can mean the corresponding operator or
+path was not used. Detailed sub-index timers currently cover IVF flat sub-indices;
+other sub-indices are visible through the encompassing search timer.
+
+`LancePrefilterLoadTime` includes `LancePrefilterInputTime` and
+`LancePrefilterBuildTime`; do not add these three together. A segment-scoped
+search without a predicate can avoid constructing the row-ID allowlist, while
+still respecting deletions and fragment visibility. Filter-readiness waiting can
+therefore remain nonzero even when the row-ID materialization counters are zero.
+
+For a warm query with no execution I/O and no prefilter materialization, inspect
+CPU queue wait, query preparation, distance/TopK, and result/sort timers. For cold
+queries, inspect partition load together with execution bytes, requests, and
+partition cache misses. Use repeated queries and the operator-level elapsed
+times to assess latency; cumulative parallel stage times alone are not a critical
+path trace.
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index e40a6f3d8b8..431d5a5d013 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -718,30 +718,23 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then
echo "Finished patching ${AZURE_SOURCE}"
fi
-# Apply Doris lance-c patches as one chain to the pinned release archive.
+# Foyer remains a local patch until its cache interface is accepted upstream.
+# All search fixes are supplied by the immutable lance-c dependency revision.
if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then
- cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}"
- LANCE_C_PATCHED_MARK="${PATCHED_MARK}_community_pr83_prefilter"
- # Older source caches carry a different PR #73 and cannot accept this chain incrementally.
- if [[ -f "${PATCHED_MARK}" && ! -f "${LANCE_C_PATCHED_MARK}" ]]; then
- echo "The lance-c patch chain changed; remove ${TP_SOURCE_DIR}/${LANCE_C_SOURCE} and rebuild."
- exit 1
+ foyer_patch_checksum="$(cksum < "${TP_PATCH_DIR}/lance-c-foyer.patch")"
+ foyer_patch_marker="${TP_SOURCE_DIR}/${LANCE_C_SOURCE}/${PATCHED_MARK}_foyer"
+ # A new local patch must also replace previously patched cached sources.
+ # Empty markers from older builds cannot identify the applied patch version.
+ if [[ -f "${foyer_patch_marker}" ]] &&
+ [[ "$(cat "${foyer_patch_marker}")" != "${foyer_patch_checksum}" ]]; then
+ rm -rf "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}"
+ "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${LANCE_C_NAME}" -C "${TP_SOURCE_DIR}/"
fi
- if [[ ! -f "${LANCE_C_PATCHED_MARK}" ]]; then
- # PR #77 provides Lance v11 for the following community patches. PR #83
- # retains PR #79's scalar-segment path when adding multi-vector execution.
- # The final patch pins the full-snapshot prefilter fix and its execution metrics.
- for lance_patch in pr-74 pr-75-pr-78 pr-77 pr-73 pr-79 pr-80 pr-83 prefilter; do
- patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \
- -p1 <"${TP_PATCH_DIR}/${LANCE_C_SOURCE}-${lance_patch}.patch"
- done
- touch "${PATCHED_MARK}" "${LANCE_C_PATCHED_MARK}"
- fi
- # Cached sources may carry the earlier prefilter pin; upgrade FTS metrics independently.
- if [[ ! -f "${PATCHED_MARK}_prefilter_fts" ]]; then
+ cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}"
+ if [[ ! -f "${PATCHED_MARK}_foyer" ]]; then
patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \
- -p1 <"${TP_PATCH_DIR}/${LANCE_C_SOURCE}-prefilter-fts.patch"
- touch "${PATCHED_MARK}_prefilter_fts"
+ -p1 <"${TP_PATCH_DIR}/lance-c-foyer.patch"
+ printf '%s\n' "${foyer_patch_checksum}" > "${PATCHED_MARK}_foyer"
fi
cd -
echo "Finished patching ${LANCE_C_SOURCE}"
diff --git a/thirdparty/patches/lance-c-0.1.9-pr-74.patch b/thirdparty/patches/lance-c-0.1.9-pr-74.patch
deleted file mode 100644
index 24c6d33457d..00000000000
--- a/thirdparty/patches/lance-c-0.1.9-pr-74.patch
+++ /dev/null
@@ -1,1159 +0,0 @@
-From b07f970bf2cf3f983cc6043bc72a0fe444c0fd4c Mon Sep 17 00:00:00 2001
-From: zhangstar333 <zh...@selectdb.com>
-Date: Thu, 3 Sep 2026 17:16:34 +0800
-Subject: [PATCH 1/2] fts
-
----
- include/lance/lance.h | 74 ++++++++---
- include/lance/lance.hpp | 44 +++++--
- src/fts_query.rs | 235 +++++++++++++++++++++++++++++------
- src/scanner.rs | 72 ++++++++---
- tests/c_api_test.rs | 264 +++++++++++++++++++++++++++++++++++++---
- 5 files changed, 595 insertions(+), 94 deletions(-)
-
-diff --git a/include/lance/lance.h b/include/lance/lance.h
-index 3bf291f..0c76edc 100644
---- a/include/lance/lance.h
-+++ b/include/lance/lance.h
-@@ -1695,39 +1695,85 @@ typedef enum {
- LANCE_FTS_COVERAGE_INDEX_ONLY = 1,
- } LanceFtsCoverageMode;
-
-+/** How the analyzed terms of one Match query are combined. */
-+typedef enum {
-+ /** At least one analyzed term must match. */
-+ LANCE_FTS_MATCH_OPERATOR_OR = 0,
-+ /** Every analyzed term must match. */
-+ LANCE_FTS_MATCH_OPERATOR_AND = 1,
-+} LanceFtsMatchOperator;
-+
-+/**
-+ * Prepare an OR Match query context for one column.
-+ *
-+ * @deprecated Use lance_dataset_prepare_fts_match_query() to select the Match
-+ * operator explicitly. This compatibility API is equivalent to
-+ * LANCE_FTS_MATCH_OPERATOR_OR.
-+ */
-+LanceFtsQueryContext* lance_dataset_prepare_fts_query(
-+ const LanceDataset* dataset,
-+ const char* column,
-+ const char* query,
-+ uint32_t max_fuzzy_distance,
-+ int32_t coverage_mode
-+);
-+
- /**
-- * Prepare an immutable, process-local FTS query context for one column.
-+ * Prepare an immutable, process-local Match query context for one column.
- *
- * Preparation pins the dataset handle's current snapshot, enumerates all
- * committed FTS segments for `column`, checks fragment coverage, opens those
-- * segments, and computes one query-specific global BM25 scorer across their
-- * indexed documents. The context can then be shared by any number of scanners
-- * created from the exact same process-local dataset snapshot. It has no
-- * serialization or cross-process transport format. Reopening the same URI and
-- * manifest version creates a different identity and cannot reuse the context,
-- * because storage options and object-store endpoints may differ.
-+ * segments, and prepares one global BM25 scorer across their indexed
-+ * documents. `match_operator` supports both AND and OR.
-+ *
-+ * The context can be shared by scanners created from the exact same
-+ * process-local dataset snapshot. It has no serialization or cross-process
-+ * transport format. Reopening the same URI and manifest version creates a
-+ * different identity and cannot reuse the context because storage options and
-+ * object-store endpoints may differ.
- *
- * In LANCE_FTS_COVERAGE_INDEX_ONLY mode, unindexed fragments are allowed and
- * excluded from both the scorer corpus and query results. In STRICT mode any
- * unindexed fragment makes this call fail.
- *
-- * Prepared contexts currently support exact Match queries only.
-- * `max_fuzzy_distance` must be zero because fuzzy execution requires its
-- * canonical expanded vocabulary to be prepared together with the scorer.
-- * This restriction does not apply to lance_scanner_full_text_search().
-- *
-- * @param max_fuzzy_distance Must be zero for prepared query contexts.
-+ * @param match_operator Fixed-width LanceFtsMatchOperator discriminant.
-+ * @param max_fuzzy_distance Reserved for prepared fuzzy matching and currently
-+ * must be 0. The parameter is retained so enabling
-+ * canonical cross-segment fuzzy vocabulary injection
-+ * later does not require another C ABI change.
- * @param coverage_mode Fixed-width LanceFtsCoverageMode discriminant.
- * @return Context handle on success, or NULL on error.
- */
--LanceFtsQueryContext* lance_dataset_prepare_fts_query(
-+LanceFtsQueryContext* lance_dataset_prepare_fts_match_query(
- const LanceDataset* dataset,
- const char* column,
- const char* query,
-+ int32_t match_operator,
- uint32_t max_fuzzy_distance,
- int32_t coverage_mode
- );
-
-+/**
-+ * Prepare an immutable, process-local Phrase query context for one column.
-+ *
-+ * The selected FTS index must store token positions. `slop == 0` requires an
-+ * exact phrase; a positive value permits that many intervening positions.
-+ * Dataset identity, coverage, sharing, and segment-scoped execution follow the
-+ * same contract as lance_dataset_prepare_fts_match_query().
-+ *
-+ * @param slop Maximum number of intervening token positions permitted between
-+ * adjacent phrase terms.
-+ * @param coverage_mode Fixed-width LanceFtsCoverageMode discriminant.
-+ * @return Context handle on success, or NULL on error.
-+ */
-+LanceFtsQueryContext* lance_dataset_prepare_fts_phrase_query(
-+ const LanceDataset* dataset,
-+ const char* column,
-+ const char* query,
-+ uint32_t slop,
-+ int32_t coverage_mode
-+);
-+
- /**
- * Close a context handle. NULL-safe. Scanners that already attached this
- * context retain shared ownership and remain valid.
-diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp
-index 6cf245f..973216a 100644
---- a/include/lance/lance.hpp
-+++ b/include/lance/lance.hpp
-@@ -128,6 +128,11 @@ enum class FtsCoverageMode : int32_t {
- IndexOnly = LANCE_FTS_COVERAGE_INDEX_ONLY,
- };
-
-+enum class FtsMatchOperator : int32_t {
-+ Or = LANCE_FTS_MATCH_OPERATOR_OR,
-+ And = LANCE_FTS_MATCH_OPERATOR_AND,
-+};
-+
- /// Tunable parameters for Dataset::write. Numeric fields default-out via 0;
- /// `data_storage_version` defaults out via `std::nullopt`.
- ///
-@@ -764,18 +769,43 @@ class Dataset {
- /// Create a Scanner builder for this dataset.
- Scanner scan() const;
-
-- /// Prepare a query-specific global BM25 scorer over the committed FTS
-- /// segments of this pinned snapshot. IndexOnly permits unindexed fragments;
-- /// Strict rejects them. Prepared contexts currently require
-- /// `max_fuzzy_distance == 0`. The context can only be attached to scanners
-- /// created from this exact process-local dataset snapshot.
-+ /// Compatibility wrapper for an OR Match query.
-+ [[deprecated("Use prepare_fts_match_query() to select the Match operator")]]
- FtsQueryContext prepare_fts_query(
- const std::string& column,
- const std::string& query,
- uint32_t max_fuzzy_distance = 0,
- FtsCoverageMode coverage_mode = FtsCoverageMode::Strict) const {
-- auto* context = lance_dataset_prepare_fts_query(
-- handle_.get(), column.c_str(), query.c_str(), max_fuzzy_distance,
-+ return prepare_fts_match_query(column, query, FtsMatchOperator::Or,
-+ max_fuzzy_distance, coverage_mode);
-+ }
-+
-+ /// Prepare a Match query with a global BM25 scorer. AND and OR are
-+ /// supported. `max_fuzzy_distance` is reserved and currently must be zero;
-+ /// keeping it here avoids another API change when canonical cross-segment
-+ /// fuzzy vocabulary injection becomes available.
-+ FtsQueryContext prepare_fts_match_query(
-+ const std::string& column,
-+ const std::string& query,
-+ FtsMatchOperator match_operator = FtsMatchOperator::Or,
-+ uint32_t max_fuzzy_distance = 0,
-+ FtsCoverageMode coverage_mode = FtsCoverageMode::Strict) const {
-+ auto* context = lance_dataset_prepare_fts_match_query(
-+ handle_.get(), column.c_str(), query.c_str(),
-+ static_cast<int32_t>(match_operator), max_fuzzy_distance,
-+ static_cast<int32_t>(coverage_mode));
-+ if (!context) check_error();
-+ return FtsQueryContext(context);
-+ }
-+
-+ /// Prepare a Phrase query. Its FTS index must store token positions.
-+ FtsQueryContext prepare_fts_phrase_query(
-+ const std::string& column,
-+ const std::string& query,
-+ uint32_t slop = 0,
-+ FtsCoverageMode coverage_mode = FtsCoverageMode::Strict) const {
-+ auto* context = lance_dataset_prepare_fts_phrase_query(
-+ handle_.get(), column.c_str(), query.c_str(), slop,
- static_cast<int32_t>(coverage_mode));
- if (!context) check_error();
- return FtsQueryContext(context);
-diff --git a/src/fts_query.rs b/src/fts_query.rs
-index cd194c7..3bf7f3e 100644
---- a/src/fts_query.rs
-+++ b/src/fts_query.rs
-@@ -14,7 +14,9 @@ use lance_core::{Error, Result};
- use lance_index::IndexCriteria;
- use lance_index::metrics::NoOpMetricsCollector;
- use lance_index::scalar::FullTextSearchQuery;
--use lance_index::scalar::inverted::query::{FtsQuery, collect_query_tokens};
-+use lance_index::scalar::inverted::query::{
-+ FtsQuery, MatchQuery, Operator, PhraseQuery, collect_query_tokens,
-+};
- use lance_index::scalar::inverted::{InvertedIndex, MemBM25Scorer, build_global_bm25_scorer};
- use lance_table::format::IndexMetadata;
- use uuid::Uuid;
-@@ -48,12 +50,53 @@ impl TryFrom<i32> for LanceFtsCoverageMode {
- }
- }
-
-+/// Operator used to combine the analyzed terms of a Match query.
-+#[repr(i32)]
-+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-+pub enum LanceFtsMatchOperator {
-+ /// At least one analyzed term must match.
-+ Or = 0,
-+ /// Every analyzed term must match.
-+ And = 1,
-+}
-+
-+impl TryFrom<i32> for LanceFtsMatchOperator {
-+ type Error = Error;
-+
-+ fn try_from(value: i32) -> Result<Self> {
-+ match value {
-+ 0 => Ok(Self::Or),
-+ 1 => Ok(Self::And),
-+ _ => Err(Error::invalid_input(format!(
-+ "invalid match_operator {value}; expected 0 (OR) or 1 (AND)"
-+ ))),
-+ }
-+ }
-+}
-+
-+impl From<LanceFtsMatchOperator> for Operator {
-+ fn from(value: LanceFtsMatchOperator) -> Self {
-+ match value {
-+ LanceFtsMatchOperator::Or => Self::Or,
-+ LanceFtsMatchOperator::And => Self::And,
-+ }
-+ }
-+}
-+
-+/// Query-specific state that must be shared by every segment-scoped scan.
-+pub(crate) enum PreparedFtsQuery {
-+ /// Exact Match queries share one corpus-wide scorer.
-+ Match(Arc<MemBM25Scorer>),
-+ /// Phrase does not expand terms, so a shared global scorer is sufficient.
-+ Phrase(Arc<MemBM25Scorer>),
-+}
-+
- /// Rust-owned immutable state behind [`LanceFtsQueryContext`].
- pub(crate) struct FtsQueryContextInner {
- pub(crate) dataset: Arc<lance::Dataset>,
- pub(crate) query: FullTextSearchQuery,
- pub(crate) segments: Vec<IndexMetadata>,
-- pub(crate) scorer: Arc<MemBM25Scorer>,
-+ pub(crate) prepared: PreparedFtsQuery,
- }
-
- impl FtsQueryContextInner {
-@@ -87,7 +130,7 @@ fn invalid_input(message: impl Into<String>) -> Error {
- async fn prepare_fts_query_context(
- dataset: Arc<lance::Dataset>,
- column: String,
-- query_text: String,
-+ query: FullTextSearchQuery,
- coverage_mode: LanceFtsCoverageMode,
- ) -> Result<FtsQueryContextInner> {
- let logical_index = dataset
-@@ -193,34 +236,78 @@ async fn prepare_fts_query_context(
- )));
- }
-
-- let query = FullTextSearchQuery::new(query_text).with_column(column.clone())?;
-- let match_query = match &query.query {
-- FtsQuery::Match(query) => query,
-+ let prepared = match &query.query {
-+ FtsQuery::Match(match_query) => {
-+ let mut tokenizer = indices[0].tokenizer();
-+ let query_tokens = collect_query_tokens(&match_query.terms, &mut tokenizer);
-+ let params = query
-+ .params()
-+ .with_fuzziness(match_query.fuzziness)
-+ .with_max_expansions(match_query.max_expansions)
-+ .with_prefix_length(match_query.prefix_length);
-+ PreparedFtsQuery::Match(Arc::new(
-+ build_global_bm25_scorer(&indices, &query_tokens, ¶ms).await?,
-+ ))
-+ }
-+ FtsQuery::Phrase(phrase_query) => {
-+ if !expected_params.has_positions() {
-+ return Err(invalid_input(format!(
-+ "FTS index '{}' for column '{column}' does not store token positions required by Phrase queries; recreate the index with positions enabled",
-+ logical_index.name
-+ )));
-+ }
-+ let mut tokenizer = indices[0].tokenizer();
-+ let query_tokens = collect_query_tokens(&phrase_query.terms, &mut tokenizer);
-+ let params = query.params().with_phrase_slop(Some(phrase_query.slop));
-+ PreparedFtsQuery::Phrase(Arc::new(
-+ build_global_bm25_scorer(&indices, &query_tokens, ¶ms).await?,
-+ ))
-+ }
- _ => {
- return Err(Error::internal(
-- "prepared FTS query unexpectedly produced a non-Match query".to_string(),
-+ "prepared FTS query must be a single-column Match or Phrase query".to_string(),
- ));
- }
- };
-- let mut tokenizer = indices[0].tokenizer();
-- let query_tokens = collect_query_tokens(&match_query.terms, &mut tokenizer);
-- let params = query
-- .params()
-- .with_fuzziness(match_query.fuzziness)
-- .with_max_expansions(match_query.max_expansions)
-- .with_prefix_length(match_query.prefix_length);
-- let scorer = Arc::new(build_global_bm25_scorer(&indices, &query_tokens, ¶ms).await?);
-
- Ok(FtsQueryContextInner {
- dataset,
- query,
- segments,
-- scorer,
-+ prepared,
- })
- }
-
--/// Prepare a process-local global BM25 scorer and the committed segment list
--/// for one single-column Match query against the dataset's pinned snapshot.
-+unsafe fn parse_query_inputs(
-+ dataset: *const LanceDataset,
-+ column: *const c_char,
-+ query: *const c_char,
-+ coverage_mode: i32,
-+) -> Result<(Arc<lance::Dataset>, String, String, LanceFtsCoverageMode)> {
-+ if dataset.is_null() || column.is_null() || query.is_null() {
-+ return Err(invalid_input("dataset, column, and query must not be NULL"));
-+ }
-+ let column = unsafe { helpers::parse_c_string(column)? }
-+ .filter(|value| !value.is_empty())
-+ .ok_or_else(|| invalid_input("column must not be empty"))?
-+ .to_string();
-+ let query = unsafe { helpers::parse_c_string(query)? }
-+ .filter(|value| !value.is_empty())
-+ .ok_or_else(|| invalid_input("query must not be empty"))?
-+ .to_string();
-+ let coverage_mode = LanceFtsCoverageMode::try_from(coverage_mode)?;
-+ let snapshot = unsafe { &*dataset }.snapshot();
-+ Ok((snapshot, column, query, coverage_mode))
-+}
-+
-+fn into_context(inner: FtsQueryContextInner) -> *mut LanceFtsQueryContext {
-+ Box::into_raw(Box::new(LanceFtsQueryContext {
-+ inner: Arc::new(inner),
-+ }))
-+}
-+
-+/// Compatibility API for an OR Match query.
-+#[deprecated(note = "use lance_dataset_prepare_fts_match_query to select the Match operator")]
- #[unsafe(no_mangle)]
- pub unsafe extern "C" fn lance_dataset_prepare_fts_query(
- dataset: *const LanceDataset,
-@@ -231,46 +318,120 @@ pub unsafe extern "C" fn lance_dataset_prepare_fts_query(
- ) -> *mut LanceFtsQueryContext {
- ffi_try!(
- unsafe {
-- prepare_fts_query_inner(dataset, column, query, max_fuzzy_distance, coverage_mode)
-+ prepare_fts_match_query_inner(
-+ dataset,
-+ column,
-+ query,
-+ LanceFtsMatchOperator::Or as i32,
-+ max_fuzzy_distance,
-+ coverage_mode,
-+ )
-+ },
-+ null
-+ )
-+}
-+
-+/// Prepare a process-local Match query context. AND and OR are supported.
-+/// `max_fuzzy_distance` is retained for the future prepared-fuzzy path but
-+/// must be zero with the currently pinned Lance revision.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_dataset_prepare_fts_match_query(
-+ dataset: *const LanceDataset,
-+ column: *const c_char,
-+ query: *const c_char,
-+ match_operator: i32,
-+ max_fuzzy_distance: u32,
-+ coverage_mode: i32,
-+) -> *mut LanceFtsQueryContext {
-+ ffi_try!(
-+ unsafe {
-+ prepare_fts_match_query_inner(
-+ dataset,
-+ column,
-+ query,
-+ match_operator,
-+ max_fuzzy_distance,
-+ coverage_mode,
-+ )
- },
- null
- )
- }
-
--unsafe fn prepare_fts_query_inner(
-+unsafe fn prepare_fts_match_query_inner(
- dataset: *const LanceDataset,
- column: *const c_char,
- query: *const c_char,
-+ match_operator: i32,
- max_fuzzy_distance: u32,
- coverage_mode: i32,
- ) -> Result<*mut LanceFtsQueryContext> {
-- if dataset.is_null() || column.is_null() || query.is_null() {
-- return Err(invalid_input("dataset, column, and query must not be NULL"));
-- }
-- let column = unsafe { helpers::parse_c_string(column)? }
-- .filter(|value| !value.is_empty())
-- .ok_or_else(|| invalid_input("column must not be empty"))?
-- .to_string();
-- let query = unsafe { helpers::parse_c_string(query)? }
-- .filter(|value| !value.is_empty())
-- .ok_or_else(|| invalid_input("query must not be empty"))?
-- .to_string();
-- let coverage_mode = LanceFtsCoverageMode::try_from(coverage_mode)?;
-+ let (snapshot, column, query_text, coverage_mode) =
-+ unsafe { parse_query_inputs(dataset, column, query, coverage_mode)? };
-+ let operator: Operator = LanceFtsMatchOperator::try_from(match_operator)?.into();
-+ // The parameter remains in the public API so callers do not need another
-+ // ABI change when Lance-C moves to a Lance revision that can inject the
-+ // same canonical fuzzy vocabulary into every segment-scoped scan. The
-+ // pinned Lance revision can share only the scorer, so accepting fuzzy here
-+ // would allow different segments to choose different capped expansions.
- if max_fuzzy_distance != 0 {
- return Err(invalid_input(format!(
-- "max_fuzzy_distance must be 0 for prepared FTS query contexts, got {max_fuzzy_distance}; fuzzy queries require a canonical prepared BM25 vocabulary"
-+ "max_fuzzy_distance must be 0 for prepared FTS with the pinned Lance revision, got {max_fuzzy_distance}; the parameter is reserved until canonical fuzzy vocabulary injection is available"
- )));
- }
-- let snapshot = unsafe { &*dataset }.snapshot();
-+ let query = FullTextSearchQuery::new_query(
-+ MatchQuery::new(query_text)
-+ .with_column(Some(column.clone()))
-+ .with_operator(operator)
-+ .with_fuzziness(Some(0))
-+ .into(),
-+ );
- let inner = block_on(prepare_fts_query_context(
- snapshot,
- column,
- query,
- coverage_mode,
- ))?;
-- Ok(Box::into_raw(Box::new(LanceFtsQueryContext {
-- inner: Arc::new(inner),
-- })))
-+ Ok(into_context(inner))
-+}
-+
-+/// Prepare a process-local Phrase query context.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_dataset_prepare_fts_phrase_query(
-+ dataset: *const LanceDataset,
-+ column: *const c_char,
-+ query: *const c_char,
-+ slop: u32,
-+ coverage_mode: i32,
-+) -> *mut LanceFtsQueryContext {
-+ ffi_try!(
-+ unsafe { prepare_fts_phrase_query_inner(dataset, column, query, slop, coverage_mode) },
-+ null
-+ )
-+}
-+
-+unsafe fn prepare_fts_phrase_query_inner(
-+ dataset: *const LanceDataset,
-+ column: *const c_char,
-+ query: *const c_char,
-+ slop: u32,
-+ coverage_mode: i32,
-+) -> Result<*mut LanceFtsQueryContext> {
-+ let (snapshot, column, query_text, coverage_mode) =
-+ unsafe { parse_query_inputs(dataset, column, query, coverage_mode)? };
-+ let query = FullTextSearchQuery::new_query(
-+ PhraseQuery::new(query_text)
-+ .with_column(Some(column.clone()))
-+ .with_slop(slop)
-+ .into(),
-+ );
-+ let inner = block_on(prepare_fts_query_context(
-+ snapshot,
-+ column,
-+ query,
-+ coverage_mode,
-+ ))?;
-+ Ok(into_context(inner))
- }
-
- /// Close a context handle. NULL-safe. Scanners that already attached the
-diff --git a/src/scanner.rs b/src/scanner.rs
-index 0c29b17..cbf1b13 100644
---- a/src/scanner.rs
-+++ b/src/scanner.rs
-@@ -18,7 +18,7 @@ use lance::Dataset;
- use lance::dataset::scanner::{
- DatasetRecordBatchStream, ExecutionStatsCallback, ExecutionSummaryCounts,
- };
--use lance::io::exec::fts::{FlatMatchQueryExec, MatchQueryExec};
-+use lance::io::exec::fts::{FlatMatchQueryExec, MatchQueryExec, PhraseQueryExec};
- use lance_core::Result;
- use lance_index::scalar::FullTextSearchQuery;
- use lance_io::stream::RecordBatchStream;
-@@ -33,7 +33,8 @@ use crate::error::{
- panic_payload_message, set_lance_error, set_last_error, swallow_unwind,
- };
- use crate::fts_query::{
-- FtsQueryContextInner, LanceFtsQueryContext, clone_context, parse_segment_uuids,
-+ FtsQueryContextInner, LanceFtsQueryContext, PreparedFtsQuery, clone_context,
-+ parse_segment_uuids,
- };
- use crate::helpers;
- use crate::runtime::{RT, block_on};
-@@ -328,17 +329,17 @@ impl PreparedScanner {
- let (plan, rewritten) = rewrite_prepared_fts_plan(
- plan,
- &distributed_fts.segments,
-- &distributed_fts.context.scorer,
-+ &distributed_fts.context.prepared,
- selected_segments_have_current_fragments,
- )?;
-- if rewritten.match_query_execs > 1
-+ if rewritten.indexed_query_execs > 1
- || rewritten.flat_match_query_execs > 1
-- || rewritten.match_query_execs + rewritten.flat_match_query_execs == 0
-- || (selected_segments_have_current_fragments && rewritten.match_query_execs != 1)
-+ || rewritten.indexed_query_execs + rewritten.flat_match_query_execs == 0
-+ || (selected_segments_have_current_fragments && rewritten.indexed_query_execs != 1)
- {
- return Err(lance_core::Error::internal(format!(
-- "unexpected prepared FTS plan for selected segments with current fragment coverage {selected_segments_have_current_fragments}: rewrote {} MatchQueryExec node(s) and removed {} FlatMatchQueryExec node(s)",
-- rewritten.match_query_execs, rewritten.flat_match_query_execs
-+ "unexpected prepared FTS plan for selected segments with current fragment coverage {selected_segments_have_current_fragments}: rewrote {} indexed FTS query node(s) and removed {} FlatMatchQueryExec node(s)",
-+ rewritten.indexed_query_execs, rewritten.flat_match_query_execs
- )));
- }
- let stream = lance_datafusion::exec::execute_plan(
-@@ -420,14 +421,14 @@ fn segments_have_current_fragments(
-
- #[derive(Default)]
- struct PreparedFtsPlanRewriteCounts {
-- match_query_execs: usize,
-+ indexed_query_execs: usize,
- flat_match_query_execs: usize,
- }
-
- fn rewrite_prepared_fts_plan(
- plan: Arc<dyn ExecutionPlan>,
- segments: &[IndexMetadata],
-- scorer: &Arc<lance_index::scalar::inverted::MemBM25Scorer>,
-+ prepared: &PreparedFtsQuery,
- selected_segments_have_current_fragments: bool,
- ) -> Result<(Arc<dyn ExecutionPlan>, PreparedFtsPlanRewriteCounts)> {
- // Lance's ordinary FTS planner adds a flat-search branch for fragments not
-@@ -438,7 +439,7 @@ fn rewrite_prepared_fts_plan(
- return Ok((
- Arc::new(EmptyExec::new(plan.schema())),
- PreparedFtsPlanRewriteCounts {
-- match_query_execs: 0,
-+ indexed_query_execs: 0,
- flat_match_query_execs: 1,
- },
- ));
-@@ -454,11 +455,11 @@ fn rewrite_prepared_fts_plan(
- let (new_child, child_rewritten) = rewrite_prepared_fts_plan(
- Arc::clone(child),
- segments,
-- scorer,
-+ prepared,
- selected_segments_have_current_fragments,
- )?;
- new_children.push(new_child);
-- rewritten.match_query_execs += child_rewritten.match_query_execs;
-+ rewritten.indexed_query_execs += child_rewritten.indexed_query_execs;
- rewritten.flat_match_query_execs += child_rewritten.flat_match_query_execs;
- }
- plan.with_new_children(new_children).map_err(|error| {
-@@ -469,10 +470,15 @@ fn rewrite_prepared_fts_plan(
- };
-
- if let Some(exec) = rebuilt.downcast_ref::<MatchQueryExec>() {
-- rewritten.match_query_execs += 1;
-+ rewritten.indexed_query_execs += 1;
- if !selected_segments_have_current_fragments {
- return Ok((Arc::new(EmptyExec::new(rebuilt.schema())), rewritten));
- }
-+ let PreparedFtsQuery::Match(scorer) = prepared else {
-+ return Err(lance_core::Error::internal(
-+ "prepared Phrase state cannot be attached to MatchQueryExec".to_string(),
-+ ));
-+ };
- let replacement = MatchQueryExec::new_with_segments(
- Arc::clone(exec.dataset()),
- exec.query().clone(),
-@@ -483,6 +489,26 @@ fn rewrite_prepared_fts_plan(
- .with_base_scorer(Arc::clone(scorer));
- return Ok((Arc::new(replacement), rewritten));
- }
-+ if let Some(exec) = rebuilt.downcast_ref::<PhraseQueryExec>() {
-+ rewritten.indexed_query_execs += 1;
-+ if !selected_segments_have_current_fragments {
-+ return Ok((Arc::new(EmptyExec::new(rebuilt.schema())), rewritten));
-+ }
-+ let PreparedFtsQuery::Phrase(scorer) = prepared else {
-+ return Err(lance_core::Error::internal(
-+ "prepared Match state cannot be attached to PhraseQueryExec".to_string(),
-+ ));
-+ };
-+ let replacement = PhraseQueryExec::new_with_segments(
-+ Arc::clone(exec.dataset()),
-+ exec.query().clone(),
-+ exec.params().clone(),
-+ exec.prefilter_source().clone(),
-+ segments.to_vec(),
-+ )
-+ .with_base_scorer(Arc::clone(scorer));
-+ return Ok((Arc::new(replacement), rewritten));
-+ }
- Ok((rebuilt, rewritten))
- }
-
-@@ -2152,7 +2178,8 @@ mod tests {
- use crate::dataset::{lance_dataset_close, lance_dataset_open};
- use crate::error::{lance_last_error_code, lance_last_error_message};
- use crate::fts_query::{
-- LanceFtsCoverageMode, lance_dataset_prepare_fts_query, lance_fts_query_context_close,
-+ LanceFtsCoverageMode, LanceFtsMatchOperator, lance_dataset_prepare_fts_match_query,
-+ lance_fts_query_context_close,
- };
- use std::ffi::{CStr, CString};
- use std::sync::atomic::{AtomicI32, AtomicUsize};
-@@ -2276,10 +2303,11 @@ mod tests {
- let column = CString::new("name").unwrap();
- let query = CString::new("a").unwrap();
- let context = unsafe {
-- lance_dataset_prepare_fts_query(
-+ lance_dataset_prepare_fts_match_query(
- dataset,
- column.as_ptr(),
- query.as_ptr(),
-+ LanceFtsMatchOperator::Or as i32,
- 0,
- LanceFtsCoverageMode::IndexOnly as i32,
- )
-@@ -2293,7 +2321,6 @@ mod tests {
- let prepared = unsafe { &*scanner }.build_scanner().unwrap();
- let distributed = prepared.distributed_fts.as_ref().unwrap();
- let segments = distributed.segments.clone();
-- let scorer = Arc::clone(&distributed.context.scorer);
- let plan = block_on(prepared.scanner.create_plan()).unwrap();
- assert_eq!(
- prepared_fts_plan_shape(&plan),
-@@ -2303,9 +2330,14 @@ mod tests {
-
- let has_current_fragments =
- segments_have_current_fragments(&distributed.context.dataset, &segments).unwrap();
-- let (rewritten, counts) =
-- rewrite_prepared_fts_plan(plan, &segments, &scorer, has_current_fragments).unwrap();
-- assert_eq!(counts.match_query_execs, 1);
-+ let (rewritten, counts) = rewrite_prepared_fts_plan(
-+ plan,
-+ &segments,
-+ &distributed.context.prepared,
-+ has_current_fragments,
-+ )
-+ .unwrap();
-+ assert_eq!(counts.indexed_query_execs, 1);
- assert_eq!(counts.flat_match_query_execs, 1);
- assert_eq!(prepared_fts_plan_shape(&rewritten), (1, 0, 0));
-
-diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index 8805764..a3e4ef7 100644
---- a/tests/c_api_test.rs
-+++ b/tests/c_api_test.rs
-@@ -5762,6 +5762,203 @@ fn load_fts_segment_uuids(uri: &str, column: &str) -> Vec<[u8; 16]> {
- })
- }
-
-+#[test]
-+#[allow(deprecated)]
-+fn test_prepared_fts_match_phrase_and_legacy_compatibility() {
-+ let tmp = tempfile::tempdir().unwrap();
-+ let uri = tmp
-+ .path()
-+ .join("prepared_fts_queries")
-+ .to_str()
-+ .unwrap()
-+ .to_string();
-+ let schema = Arc::new(Schema::new(vec![
-+ Field::new("id", DataType::Int32, false),
-+ Field::new("text", DataType::Utf8, false),
-+ ]));
-+ let batch = RecordBatch::try_new(
-+ schema.clone(),
-+ vec![
-+ Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])),
-+ Arc::new(StringArray::from(vec![
-+ "quick brown fox",
-+ "quick blue fox",
-+ "slow brown fox",
-+ "quik brown fox",
-+ "quick red brown fox",
-+ ])),
-+ ],
-+ )
-+ .unwrap();
-+ lance_c::runtime::block_on(async {
-+ Dataset::write(
-+ arrow::record_batch::RecordBatchIterator::new(vec![Ok(batch)], schema),
-+ &uri,
-+ None,
-+ )
-+ .await
-+ .unwrap();
-+ });
-+
-+ let uri_c = c_str(&uri);
-+ let column = c_str("text");
-+ let index_params =
-+ c_str(r#"{"base_tokenizer":"simple","language":"English","with_position":true}"#);
-+ let dataset = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) };
-+ assert_eq!(
-+ unsafe {
-+ lance_dataset_create_scalar_index(
-+ dataset,
-+ column.as_ptr(),
-+ ptr::null(),
-+ LanceScalarIndexType::Inverted as i32,
-+ index_params.as_ptr(),
-+ false,
-+ )
-+ },
-+ 0
-+ );
-+
-+ let query = c_str("quick brown");
-+ let exact_or = unsafe {
-+ lance_dataset_prepare_fts_match_query(
-+ dataset,
-+ column.as_ptr(),
-+ query.as_ptr(),
-+ LanceFtsMatchOperator::Or as i32,
-+ 0,
-+ LanceFtsCoverageMode::Strict as i32,
-+ )
-+ };
-+ assert!(!exact_or.is_null());
-+ assert_eq!(collect_context_fts_scores(dataset, exact_or, None).len(), 5);
-+ unsafe { lance_fts_query_context_close(exact_or) };
-+
-+ let legacy_or = unsafe {
-+ lance_dataset_prepare_fts_query(
-+ dataset,
-+ column.as_ptr(),
-+ query.as_ptr(),
-+ 0,
-+ LanceFtsCoverageMode::Strict as i32,
-+ )
-+ };
-+ assert!(!legacy_or.is_null());
-+ assert_eq!(
-+ collect_context_fts_scores(dataset, legacy_or, None).len(),
-+ 5
-+ );
-+ unsafe { lance_fts_query_context_close(legacy_or) };
-+
-+ let exact_and = unsafe {
-+ lance_dataset_prepare_fts_match_query(
-+ dataset,
-+ column.as_ptr(),
-+ query.as_ptr(),
-+ LanceFtsMatchOperator::And as i32,
-+ 0,
-+ LanceFtsCoverageMode::Strict as i32,
-+ )
-+ };
-+ assert!(!exact_and.is_null());
-+ let exact_and_scores = collect_context_fts_scores(dataset, exact_and, None);
-+ let mut exact_and_ids = exact_and_scores.keys().copied().collect::<Vec<_>>();
-+ exact_and_ids.sort_unstable();
-+ assert_eq!(exact_and_ids, vec![1, 5]);
-+ unsafe { lance_fts_query_context_close(exact_and) };
-+
-+ let fuzzy_and = unsafe {
-+ lance_dataset_prepare_fts_match_query(
-+ dataset,
-+ column.as_ptr(),
-+ query.as_ptr(),
-+ LanceFtsMatchOperator::And as i32,
-+ 1,
-+ LanceFtsCoverageMode::Strict as i32,
-+ )
-+ };
-+ assert!(fuzzy_and.is_null());
-+ let message = take_last_error_message();
-+ assert!(
-+ message.contains("max_fuzzy_distance must be 0"),
-+ "{message}"
-+ );
-+
-+ let phrase = unsafe {
-+ lance_dataset_prepare_fts_phrase_query(
-+ dataset,
-+ column.as_ptr(),
-+ query.as_ptr(),
-+ 0,
-+ LanceFtsCoverageMode::Strict as i32,
-+ )
-+ };
-+ assert!(!phrase.is_null(), "{}", take_last_error_message());
-+ let phrase_scores = collect_context_fts_scores(dataset, phrase, None);
-+ assert_eq!(phrase_scores.keys().copied().collect::<Vec<_>>(), vec![1]);
-+ unsafe { lance_fts_query_context_close(phrase) };
-+
-+ let phrase_with_slop = unsafe {
-+ lance_dataset_prepare_fts_phrase_query(
-+ dataset,
-+ column.as_ptr(),
-+ query.as_ptr(),
-+ 1,
-+ LanceFtsCoverageMode::Strict as i32,
-+ )
-+ };
-+ assert!(!phrase_with_slop.is_null(), "{}", take_last_error_message());
-+ let phrase_with_slop_scores = collect_context_fts_scores(dataset, phrase_with_slop, None);
-+ let mut phrase_with_slop_ids = phrase_with_slop_scores.keys().copied().collect::<Vec<_>>();
-+ phrase_with_slop_ids.sort_unstable();
-+ assert_eq!(phrase_with_slop_ids, vec![1, 5]);
-+ unsafe { lance_fts_query_context_close(phrase_with_slop) };
-+
-+ unsafe { lance_dataset_close(dataset) };
-+}
-+
-+#[test]
-+fn test_prepared_fts_phrase_requires_positions() {
-+ let (_tmp, uri) = create_test_dataset();
-+ let uri_c = c_str(&uri);
-+ let column = c_str("name");
-+ let query = c_str("alice smith");
-+ let index_params = c_str(r#"{"base_tokenizer":"simple","language":"English"}"#);
-+ let dataset = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) };
-+ assert_eq!(
-+ unsafe {
-+ lance_dataset_create_scalar_index(
-+ dataset,
-+ column.as_ptr(),
-+ ptr::null(),
-+ LanceScalarIndexType::Inverted as i32,
-+ index_params.as_ptr(),
-+ false,
-+ )
-+ },
-+ 0
-+ );
-+
-+ let context = unsafe {
-+ lance_dataset_prepare_fts_phrase_query(
-+ dataset,
-+ column.as_ptr(),
-+ query.as_ptr(),
-+ 0,
-+ LanceFtsCoverageMode::Strict as i32,
-+ )
-+ };
-+ assert!(context.is_null());
-+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument);
-+ let message = take_last_error_message();
-+ assert!(
-+ message.contains("does not store token positions"),
-+ "{message}"
-+ );
-+
-+ unsafe { lance_dataset_close(dataset) };
-+}
-+
- #[test]
- fn test_prepared_fts_row_id_output_is_explicit() {
- let (_tmp, uri) = create_test_dataset();
-@@ -5785,10 +5982,11 @@ fn test_prepared_fts_row_id_output_is_explicit() {
- 0
- );
- let context = unsafe {
-- lance_dataset_prepare_fts_query(
-+ lance_dataset_prepare_fts_match_query(
- dataset,
- column.as_ptr(),
- query.as_ptr(),
-+ LanceFtsMatchOperator::Or as i32,
- 0,
- LanceFtsCoverageMode::Strict as i32,
- )
-@@ -5880,10 +6078,11 @@ fn test_prepare_fts_query_index_only_allows_unindexed_fragment() {
-
- let dataset = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) };
- let strict = unsafe {
-- lance_dataset_prepare_fts_query(
-+ lance_dataset_prepare_fts_match_query(
- dataset,
- column.as_ptr(),
- query.as_ptr(),
-+ LanceFtsMatchOperator::Or as i32,
- 0,
- LanceFtsCoverageMode::Strict as i32,
- )
-@@ -5898,10 +6097,11 @@ fn test_prepare_fts_query_index_only_allows_unindexed_fragment() {
- assert!(message.contains("unindexed fragments"), "{message}");
-
- let context = unsafe {
-- lance_dataset_prepare_fts_query(
-+ lance_dataset_prepare_fts_match_query(
- dataset,
- column.as_ptr(),
- query.as_ptr(),
-+ LanceFtsMatchOperator::Or as i32,
- 0,
- LanceFtsCoverageMode::IndexOnly as i32,
- )
-@@ -5976,10 +6176,11 @@ fn test_prepared_fts_index_only_empty_segment_returns_empty_shard() {
- let query = c_str("alice");
- let dataset = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) };
- let context = unsafe {
-- lance_dataset_prepare_fts_query(
-+ lance_dataset_prepare_fts_match_query(
- dataset,
- column.as_ptr(),
- query.as_ptr(),
-+ LanceFtsMatchOperator::Or as i32,
- 0,
- LanceFtsCoverageMode::IndexOnly as i32,
- )
-@@ -6075,10 +6276,11 @@ fn test_prepared_fts_global_scorer_is_shared_across_segment_splits() {
-
- let dataset = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) };
- let context = unsafe {
-- lance_dataset_prepare_fts_query(
-+ lance_dataset_prepare_fts_match_query(
- dataset,
- column.as_ptr(),
- query.as_ptr(),
-+ LanceFtsMatchOperator::Or as i32,
- 0,
- LanceFtsCoverageMode::Strict as i32,
- )
-@@ -6186,7 +6388,7 @@ fn test_prepared_fts_global_scorer_is_shared_across_segment_splits() {
- }
-
- #[test]
--fn test_prepare_fts_query_rejects_null_empty_invalid_mode_and_fuzzy() {
-+fn test_prepare_fts_queries_reject_invalid_inputs() {
- let (_tmp, uri) = create_test_dataset();
- let uri_c = c_str(&uri);
- let dataset = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) };
-@@ -6196,10 +6398,11 @@ fn test_prepare_fts_query_rejects_null_empty_invalid_mode_and_fuzzy() {
-
- assert!(
- unsafe {
-- lance_dataset_prepare_fts_query(
-+ lance_dataset_prepare_fts_match_query(
- ptr::null(),
- column.as_ptr(),
- query.as_ptr(),
-+ LanceFtsMatchOperator::Or as i32,
- 0,
- LanceFtsCoverageMode::Strict as i32,
- )
-@@ -6208,10 +6411,11 @@ fn test_prepare_fts_query_rejects_null_empty_invalid_mode_and_fuzzy() {
- );
- assert!(
- unsafe {
-- lance_dataset_prepare_fts_query(
-+ lance_dataset_prepare_fts_match_query(
- dataset,
- empty.as_ptr(),
- query.as_ptr(),
-+ LanceFtsMatchOperator::Or as i32,
- 0,
- LanceFtsCoverageMode::Strict as i32,
- )
-@@ -6219,20 +6423,39 @@ fn test_prepare_fts_query_rejects_null_empty_invalid_mode_and_fuzzy() {
- .is_null()
- );
- assert!(
-- unsafe { lance_dataset_prepare_fts_query(dataset, column.as_ptr(), empty.as_ptr(), 0, 0) }
-- .is_null()
-+ unsafe {
-+ lance_dataset_prepare_fts_match_query(
-+ dataset,
-+ column.as_ptr(),
-+ empty.as_ptr(),
-+ LanceFtsMatchOperator::Or as i32,
-+ 0,
-+ LanceFtsCoverageMode::Strict as i32,
-+ )
-+ }
-+ .is_null()
- );
- assert!(
-- unsafe { lance_dataset_prepare_fts_query(dataset, column.as_ptr(), query.as_ptr(), 0, 99) }
-- .is_null()
-+ unsafe {
-+ lance_dataset_prepare_fts_match_query(
-+ dataset,
-+ column.as_ptr(),
-+ query.as_ptr(),
-+ LanceFtsMatchOperator::Or as i32,
-+ 0,
-+ 99,
-+ )
-+ }
-+ .is_null()
- );
- assert!(
- unsafe {
-- lance_dataset_prepare_fts_query(
-+ lance_dataset_prepare_fts_match_query(
- dataset,
- column.as_ptr(),
- query.as_ptr(),
-- 1,
-+ 99,
-+ 0,
- LanceFtsCoverageMode::Strict as i32,
- )
- }
-@@ -6244,9 +6467,18 @@ fn test_prepare_fts_query_rejects_null_empty_invalid_mode_and_fuzzy() {
- .to_string_lossy()
- .into_owned()
- };
-+ assert!(message.contains("invalid match_operator"), "{message}");
- assert!(
-- message.contains("max_fuzzy_distance must be 0"),
-- "{message}"
-+ unsafe {
-+ lance_dataset_prepare_fts_phrase_query(
-+ dataset,
-+ column.as_ptr(),
-+ ptr::null(),
-+ 0,
-+ LanceFtsCoverageMode::Strict as i32,
-+ )
-+ }
-+ .is_null()
- );
- let scanner = unsafe { lance_scanner_new(dataset, ptr::null(), ptr::null()) };
- assert_eq!(
-
-From 9bb38749c9de185664511def3cbeb699ee60a38b Mon Sep 17 00:00:00 2001
-From: zhangstar333 <zh...@selectdb.com>
-Date: Thu, 3 Sep 2026 17:49:02 +0800
-Subject: [PATCH 2/2] update slop i32
-
----
- include/lance/lance.h | 6 +++---
- include/lance/lance.hpp | 5 +++--
- src/fts_query.rs | 6 ++++--
- tests/c_api_test.rs | 14 ++++++++++++++
- 4 files changed, 24 insertions(+), 7 deletions(-)
-
-diff --git a/include/lance/lance.h b/include/lance/lance.h
-index 0c76edc..25c0430 100644
---- a/include/lance/lance.h
-+++ b/include/lance/lance.h
-@@ -1761,8 +1761,8 @@ LanceFtsQueryContext* lance_dataset_prepare_fts_match_query(
- * Dataset identity, coverage, sharing, and segment-scoped execution follow the
- * same contract as lance_dataset_prepare_fts_match_query().
- *
-- * @param slop Maximum number of intervening token positions permitted between
-- * adjacent phrase terms.
-+ * @param slop Maximum non-negative number of intervening token positions
-+ * permitted between adjacent phrase terms.
- * @param coverage_mode Fixed-width LanceFtsCoverageMode discriminant.
- * @return Context handle on success, or NULL on error.
- */
-@@ -1770,7 +1770,7 @@ LanceFtsQueryContext* lance_dataset_prepare_fts_phrase_query(
- const LanceDataset* dataset,
- const char* column,
- const char* query,
-- uint32_t slop,
-+ int32_t slop,
- int32_t coverage_mode
- );
-
-diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp
-index 973216a..1e69a06 100644
---- a/include/lance/lance.hpp
-+++ b/include/lance/lance.hpp
-@@ -798,11 +798,12 @@ class Dataset {
- return FtsQueryContext(context);
- }
-
-- /// Prepare a Phrase query. Its FTS index must store token positions.
-+ /// Prepare a Phrase query. Its FTS index must store token positions and
-+ /// slop must be non-negative.
- FtsQueryContext prepare_fts_phrase_query(
- const std::string& column,
- const std::string& query,
-- uint32_t slop = 0,
-+ int32_t slop = 0,
- FtsCoverageMode coverage_mode = FtsCoverageMode::Strict) const {
- auto* context = lance_dataset_prepare_fts_phrase_query(
- handle_.get(), column.c_str(), query.c_str(), slop,
-diff --git a/src/fts_query.rs b/src/fts_query.rs
-index 3bf7f3e..c9d3a43 100644
---- a/src/fts_query.rs
-+++ b/src/fts_query.rs
-@@ -401,7 +401,7 @@ pub unsafe extern "C" fn lance_dataset_prepare_fts_phrase_query(
- dataset: *const LanceDataset,
- column: *const c_char,
- query: *const c_char,
-- slop: u32,
-+ slop: i32,
- coverage_mode: i32,
- ) -> *mut LanceFtsQueryContext {
- ffi_try!(
-@@ -414,9 +414,11 @@ unsafe fn prepare_fts_phrase_query_inner(
- dataset: *const LanceDataset,
- column: *const c_char,
- query: *const c_char,
-- slop: u32,
-+ slop: i32,
- coverage_mode: i32,
- ) -> Result<*mut LanceFtsQueryContext> {
-+ let slop = u32::try_from(slop)
-+ .map_err(|_| invalid_input(format!("slop must be non-negative, got {slop}")))?;
- let (snapshot, column, query_text, coverage_mode) =
- unsafe { parse_query_inputs(dataset, column, query, coverage_mode)? };
- let query = FullTextSearchQuery::new_query(
-diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index a3e4ef7..9411e15 100644
---- a/tests/c_api_test.rs
-+++ b/tests/c_api_test.rs
-@@ -5914,6 +5914,20 @@ fn test_prepared_fts_match_phrase_and_legacy_compatibility() {
- assert_eq!(phrase_with_slop_ids, vec![1, 5]);
- unsafe { lance_fts_query_context_close(phrase_with_slop) };
-
-+ let negative_phrase_slop = unsafe {
-+ lance_dataset_prepare_fts_phrase_query(
-+ dataset,
-+ column.as_ptr(),
-+ query.as_ptr(),
-+ -1,
-+ LanceFtsCoverageMode::Strict as i32,
-+ )
-+ };
-+ assert!(negative_phrase_slop.is_null());
-+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument);
-+ let message = take_last_error_message();
-+ assert!(message.contains("slop must be non-negative"), "{message}");
-+
- unsafe { lance_dataset_close(dataset) };
- }
-
diff --git a/thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch b/thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch
deleted file mode 100644
index 08687627e7e..00000000000
--- a/thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch
+++ /dev/null
@@ -1,2441 +0,0 @@
-Lance-C v0.1.9 scanner options: PR #75 followed by PR #78.
-Apply after lance-c-0.1.9-pr-74.patch, before lance-c-0.1.9-pr-77.patch.
-The upstream mail patches below are concatenated without modification.
-
-PR #75: https://github.com/lance-format/lance-c/pull/75
-Head: 043a1f7eac253d8ac6be3f970b60fcbf615295aa
-PR #78: https://github.com/lance-format/lance-c/pull/78
-Head: e894f591aef358cd36fdbf915c4d5b95fd0e8348
-
-From 0752592cc4bbc69f8f9333362a00f3bceee73100 Mon Sep 17 00:00:00 2001
-From: zhangstar333 <zh...@selectdb.com>
-Date: Fri, 4 Sep 2026 23:35:29 +0800
-Subject: [PATCH 1/2] add some setter
-
----
- include/lance/lance.h | 81 +++++++++++
- include/lance/lance.hpp | 47 +++++++
- src/scanner.rs | 279 +++++++++++++++++++++++++++++++++++++
- tests/c_api_test.rs | 230 ++++++++++++++++++++++++++++++
- tests/cpp/test_cpp_api.cpp | 7 +
- 5 files changed, 644 insertions(+)
-
-diff --git a/include/lance/lance.h b/include/lance/lance.h
-index 3bf291f..00f415b 100644
---- a/include/lance/lance.h
-+++ b/include/lance/lance.h
-@@ -934,6 +934,74 @@ LanceScanner* lance_scanner_new(
- int32_t lance_scanner_set_limit(LanceScanner* scanner, int64_t limit);
- int32_t lance_scanner_set_offset(LanceScanner* scanner, int64_t offset);
- int32_t lance_scanner_set_batch_size(LanceScanner* scanner, int64_t batch_size);
-+
-+/**
-+ * Set the target output batch size in bytes.
-+ *
-+ * When set, this takes precedence over the row-based batch size. The value
-+ * must be greater than zero and must be set before scanning starts.
-+ */
-+int32_t lance_scanner_set_batch_size_bytes(
-+ LanceScanner* scanner,
-+ uint64_t batch_size_bytes
-+);
-+
-+/**
-+ * Set the scanner I/O buffer size in bytes.
-+ *
-+ * The value must be greater than zero and must be set before scanning starts.
-+ * This bounds buffered I/O received from storage, but is not a hard limit on
-+ * all memory used by the scanner.
-+ *
-+ * @param scanner Scanner handle. Must not be NULL.
-+ * @param io_buffer_size_bytes I/O buffer size in bytes. Must be greater than zero.
-+ * @return 0 on success, -1 on error.
-+ */
-+int32_t lance_scanner_set_io_buffer_size(
-+ LanceScanner* scanner,
-+ uint64_t io_buffer_size_bytes
-+);
-+
-+/**
-+ * Set the maximum number of batches decoded concurrently.
-+ *
-+ * @param batch_readahead Number of in-flight batch decode tasks. Must be greater than zero.
-+ */
-+int32_t lance_scanner_set_batch_readahead(
-+ LanceScanner* scanner,
-+ size_t batch_readahead
-+);
-+
-+/**
-+ * Set fragment readahead for unordered scans.
-+ *
-+ * This setting is only used when scan-in-order is disabled. The value must be
-+ * greater than zero.
-+ */
-+int32_t lance_scanner_set_fragment_readahead(
-+ LanceScanner* scanner,
-+ size_t fragment_readahead
-+);
-+
-+/**
-+ * Set the target number of physical execution partitions.
-+ *
-+ * This controls the partition count used by the physical optimizer and can be
-+ * used to bound scan CPU parallelism. The value must be greater than zero and
-+ * must be set before scanning starts.
-+ */
-+int32_t lance_scanner_set_target_parallelism(
-+ LanceScanner* scanner,
-+ size_t target_parallelism
-+);
-+
-+/**
-+ * Configure whether batches are returned in storage order (default: true).
-+ *
-+ * Disabling ordering can improve throughput by returning batches as soon as
-+ * they are ready.
-+ */
-+int32_t lance_scanner_set_scan_in_order(LanceScanner* scanner, bool scan_in_order);
- int32_t lance_scanner_with_row_id(LanceScanner* scanner, bool enable);
-
- /**
-@@ -1656,6 +1724,19 @@ int32_t lance_scanner_nearest(
- );
-
- int32_t lance_scanner_set_nprobes(LanceScanner* scanner, uint32_t n);
-+
-+/**
-+ * Set vector index partition-search concurrency for each query.
-+ *
-+ * A value of -1 uses the CPU pool size, 0 selects Lance's automatic policy,
-+ * 1 uses the sequential path, and values greater than 1 request parallel
-+ * partition search. The effective value is capped by available parallelism.
-+ * Values below -1 are rejected. Must be set before scanning starts.
-+ */
-+int32_t lance_scanner_set_query_parallelism(
-+ LanceScanner* scanner,
-+ int32_t query_parallelism
-+);
- int32_t lance_scanner_set_refine_factor(LanceScanner* scanner, uint32_t f);
- int32_t lance_scanner_set_ef(LanceScanner* scanner, uint32_t e);
- int32_t lance_scanner_set_metric(LanceScanner* scanner, LanceMetricType metric);
-diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp
-index 6cf245f..3c03f86 100644
---- a/include/lance/lance.hpp
-+++ b/include/lance/lance.hpp
-@@ -1196,6 +1196,48 @@ class Scanner {
- return *this;
- }
-
-+ /// Set the target output batch size in bytes.
-+ Scanner& batch_size_bytes(uint64_t bytes) {
-+ if (lance_scanner_set_batch_size_bytes(handle_.get(), bytes) != 0)
-+ check_error();
-+ return *this;
-+ }
-+
-+ /// Set the scanner I/O buffer size in bytes.
-+ Scanner& io_buffer_size(uint64_t bytes) {
-+ if (lance_scanner_set_io_buffer_size(handle_.get(), bytes) != 0)
-+ check_error();
-+ return *this;
-+ }
-+
-+ /// Set the number of batches decoded concurrently.
-+ Scanner& batch_readahead(size_t batches) {
-+ if (lance_scanner_set_batch_readahead(handle_.get(), batches) != 0)
-+ check_error();
-+ return *this;
-+ }
-+
-+ /// Set fragment readahead for unordered scans.
-+ Scanner& fragment_readahead(size_t fragments) {
-+ if (lance_scanner_set_fragment_readahead(handle_.get(), fragments) != 0)
-+ check_error();
-+ return *this;
-+ }
-+
-+ /// Set the target number of physical execution partitions.
-+ Scanner& target_parallelism(size_t partitions) {
-+ if (lance_scanner_set_target_parallelism(handle_.get(), partitions) != 0)
-+ check_error();
-+ return *this;
-+ }
-+
-+ /// Configure whether batches are returned in storage order.
-+ Scanner& scan_in_order(bool ordered = true) {
-+ if (lance_scanner_set_scan_in_order(handle_.get(), ordered) != 0)
-+ check_error();
-+ return *this;
-+ }
-+
- /// Enable/disable row ID in output.
- Scanner& with_row_id(bool enable = true) {
- if (lance_scanner_with_row_id(handle_.get(), enable) != 0)
-@@ -1313,6 +1355,11 @@ class Scanner {
- if (lance_scanner_set_nprobes(handle_.get(), n) != 0) check_error();
- return *this;
- }
-+ Scanner& query_parallelism(int32_t parallelism) {
-+ if (lance_scanner_set_query_parallelism(handle_.get(), parallelism) != 0)
-+ check_error();
-+ return *this;
-+ }
- Scanner& refine_factor(uint32_t f) {
- if (lance_scanner_set_refine_factor(handle_.get(), f) != 0) check_error();
- return *this;
-diff --git a/src/scanner.rs b/src/scanner.rs
-index 0c29b17..ebedafc 100644
---- a/src/scanner.rs
-+++ b/src/scanner.rs
-@@ -60,11 +60,18 @@ pub struct LanceScanner {
- limit: Option<i64>,
- offset: Option<i64>,
- batch_size: Option<usize>,
-+ batch_size_bytes: Option<u64>,
-+ io_buffer_size: Option<u64>,
-+ batch_readahead: Option<usize>,
-+ fragment_readahead: Option<usize>,
-+ target_parallelism: Option<usize>,
-+ scan_in_order: Option<bool>,
- with_row_id: bool,
- fragment_ids: Option<Vec<u64>>,
- index_segments: Option<Vec<Uuid>>,
- nearest: Option<NearestQuery>,
- nprobes: Option<u32>,
-+ query_parallelism: Option<i32>,
- refine_factor: Option<u32>,
- ef: Option<u32>,
- metric_override: Option<crate::index::LanceMetricType>,
-@@ -129,11 +136,18 @@ impl LanceScanner {
- limit: None,
- offset: None,
- batch_size: None,
-+ batch_size_bytes: None,
-+ io_buffer_size: None,
-+ batch_readahead: None,
-+ fragment_readahead: None,
-+ target_parallelism: None,
-+ scan_in_order: None,
- with_row_id: false,
- fragment_ids: None,
- index_segments: None,
- nearest: None,
- nprobes: None,
-+ query_parallelism: None,
- refine_factor: None,
- ef: None,
- metric_override: None,
-@@ -164,6 +178,15 @@ impl LanceScanner {
- Arc::clone(&self.poisoned)
- }
-
-+ fn ensure_scan_not_started(&self, setting_name: &str) -> Result<()> {
-+ if self.scan_started.load(Ordering::Acquire) {
-+ return Err(lance_core::Error::invalid_input_source(
-+ format!("{setting_name} must be set before the scan starts").into(),
-+ ));
-+ }
-+ Ok(())
-+ }
-+
- /// Apply fragment selection to a scanner builder if fragment_ids is set.
- fn apply_fragment_filter(&self, scanner: &mut lance::dataset::scanner::Scanner) -> Result<()> {
- if let Some(ids) = &self.fragment_ids {
-@@ -231,6 +254,24 @@ impl LanceScanner {
- if let Some(bs) = self.batch_size {
- scanner.batch_size(bs);
- }
-+ if let Some(batch_size_bytes) = self.batch_size_bytes {
-+ scanner.batch_size_bytes(batch_size_bytes);
-+ }
-+ if let Some(io_buffer_size) = self.io_buffer_size {
-+ scanner.io_buffer_size(io_buffer_size);
-+ }
-+ if let Some(batch_readahead) = self.batch_readahead {
-+ scanner.batch_readahead(batch_readahead);
-+ }
-+ if let Some(fragment_readahead) = self.fragment_readahead {
-+ scanner.fragment_readahead(fragment_readahead);
-+ }
-+ if let Some(target_parallelism) = self.target_parallelism {
-+ scanner.target_parallelism(target_parallelism);
-+ }
-+ if let Some(scan_in_order) = self.scan_in_order {
-+ scanner.scan_in_order(scan_in_order);
-+ }
- if self.with_row_id {
- scanner.with_row_id();
- }
-@@ -260,6 +301,9 @@ impl LanceScanner {
- if let Some(np) = self.nprobes {
- scanner.nprobes(np as usize);
- }
-+ if let Some(query_parallelism) = self.query_parallelism {
-+ scanner.query_parallelism(query_parallelism);
-+ }
- if let Some(rf) = self.refine_factor {
- scanner.refine(rf);
- }
-@@ -757,6 +801,205 @@ unsafe fn scanner_set_batch_size_inner(scanner: *mut LanceScanner, batch_size: i
- Ok(0)
- }
-
-+/// Set the target output batch size in bytes. Returns 0 on success.
-+///
-+/// The size must be greater than zero and must be set before the scan starts.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_batch_size_bytes(
-+ scanner: *mut LanceScanner,
-+ batch_size_bytes: u64,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_batch_size_bytes_inner(scanner, batch_size_bytes)
-+ })
-+}
-+
-+unsafe fn scanner_set_batch_size_bytes_inner(
-+ scanner: *mut LanceScanner,
-+ batch_size_bytes: u64,
-+) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ if batch_size_bytes == 0 {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "batch_size_bytes must be greater than 0, got 0".into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("batch_size_bytes")?;
-+ scanner.batch_size_bytes = Some(batch_size_bytes);
-+ Ok(0)
-+}
-+
-+/// Set the scanner I/O buffer size in bytes. Returns 0 on success.
-+///
-+/// The size must be greater than zero and must be set before the scan starts.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_io_buffer_size(
-+ scanner: *mut LanceScanner,
-+ io_buffer_size_bytes: u64,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_io_buffer_size_inner(scanner, io_buffer_size_bytes)
-+ })
-+}
-+
-+unsafe fn scanner_set_io_buffer_size_inner(
-+ scanner: *mut LanceScanner,
-+ io_buffer_size_bytes: u64,
-+) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ if io_buffer_size_bytes == 0 {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "io_buffer_size_bytes must be greater than 0, got 0".into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("io_buffer_size_bytes")?;
-+ scanner.io_buffer_size = Some(io_buffer_size_bytes);
-+ Ok(0)
-+}
-+
-+/// Set the number of batches to decode concurrently. Returns 0 on success.
-+///
-+/// The value must be greater than zero and must be set before the scan starts.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_batch_readahead(
-+ scanner: *mut LanceScanner,
-+ batch_readahead: usize,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_batch_readahead_inner(scanner, batch_readahead)
-+ })
-+}
-+
-+unsafe fn scanner_set_batch_readahead_inner(
-+ scanner: *mut LanceScanner,
-+ batch_readahead: usize,
-+) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ if batch_readahead == 0 {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "batch_readahead must be greater than 0, got 0".into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("batch_readahead")?;
-+ scanner.batch_readahead = Some(batch_readahead);
-+ Ok(0)
-+}
-+
-+/// Set the number of fragments to read ahead for unordered scans.
-+///
-+/// The value must be greater than zero and must be set before the scan starts.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_fragment_readahead(
-+ scanner: *mut LanceScanner,
-+ fragment_readahead: usize,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_fragment_readahead_inner(scanner, fragment_readahead)
-+ })
-+}
-+
-+unsafe fn scanner_set_fragment_readahead_inner(
-+ scanner: *mut LanceScanner,
-+ fragment_readahead: usize,
-+) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ if fragment_readahead == 0 {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "fragment_readahead must be greater than 0, got 0".into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("fragment_readahead")?;
-+ scanner.fragment_readahead = Some(fragment_readahead);
-+ Ok(0)
-+}
-+
-+/// Set the target number of physical execution partitions.
-+///
-+/// The value must be greater than zero and must be set before the scan starts.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_target_parallelism(
-+ scanner: *mut LanceScanner,
-+ target_parallelism: usize,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_target_parallelism_inner(scanner, target_parallelism)
-+ })
-+}
-+
-+unsafe fn scanner_set_target_parallelism_inner(
-+ scanner: *mut LanceScanner,
-+ target_parallelism: usize,
-+) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ if target_parallelism == 0 {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "target_parallelism must be greater than 0, got 0".into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("target_parallelism")?;
-+ scanner.target_parallelism = Some(target_parallelism);
-+ Ok(0)
-+}
-+
-+/// Configure whether scan results are returned in storage order.
-+///
-+/// Must be set before the scan starts.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_scan_in_order(
-+ scanner: *mut LanceScanner,
-+ scan_in_order: bool,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_scan_in_order_inner(scanner, scan_in_order)
-+ })
-+}
-+
-+unsafe fn scanner_set_scan_in_order_inner(
-+ scanner: *mut LanceScanner,
-+ scan_in_order: bool,
-+) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("scan_in_order")?;
-+ scanner.scan_in_order = Some(scan_in_order);
-+ Ok(0)
-+}
-+
- /// Enable or disable row ID in scan output. Returns 0.
- #[unsafe(no_mangle)]
- pub unsafe extern "C" fn lance_scanner_with_row_id(
-@@ -1766,6 +2009,42 @@ scanner_set_u32!(lance_scanner_set_nprobes, nprobes);
- scanner_set_u32!(lance_scanner_set_refine_factor, refine_factor);
- scanner_set_u32!(lance_scanner_set_ef, ef);
-
-+/// Set vector index partition-search concurrency for each query.
-+///
-+/// `-1` uses the CPU pool size, `0` selects Lance's automatic policy, and
-+/// positive values request that many workers. Values below `-1` are invalid.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_query_parallelism(
-+ scanner: *mut LanceScanner,
-+ query_parallelism: i32,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_query_parallelism_inner(scanner, query_parallelism)
-+ })
-+}
-+
-+unsafe fn scanner_set_query_parallelism_inner(
-+ scanner: *mut LanceScanner,
-+ query_parallelism: i32,
-+) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ if query_parallelism < -1 {
-+ return Err(lance_core::Error::invalid_input_source(
-+ format!("query_parallelism must be -1, 0, or greater than 0, got {query_parallelism}")
-+ .into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("query_parallelism")?;
-+ scanner.query_parallelism = Some(query_parallelism);
-+ Ok(0)
-+}
-+
- #[unsafe(no_mangle)]
- pub unsafe extern "C" fn lance_scanner_set_metric(scanner: *mut LanceScanner, metric: i32) -> i32 {
- scanner_poison_check!(scanner, -1);
-diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index 8805764..42f842d 100644
---- a/tests/c_api_test.rs
-+++ b/tests/c_api_test.rs
-@@ -1207,6 +1207,154 @@ fn test_scanner_batch_size() {
- unsafe { lance_dataset_close(ds) };
- }
-
-+#[test]
-+fn test_scanner_execution_tuning_options() {
-+ let (_tmp, uri) = create_multi_fragment_dataset();
-+ let c_uri = c_str(&uri);
-+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) };
-+ assert!(!ds.is_null());
-+
-+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) };
-+ assert!(!scanner.is_null());
-+ assert_eq!(
-+ unsafe { lance_scanner_set_batch_size_bytes(scanner, 1024) },
-+ 0
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_io_buffer_size(scanner, 64 * 1024) },
-+ 0
-+ );
-+ assert_eq!(unsafe { lance_scanner_set_batch_readahead(scanner, 1) }, 0);
-+ assert_eq!(
-+ unsafe { lance_scanner_set_fragment_readahead(scanner, 1) },
-+ 0
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_target_parallelism(scanner, 1) },
-+ 0
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_scan_in_order(scanner, false) },
-+ 0
-+ );
-+
-+ let mut ffi_stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(
-+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut ffi_stream) },
-+ 0
-+ );
-+ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut ffi_stream) }.unwrap();
-+ let total_rows: usize = reader.map(|batch| batch.unwrap().num_rows()).sum();
-+ assert_eq!(total_rows, 10);
-+
-+ unsafe { lance_scanner_close(scanner) };
-+ unsafe { lance_dataset_close(ds) };
-+}
-+
-+#[test]
-+fn test_scanner_execution_tuning_options_reject_zero() {
-+ let (_tmp, uri) = create_test_dataset();
-+ let c_uri = c_str(&uri);
-+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) };
-+ assert!(!ds.is_null());
-+
-+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) };
-+ assert!(!scanner.is_null());
-+
-+ assert_eq!(
-+ unsafe { lance_scanner_set_batch_size_bytes(scanner, 0) },
-+ -1
-+ );
-+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument);
-+ assert!(take_last_error_message().contains("batch_size_bytes must be greater than 0, got 0"));
-+
-+ assert_eq!(unsafe { lance_scanner_set_io_buffer_size(scanner, 0) }, -1);
-+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument);
-+ assert!(
-+ take_last_error_message().contains("io_buffer_size_bytes must be greater than 0, got 0")
-+ );
-+
-+ assert_eq!(unsafe { lance_scanner_set_batch_readahead(scanner, 0) }, -1);
-+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument);
-+ assert!(take_last_error_message().contains("batch_readahead must be greater than 0, got 0"));
-+
-+ assert_eq!(
-+ unsafe { lance_scanner_set_fragment_readahead(scanner, 0) },
-+ -1
-+ );
-+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument);
-+ assert!(take_last_error_message().contains("fragment_readahead must be greater than 0, got 0"));
-+
-+ assert_eq!(
-+ unsafe { lance_scanner_set_target_parallelism(scanner, 0) },
-+ -1
-+ );
-+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument);
-+ assert!(take_last_error_message().contains("target_parallelism must be greater than 0, got 0"));
-+
-+ unsafe { lance_scanner_close(scanner) };
-+ unsafe { lance_dataset_close(ds) };
-+}
-+
-+#[test]
-+fn test_scanner_execution_tuning_options_reject_after_scan_start() {
-+ let (_tmp, uri) = create_test_dataset();
-+ let c_uri = c_str(&uri);
-+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) };
-+ assert!(!ds.is_null());
-+
-+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) };
-+ assert!(!scanner.is_null());
-+
-+ let mut ffi_stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(
-+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut ffi_stream) },
-+ 0
-+ );
-+
-+ assert_eq!(
-+ unsafe { lance_scanner_set_batch_size_bytes(scanner, 1024) },
-+ -1
-+ );
-+ assert!(take_last_error_message().contains("batch_size_bytes must be set before"));
-+
-+ assert_eq!(
-+ unsafe { lance_scanner_set_io_buffer_size(scanner, 64 * 1024) },
-+ -1
-+ );
-+ assert!(take_last_error_message().contains("io_buffer_size_bytes must be set before"));
-+
-+ assert_eq!(unsafe { lance_scanner_set_batch_readahead(scanner, 1) }, -1);
-+ assert!(take_last_error_message().contains("batch_readahead must be set before"));
-+
-+ assert_eq!(
-+ unsafe { lance_scanner_set_fragment_readahead(scanner, 1) },
-+ -1
-+ );
-+ assert!(take_last_error_message().contains("fragment_readahead must be set before"));
-+
-+ assert_eq!(
-+ unsafe { lance_scanner_set_target_parallelism(scanner, 1) },
-+ -1
-+ );
-+ assert!(take_last_error_message().contains("target_parallelism must be set before"));
-+
-+ assert_eq!(
-+ unsafe { lance_scanner_set_scan_in_order(scanner, false) },
-+ -1
-+ );
-+ assert!(take_last_error_message().contains("scan_in_order must be set before"));
-+
-+ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut ffi_stream) }.unwrap();
-+ assert_eq!(
-+ reader.map(|batch| batch.unwrap().num_rows()).sum::<usize>(),
-+ 5
-+ );
-+
-+ unsafe { lance_scanner_close(scanner) };
-+ unsafe { lance_dataset_close(ds) };
-+}
-+
- // ---------------------------------------------------------------------------
- // Combined filter + projection + limit
- // ---------------------------------------------------------------------------
-@@ -1441,6 +1589,34 @@ fn test_null_safety_comprehensive() {
- unsafe { lance_scanner_set_batch_size(ptr::null_mut(), 10) },
- -1
- );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_batch_size_bytes(ptr::null_mut(), 1024) },
-+ -1
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_io_buffer_size(ptr::null_mut(), 64 * 1024) },
-+ -1
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_batch_readahead(ptr::null_mut(), 1) },
-+ -1
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_fragment_readahead(ptr::null_mut(), 1) },
-+ -1
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_target_parallelism(ptr::null_mut(), 1) },
-+ -1
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_query_parallelism(ptr::null_mut(), 1) },
-+ -1
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_scan_in_order(ptr::null_mut(), true) },
-+ -1
-+ );
- assert_eq!(
- unsafe { lance_scanner_with_row_id(ptr::null_mut(), true) },
- -1
-@@ -5216,6 +5392,7 @@ fn test_scanner_nearest_with_ivf_pq_index() {
- 10,
- );
- lance_scanner_set_nprobes(scanner, 4);
-+ assert_eq!(lance_scanner_set_query_parallelism(scanner, 4), 0);
- }
-
- let mut stream = FFI_ArrowArrayStream::empty();
-@@ -5234,6 +5411,59 @@ fn test_scanner_nearest_with_ivf_pq_index() {
- unsafe { lance_dataset_close(ds) };
- }
-
-+#[test]
-+fn test_scanner_query_parallelism_validation_and_lifecycle() {
-+ let (_tmp, uri) = create_test_dataset();
-+ let uri_c = c_str(&uri);
-+ let ds = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) };
-+ assert!(!ds.is_null());
-+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) };
-+ assert!(!scanner.is_null());
-+
-+ assert_eq!(
-+ unsafe { lance_scanner_set_query_parallelism(scanner, -1) },
-+ 0
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_query_parallelism(scanner, 0) },
-+ 0
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_query_parallelism(scanner, 2) },
-+ 0
-+ );
-+
-+ assert_eq!(
-+ unsafe { lance_scanner_set_query_parallelism(scanner, -2) },
-+ -1
-+ );
-+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument);
-+ assert!(
-+ take_last_error_message()
-+ .contains("query_parallelism must be -1, 0, or greater than 0, got -2")
-+ );
-+
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(
-+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) },
-+ 0
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_query_parallelism(scanner, 1) },
-+ -1
-+ );
-+ assert!(take_last_error_message().contains("query_parallelism must be set before"));
-+
-+ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap();
-+ assert_eq!(
-+ reader.map(|batch| batch.unwrap().num_rows()).sum::<usize>(),
-+ 5
-+ );
-+
-+ unsafe { lance_scanner_close(scanner) };
-+ unsafe { lance_dataset_close(ds) };
-+}
-+
- #[test]
- fn test_scanner_nearest_dim_mismatch() {
- let (_tmp, uri) = create_vector_dataset(64, 8);
-diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp
-index 17b1ab6..28762b8 100644
---- a/tests/cpp/test_cpp_api.cpp
-+++ b/tests/cpp/test_cpp_api.cpp
-@@ -134,6 +134,12 @@ static void test_scanner_fluent(const std::string& uri) {
- scanner.limit(5)
- .offset(0)
- .batch_size(2)
-+ .batch_size_bytes(1024)
-+ .io_buffer_size(64 * 1024)
-+ .batch_readahead(1)
-+ .fragment_readahead(1)
-+ .target_parallelism(1)
-+ .scan_in_order(false)
- .statistics_callback(capture_scan_statistics, &captured);
-
- ArrowArrayStream stream;
-@@ -370,6 +376,7 @@ static void test_nearest_smoke(const std::string& uri) {
- try {
- scanner.nearest("embedding", q, 8, 5)
- .nprobes(2)
-+ .query_parallelism(2)
- .refine_factor(1)
- .ef(50)
- .metric(LANCE_METRIC_L2)
-
-From 043a1f7eac253d8ac6be3f970b60fcbf615295aa Mon Sep 17 00:00:00 2001
-From: zhangstar333 <zh...@selectdb.com>
-Date: Sat, 5 Sep 2026 00:07:06 +0800
-Subject: [PATCH 2/2] add check
-
----
- include/lance/lance.h | 8 ++++----
- include/lance/lance.hpp | 2 +-
- src/scanner.rs | 11 ++++++++++-
- tests/c_api_test.rs | 17 ++++++++++++++++-
- 4 files changed, 31 insertions(+), 7 deletions(-)
-
-diff --git a/include/lance/lance.h b/include/lance/lance.h
-index 00f415b..541134f 100644
---- a/include/lance/lance.h
-+++ b/include/lance/lance.h
-@@ -949,12 +949,12 @@ int32_t lance_scanner_set_batch_size_bytes(
- /**
- * Set the scanner I/O buffer size in bytes.
- *
-- * The value must be greater than zero and must be set before scanning starts.
-- * This bounds buffered I/O received from storage, but is not a hard limit on
-- * all memory used by the scanner.
-+ * The value must be between 1 and INT64_MAX, inclusive, and must be set before
-+ * scanning starts. This bounds buffered I/O received from storage, but is not
-+ * a hard limit on all memory used by the scanner.
- *
- * @param scanner Scanner handle. Must not be NULL.
-- * @param io_buffer_size_bytes I/O buffer size in bytes. Must be greater than zero.
-+ * @param io_buffer_size_bytes I/O buffer size in bytes, in the range [1, INT64_MAX].
- * @return 0 on success, -1 on error.
- */
- int32_t lance_scanner_set_io_buffer_size(
-diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp
-index 3c03f86..a60dcd4 100644
---- a/include/lance/lance.hpp
-+++ b/include/lance/lance.hpp
-@@ -1203,7 +1203,7 @@ class Scanner {
- return *this;
- }
-
-- /// Set the scanner I/O buffer size in bytes.
-+ /// Set the scanner I/O buffer size in bytes, in the range [1, INT64_MAX].
- Scanner& io_buffer_size(uint64_t bytes) {
- if (lance_scanner_set_io_buffer_size(handle_.get(), bytes) != 0)
- check_error();
-diff --git a/src/scanner.rs b/src/scanner.rs
-index ebedafc..6414cc0 100644
---- a/src/scanner.rs
-+++ b/src/scanner.rs
-@@ -837,7 +837,7 @@ unsafe fn scanner_set_batch_size_bytes_inner(
-
- /// Set the scanner I/O buffer size in bytes. Returns 0 on success.
- ///
--/// The size must be greater than zero and must be set before the scan starts.
-+/// The size must be between 1 and [`i64::MAX`] and must be set before the scan starts.
- #[unsafe(no_mangle)]
- pub unsafe extern "C" fn lance_scanner_set_io_buffer_size(
- scanner: *mut LanceScanner,
-@@ -863,6 +863,15 @@ unsafe fn scanner_set_io_buffer_size_inner(
- "io_buffer_size_bytes must be greater than 0, got 0".into(),
- ));
- }
-+ if io_buffer_size_bytes > i64::MAX as u64 {
-+ return Err(lance_core::Error::invalid_input_source(
-+ format!(
-+ "io_buffer_size_bytes must be at most {}, got {io_buffer_size_bytes}",
-+ i64::MAX
-+ )
-+ .into(),
-+ ));
-+ }
- let scanner = unsafe { &mut *scanner };
- scanner.ensure_scan_not_started("io_buffer_size_bytes")?;
- scanner.io_buffer_size = Some(io_buffer_size_bytes);
-diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index 42f842d..ffdd915 100644
---- a/tests/c_api_test.rs
-+++ b/tests/c_api_test.rs
-@@ -1252,7 +1252,7 @@ fn test_scanner_execution_tuning_options() {
- }
-
- #[test]
--fn test_scanner_execution_tuning_options_reject_zero() {
-+fn test_scanner_execution_tuning_options_reject_invalid_values() {
- let (_tmp, uri) = create_test_dataset();
- let c_uri = c_str(&uri);
- let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) };
-@@ -1274,6 +1274,21 @@ fn test_scanner_execution_tuning_options_reject_zero() {
- take_last_error_message().contains("io_buffer_size_bytes must be greater than 0, got 0")
- );
-
-+ assert_eq!(
-+ unsafe { lance_scanner_set_io_buffer_size(scanner, i64::MAX as u64) },
-+ 0
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_io_buffer_size(scanner, u64::MAX) },
-+ -1
-+ );
-+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument);
-+ assert!(take_last_error_message().contains(&format!(
-+ "io_buffer_size_bytes must be at most {}, got {}",
-+ i64::MAX,
-+ u64::MAX
-+ )));
-+
- assert_eq!(unsafe { lance_scanner_set_batch_readahead(scanner, 0) }, -1);
- assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument);
- assert!(take_last_error_message().contains("batch_readahead must be greater than 0, got 0"));
-From 057135cdd4ac6ac7b5348a1832caf7081a8541fb Mon Sep 17 00:00:00 2001
-From: zhangstar333 <zh...@selectdb.com>
-Date: Mon, 7 Sep 2026 12:35:45 +0800
-Subject: [PATCH 1/2] feat: expose scanner use_scalar_index options
-
----
- include/lance/lance.h | 83 +++++++++
- include/lance/lance.hpp | 47 ++++++
- src/scanner.rs | 335 +++++++++++++++++++++++++++++++++++++
- tests/c_api_test.rs | 296 ++++++++++++++++++++++++++++++++
- tests/cpp/test_cpp_api.cpp | 8 +
- 5 files changed, 769 insertions(+)
-
-diff --git a/include/lance/lance.h b/include/lance/lance.h
-index 31213da..6ace2cb 100644
---- a/include/lance/lance.h
-+++ b/include/lance/lance.h
-@@ -147,6 +147,13 @@ typedef enum {
- LANCE_METRIC_HAMMING = 3,
- } LanceMetricType;
-
-+/** Speed / accuracy tradeoff for approximate vector search. */
-+typedef enum {
-+ LANCE_APPROX_MODE_FAST = 0,
-+ LANCE_APPROX_MODE_NORMAL = 1,
-+ LANCE_APPROX_MODE_ACCURATE = 2,
-+} LanceApproxMode;
-+
- typedef enum {
- LANCE_DTYPE_FLOAT32 = 0,
- LANCE_DTYPE_FLOAT16 = 1,
-@@ -1002,8 +1009,54 @@ int32_t lance_scanner_set_target_parallelism(
- * they are ready.
- */
- int32_t lance_scanner_set_scan_in_order(LanceScanner* scanner, bool scan_in_order);
-+
-+/**
-+ * Configure whether scalar indices may be used to optimize filters.
-+ *
-+ * Scalar indices are enabled by default. Disable this to force filter
-+ * evaluation without scalar indices. This setting is independent of
-+ * `lance_scanner_set_use_index`, which controls vector ANN index usage.
-+ * Must be set before scanning starts.
-+ */
-+int32_t lance_scanner_set_use_scalar_index(
-+ LanceScanner* scanner,
-+ bool use_scalar_index
-+);
-+
-+/**
-+ * Configure whether row-based output batches are strict.
-+ *
-+ * When enabled, every batch except the last has exactly the configured row
-+ * batch size. This may require copying and cannot be combined with a byte-based
-+ * batch-size limit. Must be set before scanning starts.
-+ */
-+int32_t lance_scanner_set_strict_batch_size(
-+ LanceScanner* scanner,
-+ bool strict_batch_size
-+);
-+
-+/**
-+ * Configure whether file statistics may optimize the scan (default: true).
-+ * Intended primarily for debugging and benchmarking. Must be set before
-+ * scanning starts.
-+ */
-+int32_t lance_scanner_set_use_stats(LanceScanner* scanner, bool use_stats);
-+
- int32_t lance_scanner_with_row_id(LanceScanner* scanner, bool enable);
-
-+/** Include or omit the `_rowaddr` metadata column. Must be set before scanning. */
-+int32_t lance_scanner_with_row_address(LanceScanner* scanner, bool enable);
-+
-+/**
-+ * Configure whether deleted rows still present in storage are returned.
-+ * Deleted rows have a NULL `_rowid`; callers should also enable row IDs.
-+ * Must be set before scanning starts.
-+ */
-+int32_t lance_scanner_set_include_deleted_rows(
-+ LanceScanner* scanner,
-+ bool include_deleted_rows
-+);
-+
- /**
- * Restrict scan to the given fragment IDs. Must be called before iteration.
- * @param ids Array of fragment IDs
-@@ -1725,6 +1778,36 @@ int32_t lance_scanner_nearest(
-
- int32_t lance_scanner_set_nprobes(LanceScanner* scanner, uint32_t n);
-
-+/**
-+ * Set the minimum number of vector-index partitions to search.
-+ * Must be greater than zero and no greater than `maximum_nprobes` when set.
-+ * Must be set before scanning starts.
-+ */
-+int32_t lance_scanner_set_minimum_nprobes(
-+ LanceScanner* scanner,
-+ uint32_t minimum_nprobes
-+);
-+
-+/**
-+ * Set the maximum number of vector-index partitions to search.
-+ * Must be greater than zero and no less than `minimum_nprobes` when set.
-+ * This only affects prefiltered searches that need more candidates.
-+ * Must be set before scanning starts.
-+ */
-+int32_t lance_scanner_set_maximum_nprobes(
-+ LanceScanner* scanner,
-+ uint32_t maximum_nprobes
-+);
-+
-+/**
-+ * Configure the speed / accuracy tradeoff for approximate vector search.
-+ * Must be set before scanning starts.
-+ */
-+int32_t lance_scanner_set_approx_mode(
-+ LanceScanner* scanner,
-+ LanceApproxMode approx_mode
-+);
-+
- /**
- * Set vector index partition-search concurrency for each query.
- *
-diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp
-index 404d2df..e08f76a 100644
---- a/include/lance/lance.hpp
-+++ b/include/lance/lance.hpp
-@@ -1269,6 +1269,27 @@ class Scanner {
- return *this;
- }
-
-+ /// Configure whether scalar indices may be used to optimize filters.
-+ Scanner& use_scalar_index(bool enable = true) {
-+ if (lance_scanner_set_use_scalar_index(handle_.get(), enable) != 0)
-+ check_error();
-+ return *this;
-+ }
-+
-+ /// Configure whether row-based output batches are strict.
-+ Scanner& strict_batch_size(bool strict_batch_size = true) {
-+ if (lance_scanner_set_strict_batch_size(handle_.get(), strict_batch_size) != 0)
-+ check_error();
-+ return *this;
-+ }
-+
-+ /// Configure whether file statistics may optimize the scan.
-+ Scanner& use_stats(bool use_stats = true) {
-+ if (lance_scanner_set_use_stats(handle_.get(), use_stats) != 0)
-+ check_error();
-+ return *this;
-+ }
-+
- /// Enable/disable row ID in output.
- Scanner& with_row_id(bool enable = true) {
- if (lance_scanner_with_row_id(handle_.get(), enable) != 0)
-@@ -1276,6 +1297,20 @@ class Scanner {
- return *this;
- }
-
-+ /// Include or omit the `_rowaddr` metadata column.
-+ Scanner& with_row_address(bool enable = true) {
-+ if (lance_scanner_with_row_address(handle_.get(), enable) != 0)
-+ check_error();
-+ return *this;
-+ }
-+
-+ /// Configure whether deleted rows still present in storage are returned.
-+ Scanner& include_deleted_rows(bool include_deleted_rows = true) {
-+ if (lance_scanner_set_include_deleted_rows(handle_.get(), include_deleted_rows) != 0)
-+ check_error();
-+ return *this;
-+ }
-+
- /// Restrict scan to specific fragment IDs.
- Scanner& fragment_ids(const uint64_t* ids, size_t len) {
- if (lance_scanner_set_fragment_ids(handle_.get(), ids, len) != 0)
-@@ -1386,6 +1421,18 @@ class Scanner {
- if (lance_scanner_set_nprobes(handle_.get(), n) != 0) check_error();
- return *this;
- }
-+ Scanner& minimum_nprobes(uint32_t minimum_nprobes) {
-+ if (lance_scanner_set_minimum_nprobes(handle_.get(), minimum_nprobes) != 0) check_error();
-+ return *this;
-+ }
-+ Scanner& maximum_nprobes(uint32_t maximum_nprobes) {
-+ if (lance_scanner_set_maximum_nprobes(handle_.get(), maximum_nprobes) != 0) check_error();
-+ return *this;
-+ }
-+ Scanner& approx_mode(LanceApproxMode approx_mode) {
-+ if (lance_scanner_set_approx_mode(handle_.get(), approx_mode) != 0) check_error();
-+ return *this;
-+ }
- Scanner& query_parallelism(int32_t parallelism) {
- if (lance_scanner_set_query_parallelism(handle_.get(), parallelism) != 0)
- check_error();
-diff --git a/src/scanner.rs b/src/scanner.rs
-index d3ef3be..53cd5b6 100644
---- a/src/scanner.rs
-+++ b/src/scanner.rs
-@@ -21,6 +21,7 @@ use lance::dataset::scanner::{
- use lance::io::exec::fts::{FlatMatchQueryExec, MatchQueryExec, PhraseQueryExec};
- use lance_core::Result;
- use lance_index::scalar::FullTextSearchQuery;
-+use lance_index::vector::ApproxMode;
- use lance_io::stream::RecordBatchStream;
- use lance_table::format::IndexMetadata;
- use uuid::Uuid;
-@@ -51,6 +52,38 @@ pub enum LanceDataType {
- Int8 = 4,
- }
-
-+/// Speed / accuracy tradeoff for approximate vector search, mirroring the C
-+/// enum `LanceApproxMode`.
-+#[repr(i32)]
-+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-+pub enum LanceApproxMode {
-+ Fast = 0,
-+ Normal = 1,
-+ Accurate = 2,
-+}
-+
-+impl LanceApproxMode {
-+ fn from_i32(value: i32) -> Result<Self> {
-+ match value {
-+ 0 => Ok(Self::Fast),
-+ 1 => Ok(Self::Normal),
-+ 2 => Ok(Self::Accurate),
-+ _ => Err(lance_core::Error::invalid_input_source(
-+ format!("approx_mode must be 0 (FAST), 1 (NORMAL), or 2 (ACCURATE), got {value}")
-+ .into(),
-+ )),
-+ }
-+ }
-+
-+ fn to_approx_mode(self) -> ApproxMode {
-+ match self {
-+ Self::Fast => ApproxMode::Fast,
-+ Self::Normal => ApproxMode::Normal,
-+ Self::Accurate => ApproxMode::Accurate,
-+ }
-+ }
-+}
-+
- /// Opaque scanner handle. Stores configuration until stream materialization.
- pub struct LanceScanner {
- dataset: Arc<Dataset>,
-@@ -62,16 +95,24 @@ pub struct LanceScanner {
- offset: Option<i64>,
- batch_size: Option<usize>,
- batch_size_bytes: Option<u64>,
-+ strict_batch_size: Option<bool>,
- io_buffer_size: Option<u64>,
- batch_readahead: Option<usize>,
- fragment_readahead: Option<usize>,
- target_parallelism: Option<usize>,
- scan_in_order: Option<bool>,
-+ use_scalar_index: Option<bool>,
-+ use_stats: Option<bool>,
- with_row_id: bool,
-+ with_row_address: bool,
-+ include_deleted_rows: bool,
- fragment_ids: Option<Vec<u64>>,
- index_segments: Option<Vec<Uuid>>,
- nearest: Option<NearestQuery>,
- nprobes: Option<u32>,
-+ minimum_nprobes: Option<u32>,
-+ maximum_nprobes: Option<u32>,
-+ approx_mode: Option<LanceApproxMode>,
- query_parallelism: Option<i32>,
- refine_factor: Option<u32>,
- ef: Option<u32>,
-@@ -138,16 +179,24 @@ impl LanceScanner {
- offset: None,
- batch_size: None,
- batch_size_bytes: None,
-+ strict_batch_size: None,
- io_buffer_size: None,
- batch_readahead: None,
- fragment_readahead: None,
- target_parallelism: None,
- scan_in_order: None,
-+ use_scalar_index: None,
-+ use_stats: None,
- with_row_id: false,
-+ with_row_address: false,
-+ include_deleted_rows: false,
- fragment_ids: None,
- index_segments: None,
- nearest: None,
- nprobes: None,
-+ minimum_nprobes: None,
-+ maximum_nprobes: None,
-+ approx_mode: None,
- query_parallelism: None,
- refine_factor: None,
- ef: None,
-@@ -258,6 +307,9 @@ impl LanceScanner {
- if let Some(batch_size_bytes) = self.batch_size_bytes {
- scanner.batch_size_bytes(batch_size_bytes);
- }
-+ if let Some(strict_batch_size) = self.strict_batch_size {
-+ scanner.strict_batch_size(strict_batch_size);
-+ }
- if let Some(io_buffer_size) = self.io_buffer_size {
- scanner.io_buffer_size(io_buffer_size);
- }
-@@ -273,9 +325,21 @@ impl LanceScanner {
- if let Some(scan_in_order) = self.scan_in_order {
- scanner.scan_in_order(scan_in_order);
- }
-+ if let Some(use_scalar_index) = self.use_scalar_index {
-+ scanner.use_scalar_index(use_scalar_index);
-+ }
-+ if let Some(use_stats) = self.use_stats {
-+ scanner.use_stats(use_stats);
-+ }
- if self.with_row_id {
- scanner.with_row_id();
- }
-+ if self.with_row_address {
-+ scanner.with_row_address();
-+ }
-+ if self.include_deleted_rows {
-+ scanner.include_deleted_rows();
-+ }
- self.apply_fragment_filter(&mut scanner)?;
- if self.index_segments.is_some() && self.nearest.is_none() {
- return Err(lance_core::Error::invalid_input_source(
-@@ -302,6 +366,15 @@ impl LanceScanner {
- if let Some(np) = self.nprobes {
- scanner.nprobes(np as usize);
- }
-+ if let Some(minimum_nprobes) = self.minimum_nprobes {
-+ scanner.minimum_nprobes(minimum_nprobes as usize);
-+ }
-+ if let Some(maximum_nprobes) = self.maximum_nprobes {
-+ scanner.maximum_nprobes(maximum_nprobes as usize);
-+ }
-+ if let Some(approx_mode) = self.approx_mode {
-+ scanner.approx_mode(approx_mode.to_approx_mode());
-+ }
- if let Some(query_parallelism) = self.query_parallelism {
- scanner.query_parallelism(query_parallelism);
- }
-@@ -1035,6 +1108,92 @@ unsafe fn scanner_set_scan_in_order_inner(
- Ok(0)
- }
-
-+/// Configure whether scalar indices may be used to optimize filters.
-+///
-+/// Scalar indices are enabled by default in Lance. Must be set before the scan
-+/// starts.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_use_scalar_index(
-+ scanner: *mut LanceScanner,
-+ use_scalar_index: bool,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_use_scalar_index_inner(scanner, use_scalar_index)
-+ })
-+}
-+
-+unsafe fn scanner_set_use_scalar_index_inner(
-+ scanner: *mut LanceScanner,
-+ use_scalar_index: bool,
-+) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("use_scalar_index")?;
-+ scanner.use_scalar_index = Some(use_scalar_index);
-+ Ok(0)
-+}
-+
-+/// Configure whether output batches use the exact row-based batch size.
-+///
-+/// Must be set before the scan starts. Lance rejects enabling this together
-+/// with a byte-based batch-size limit when the scan is materialized.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_strict_batch_size(
-+ scanner: *mut LanceScanner,
-+ strict_batch_size: bool,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_strict_batch_size_inner(scanner, strict_batch_size)
-+ })
-+}
-+
-+unsafe fn scanner_set_strict_batch_size_inner(
-+ scanner: *mut LanceScanner,
-+ strict_batch_size: bool,
-+) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("strict_batch_size")?;
-+ scanner.strict_batch_size = Some(strict_batch_size);
-+ Ok(0)
-+}
-+
-+/// Configure whether file statistics may be used to optimize the scan.
-+///
-+/// Statistics are enabled by default. Must be set before the scan starts.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_use_stats(
-+ scanner: *mut LanceScanner,
-+ use_stats: bool,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_use_stats_inner(scanner, use_stats)
-+ })
-+}
-+
-+unsafe fn scanner_set_use_stats_inner(scanner: *mut LanceScanner, use_stats: bool) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("use_stats")?;
-+ scanner.use_stats = Some(use_stats);
-+ Ok(0)
-+}
-+
- /// Enable or disable row ID in scan output. Returns 0.
- #[unsafe(no_mangle)]
- pub unsafe extern "C" fn lance_scanner_with_row_id(
-@@ -1058,6 +1217,62 @@ unsafe fn scanner_with_row_id_inner(scanner: *mut LanceScanner, enable: bool) ->
- Ok(0)
- }
-
-+/// Enable or disable the `_rowaddr` metadata column in scan output.
-+///
-+/// Must be set before the scan starts.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_with_row_address(
-+ scanner: *mut LanceScanner,
-+ enable: bool,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_with_row_address_inner(scanner, enable)
-+ })
-+}
-+
-+unsafe fn scanner_with_row_address_inner(scanner: *mut LanceScanner, enable: bool) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("with_row_address")?;
-+ scanner.with_row_address = enable;
-+ Ok(0)
-+}
-+
-+/// Configure whether deleted rows still present in storage are returned.
-+///
-+/// Deleted rows have a NULL `_rowid`, so callers should also enable row IDs.
-+/// Must be set before the scan starts.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_include_deleted_rows(
-+ scanner: *mut LanceScanner,
-+ include_deleted_rows: bool,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_include_deleted_rows_inner(scanner, include_deleted_rows)
-+ })
-+}
-+
-+unsafe fn scanner_set_include_deleted_rows_inner(
-+ scanner: *mut LanceScanner,
-+ include_deleted_rows: bool,
-+) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("include_deleted_rows")?;
-+ scanner.include_deleted_rows = include_deleted_rows;
-+ Ok(0)
-+}
-+
- /// Restrict the scan to the given fragment IDs.
- /// Must be called before any iteration method.
- ///
-@@ -2044,6 +2259,126 @@ scanner_set_u32!(lance_scanner_set_nprobes, nprobes);
- scanner_set_u32!(lance_scanner_set_refine_factor, refine_factor);
- scanner_set_u32!(lance_scanner_set_ef, ef);
-
-+/// Set the minimum number of vector-index partitions to search.
-+///
-+/// The value must be greater than zero, no greater than a configured
-+/// `maximum_nprobes`, and must be set before the scan starts.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_minimum_nprobes(
-+ scanner: *mut LanceScanner,
-+ minimum_nprobes: u32,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_minimum_nprobes_inner(scanner, minimum_nprobes)
-+ })
-+}
-+
-+unsafe fn scanner_set_minimum_nprobes_inner(
-+ scanner: *mut LanceScanner,
-+ minimum_nprobes: u32,
-+) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ if minimum_nprobes == 0 {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "minimum_nprobes must be greater than 0, got 0".into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("minimum_nprobes")?;
-+ if let Some(maximum_nprobes) = scanner.maximum_nprobes
-+ && minimum_nprobes > maximum_nprobes
-+ {
-+ return Err(lance_core::Error::invalid_input_source(
-+ format!(
-+ "minimum_nprobes ({minimum_nprobes}) must not exceed maximum_nprobes ({maximum_nprobes})"
-+ )
-+ .into(),
-+ ));
-+ }
-+ scanner.minimum_nprobes = Some(minimum_nprobes);
-+ Ok(0)
-+}
-+
-+/// Set the maximum number of vector-index partitions to search.
-+///
-+/// The value must be greater than zero, no less than a configured
-+/// `minimum_nprobes`, and must be set before the scan starts.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_maximum_nprobes(
-+ scanner: *mut LanceScanner,
-+ maximum_nprobes: u32,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_maximum_nprobes_inner(scanner, maximum_nprobes)
-+ })
-+}
-+
-+unsafe fn scanner_set_maximum_nprobes_inner(
-+ scanner: *mut LanceScanner,
-+ maximum_nprobes: u32,
-+) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ if maximum_nprobes == 0 {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "maximum_nprobes must be greater than 0, got 0".into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("maximum_nprobes")?;
-+ if let Some(minimum_nprobes) = scanner.minimum_nprobes
-+ && maximum_nprobes < minimum_nprobes
-+ {
-+ return Err(lance_core::Error::invalid_input_source(
-+ format!(
-+ "maximum_nprobes ({maximum_nprobes}) must not be less than minimum_nprobes ({minimum_nprobes})"
-+ )
-+ .into(),
-+ ));
-+ }
-+ scanner.maximum_nprobes = Some(maximum_nprobes);
-+ Ok(0)
-+}
-+
-+/// Configure the speed / accuracy tradeoff for approximate vector search.
-+///
-+/// Must be set before the scan starts.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_approx_mode(
-+ scanner: *mut LanceScanner,
-+ approx_mode: i32,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_approx_mode_inner(scanner, approx_mode)
-+ })
-+}
-+
-+unsafe fn scanner_set_approx_mode_inner(
-+ scanner: *mut LanceScanner,
-+ approx_mode: i32,
-+) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ let approx_mode = LanceApproxMode::from_i32(approx_mode)?;
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("approx_mode")?;
-+ scanner.approx_mode = Some(approx_mode);
-+ Ok(0)
-+}
-+
- /// Set vector index partition-search concurrency for each query.
- ///
- /// `-1` uses the CPU pool size, `0` selects Lance's automatic policy, and
-diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index bde742d..a550a93 100644
---- a/tests/c_api_test.rs
-+++ b/tests/c_api_test.rs
-@@ -1237,6 +1237,12 @@ fn test_scanner_execution_tuning_options() {
- unsafe { lance_scanner_set_scan_in_order(scanner, false) },
- 0
- );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_use_scalar_index(scanner, false) },
-+ 0
-+ );
-+ assert_eq!(unsafe { lance_scanner_set_use_stats(scanner, false) }, 0);
-+ assert_eq!(unsafe { lance_scanner_with_row_address(scanner, true) }, 0);
-
- let mut ffi_stream = FFI_ArrowArrayStream::empty();
- assert_eq!(
-@@ -1244,6 +1250,7 @@ fn test_scanner_execution_tuning_options() {
- 0
- );
- let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut ffi_stream) }.unwrap();
-+ assert!(reader.schema().field_with_name("_rowaddr").is_ok());
- let total_rows: usize = reader.map(|batch| batch.unwrap().num_rows()).sum();
- assert_eq!(total_rows, 10);
-
-@@ -1360,6 +1367,30 @@ fn test_scanner_execution_tuning_options_reject_after_scan_start() {
- );
- assert!(take_last_error_message().contains("scan_in_order must be set before"));
-
-+ assert_eq!(
-+ unsafe { lance_scanner_set_use_scalar_index(scanner, false) },
-+ -1
-+ );
-+ assert!(take_last_error_message().contains("use_scalar_index must be set before"));
-+
-+ assert_eq!(
-+ unsafe { lance_scanner_set_strict_batch_size(scanner, true) },
-+ -1
-+ );
-+ assert!(take_last_error_message().contains("strict_batch_size must be set before"));
-+
-+ assert_eq!(unsafe { lance_scanner_set_use_stats(scanner, false) }, -1);
-+ assert!(take_last_error_message().contains("use_stats must be set before"));
-+
-+ assert_eq!(unsafe { lance_scanner_with_row_address(scanner, true) }, -1);
-+ assert!(take_last_error_message().contains("with_row_address must be set before"));
-+
-+ assert_eq!(
-+ unsafe { lance_scanner_set_include_deleted_rows(scanner, true) },
-+ -1
-+ );
-+ assert!(take_last_error_message().contains("include_deleted_rows must be set before"));
-+
- let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut ffi_stream) }.unwrap();
- assert_eq!(
- reader.map(|batch| batch.unwrap().num_rows()).sum::<usize>(),
-@@ -1370,6 +1401,79 @@ fn test_scanner_execution_tuning_options_reject_after_scan_start() {
- unsafe { lance_dataset_close(ds) };
- }
-
-+#[test]
-+fn test_scanner_strict_batch_size_across_fragments() {
-+ let (_tmp, uri) = create_multi_fragment_dataset();
-+ let c_uri = c_str(&uri);
-+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) };
-+ assert!(!ds.is_null());
-+
-+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) };
-+ assert!(!scanner.is_null());
-+ assert_eq!(unsafe { lance_scanner_set_batch_size(scanner, 3) }, 0);
-+ assert_eq!(
-+ unsafe { lance_scanner_set_strict_batch_size(scanner, true) },
-+ 0
-+ );
-+
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(
-+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) },
-+ 0
-+ );
-+ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap();
-+ let batch_sizes = reader
-+ .map(|batch| batch.unwrap().num_rows())
-+ .collect::<Vec<_>>();
-+ assert_eq!(batch_sizes, vec![3, 3, 3, 1]);
-+
-+ unsafe { lance_scanner_close(scanner) };
-+ unsafe { lance_dataset_close(ds) };
-+}
-+
-+#[test]
-+fn test_scanner_include_deleted_rows() {
-+ let (_tmp, uri) = create_multi_fragment_dataset();
-+ let c_uri = c_str(&uri);
-+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) };
-+ assert!(!ds.is_null());
-+
-+ let predicate = c_str("id >= 8");
-+ let mut num_deleted = 0;
-+ assert_eq!(
-+ unsafe { lance_dataset_delete(ds, predicate.as_ptr(), &mut num_deleted) },
-+ 0
-+ );
-+ assert_eq!(num_deleted, 2);
-+
-+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) };
-+ assert!(!scanner.is_null());
-+ assert_eq!(unsafe { lance_scanner_with_row_id(scanner, true) }, 0);
-+ assert_eq!(
-+ unsafe { lance_scanner_set_include_deleted_rows(scanner, true) },
-+ 0
-+ );
-+
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(
-+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) },
-+ 0
-+ );
-+ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap();
-+ let batches = reader.map(|batch| batch.unwrap()).collect::<Vec<_>>();
-+ assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 10);
-+ assert_eq!(
-+ batches
-+ .iter()
-+ .map(|batch| batch.column_by_name("_rowid").unwrap().null_count())
-+ .sum::<usize>(),
-+ 2
-+ );
-+
-+ unsafe { lance_scanner_close(scanner) };
-+ unsafe { lance_dataset_close(ds) };
-+}
-+
- // ---------------------------------------------------------------------------
- // Combined filter + projection + limit
- // ---------------------------------------------------------------------------
-@@ -1632,10 +1736,42 @@ fn test_null_safety_comprehensive() {
- unsafe { lance_scanner_set_scan_in_order(ptr::null_mut(), true) },
- -1
- );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_use_scalar_index(ptr::null_mut(), false) },
-+ -1
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_strict_batch_size(ptr::null_mut(), true) },
-+ -1
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_use_stats(ptr::null_mut(), false) },
-+ -1
-+ );
- assert_eq!(
- unsafe { lance_scanner_with_row_id(ptr::null_mut(), true) },
- -1
- );
-+ assert_eq!(
-+ unsafe { lance_scanner_with_row_address(ptr::null_mut(), true) },
-+ -1
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_include_deleted_rows(ptr::null_mut(), true) },
-+ -1
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_minimum_nprobes(ptr::null_mut(), 1) },
-+ -1
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_maximum_nprobes(ptr::null_mut(), 1) },
-+ -1
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_approx_mode(ptr::null_mut(), LanceApproxMode::Normal as i32,) },
-+ -1
-+ );
-
- // Scanner iteration with NULL.
- let mut ffi_stream2 = FFI_ArrowArrayStream::empty();
-@@ -3077,6 +3213,90 @@ fn test_create_scalar_index_btree() {
- unsafe { lance_dataset_close(ds) };
- }
-
-+#[test]
-+fn test_scanner_set_use_scalar_index_controls_filter_planning() {
-+ let (_tmp, uri) = create_test_dataset();
-+ let uri_c = c_str(&uri);
-+ let ds = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) };
-+ assert!(!ds.is_null());
-+
-+ let column = c_str("id");
-+ assert_eq!(
-+ unsafe {
-+ lance_dataset_create_scalar_index(
-+ ds,
-+ column.as_ptr(),
-+ ptr::null(),
-+ LanceScalarIndexType::BTree as i32,
-+ ptr::null(),
-+ false,
-+ )
-+ },
-+ 0
-+ );
-+
-+ let run_scan = |use_scalar_index: bool| {
-+ let filter = c_str("id = 3");
-+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), filter.as_ptr()) };
-+ assert!(!scanner.is_null());
-+ assert_eq!(
-+ unsafe { lance_scanner_set_use_scalar_index(scanner, use_scalar_index) },
-+ 0
-+ );
-+
-+ let mut captured = CapturedScanStatistics::default();
-+ assert_eq!(
-+ unsafe {
-+ lance_scanner_set_statistics_callback(
-+ scanner,
-+ Some(capture_scan_statistics),
-+ (&mut captured as *mut CapturedScanStatistics).cast(),
-+ )
-+ },
-+ 0
-+ );
-+
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(
-+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) },
-+ 0
-+ );
-+ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap();
-+ let ids = reader
-+ .flat_map(|batch| {
-+ let batch = batch.unwrap();
-+ batch
-+ .column_by_name("id")
-+ .unwrap()
-+ .as_any()
-+ .downcast_ref::<Int32Array>()
-+ .unwrap()
-+ .values()
-+ .to_vec()
-+ })
-+ .collect::<Vec<_>>();
-+
-+ assert_eq!(captured.calls, 1);
-+ unsafe { lance_scanner_close(scanner) };
-+ (ids, captured)
-+ };
-+
-+ let (indexed_ids, indexed_statistics) = run_scan(true);
-+ let (unindexed_ids, unindexed_statistics) = run_scan(false);
-+ assert_eq!(indexed_ids, vec![3]);
-+ assert_eq!(unindexed_ids, indexed_ids);
-+ assert!(
-+ indexed_statistics.indices_loaded > 0,
-+ "enabled scan should load the scalar index"
-+ );
-+ assert_eq!(
-+ unindexed_statistics.indices_loaded, 0,
-+ "disabled scan should bypass the scalar index"
-+ );
-+
-+ unsafe { lance_dataset_close(ds) };
-+}
-+
- #[test]
- fn test_scalar_index_segment_build_is_fragment_scoped_and_uncommitted() {
- let (_tmp, uri) = create_many_small_fragments(2);
-@@ -5409,6 +5629,12 @@ fn test_scanner_nearest_with_ivf_pq_index() {
- 10,
- );
- lance_scanner_set_nprobes(scanner, 4);
-+ assert_eq!(lance_scanner_set_minimum_nprobes(scanner, 2), 0);
-+ assert_eq!(lance_scanner_set_maximum_nprobes(scanner, 6), 0);
-+ assert_eq!(
-+ lance_scanner_set_approx_mode(scanner, LanceApproxMode::Accurate as i32),
-+ 0
-+ );
- assert_eq!(lance_scanner_set_query_parallelism(scanner, 4), 0);
- }
-
-@@ -5428,6 +5654,76 @@ fn test_scanner_nearest_with_ivf_pq_index() {
- unsafe { lance_dataset_close(ds) };
- }
-
-+#[test]
-+fn test_scanner_adaptive_nprobes_and_approx_mode_validation_and_lifecycle() {
-+ let (_tmp, uri) = create_test_dataset();
-+ let uri_c = c_str(&uri);
-+ let ds = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) };
-+ assert!(!ds.is_null());
-+ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) };
-+ assert!(!scanner.is_null());
-+
-+ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 0) }, -1);
-+ assert!(take_last_error_message().contains("minimum_nprobes must be greater than 0, got 0"));
-+ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 0) }, -1);
-+ assert!(take_last_error_message().contains("maximum_nprobes must be greater than 0, got 0"));
-+ assert_eq!(unsafe { lance_scanner_set_approx_mode(scanner, 3) }, -1);
-+ assert!(
-+ take_last_error_message()
-+ .contains("approx_mode must be 0 (FAST), 1 (NORMAL), or 2 (ACCURATE), got 3")
-+ );
-+
-+ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 2) }, 0);
-+ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 3) }, -1);
-+ assert!(
-+ take_last_error_message()
-+ .contains("minimum_nprobes (3) must not exceed maximum_nprobes (2)")
-+ );
-+ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 1) }, 0);
-+ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 1) }, 0);
-+ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 2) }, -1);
-+ assert!(
-+ take_last_error_message()
-+ .contains("minimum_nprobes (2) must not exceed maximum_nprobes (1)")
-+ );
-+ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 2) }, 0);
-+ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 2) }, 0);
-+ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 1) }, -1);
-+ assert!(
-+ take_last_error_message()
-+ .contains("maximum_nprobes (1) must not be less than minimum_nprobes (2)")
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_approx_mode(scanner, LanceApproxMode::Fast as i32) },
-+ 0
-+ );
-+
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(
-+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) },
-+ 0
-+ );
-+
-+ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 1) }, -1);
-+ assert!(take_last_error_message().contains("minimum_nprobes must be set before"));
-+ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 1) }, -1);
-+ assert!(take_last_error_message().contains("maximum_nprobes must be set before"));
-+ assert_eq!(
-+ unsafe { lance_scanner_set_approx_mode(scanner, LanceApproxMode::Normal as i32) },
-+ -1
-+ );
-+ assert!(take_last_error_message().contains("approx_mode must be set before"));
-+
-+ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap();
-+ assert_eq!(
-+ reader.map(|batch| batch.unwrap().num_rows()).sum::<usize>(),
-+ 5
-+ );
-+
-+ unsafe { lance_scanner_close(scanner) };
-+ unsafe { lance_dataset_close(ds) };
-+}
-+
- #[test]
- fn test_scanner_query_parallelism_validation_and_lifecycle() {
- let (_tmp, uri) = create_test_dataset();
-diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp
-index 28762b8..60b8fc4 100644
---- a/tests/cpp/test_cpp_api.cpp
-+++ b/tests/cpp/test_cpp_api.cpp
-@@ -140,6 +140,11 @@ static void test_scanner_fluent(const std::string& uri) {
- .fragment_readahead(1)
- .target_parallelism(1)
- .scan_in_order(false)
-+ .use_scalar_index(false)
-+ .strict_batch_size(false)
-+ .use_stats(false)
-+ .with_row_address(true)
-+ .include_deleted_rows(false)
- .statistics_callback(capture_scan_statistics, &captured);
-
- ArrowArrayStream stream;
-@@ -376,6 +381,9 @@ static void test_nearest_smoke(const std::string& uri) {
- try {
- scanner.nearest("embedding", q, 8, 5)
- .nprobes(2)
-+ .minimum_nprobes(1)
-+ .maximum_nprobes(2)
-+ .approx_mode(LANCE_APPROX_MODE_NORMAL)
- .query_parallelism(2)
- .refine_factor(1)
- .ef(50)
-
-From e894f591aef358cd36fdbf915c4d5b95fd0e8348 Mon Sep 17 00:00:00 2001
-From: zhangstar333 <zh...@selectdb.com>
-Date: Mon, 7 Sep 2026 14:27:10 +0800
-Subject: [PATCH 2/2] update
-
----
- include/lance/lance.h | 18 +++-
- include/lance/lance.hpp | 7 +-
- src/scanner.rs | 210 +++++++++++++++++++++++++++++++---------
- tests/c_api_test.rs | 77 +++++++++++++++
- 4 files changed, 261 insertions(+), 51 deletions(-)
-
-diff --git a/include/lance/lance.h b/include/lance/lance.h
-index 6ace2cb..8173ae5 100644
---- a/include/lance/lance.h
-+++ b/include/lance/lance.h
-@@ -946,7 +946,8 @@ int32_t lance_scanner_set_batch_size(LanceScanner* scanner, int64_t batch_size);
- * Set the target output batch size in bytes.
- *
- * When set, this takes precedence over the row-based batch size. The value
-- * must be greater than zero and must be set before scanning starts.
-+ * must be greater than zero and must be set before scanning starts. The call
-+ * is rejected without changing scanner state if strict batch sizing is enabled.
- */
- int32_t lance_scanner_set_batch_size_bytes(
- LanceScanner* scanner,
-@@ -1028,7 +1029,8 @@ int32_t lance_scanner_set_use_scalar_index(
- *
- * When enabled, every batch except the last has exactly the configured row
- * batch size. This may require copying and cannot be combined with a byte-based
-- * batch-size limit. Must be set before scanning starts.
-+ * batch-size limit. The call is rejected without changing scanner state if a
-+ * byte limit is already set. Must be set before scanning starts.
- */
- int32_t lance_scanner_set_strict_batch_size(
- LanceScanner* scanner,
-@@ -1776,11 +1778,19 @@ int32_t lance_scanner_nearest(
- uint32_t k
- );
-
--int32_t lance_scanner_set_nprobes(LanceScanner* scanner, uint32_t n);
-+/**
-+ * Set both the minimum and maximum vector-index partition-search bounds.
-+ *
-+ * This replaces both bounds configured by earlier calls to any nprobes
-+ * setter. The value must be greater than zero. Must be set before scanning.
-+ */
-+int32_t lance_scanner_set_nprobes(LanceScanner* scanner, uint32_t nprobes);
-
- /**
- * Set the minimum number of vector-index partitions to search.
-+ * This replaces only the minimum bound; the current maximum is preserved.
- * Must be greater than zero and no greater than `maximum_nprobes` when set.
-+ * An invalid resulting range is rejected without changing either bound.
- * Must be set before scanning starts.
- */
- int32_t lance_scanner_set_minimum_nprobes(
-@@ -1790,8 +1800,10 @@ int32_t lance_scanner_set_minimum_nprobes(
-
- /**
- * Set the maximum number of vector-index partitions to search.
-+ * This replaces only the maximum bound; the current minimum is preserved.
- * Must be greater than zero and no less than `minimum_nprobes` when set.
- * This only affects prefiltered searches that need more candidates.
-+ * An invalid resulting range is rejected without changing either bound.
- * Must be set before scanning starts.
- */
- int32_t lance_scanner_set_maximum_nprobes(
-diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp
-index e08f76a..c12c0c6 100644
---- a/include/lance/lance.hpp
-+++ b/include/lance/lance.hpp
-@@ -1417,14 +1417,17 @@ class Scanner {
- return *this;
- }
-
-- Scanner& nprobes(uint32_t n) {
-- if (lance_scanner_set_nprobes(handle_.get(), n) != 0) check_error();
-+ /// Replace both minimum and maximum partition-search bounds.
-+ Scanner& nprobes(uint32_t nprobes) {
-+ if (lance_scanner_set_nprobes(handle_.get(), nprobes) != 0) check_error();
- return *this;
- }
-+ /// Replace only the minimum partition-search bound.
- Scanner& minimum_nprobes(uint32_t minimum_nprobes) {
- if (lance_scanner_set_minimum_nprobes(handle_.get(), minimum_nprobes) != 0) check_error();
- return *this;
- }
-+ /// Replace only the maximum partition-search bound.
- Scanner& maximum_nprobes(uint32_t maximum_nprobes) {
- if (lance_scanner_set_maximum_nprobes(handle_.get(), maximum_nprobes) != 0) check_error();
- return *this;
-diff --git a/src/scanner.rs b/src/scanner.rs
-index 53cd5b6..4ceeb0e 100644
---- a/src/scanner.rs
-+++ b/src/scanner.rs
-@@ -109,9 +109,7 @@ pub struct LanceScanner {
- fragment_ids: Option<Vec<u64>>,
- index_segments: Option<Vec<Uuid>>,
- nearest: Option<NearestQuery>,
-- nprobes: Option<u32>,
-- minimum_nprobes: Option<u32>,
-- maximum_nprobes: Option<u32>,
-+ nprobes: NprobesRange,
- approx_mode: Option<LanceApproxMode>,
- query_parallelism: Option<i32>,
- refine_factor: Option<u32>,
-@@ -148,6 +146,72 @@ struct NearestQuery {
- k: u32,
- }
-
-+/// The effective adaptive partition-search range shared by all three nprobes
-+/// setters. Updates are computed and validated before replacing this state.
-+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
-+struct NprobesRange {
-+ minimum: Option<u32>,
-+ maximum: Option<u32>,
-+}
-+
-+impl NprobesRange {
-+ fn exact(nprobes: u32) -> Result<Self> {
-+ if nprobes == 0 {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "nprobes must be greater than 0, got 0".into(),
-+ ));
-+ }
-+ Ok(Self {
-+ minimum: Some(nprobes),
-+ maximum: Some(nprobes),
-+ })
-+ }
-+
-+ fn with_minimum(self, minimum_nprobes: u32) -> Result<Self> {
-+ if minimum_nprobes == 0 {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "minimum_nprobes must be greater than 0, got 0".into(),
-+ ));
-+ }
-+ if let Some(maximum_nprobes) = self.maximum
-+ && minimum_nprobes > maximum_nprobes
-+ {
-+ return Err(lance_core::Error::invalid_input_source(
-+ format!(
-+ "minimum_nprobes ({minimum_nprobes}) must not exceed maximum_nprobes ({maximum_nprobes})"
-+ )
-+ .into(),
-+ ));
-+ }
-+ Ok(Self {
-+ minimum: Some(minimum_nprobes),
-+ ..self
-+ })
-+ }
-+
-+ fn with_maximum(self, maximum_nprobes: u32) -> Result<Self> {
-+ if maximum_nprobes == 0 {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "maximum_nprobes must be greater than 0, got 0".into(),
-+ ));
-+ }
-+ if let Some(minimum_nprobes) = self.minimum
-+ && maximum_nprobes < minimum_nprobes
-+ {
-+ return Err(lance_core::Error::invalid_input_source(
-+ format!(
-+ "maximum_nprobes ({maximum_nprobes}) must not be less than minimum_nprobes ({minimum_nprobes})"
-+ )
-+ .into(),
-+ ));
-+ }
-+ Ok(Self {
-+ maximum: Some(maximum_nprobes),
-+ ..self
-+ })
-+ }
-+}
-+
- /// Poll status for `lance_scanner_poll_next`.
- #[repr(C)]
- #[derive(Debug, PartialEq, Eq)]
-@@ -193,9 +257,7 @@ impl LanceScanner {
- fragment_ids: None,
- index_segments: None,
- nearest: None,
-- nprobes: None,
-- minimum_nprobes: None,
-- maximum_nprobes: None,
-+ nprobes: NprobesRange::default(),
- approx_mode: None,
- query_parallelism: None,
- refine_factor: None,
-@@ -363,13 +425,10 @@ impl LanceScanner {
- }
- if let Some(n) = &self.nearest {
- scanner.nearest(&n.column, n.query.as_ref(), n.k as usize)?;
-- if let Some(np) = self.nprobes {
-- scanner.nprobes(np as usize);
-- }
-- if let Some(minimum_nprobes) = self.minimum_nprobes {
-+ if let Some(minimum_nprobes) = self.nprobes.minimum {
- scanner.minimum_nprobes(minimum_nprobes as usize);
- }
-- if let Some(maximum_nprobes) = self.maximum_nprobes {
-+ if let Some(maximum_nprobes) = self.nprobes.maximum {
- scanner.maximum_nprobes(maximum_nprobes as usize);
- }
- if let Some(approx_mode) = self.approx_mode {
-@@ -930,6 +989,14 @@ unsafe fn scanner_set_batch_size_bytes_inner(
- }
- let scanner = unsafe { &mut *scanner };
- scanner.ensure_scan_not_started("batch_size_bytes")?;
-+ if scanner.strict_batch_size == Some(true) {
-+ return Err(lance_core::Error::invalid_input_source(
-+ format!(
-+ "strict_batch_size=true cannot be combined with batch_size_bytes={batch_size_bytes}"
-+ )
-+ .into(),
-+ ));
-+ }
- scanner.batch_size_bytes = Some(batch_size_bytes);
- Ok(0)
- }
-@@ -1140,8 +1207,8 @@ unsafe fn scanner_set_use_scalar_index_inner(
-
- /// Configure whether output batches use the exact row-based batch size.
- ///
--/// Must be set before the scan starts. Lance rejects enabling this together
--/// with a byte-based batch-size limit when the scan is materialized.
-+/// Must be set before the scan starts. Enabling this together with a
-+/// byte-based batch-size limit is rejected without changing scanner state.
- #[unsafe(no_mangle)]
- pub unsafe extern "C" fn lance_scanner_set_strict_batch_size(
- scanner: *mut LanceScanner,
-@@ -1164,6 +1231,14 @@ unsafe fn scanner_set_strict_batch_size_inner(
- }
- let scanner = unsafe { &mut *scanner };
- scanner.ensure_scan_not_started("strict_batch_size")?;
-+ if strict_batch_size && let Some(batch_size_bytes) = scanner.batch_size_bytes {
-+ return Err(lance_core::Error::invalid_input_source(
-+ format!(
-+ "strict_batch_size=true cannot be combined with batch_size_bytes={batch_size_bytes}"
-+ )
-+ .into(),
-+ ));
-+ }
- scanner.strict_batch_size = Some(strict_batch_size);
- Ok(0)
- }
-@@ -2255,10 +2330,38 @@ macro_rules! scanner_set_u32 {
- };
- }
-
--scanner_set_u32!(lance_scanner_set_nprobes, nprobes);
- scanner_set_u32!(lance_scanner_set_refine_factor, refine_factor);
- scanner_set_u32!(lance_scanner_set_ef, ef);
-
-+/// Set both vector-index partition-search bounds to the same value.
-+///
-+/// This replaces any values previously configured through
-+/// `minimum_nprobes` or `maximum_nprobes`. The value must be greater than zero
-+/// and must be set before the scan starts.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_nprobes(
-+ scanner: *mut LanceScanner,
-+ nprobes: u32,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ scanner_set_nprobes_inner(scanner, nprobes)
-+ })
-+}
-+
-+unsafe fn scanner_set_nprobes_inner(scanner: *mut LanceScanner, nprobes: u32) -> Result<i32> {
-+ if scanner.is_null() {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scanner is NULL".into(),
-+ ));
-+ }
-+ let scanner = unsafe { &mut *scanner };
-+ scanner.ensure_scan_not_started("nprobes")?;
-+ let next = NprobesRange::exact(nprobes)?;
-+ scanner.nprobes = next;
-+ Ok(0)
-+}
-+
- /// Set the minimum number of vector-index partitions to search.
- ///
- /// The value must be greater than zero, no greater than a configured
-@@ -2283,24 +2386,10 @@ unsafe fn scanner_set_minimum_nprobes_inner(
- "scanner is NULL".into(),
- ));
- }
-- if minimum_nprobes == 0 {
-- return Err(lance_core::Error::invalid_input_source(
-- "minimum_nprobes must be greater than 0, got 0".into(),
-- ));
-- }
- let scanner = unsafe { &mut *scanner };
- scanner.ensure_scan_not_started("minimum_nprobes")?;
-- if let Some(maximum_nprobes) = scanner.maximum_nprobes
-- && minimum_nprobes > maximum_nprobes
-- {
-- return Err(lance_core::Error::invalid_input_source(
-- format!(
-- "minimum_nprobes ({minimum_nprobes}) must not exceed maximum_nprobes ({maximum_nprobes})"
-- )
-- .into(),
-- ));
-- }
-- scanner.minimum_nprobes = Some(minimum_nprobes);
-+ let next = scanner.nprobes.with_minimum(minimum_nprobes)?;
-+ scanner.nprobes = next;
- Ok(0)
- }
-
-@@ -2328,24 +2417,10 @@ unsafe fn scanner_set_maximum_nprobes_inner(
- "scanner is NULL".into(),
- ));
- }
-- if maximum_nprobes == 0 {
-- return Err(lance_core::Error::invalid_input_source(
-- "maximum_nprobes must be greater than 0, got 0".into(),
-- ));
-- }
- let scanner = unsafe { &mut *scanner };
- scanner.ensure_scan_not_started("maximum_nprobes")?;
-- if let Some(minimum_nprobes) = scanner.minimum_nprobes
-- && maximum_nprobes < minimum_nprobes
-- {
-- return Err(lance_core::Error::invalid_input_source(
-- format!(
-- "maximum_nprobes ({maximum_nprobes}) must not be less than minimum_nprobes ({minimum_nprobes})"
-- )
-- .into(),
-- ));
-- }
-- scanner.maximum_nprobes = Some(maximum_nprobes);
-+ let next = scanner.nprobes.with_maximum(maximum_nprobes)?;
-+ scanner.nprobes = next;
- Ok(0)
- }
-
-@@ -2885,6 +2960,49 @@ mod tests {
- )
- }
-
-+ #[test]
-+ fn nprobes_setters_share_one_validated_range() {
-+ let (_tmp, uri) = create_test_dataset();
-+ let (dataset, scanner) = open_dataset_and_scanner(&uri);
-+ let assert_range = |minimum, maximum| {
-+ assert_eq!(
-+ unsafe { &*scanner }.nprobes,
-+ NprobesRange { minimum, maximum }
-+ );
-+ };
-+
-+ assert_range(None, None);
-+ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 2) }, 0);
-+ assert_range(Some(2), None);
-+ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 5) }, 0);
-+ assert_range(Some(2), Some(5));
-+
-+ // The combined setter replaces both bounds.
-+ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 4) }, 0);
-+ assert_range(Some(4), Some(4));
-+
-+ // A failed partial update leaves both bounds unchanged.
-+ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 5) }, -1);
-+ assert_range(Some(4), Some(4));
-+
-+ // Widening the maximum first makes the new minimum valid.
-+ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 6) }, 0);
-+ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 5) }, 0);
-+ assert_range(Some(5), Some(6));
-+
-+ // A later combined call deterministically replaces the widened range.
-+ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 3) }, 0);
-+ assert_range(Some(3), Some(3));
-+ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 2) }, -1);
-+ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 0) }, -1);
-+ assert_range(Some(3), Some(3));
-+
-+ unsafe {
-+ lance_scanner_close(scanner);
-+ lance_dataset_close(dataset);
-+ }
-+ }
-+
- #[test]
- fn prepared_fts_index_only_plan_does_not_scan_indexed_fragment_row_ids() {
- let (_tmp, uri) = create_test_dataset();
-diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index a550a93..3b3424b 100644
---- a/tests/c_api_test.rs
-+++ b/tests/c_api_test.rs
-@@ -1318,6 +1318,78 @@ fn test_scanner_execution_tuning_options_reject_invalid_values() {
- unsafe { lance_dataset_close(ds) };
- }
-
-+#[test]
-+fn test_scanner_strict_batch_size_and_bytes_conflict_is_recoverable() {
-+ let (_tmp, uri) = create_test_dataset();
-+ let c_uri = c_str(&uri);
-+ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) };
-+ assert!(!ds.is_null());
-+
-+ let consume = |scanner| {
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(
-+ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) },
-+ 0
-+ );
-+ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap();
-+ assert_eq!(
-+ reader.map(|batch| batch.unwrap().num_rows()).sum::<usize>(),
-+ 5
-+ );
-+ };
-+
-+ // A byte limit already exists: strict=true is rejected without starting
-+ // the scan or replacing the prior strict setting.
-+ let bytes_first = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) };
-+ assert_eq!(
-+ unsafe { lance_scanner_set_batch_size_bytes(bytes_first, 1024) },
-+ 0
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_strict_batch_size(bytes_first, true) },
-+ -1
-+ );
-+ assert!(
-+ take_last_error_message()
-+ .contains("strict_batch_size=true cannot be combined with batch_size_bytes=1024")
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_use_stats(bytes_first, false) },
-+ 0,
-+ "the rejected setter must not mark the scan as started"
-+ );
-+ consume(bytes_first);
-+
-+ // Strict sizing already exists: the byte limit is rejected without
-+ // mutation. The caller can disable strict sizing and retry on this handle.
-+ let strict_first = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) };
-+ assert_eq!(
-+ unsafe { lance_scanner_set_strict_batch_size(strict_first, true) },
-+ 0
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_batch_size_bytes(strict_first, 1024) },
-+ -1
-+ );
-+ assert!(
-+ take_last_error_message()
-+ .contains("strict_batch_size=true cannot be combined with batch_size_bytes=1024")
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_strict_batch_size(strict_first, false) },
-+ 0
-+ );
-+ assert_eq!(
-+ unsafe { lance_scanner_set_batch_size_bytes(strict_first, 1024) },
-+ 0
-+ );
-+ consume(strict_first);
-+
-+ unsafe { lance_scanner_close(bytes_first) };
-+ unsafe { lance_scanner_close(strict_first) };
-+ unsafe { lance_dataset_close(ds) };
-+}
-+
- #[test]
- fn test_scanner_execution_tuning_options_reject_after_scan_start() {
- let (_tmp, uri) = create_test_dataset();
-@@ -1760,6 +1832,7 @@ fn test_null_safety_comprehensive() {
- unsafe { lance_scanner_set_include_deleted_rows(ptr::null_mut(), true) },
- -1
- );
-+ assert_eq!(unsafe { lance_scanner_set_nprobes(ptr::null_mut(), 1) }, -1);
- assert_eq!(
- unsafe { lance_scanner_set_minimum_nprobes(ptr::null_mut(), 1) },
- -1
-@@ -5663,6 +5736,8 @@ fn test_scanner_adaptive_nprobes_and_approx_mode_validation_and_lifecycle() {
- let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) };
- assert!(!scanner.is_null());
-
-+ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 0) }, -1);
-+ assert!(take_last_error_message().contains("nprobes must be greater than 0, got 0"));
- assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 0) }, -1);
- assert!(take_last_error_message().contains("minimum_nprobes must be greater than 0, got 0"));
- assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 0) }, -1);
-@@ -5704,6 +5779,8 @@ fn test_scanner_adaptive_nprobes_and_approx_mode_validation_and_lifecycle() {
- 0
- );
-
-+ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 1) }, -1);
-+ assert!(take_last_error_message().contains("nprobes must be set before"));
- assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 1) }, -1);
- assert!(take_last_error_message().contains("minimum_nprobes must be set before"));
- assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 1) }, -1);
diff --git a/thirdparty/patches/lance-c-0.1.9-pr-77.patch b/thirdparty/patches/lance-c-0.1.9-pr-77.patch
deleted file mode 100644
index 341a1c692dc..00000000000
--- a/thirdparty/patches/lance-c-0.1.9-pr-77.patch
+++ /dev/null
@@ -1,1863 +0,0 @@
-From eaf06c0374e62de6b519f55efe17de0c58e88c0a Mon Sep 17 00:00:00 2001
-From: "jianjian.xie" <jj...@gmail.com>
-Date: Fri, 4 Sep 2026 22:04:55 -0700
-Subject: [PATCH] build: bump lance to v11.0.0
-
-Summary:
-Intent:
-- Move the lance git pins from e934cc2c to ab6b5bbe (lance v11.0.0 release tag)
- so lance-c tracks a released upstream version instead of an arbitrary commit.
-- Pick up the v11 blob APIs (read_blob_ranges, Option-based take_blobs results)
- needed to answer #76 without a second pin bump.
-
-Changes:
-- Point all lance, lance-core, lance-file, lance-index, lance-io, lance-linalg,
- lance-table, lance-datafusion, and lance-datagen dependencies at ab6b5bbe.
-- Re-resolve Cargo.lock; blake3, jiff, and reqwest 0.13 were unlocked explicitly
- because v11 raised their minimum versions, the rest follows from lance v11
- (opendal 0.58, lance-namespace-reqwest-client 0.11, etc.).
-- Adapt to upstream signature changes: build_global_bm25_scorer takes an optional
- metrics collector, MatchQueryExec/PhraseQueryExec::new_with_segments are now
- fallible, DataFile::new takes a ConcreteFileVersion, and pb::IndexMetadata
- gained covering_fields.
-- Keep the DOT PQ strict-subset guard; upstream make_global_pq is unchanged at
- v11, so only the referenced revision in the comment and error text moved.
-
-Test Plan:
-- cargo fmt, cargo check --all-targets, cargo clippy --all-targets -D warnings.
-- cargo test: 367 passed, 0 failed, 2 ignored.
-- cargo test --test compile_and_run_test -- --ignored: 2 passed (C and C++
- compile-and-run against the rebuilt library).
-
-Co-Authored-By: Claude Fable 5.1 <no...@anthropic.com>
----
- Cargo.lock | 653 ++++++++++++++++++++++++-------------------
- Cargo.toml | 22 +-
- src/fts_query.rs | 4 +-
- src/index_segment.rs | 4 +-
- src/scanner.rs | 4 +-
- tests/c_api_test.rs | 8 +-
- 6 files changed, 391 insertions(+), 304 deletions(-)
-
-diff --git a/Cargo.lock b/Cargo.lock
-index 60c1caf..bc37cb9 100644
---- a/Cargo.lock
-+++ b/Cargo.lock
-@@ -222,7 +222,7 @@ dependencies = [
- "arrow-schema",
- "arrow-select",
- "atoi",
-- "base64",
-+ "base64 0.22.1",
- "chrono",
- "comfy-table",
- "half",
-@@ -332,7 +332,7 @@ version = "58.3.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"
- dependencies = [
-- "bitflags",
-+ "bitflags 2.11.0",
- "serde_core",
- "serde_json",
- ]
-@@ -434,6 +434,16 @@ dependencies = [
- "loom",
- ]
-
-+[[package]]
-+name = "asyncband"
-+version = "0.6.7"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0"
-+dependencies = [
-+ "hashbrown 0.17.1",
-+ "slab",
-+]
-+
- [[package]]
- name = "atoi"
- version = "2.0.0"
-@@ -504,7 +514,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc"
- dependencies = [
- "aws-lc-sys",
-- "untrusted 0.7.1",
- "zeroize",
- ]
-
-@@ -631,7 +640,7 @@ dependencies = [
- "bytes",
- "form_urlencoded",
- "hex",
-- "hmac",
-+ "hmac 0.12.1",
- "http 0.2.12",
- "http 1.4.0",
- "percent-encoding",
-@@ -829,6 +838,12 @@ version = "0.22.1"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
-
-+[[package]]
-+name = "base64"
-+version = "0.23.1"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
-+
- [[package]]
- name = "base64-simd"
- version = "0.8.0"
-@@ -858,6 +873,12 @@ dependencies = [
- "num-traits",
- ]
-
-+[[package]]
-+name = "bitflags"
-+version = "1.3.2"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-+
- [[package]]
- name = "bitflags"
- version = "2.11.0"
-@@ -887,16 +908,15 @@ dependencies = [
-
- [[package]]
- name = "blake3"
--version = "1.8.3"
-+version = "1.8.7"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d"
-+checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae"
- dependencies = [
-- "arrayref",
- "arrayvec",
- "cc",
- "cfg-if 1.0.4",
- "constant_time_eq",
-- "cpufeatures 0.2.17",
-+ "cpufeatures 0.3.0",
- ]
-
- [[package]]
-@@ -1127,6 +1147,12 @@ dependencies = [
- "cc",
- ]
-
-+[[package]]
-+name = "cmov"
-+version = "0.5.4"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
-+
- [[package]]
- name = "colorchoice"
- version = "1.0.5"
-@@ -1295,12 +1321,13 @@ dependencies = [
- ]
-
- [[package]]
--name = "crc32c"
--version = "0.6.8"
-+name = "crc-fast"
-+version = "1.10.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47"
-+checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5"
- dependencies = [
-- "rustc_version",
-+ "digest 0.10.7",
-+ "spin 0.10.1",
- ]
-
- [[package]]
-@@ -1437,6 +1464,15 @@ version = "0.0.7"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1"
-
-+[[package]]
-+name = "ctutils"
-+version = "0.4.2"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e"
-+dependencies = [
-+ "cmov",
-+]
-+
- [[package]]
- name = "darling"
- version = "0.23.0"
-@@ -1785,7 +1821,7 @@ checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7"
- dependencies = [
- "arrow",
- "arrow-buffer",
-- "base64",
-+ "base64 0.22.1",
- "blake2",
- "blake3",
- "chrono",
-@@ -2112,6 +2148,37 @@ dependencies = [
- "url",
- ]
-
-+[[package]]
-+name = "defmt"
-+version = "1.1.1"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1"
-+dependencies = [
-+ "bitflags 1.3.2",
-+ "defmt-macros",
-+]
-+
-+[[package]]
-+name = "defmt-macros"
-+version = "1.1.1"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8"
-+dependencies = [
-+ "defmt-parser",
-+ "proc-macro2",
-+ "quote",
-+ "syn 2.0.117",
-+]
-+
-+[[package]]
-+name = "defmt-parser"
-+version = "1.0.0"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e"
-+dependencies = [
-+ "thiserror 2.0.18",
-+]
-+
- [[package]]
- name = "der"
- version = "0.7.10"
-@@ -2154,6 +2221,7 @@ dependencies = [
- "block-buffer 0.12.1",
- "const-oid 0.10.2",
- "crypto-common 0.2.2",
-+ "ctutils",
- ]
-
- [[package]]
-@@ -2322,7 +2390,7 @@ version = "25.12.19"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
- dependencies = [
-- "bitflags",
-+ "bitflags 2.11.0",
- "rustc_version",
- ]
-
-@@ -2369,6 +2437,12 @@ dependencies = [
- "percent-encoding",
- ]
-
-+[[package]]
-+name = "frostem"
-+version = "1.20260821.5"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "36a80a7406da302e04bfd2ca987907590d3a1f3c69958947c43890abd7426b2f"
-+
- [[package]]
- name = "fs_extra"
- version = "1.3.0"
-@@ -2377,8 +2451,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
-
- [[package]]
- name = "fsst"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow-array",
- "rand 0.9.2",
-@@ -2727,14 +2801,23 @@ dependencies = [
-
- [[package]]
- name = "goosefs-sdk"
--version = "0.1.5"
-+version = "0.1.9"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f"
-+checksum = "e1ea4eee6dcbc31b25ab4fd577adc55b677d2bed3aa3016c44c58fbe1b2298a5"
- dependencies = [
-+ "arc-swap",
- "async-trait",
- "bytes",
- "dashmap",
-+ "fastrand",
-+ "futures",
- "hostname",
-+ "io-uring",
-+ "itoa",
-+ "libc",
-+ "lru",
-+ "memmap2",
-+ "moka",
- "prost",
- "prost-types",
- "rand 0.9.2",
-@@ -2747,6 +2830,7 @@ dependencies = [
- "tonic-prost",
- "tracing",
- "uuid",
-+ "xxhash-rust",
- ]
-
- [[package]]
-@@ -2900,6 +2984,15 @@ dependencies = [
- "digest 0.10.7",
- ]
-
-+[[package]]
-+name = "hmac"
-+version = "0.13.0"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f"
-+dependencies = [
-+ "digest 0.11.3",
-+]
-+
- [[package]]
- name = "hostname"
- version = "0.4.2"
-@@ -3053,7 +3146,7 @@ version = "0.1.20"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
- dependencies = [
-- "base64",
-+ "base64 0.22.1",
- "bytes",
- "futures-channel",
- "futures-util",
-@@ -3347,7 +3440,7 @@ version = "0.7.12"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62"
- dependencies = [
-- "bitflags",
-+ "bitflags 2.11.0",
- "cfg-if 1.0.4",
- "libc",
- ]
-@@ -3400,10 +3493,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
-
- [[package]]
- name = "jiff"
--version = "0.2.23"
-+version = "0.2.35"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359"
-+checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc"
- dependencies = [
-+ "defmt",
-+ "jiff-core",
- "jiff-static",
- "jiff-tzdb-platform",
- "js-sys",
-@@ -3412,15 +3507,25 @@ dependencies = [
- "portable-atomic-util",
- "serde_core",
- "wasm-bindgen",
-- "windows-sys 0.61.2",
-+ "windows-link",
-+]
-+
-+[[package]]
-+name = "jiff-core"
-+version = "0.1.0"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09"
-+dependencies = [
-+ "defmt",
- ]
-
- [[package]]
- name = "jiff-static"
--version = "0.2.23"
-+version = "0.2.35"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4"
-+checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204"
- dependencies = [
-+ "jiff-core",
- "proc-macro2",
- "quote",
- "syn 2.0.117",
-@@ -3530,24 +3635,6 @@ dependencies = [
- "serde_json",
- ]
-
--[[package]]
--name = "jsonwebtoken"
--version = "10.4.0"
--source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc"
--dependencies = [
-- "aws-lc-rs",
-- "base64",
-- "getrandom 0.2.17",
-- "js-sys",
-- "pem",
-- "serde",
-- "serde_json",
-- "signature",
-- "simple_asn1",
-- "zeroize",
--]
--
- [[package]]
- name = "konst"
- version = "0.4.3"
-@@ -3567,8 +3654,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
-
- [[package]]
- name = "lance"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arc-swap",
- "arrow",
-@@ -3584,7 +3671,6 @@ dependencies = [
- "async-recursion",
- "async-trait",
- "async_cell",
-- "aws-credential-types",
- "byteorder",
- "bytes",
- "chrono",
-@@ -3599,7 +3685,6 @@ dependencies = [
- "either",
- "fst",
- "futures",
-- "half",
- "humantime",
- "itertools 0.14.0",
- "lance-arrow",
-@@ -3641,8 +3726,8 @@ dependencies = [
-
- [[package]]
- name = "lance-arrow"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow-array",
- "arrow-buffer",
-@@ -3664,7 +3749,7 @@ dependencies = [
- [[package]]
- name = "lance-arrow-scalar"
- version = "58.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow-array",
- "arrow-buffer",
-@@ -3678,7 +3763,7 @@ dependencies = [
- [[package]]
- name = "lance-arrow-stats"
- version = "58.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow-array",
- "arrow-schema",
-@@ -3687,8 +3772,8 @@ dependencies = [
-
- [[package]]
- name = "lance-bitpacking"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrayref",
- "crunchy",
-@@ -3728,20 +3813,19 @@ dependencies = [
-
- [[package]]
- name = "lance-core"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "async-trait",
-- "byteorder",
-+ "blake3",
- "bytes",
- "datafusion-common",
- "datafusion-sql",
- "futures",
-- "itertools 0.14.0",
- "lance-arrow",
- "lance-derive",
- "libc",
-@@ -3752,13 +3836,13 @@ dependencies = [
- "object_store",
- "pin-project",
- "prost",
-+ "quick_cache",
- "rand 0.9.2",
- "roaring",
- "serde_json",
- "snafu",
- "tempfile",
- "tokio",
-- "tokio-stream",
- "tokio-util",
- "tracing",
- "twox-hash",
-@@ -3767,8 +3851,8 @@ dependencies = [
-
- [[package]]
- name = "lance-datafusion"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow",
- "arrow-array",
-@@ -3788,7 +3872,6 @@ dependencies = [
- "jsonb",
- "lance-arrow",
- "lance-core",
-- "lance-datagen",
- "lance-geo",
- "log",
- "pin-project",
-@@ -3800,8 +3883,8 @@ dependencies = [
-
- [[package]]
- name = "lance-datagen"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow",
- "arrow-array",
-@@ -3818,8 +3901,8 @@ dependencies = [
-
- [[package]]
- name = "lance-derive"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "proc-macro2",
- "quote",
-@@ -3828,8 +3911,8 @@ dependencies = [
-
- [[package]]
- name = "lance-encoding"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow-arith",
- "arrow-array",
-@@ -3854,8 +3937,6 @@ dependencies = [
- "num-traits",
- "prost",
- "prost-build",
-- "rand 0.9.2",
-- "strum",
- "tokio",
- "tracing",
- "xxhash-rust",
-@@ -3864,12 +3945,13 @@ dependencies = [
-
- [[package]]
- name = "lance-file"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow-arith",
- "arrow-array",
- "arrow-buffer",
-+ "arrow-cast",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
-@@ -3895,8 +3977,8 @@ dependencies = [
-
- [[package]]
- name = "lance-geo"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "datafusion",
- "geo-traits",
-@@ -3910,13 +3992,14 @@ dependencies = [
-
- [[package]]
- name = "lance-index"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arc-swap",
- "arrow",
- "arrow-arith",
- "arrow-array",
-+ "arrow-ipc",
- "arrow-ord",
- "arrow-schema",
- "arrow-select",
-@@ -3925,7 +4008,6 @@ dependencies = [
- "async-trait",
- "bitvec",
- "bytes",
-- "chrono",
- "crossbeam-queue",
- "datafusion",
- "datafusion-common",
-@@ -3945,7 +4027,6 @@ dependencies = [
- "lance-bitpacking",
- "lance-core",
- "lance-datafusion",
-- "lance-datagen",
- "lance-encoding",
- "lance-file",
- "lance-geo",
-@@ -3975,13 +4056,12 @@ dependencies = [
- "tempfile",
- "tokio",
- "tracing",
-- "uuid",
- ]
-
- [[package]]
- name = "lance-index-core"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow-array",
- "arrow-schema",
-@@ -4003,18 +4083,12 @@ dependencies = [
-
- [[package]]
- name = "lance-io"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow",
-- "arrow-arith",
- "arrow-array",
-- "arrow-buffer",
-- "arrow-cast",
-- "arrow-data",
- "arrow-schema",
-- "arrow-select",
-- "async-recursion",
- "async-trait",
- "aws-config",
- "aws-credential-types",
-@@ -4022,10 +4096,8 @@ dependencies = [
- "bytes",
- "chrono",
- "futures",
-- "goosefs-sdk",
- "http 1.4.0",
- "io-uring",
-- "lance-arrow",
- "lance-core",
- "lance-namespace",
- "log",
-@@ -4037,34 +4109,37 @@ dependencies = [
- "pin-project",
- "prost",
- "rand 0.9.2",
-+ "reqsign-core",
-+ "reqsign-file-read-tokio",
-+ "reqsign-google",
- "serde",
-+ "serde_json",
- "tempfile",
- "tokio",
- "tracing",
- "url",
-+ "uuid",
- ]
-
- [[package]]
- name = "lance-linalg"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow-array",
-- "arrow-buffer",
- "arrow-schema",
- "cc",
- "half",
- "lance-arrow",
- "lance-core",
- "num-traits",
-- "rand 0.9.2",
- "rayon",
- ]
-
- [[package]]
- name = "lance-namespace"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow",
- "async-trait",
-@@ -4076,9 +4151,9 @@ dependencies = [
-
- [[package]]
- name = "lance-namespace-reqwest-client"
--version = "0.8.6"
-+version = "0.11.1"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d"
-+checksum = "1d06b1fbb5d41f93bc652b61e2872af92e8a6c5f6b4ce8839a8ecfa05365d359"
- dependencies = [
- "reqwest 0.12.28",
- "serde",
-@@ -4090,14 +4165,13 @@ dependencies = [
-
- [[package]]
- name = "lance-select"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-schema",
- "byteorder",
-- "bytes",
- "itertools 0.14.0",
- "lance-core",
- "roaring",
-@@ -4106,8 +4180,8 @@ dependencies = [
-
- [[package]]
- name = "lance-table"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
- "arrow",
- "arrow-array",
-@@ -4115,6 +4189,7 @@ dependencies = [
- "arrow-ipc",
- "arrow-schema",
- "async-trait",
-+ "blake3",
- "byteorder",
- "bytes",
- "chrono",
-@@ -4144,11 +4219,11 @@ dependencies = [
-
- [[package]]
- name = "lance-tokenizer"
--version = "9.1.0-beta.3"
--source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0"
-+version = "11.0.0"
-+source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
- dependencies = [
-+ "frostem",
- "icu_segmenter",
-- "rust-stemmers",
- "serde",
- "stop-words",
- "unicode-normalization",
-@@ -4160,7 +4235,7 @@ version = "1.5.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
- dependencies = [
-- "spin",
-+ "spin 0.9.8",
- ]
-
- [[package]]
-@@ -4291,9 +4366,9 @@ dependencies = [
-
- [[package]]
- name = "log"
--version = "0.4.29"
-+version = "0.4.34"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
-+checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
-
- [[package]]
- name = "loom"
-@@ -4308,6 +4383,15 @@ dependencies = [
- "tracing-subscriber",
- ]
-
-+[[package]]
-+name = "lru"
-+version = "0.18.4"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25"
-+dependencies = [
-+ "hashbrown 0.17.1",
-+]
-+
- [[package]]
- name = "lru-slab"
- version = "0.1.2"
-@@ -4399,6 +4483,15 @@ version = "2.8.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
-
-+[[package]]
-+name = "memmap2"
-+version = "0.9.11"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
-+dependencies = [
-+ "libc",
-+]
-+
- [[package]]
- name = "mime"
- version = "0.3.17"
-@@ -4619,7 +4712,7 @@ version = "0.3.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
- dependencies = [
-- "bitflags",
-+ "bitflags 2.11.0",
- ]
-
- [[package]]
-@@ -4648,7 +4741,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49"
- dependencies = [
- "async-trait",
-- "base64",
-+ "base64 0.22.1",
- "bytes",
- "chrono",
- "form_urlencoded",
-@@ -4664,7 +4757,7 @@ dependencies = [
- "md-5 0.10.6",
- "parking_lot",
- "percent-encoding",
-- "quick-xml",
-+ "quick-xml 0.39.4",
- "rand 0.10.1",
- "reqwest 0.12.28",
- "ring",
-@@ -4683,9 +4776,9 @@ dependencies = [
-
- [[package]]
- name = "object_store_opendal"
--version = "0.57.0"
-+version = "0.58.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8"
-+checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac"
- dependencies = [
- "async-trait",
- "bytes",
-@@ -4718,12 +4811,13 @@ checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
-
- [[package]]
- name = "opendal"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1"
-+checksum = "33dbff14cc9bb085224256d6a81289d2f3202e85b06f408d42534b42162a4231"
- dependencies = [
- "ctor 1.0.13",
- "opendal-core",
-+ "opendal-http-transport-reqwest",
- "opendal-layer-concurrent-limit",
- "opendal-layer-logging",
- "opendal-layer-retry",
-@@ -4741,24 +4835,22 @@ dependencies = [
-
- [[package]]
- name = "opendal-core"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309"
-+checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7"
- dependencies = [
- "anyhow",
-- "base64",
-+ "asyncband",
-+ "base64 0.23.1",
- "bytes",
- "futures",
- "http 1.4.0",
-- "http-body 1.0.1",
- "jiff",
- "log",
- "md-5 0.11.0",
-- "mea",
- "percent-encoding",
-- "quick-xml",
-+ "quick-xml 0.41.0",
- "reqsign-core",
-- "reqwest 0.13.3",
- "serde",
- "serde_json",
- "tokio",
-@@ -4767,23 +4859,37 @@ dependencies = [
- "web-time",
- ]
-
-+[[package]]
-+name = "opendal-http-transport-reqwest"
-+version = "0.58.2"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c"
-+dependencies = [
-+ "bytes",
-+ "futures",
-+ "http 1.4.0",
-+ "http-body 1.0.1",
-+ "opendal-core",
-+ "reqwest 0.13.4",
-+]
-+
- [[package]]
- name = "opendal-layer-concurrent-limit"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f"
-+checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba"
- dependencies = [
-+ "asyncband",
- "futures",
- "http 1.4.0",
-- "mea",
- "opendal-core",
- ]
-
- [[package]]
- name = "opendal-layer-logging"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2"
-+checksum = "c2de17c61cd32e9d8d7d8efb91795e714dbccbafbc3c4e219e1542f4d6324161"
- dependencies = [
- "log",
- "opendal-core",
-@@ -4791,9 +4897,9 @@ dependencies = [
-
- [[package]]
- name = "opendal-layer-retry"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7"
-+checksum = "e94db301964a25366090484d61e6da16d5979cc8faf02c3e12210dc74fafed38"
- dependencies = [
- "backon",
- "log",
-@@ -4802,9 +4908,9 @@ dependencies = [
-
- [[package]]
- name = "opendal-layer-timeout"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3"
-+checksum = "08956ddda07465449bfd48825f4f0f25e0351278ac974eaa659895d9d74f2c80"
- dependencies = [
- "opendal-core",
- "tokio",
-@@ -4812,17 +4918,17 @@ dependencies = [
-
- [[package]]
- name = "opendal-service-azblob"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a"
-+checksum = "6d278d2fb57661947d1c9fb44047e432b2782c48dc085b7abc99fe6e18c26cf8"
- dependencies = [
-- "base64",
-+ "base64 0.23.1",
- "bytes",
- "http 1.4.0",
- "log",
- "opendal-core",
- "opendal-service-azure-common",
-- "quick-xml",
-+ "quick-xml 0.41.0",
- "reqsign-azure-storage",
- "reqsign-core",
- "reqsign-file-read-tokio",
-@@ -4833,17 +4939,18 @@ dependencies = [
-
- [[package]]
- name = "opendal-service-azdls"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361"
-+checksum = "2d564484a8f7d091827e825cfc91ed45bd48e64d262451ee041fe843db81bd8a"
- dependencies = [
-- "base64",
-+ "asyncband",
-+ "base64 0.23.1",
- "bytes",
- "http 1.4.0",
- "log",
- "opendal-core",
- "opendal-service-azure-common",
-- "quick-xml",
-+ "quick-xml 0.41.0",
- "reqsign-azure-storage",
- "reqsign-core",
- "reqsign-file-read-tokio",
-@@ -4853,9 +4960,9 @@ dependencies = [
-
- [[package]]
- name = "opendal-service-azure-common"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051"
-+checksum = "cfcc1bfdac4f54d9018c462dd32ad9e2f68fdf584f825c811550c8ffc87894cd"
- dependencies = [
- "http 1.4.0",
- "opendal-core",
-@@ -4863,15 +4970,15 @@ dependencies = [
-
- [[package]]
- name = "opendal-service-cos"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901"
-+checksum = "bb021c128ebde42e6f3e719d4cfed27e994017aa503a7a1e5818bdb61fd67dc9"
- dependencies = [
- "bytes",
- "http 1.4.0",
- "log",
- "opendal-core",
-- "quick-xml",
-+ "quick-xml 0.41.0",
- "reqsign-core",
- "reqsign-file-read-tokio",
- "reqsign-tencent-cos",
-@@ -4880,9 +4987,9 @@ dependencies = [
-
- [[package]]
- name = "opendal-service-gcs"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47"
-+checksum = "da1f2a8c975fd22fea01f0409bcb8774f1ac02ad79863a3490098203121555d3"
- dependencies = [
- "async-trait",
- "bytes",
-@@ -4890,7 +4997,7 @@ dependencies = [
- "log",
- "opendal-core",
- "percent-encoding",
-- "quick-xml",
-+ "quick-xml 0.41.0",
- "reqsign-core",
- "reqsign-file-read-tokio",
- "reqsign-google",
-@@ -4901,9 +5008,9 @@ dependencies = [
-
- [[package]]
- name = "opendal-service-goosefs"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4"
-+checksum = "89fd71b80078f2983bd363e322fbebfa6a46d5fc56f17e4f23d76d5edb31226b"
- dependencies = [
- "bytes",
- "goosefs-sdk",
-@@ -4915,9 +5022,9 @@ dependencies = [
-
- [[package]]
- name = "opendal-service-hf"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848"
-+checksum = "c8a3c8ec0c2918fa23f258fa28822f222ec8c1e3a501ded665b78bc65a9d7734"
- dependencies = [
- "bytes",
- "hf-xet",
-@@ -4925,22 +5032,21 @@ dependencies = [
- "log",
- "opendal-core",
- "percent-encoding",
-- "reqwest 0.13.3",
- "serde",
- "serde_json",
- ]
-
- [[package]]
- name = "opendal-service-oss"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace"
-+checksum = "8e3ce7a2ceb925e0f28f545b169eb57d04cec6116aae94b8584d2bdbe7d456c6"
- dependencies = [
- "bytes",
- "http 1.4.0",
- "log",
- "opendal-core",
-- "quick-xml",
-+ "quick-xml 0.41.0",
- "reqsign-aliyun-oss",
- "reqsign-core",
- "reqsign-file-read-tokio",
-@@ -4949,18 +5055,18 @@ dependencies = [
-
- [[package]]
- name = "opendal-service-s3"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb"
-+checksum = "c64335f9f24ccb62ac36f1d976342b48611a75ba61979813a4f78a4ebd94de42"
- dependencies = [
-- "base64",
-+ "base64 0.23.1",
- "bytes",
-- "crc32c",
-+ "crc-fast",
- "http 1.4.0",
- "log",
- "md-5 0.11.0",
- "opendal-core",
-- "quick-xml",
-+ "quick-xml 0.41.0",
- "reqsign-aws-v4",
- "reqsign-core",
- "reqsign-file-read-tokio",
-@@ -4970,14 +5076,14 @@ dependencies = [
-
- [[package]]
- name = "opendal-service-tos"
--version = "0.57.0"
-+version = "0.58.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1"
-+checksum = "70c3c507c3a565b2feb4c7b5f653436acd34ca59ffc842a7671b5498b13b76bf"
- dependencies = [
- "bytes",
- "http 1.4.0",
- "opendal-core",
-- "quick-xml",
-+ "quick-xml 0.41.0",
- "reqsign-core",
- "reqsign-file-read-tokio",
- "reqsign-volcengine-tos",
-@@ -5084,7 +5190,7 @@ version = "0.8.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3"
- dependencies = [
-- "base64",
-+ "base64 0.22.1",
- "serde",
- ]
-
-@@ -5122,16 +5228,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
- dependencies = [
- "digest 0.10.7",
-- "hmac",
-+ "hmac 0.12.1",
- ]
-
- [[package]]
- name = "pem"
--version = "3.0.6"
-+version = "4.0.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be"
-+checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120"
- dependencies = [
-- "base64",
-+ "base64 0.23.1",
- "serde_core",
- ]
-
-@@ -5392,6 +5498,28 @@ dependencies = [
- "serde",
- ]
-
-+[[package]]
-+name = "quick-xml"
-+version = "0.41.0"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
-+dependencies = [
-+ "memchr",
-+ "serde",
-+]
-+
-+[[package]]
-+name = "quick_cache"
-+version = "0.6.24"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9"
-+dependencies = [
-+ "ahash",
-+ "equivalent",
-+ "hashbrown 0.16.1",
-+ "parking_lot",
-+]
-+
- [[package]]
- name = "quinn"
- version = "0.11.9"
-@@ -5616,7 +5744,7 @@ version = "0.5.18"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
- dependencies = [
-- "bitflags",
-+ "bitflags 2.11.0",
- ]
-
- [[package]]
-@@ -5697,9 +5825,9 @@ dependencies = [
-
- [[package]]
- name = "reqsign-aliyun-oss"
--version = "3.0.0"
-+version = "3.1.4"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95"
-+checksum = "68d24d281f734a463093b7b93aae8b16f5f8496a54fbeec4d2d10b0488295296"
- dependencies = [
- "anyhow",
- "form_urlencoded",
-@@ -5713,18 +5841,18 @@ dependencies = [
- ]
-
- [[package]]
--name = "reqsign-aws-v4"
--version = "3.0.0"
-+name = "reqsign-aws-core"
-+version = "3.1.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43"
-+checksum = "4d63b56638bb3cc7bd376a7cdce1ba3089777a08f47e4097888f2d784cc3f46c"
- dependencies = [
-- "anyhow",
- "bytes",
- "form_urlencoded",
-+ "hex",
- "http 1.4.0",
- "log",
- "percent-encoding",
-- "quick-xml",
-+ "quick-xml 0.41.0",
- "reqsign-core",
- "rust-ini",
- "serde",
-@@ -5733,18 +5861,32 @@ dependencies = [
- "sha1",
- ]
-
-+[[package]]
-+name = "reqsign-aws-v4"
-+version = "3.2.0"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "4a0c499f4ed12d04c3d4c78fe4cb01aee22c9dae22848c14db2c6313d9df9f43"
-+dependencies = [
-+ "bytes",
-+ "http 1.4.0",
-+ "log",
-+ "quick-xml 0.41.0",
-+ "reqsign-aws-core",
-+ "reqsign-core",
-+ "serde",
-+]
-+
- [[package]]
- name = "reqsign-azure-storage"
--version = "3.0.0"
-+version = "3.2.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6"
-+checksum = "e8177b4f08620ab7f2e9cab7d7ccb9da1b61c66b889fed46cc0880b0fe75eb6b"
- dependencies = [
- "anyhow",
-- "base64",
-+ "base64 0.23.1",
- "bytes",
- "form_urlencoded",
- "http 1.4.0",
-- "jsonwebtoken",
- "log",
- "pem",
- "percent-encoding",
-@@ -5757,31 +5899,33 @@ dependencies = [
-
- [[package]]
- name = "reqsign-core"
--version = "3.0.0"
-+version = "3.3.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102"
-+checksum = "f4ac1510872d9481205975d264deb39c109797e5068cc882ed9064270eaae5fa"
- dependencies = [
- "anyhow",
-- "base64",
-+ "base64 0.23.1",
- "bytes",
-- "form_urlencoded",
- "futures",
- "hex",
-- "hmac",
-+ "hmac 0.13.0",
- "http 1.4.0",
- "jiff",
- "log",
- "percent-encoding",
-+ "rsa",
-+ "serde",
-+ "serde_json",
- "sha1",
-- "sha2 0.10.9",
-+ "sha2 0.11.0",
- "windows-sys 0.61.2",
- ]
-
- [[package]]
- name = "reqsign-file-read-tokio"
--version = "3.0.0"
-+version = "3.0.5"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5"
-+checksum = "95c3371bfc7e5c7f9627a04133af3583fd6c28715e7c83f79db38f3b384f535f"
- dependencies = [
- "anyhow",
- "reqsign-core",
-@@ -5790,13 +5934,13 @@ dependencies = [
-
- [[package]]
- name = "reqsign-google"
--version = "3.0.0"
-+version = "3.1.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e"
-+checksum = "f81a9d38870892443489c0abb5332edfa81d5a14c437af9caef7c194897c92c1"
- dependencies = [
-+ "bytes",
- "form_urlencoded",
- "http 1.4.0",
-- "jsonwebtoken",
- "log",
- "percent-encoding",
- "reqsign-aws-v4",
-@@ -5804,15 +5948,14 @@ dependencies = [
- "rsa",
- "serde",
- "serde_json",
-- "sha2 0.10.9",
- "tokio",
- ]
-
- [[package]]
- name = "reqsign-tencent-cos"
--version = "3.0.0"
-+version = "3.0.5"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a"
-+checksum = "b15c5a4df7c3f16823ae242675c5ebfb52d640cc9a50d1fcf943263247fa1730"
- dependencies = [
- "anyhow",
- "http 1.4.0",
-@@ -5825,9 +5968,9 @@ dependencies = [
-
- [[package]]
- name = "reqsign-volcengine-tos"
--version = "3.0.0"
-+version = "3.1.1"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "f9d757602a7ef2b6025c0da77e6d2e23fbdef35930fa466b15ffbf0a3f13acf7"
-+checksum = "173387eb5ae4cf6a0a7098665ebcc6729862819dc3d95a81aee840767803e3d9"
- dependencies = [
- "anyhow",
- "http 1.4.0",
-@@ -5842,7 +5985,7 @@ version = "0.12.28"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
- dependencies = [
-- "base64",
-+ "base64 0.22.1",
- "bytes",
- "encoding_rs",
- "futures-core",
-@@ -5884,11 +6027,11 @@ dependencies = [
-
- [[package]]
- name = "reqwest"
--version = "0.13.3"
-+version = "0.13.4"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0"
-+checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
- dependencies = [
-- "base64",
-+ "base64 0.22.1",
- "bytes",
- "futures-core",
- "futures-util",
-@@ -5931,7 +6074,7 @@ dependencies = [
- "anyhow",
- "async-trait",
- "http 1.4.0",
-- "reqwest 0.13.3",
-+ "reqwest 0.13.4",
- "thiserror 2.0.18",
- "tower-service",
- ]
-@@ -5946,7 +6089,7 @@ dependencies = [
- "cfg-if 1.0.4",
- "getrandom 0.2.17",
- "libc",
-- "untrusted 0.9.0",
-+ "untrusted",
- "windows-sys 0.52.0",
- ]
-
-@@ -6008,16 +6151,6 @@ dependencies = [
- "ordered-multimap",
- ]
-
--[[package]]
--name = "rust-stemmers"
--version = "1.2.0"
--source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
--dependencies = [
-- "serde",
-- "serde_derive",
--]
--
- [[package]]
- name = "rustc-hash"
- version = "2.1.1"
-@@ -6039,7 +6172,7 @@ version = "1.1.4"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
- dependencies = [
-- "bitflags",
-+ "bitflags 2.11.0",
- "errno",
- "libc",
- "linux-raw-sys",
-@@ -6119,7 +6252,7 @@ dependencies = [
- "aws-lc-rs",
- "ring",
- "rustls-pki-types",
-- "untrusted 0.9.0",
-+ "untrusted",
- ]
-
- [[package]]
-@@ -6244,7 +6377,7 @@ version = "3.7.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
- dependencies = [
-- "bitflags",
-+ "bitflags 2.11.0",
- "core-foundation 0.10.1",
- "core-foundation-sys",
- "libc",
-@@ -6373,7 +6506,7 @@ version = "3.20.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2"
- dependencies = [
-- "base64",
-+ "base64 0.22.1",
- "bs58",
- "chrono",
- "hex",
-@@ -6414,13 +6547,13 @@ dependencies = [
-
- [[package]]
- name = "sha1"
--version = "0.10.6"
-+version = "0.11.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
-+checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
- dependencies = [
- "cfg-if 1.0.4",
-- "cpufeatures 0.2.17",
-- "digest 0.10.7",
-+ "cpufeatures 0.3.0",
-+ "digest 0.11.3",
- ]
-
- [[package]]
-@@ -6523,18 +6656,6 @@ version = "0.1.5"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
-
--[[package]]
--name = "simple_asn1"
--version = "0.6.4"
--source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d"
--dependencies = [
-- "num-bigint",
-- "num-traits",
-- "thiserror 2.0.18",
-- "time",
--]
--
- [[package]]
- name = "siphasher"
- version = "1.0.2"
-@@ -6602,6 +6723,12 @@ version = "0.9.8"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
-
-+[[package]]
-+name = "spin"
-+version = "0.10.1"
-+source = "registry+https://github.com/rust-lang/crates.io-index"
-+checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
-+
- [[package]]
- name = "spki"
- version = "0.7.3"
-@@ -6682,28 +6809,6 @@ version = "0.11.1"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
-
--[[package]]
--name = "strum"
--version = "0.26.3"
--source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
--dependencies = [
-- "strum_macros",
--]
--
--[[package]]
--name = "strum_macros"
--version = "0.26.4"
--source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
--dependencies = [
-- "heck",
-- "proc-macro2",
-- "quote",
-- "rustversion",
-- "syn 2.0.117",
--]
--
- [[package]]
- name = "substrait"
- version = "0.63.0"
-@@ -6804,7 +6909,7 @@ version = "0.7.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
- dependencies = [
-- "bitflags",
-+ "bitflags 2.11.0",
- "core-foundation 0.9.4",
- "system-configuration-sys",
- ]
-@@ -7078,7 +7183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
- dependencies = [
- "async-trait",
-- "base64",
-+ "base64 0.22.1",
- "bytes",
- "h2",
- "http 1.4.0",
-@@ -7136,7 +7241,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
- dependencies = [
- "async-compression",
-- "bitflags",
-+ "bitflags 2.11.0",
- "bytes",
- "futures-core",
- "futures-util",
-@@ -7370,12 +7475,6 @@ version = "0.2.11"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
-
--[[package]]
--name = "untrusted"
--version = "0.7.1"
--source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
--
- [[package]]
- name = "untrusted"
- version = "0.9.0"
-@@ -7622,7 +7721,7 @@ version = "0.244.0"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
- dependencies = [
-- "bitflags",
-+ "bitflags 2.11.0",
- "hashbrown 0.15.5",
- "indexmap 2.14.0",
- "semver",
-@@ -8054,7 +8153,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
- dependencies = [
- "anyhow",
-- "bitflags",
-+ "bitflags 2.11.0",
- "indexmap 2.14.0",
- "log",
- "serde",
-@@ -8132,7 +8231,7 @@ checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8"
- dependencies = [
- "anyhow",
- "async-trait",
-- "base64",
-+ "base64 0.22.1",
- "bytes",
- "clap",
- "crc32fast",
-@@ -8143,7 +8242,7 @@ dependencies = [
- "more-asserts",
- "rand 0.10.1",
- "redb",
-- "reqwest 0.13.3",
-+ "reqwest 0.13.4",
- "reqwest-middleware",
- "serde",
- "serde_json",
-@@ -8169,7 +8268,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee"
- dependencies = [
- "async-trait",
-- "base64",
-+ "base64 0.22.1",
- "blake3",
- "bytemuck",
- "bytes",
-@@ -8256,7 +8355,7 @@ dependencies = [
- "oneshot",
- "pin-project",
- "rand 0.10.1",
-- "reqwest 0.13.3",
-+ "reqwest 0.13.4",
- "serde",
- "serde_json",
- "shellexpand",
-@@ -8352,20 +8451,6 @@ name = "zeroize"
- version = "1.8.2"
- source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
--dependencies = [
-- "zeroize_derive",
--]
--
--[[package]]
--name = "zeroize_derive"
--version = "1.4.3"
--source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e"
--dependencies = [
-- "proc-macro2",
-- "quote",
-- "syn 2.0.117",
--]
-
- [[package]]
- name = "zerotrie"
-diff --git a/Cargo.toml b/Cargo.toml
-index d072a5d..3920a65 100644
---- a/Cargo.toml
-+++ b/Cargo.toml
-@@ -18,14 +18,14 @@ rust-version = "1.91.0"
- crate-type = ["cdylib", "staticlib", "rlib"]
-
- [dependencies]
--lance = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c", features = ["substrait"] }
--lance-core = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" }
--lance-file = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" }
--lance-index = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" }
--lance-io = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" }
--lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" }
--lance-table = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" }
--lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c", features = ["substrait"] }
-+lance = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] }
-+lance-core = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-+lance-file = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-+lance-index = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-+lance-io = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-+lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-+lance-table = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-+lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] }
- datafusion = { version = "54.0.0", default-features = false }
- arrow = { version = "58.0.0", features = ["prettyprint", "ffi"] }
- arrow-array = "58.0.0"
-@@ -45,9 +45,9 @@ snafu = "0.9"
- uuid = { version = "1", features = ["v4"] }
-
- [dev-dependencies]
--lance = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c", features = ["substrait"] }
--lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" }
--lance-file = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" }
-+lance = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] }
-+lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-+lance-file = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
- tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
- arrow-array = "58.0.0"
- arrow-schema = "58.0.0"
-diff --git a/src/fts_query.rs b/src/fts_query.rs
-index c9d3a43..71874e0 100644
---- a/src/fts_query.rs
-+++ b/src/fts_query.rs
-@@ -246,7 +246,7 @@ async fn prepare_fts_query_context(
- .with_max_expansions(match_query.max_expansions)
- .with_prefix_length(match_query.prefix_length);
- PreparedFtsQuery::Match(Arc::new(
-- build_global_bm25_scorer(&indices, &query_tokens, ¶ms).await?,
-+ build_global_bm25_scorer(&indices, &query_tokens, ¶ms, None).await?,
- ))
- }
- FtsQuery::Phrase(phrase_query) => {
-@@ -260,7 +260,7 @@ async fn prepare_fts_query_context(
- let query_tokens = collect_query_tokens(&phrase_query.terms, &mut tokenizer);
- let params = query.params().with_phrase_slop(Some(phrase_query.slop));
- PreparedFtsQuery::Phrase(Arc::new(
-- build_global_bm25_scorer(&indices, &query_tokens, ¶ms).await?,
-+ build_global_bm25_scorer(&indices, &query_tokens, ¶ms, None).await?,
- ))
- }
- _ => {
-diff --git a/src/index_segment.rs b/src/index_segment.rs
-index a46c4f3..d4e143c 100644
---- a/src/index_segment.rs
-+++ b/src/index_segment.rs
-@@ -889,7 +889,7 @@ unsafe fn new_vector_builder_inner(
- // TODO(upstream-lance): Remove this fail-fast once Lance's distributed
- // vector-index path reconstructs a supplied PQ codebook with an L2
- // ProductQuantizer, matching the ordinary full-dataset path. Pinned Lance
-- // revision e934cc2c rewraps supplied codebooks with DistanceType::Dot in
-+ // revision ab6b5bbe rewraps supplied codebooks with DistanceType::Dot in
- // `make_global_pq`, which silently switches PQ code assignment away from
- // the L2 contract shared by full-dataset builds and index readers.
- if matches!(
-@@ -910,7 +910,7 @@ unsafe fn new_vector_builder_inner(
- let selected_fragment_ids: HashSet<u32> = fragment_ids.iter().copied().collect();
- if selected_fragment_ids != all_fragment_ids {
- return Err(invalid_input(format!(
-- "pq_codebook is supplied for metric=DOT, index_type={:?}, mode={:?}, and an effective strict fragment subset ({} of {} fragments): pinned Lance revision e934cc2c reconstructs the supplied codebook with a DOT ProductQuantizer in the distributed build path (make_global_pq), silently breaking the L2 PQ-assignment contract; cover the full dataset in one segment (pass NULL fragment_ids or list every fragment) or wait for upstream Lance DOT support",
-+ "pq_codebook is supplied for metric=DOT, index_type={:?}, mode={:?}, and an effective strict fragment subset ({} of {} fragments): pinned Lance revision ab6b5bbe reconstructs the supplied codebook with a DOT ProductQuantizer in the distributed build path (make_global_pq), silently breaking the L2 PQ-assignment contract; cover the full dataset in one segment (pass NULL fragment_ids or list every fragment) or wait for upstream Lance DOT support",
- params.index_type,
- parsed.mode,
- selected_fragment_ids.len(),
-diff --git a/src/scanner.rs b/src/scanner.rs
-index 7e898ce..d3ef3be 100644
---- a/src/scanner.rs
-+++ b/src/scanner.rs
-@@ -529,7 +529,7 @@ fn rewrite_prepared_fts_plan(
- exec.params().clone(),
- exec.prefilter_source().clone(),
- segments.to_vec(),
-- )
-+ )?
- .with_base_scorer(Arc::clone(scorer));
- return Ok((Arc::new(replacement), rewritten));
- }
-@@ -549,7 +549,7 @@ fn rewrite_prepared_fts_plan(
- exec.params().clone(),
- exec.prefilter_source().clone(),
- segments.to_vec(),
-- )
-+ )?
- .with_base_scorer(Arc::clone(scorer));
- return Ok((Arc::new(replacement), rewritten));
- }
-diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index b763cef..bde742d 100644
---- a/tests/c_api_test.rs
-+++ b/tests/c_api_test.rs
-@@ -2449,8 +2449,7 @@ fn test_robotics_e2e_write_then_finalize() {
- format!("data/{}", filename),
- field_ids,
- column_indices,
-- meta.major_version as u32,
-- meta.minor_version as u32,
-+ meta.version,
- None, // file_size_bytes
- None, // base_id
- );
-@@ -3405,6 +3404,7 @@ fn test_index_segment_metadata_parse_rejects_malformed_and_dangerous_input() {
- created_at: Some(u64::MAX),
- base_id: None,
- files: Vec::new(),
-+ covering_fields: Vec::new(),
- }
- .encode_to_vec();
- assert_eq!(
-@@ -3429,6 +3429,7 @@ fn test_index_segment_metadata_parse_rejects_malformed_and_dangerous_input() {
- created_at: None,
- base_id: None,
- files: Vec::new(),
-+ covering_fields: Vec::new(),
- }
- .encode_to_vec();
- assert_eq!(
-@@ -3456,6 +3457,7 @@ fn test_index_segment_metadata_parse_rejects_malformed_and_dangerous_input() {
- created_at: None,
- base_id: None,
- files: Vec::new(),
-+ covering_fields: Vec::new(),
- }
- .encode_to_vec();
- assert_eq!(
-@@ -4033,7 +4035,7 @@ fn test_vector_index_segment_rejects_strict_subset_dot_pq() {
- let message = take_last_error_message();
- assert!(message.contains("metric=DOT"), "{message}");
- assert!(message.contains("strict fragment subset"), "{message}");
-- assert!(message.contains("e934cc2c"), "{message}");
-+ assert!(message.contains("ab6b5bbe"), "{message}");
- assert!(message.contains("1 of 2 fragments"), "{message}");
- assert!(!centroids.is_released());
- assert!(!codebook.is_released());
diff --git a/thirdparty/patches/lance-c-0.1.9-pr-79.patch b/thirdparty/patches/lance-c-0.1.9-pr-79.patch
deleted file mode 100644
index ad32a48065f..00000000000
--- a/thirdparty/patches/lance-c-0.1.9-pr-79.patch
+++ /dev/null
@@ -1,1782 +0,0 @@
-From d819fbdfa52031d84d1fa01f2d06c51a6712c40b Mon Sep 17 00:00:00 2001
-From: zhangstar333 <zh...@selectdb.com>
-Date: Tue, 8 Sep 2026 22:30:48 +0800
-Subject: [PATCH 1/5] scalar index segment
-
----
- docs/scalar-segment-scans.md | 78 +++++++++++
- include/lance/lance.h | 24 ++++
- include/lance/lance.hpp | 14 ++
- src/lib.rs | 1 +
- src/scalar_segment.rs | 224 +++++++++++++++++++++++++++++++
- src/scanner.rs | 61 +++++++++
- tests/c_api_test.rs | 249 +++++++++++++++++++++++++++++++++++
- 7 files changed, 651 insertions(+)
- create mode 100644 docs/scalar-segment-scans.md
- create mode 100644 src/scalar_segment.rs
-
-diff --git a/docs/scalar-segment-scans.md b/docs/scalar-segment-scans.md
-new file mode 100644
-index 0000000..dc7c141
---- /dev/null
-+++ b/docs/scalar-segment-scans.md
-@@ -0,0 +1,78 @@
-+# Scalar index segment scans
-+
-+An ordinary scanner can use one physical BTree/Bitmap segment to generate
-+candidates, then read those candidates with the complete scanner filter. This
-+does not run a global search of the other segments of the logical index. It does
-+not subdivide a physical segment or make its own index search incremental.
-+
-+## Configuring a task
-+
-+Open a fixed dataset version. Select the physical index UUID from that version's
-+metadata and pass the task's complete fragment domain explicitly:
-+
-+```c
-+LanceScanner *scanner = lance_scanner_new(dataset, columns, full_filter_sql);
-+/* Check every return value in production. */
-+lance_scanner_set_fragment_ids(scanner, fragment_ids, fragment_count);
-+lance_scanner_set_scalar_index_segment(scanner, segment_uuid_16_bytes);
-+lance_scanner_set_limit(scanner, 20000);
-+/* The scanner-owning thread calls lance_scanner_next as usual. */
-+```
-+
-+SQL, Substrait and additional SQL filters keep their existing precedence and AND
-+composition. The caller does not supply a separate driver predicate: Lance-C
-+uses the typed filter planner and selects a necessary indexed leaf belonging to
-+the requested logical index. It only descends through AND, never through OR or
-+NOT. It then searches the selected UUID and applies the complete filter while
-+reading candidates with automatic scalar-index planning disabled.
-+
-+Each task's fragment IDs define its result domain, including on fallback. A
-+distributed planner must assign disjoint domains whose union covers the intended
-+scan. Unindexed fragments need their own tasks, or an explicit domain including
-+them (which causes that task to use fallback). Merely listing indexed segments
-+does not include appended, unindexed data automatically.
-+
-+An unknown UUID, absent fragment or invalid option combination is an error. A
-+known segment with incomplete/unknown coverage, no suitable driver, unsupported
-+index type, nested key, overlays, fragment reuse, non-exact results or unsupported
-+row-ID domain falls back to a non-indexed scan of the entire explicit domain.
-+I/O and corruption errors are propagated, not converted to empty results or
-+successful fallback.
-+
-+The first implementation supports live-row ordinary scans and cannot be combined
-+with vector/FTS queries. Physical row-address
-+results on stable-row-ID datasets currently fall back; results already expressed
-+in the correct row-ID domain use the candidate path. Deletes and all remaining
-+predicates are handled by the ordinary reader. No candidate-count limit is
-+applied: LIMIT/OFFSET remain after the scanner's complete filter.
-+
-+If the host has additional predicates outside Lance, do not set a local limit
-+before those predicates. Never divide the global limit by the number of tasks.
-+Global OFFSET belongs to the coordinator, not independently to each task.
-+
-+## Stopping after the host limit
-+
-+This mode uses the existing scanner/stream lifecycle. Once the host has enough
-+rows, it stops requesting further batches and closes the scanner after any active
-+call has returned. Do not call `lance_scanner_close` concurrently with `next`.
-+Exported Arrow streams remain owned by the caller and must also be released after
-+their active consumers have finished.
-+
-+A host stop flag does not interrupt an in-progress `lance_scanner_next`: current
-+index evaluation or I/O may finish before the host observes stop and closes the
-+stream. No separate cancellation signal or thread is introduced. The host remains
-+responsible for enforcing the global LIMIT across concurrent tasks.
-+
-+## Memory and statistics
-+
-+Candidate masks stay in Rust, and record batches are streamed. Each active task
-+can still hold a complete segment's candidate set; scanner I/O buffer size does
-+not cap that allocation. Control task concurrency and physical segment size.
-+
-+Successful exhaustion merges segment-search metrics into the existing statistics
-+callback exactly once. New metrics include `scalar_segments_requested`,
-+`scalar_segments_searched`, `scalar_segment_candidate_rows`,
-+`scalar_segment_prepare_time`, `scalar_segment_search_time`, and
-+`scalar_segment_fallback_*` reasons. `prepare_time` includes search time. Early
-+release, cancellation and errors retain the existing callback contract: final
-+statistics are not guaranteed. Metrics do not establish global task concurrency.
-diff --git a/include/lance/lance.h b/include/lance/lance.h
-index 8173ae5..cbd8330 100644
---- a/include/lance/lance.h
-+++ b/include/lance/lance.h
-@@ -1858,6 +1858,30 @@ int32_t lance_scanner_set_index_segments(
- size_t len
- );
-
-+/**
-+ * Accelerate an ordinary scalar-filtered scan with one physical index segment.
-+ * segment_uuid points to 16 UUID bytes in RFC 4122 order; NULL clears the setting.
-+ * Must be configured before scanning. Requires explicit nonempty fragment_ids,
-+ * which define BOTH the read and fallback domain, independently of the segment.
-+ * Missing snapshot UUIDs / fragment IDs are errors. Extra segment coverage is
-+ * excluded by fragment_ids; incomplete coverage falls back to a full filtered
-+ * scan of those fragment_ids. Callers distributing work must assign disjoint
-+ * fragment domains and separately include any unindexed data they wish to read.
-+ *
-+ * BTree/Bitmap searches use a necessary AND-conjunct of the full scanner filter
-+ * on the selected logical index. All predicates are reapplied during candidate
-+ * reads; other scalar indices are disabled. OR/NOT-only filters, overlays,
-+ * fragment reuse, unsupported index types / result domains
-+ * and missing coverage use the same domain without an index. No filter also
-+ * falls back. LIMIT/OFFSET apply after the complete scanner filter, never to the
-+ * unfiltered candidate set. Vector/FTS queries are rejected.
-+ *
-+ * UUID bytes are copied. Metadata and final option compatibility are validated
-+ * when creating the stream. Index corruption or I/O failures remain errors.
-+ */
-+int32_t lance_scanner_set_scalar_index_segment(
-+ LanceScanner* scanner, const uint8_t* segment_uuid);
-+
- /* ─── Full-text search (Phase 2) ─── */
-
- /**
-diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp
-index c12c0c6..8e1c6cf 100644
---- a/include/lance/lance.hpp
-+++ b/include/lance/lance.hpp
-@@ -1311,6 +1311,20 @@ class Scanner {
- return *this;
- }
-
-+ /// Restrict scalar candidate generation to one segment; fragment_ids is
-+ /// required and defines the complete read/fallback domain. See lance.h.
-+ Scanner& scalar_index_segment(const std::array<uint8_t, 16>& segment_uuid) {
-+ if (lance_scanner_set_scalar_index_segment(handle_.get(), segment_uuid.data()) != 0)
-+ check_error();
-+ return *this;
-+ }
-+
-+ Scanner& clear_scalar_index_segment() {
-+ if (lance_scanner_set_scalar_index_segment(handle_.get(), nullptr) != 0)
-+ check_error();
-+ return *this;
-+ }
-+
- /// Restrict scan to specific fragment IDs.
- Scanner& fragment_ids(const uint64_t* ids, size_t len) {
- if (lance_scanner_set_fragment_ids(handle_.get(), ids, len) != 0)
-diff --git a/src/lib.rs b/src/lib.rs
-index 8b212f5..c7ca4cf 100644
---- a/src/lib.rs
-+++ b/src/lib.rs
-@@ -39,6 +39,7 @@ mod index_segment;
- mod merge_insert;
- mod restore;
- pub mod runtime;
-+mod scalar_segment;
- mod scanner;
- mod session;
- pub mod stream_guard;
-diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs
-new file mode 100644
-index 0000000..2ab1297
---- /dev/null
-+++ b/src/scalar_segment.rs
-@@ -0,0 +1,224 @@
-+// SPDX-License-Identifier: Apache-2.0
-+// SPDX-FileCopyrightText: Copyright The Lance Authors
-+
-+//! Segment-scoped candidate generation for ordinary scans. The explicit fragment
-+//! list is the read domain, including on fallback; a segment is only an accelerator.
-+
-+use std::collections::HashSet;
-+use std::sync::Arc;
-+use std::time::Instant;
-+
-+use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
-+use lance::Dataset;
-+use lance::dataset::scanner::{
-+ ExecutionStatsCallback, ExecutionSummaryCounts, RowAddrMask, Scanner,
-+};
-+use lance::index::{DatasetIndexExt, DatasetIndexInternalExt};
-+use lance::io::exec::utils::IndexMetrics;
-+use lance_core::{Error, Result};
-+use lance_datafusion::planner::Planner;
-+use lance_datafusion::utils::MetricsExt;
-+use lance_index::IndexType;
-+use lance_index::scalar::SearchResult;
-+use lance_index::scalar::expression::{PlannerIndexExt, ScalarIndexExpr, ScalarIndexSearch};
-+use uuid::Uuid;
-+
-+pub(crate) struct PreparedScalarSegment {
-+ pub dataset: Arc<Dataset>,
-+ pub segment_uuid: Uuid,
-+ pub fragment_ids: Vec<u64>,
-+ pub callback: Option<ExecutionStatsCallback>,
-+}
-+
-+fn invalid(message: impl Into<String>) -> Error {
-+ Error::invalid_input_source(message.into().into())
-+}
-+
-+// Only descend through AND: a leaf below OR or NOT need not contain all matches
-+// of the full expression. The original expression is always reapplied by reader.
-+fn driver<'a>(expr: &'a ScalarIndexExpr, index_name: &str) -> Option<&'a ScalarIndexSearch> {
-+ match expr {
-+ ScalarIndexExpr::Query(search) if search.index_name == index_name => Some(search),
-+ ScalarIndexExpr::And(lhs, rhs) => {
-+ driver(lhs, index_name).or_else(|| driver(rhs, index_name))
-+ }
-+ _ => None,
-+ }
-+}
-+
-+impl PreparedScalarSegment {
-+ pub async fn configure(self, mut reader: Scanner) -> Result<Scanner> {
-+ // Never let either candidate reads or fallback re-enter a global index search.
-+ reader.use_scalar_index(false);
-+ let mut stats = ExecutionSummaryCounts::default();
-+ stats
-+ .all_counts
-+ .insert("scalar_segments_requested".into(), 1);
-+ let plan_metrics = ExecutionPlanMetricsSet::new();
-+ let metrics = IndexMetrics::new(&plan_metrics, 0);
-+ let started = Instant::now();
-+ let reason = self
-+ .configure_candidates(&mut reader, &metrics, &mut stats)
-+ .await?;
-+ metrics.flush_io();
-+ stats.all_times.insert(
-+ "scalar_segment_prepare_time".into(),
-+ started.elapsed().as_nanos().min(usize::MAX as u128) as usize,
-+ );
-+ if let Some(reason) = reason {
-+ stats
-+ .all_counts
-+ .insert("scalar_segment_fallbacks".into(), 1);
-+ stats
-+ .all_counts
-+ .insert(format!("scalar_segment_fallback_{reason}"), 1);
-+ }
-+ for (name, count) in plan_metrics.clone_inner().iter_counts() {
-+ let name = name.as_ref();
-+ match name {
-+ "iops" => stats.iops += count.value(),
-+ "requests" => stats.requests += count.value(),
-+ "bytes_read" => stats.bytes_read += count.value(),
-+ "indices_loaded" => stats.indices_loaded += count.value(),
-+ "parts_loaded" => stats.parts_loaded += count.value(),
-+ "index_comparisons" => stats.index_comparisons += count.value(),
-+ _ => *stats.all_counts.entry(name.to_string()).or_default() += count.value(),
-+ }
-+ }
-+ if let Some(callback) = self.callback {
-+ // Preserve the callback's once-per-successfully-exhausted-stream contract.
-+ // Candidate work is not part of the underlying reader's plan metrics.
-+ reader.scan_stats_callback(Arc::new(move |read| {
-+ let mut combined = read.clone();
-+ combined.iops += stats.iops;
-+ combined.requests += stats.requests;
-+ combined.bytes_read += stats.bytes_read;
-+ combined.indices_loaded += stats.indices_loaded;
-+ combined.parts_loaded += stats.parts_loaded;
-+ combined.index_comparisons += stats.index_comparisons;
-+ for (name, value) in &stats.all_counts {
-+ *combined.all_counts.entry(name.clone()).or_default() += value;
-+ }
-+ for (name, value) in &stats.all_times {
-+ *combined.all_times.entry(name.clone()).or_default() += value;
-+ }
-+ callback(&combined);
-+ }));
-+ }
-+ Ok(reader)
-+ }
-+
-+ async fn configure_candidates(
-+ &self,
-+ reader: &mut Scanner,
-+ metrics: &IndexMetrics,
-+ stats: &mut ExecutionSummaryCounts,
-+ ) -> Result<Option<&'static str>> {
-+ let fragments = self.dataset.get_fragments();
-+ let visible: HashSet<u64> = fragments.iter().map(|f| f.id() as u64).collect();
-+ if self.fragment_ids.iter().any(|id| !visible.contains(id)) {
-+ return Err(invalid(
-+ "scalar segment fragment_ids contains a fragment absent from the dataset snapshot",
-+ ));
-+ }
-+ let indices = self.dataset.load_indices().await?;
-+ let index_meta = indices
-+ .iter()
-+ .find(|i| i.uuid == self.segment_uuid)
-+ .ok_or_else(|| {
-+ invalid(format!(
-+ "scalar index segment {} is absent from the dataset snapshot",
-+ self.segment_uuid
-+ ))
-+ })?;
-+ let field_id = index_meta
-+ .keyed_field()
-+ .ok_or_else(|| invalid("scalar segment must index a single key field"))?;
-+ let field =
-+ self.dataset.schema().field_by_id(field_id).ok_or_else(|| {
-+ invalid("scalar segment key field is absent from the dataset schema")
-+ })?;
-+ // Keep V1 to flat scalar fields. A dotted name is not sufficient to prove
-+ // the field path of an evolved or nested schema.
-+ if !self
-+ .dataset
-+ .schema()
-+ .fields
-+ .iter()
-+ .any(|f| f.id == field.id)
-+ {
-+ return Ok(Some("nested_field"));
-+ }
-+ let scope: HashSet<u64> = self.fragment_ids.iter().copied().collect();
-+ let Some(coverage) = index_meta.fragment_bitmap.as_ref() else {
-+ return Ok(Some("unknown_coverage"));
-+ };
-+ if self
-+ .fragment_ids
-+ .iter()
-+ .any(|id| u32::try_from(*id).map_or(true, |id| !coverage.contains(id)))
-+ {
-+ // Scan the ENTIRE explicit read domain, not just the covered part.
-+ return Ok(Some("partial_coverage"));
-+ }
-+ if fragments
-+ .iter()
-+ .filter(|f| scope.contains(&(f.id() as u64)))
-+ .any(|f| !f.metadata().overlays.is_empty() || f.metadata().physical_rows.is_none())
-+ {
-+ return Ok(Some("fragment_state"));
-+ }
-+ // Fragment reuse can change the domain of an old segment. Until its
-+ // coverage mapping is handled here, preserve correctness with a scoped scan.
-+ if self.dataset.frag_reuse_index_uuid().await.is_some() {
-+ return Ok(Some("fragment_reuse"));
-+ }
-+ let Some(filter) = reader.get_expr_filter()? else {
-+ return Ok(Some("no_filter"));
-+ };
-+ let planner = Planner::new(Arc::new(self.dataset.schema().into()));
-+ let index_info = self.dataset.scalar_index_info().await?;
-+ let filter_plan = planner.create_filter_plan(filter, &index_info, true)?;
-+ let Some(search) = filter_plan
-+ .index_query
-+ .as_ref()
-+ .and_then(|expr| driver(expr, &index_meta.name))
-+ else {
-+ return Ok(Some("no_driver"));
-+ };
-+ if search.column != field.name {
-+ return Ok(Some("field_path"));
-+ }
-+ let index = self
-+ .dataset
-+ .open_scalar_index(&search.column, &self.segment_uuid, metrics)
-+ .await?;
-+ if !matches!(index.index_type(), IndexType::BTree | IndexType::Bitmap) {
-+ return Ok(Some("index_type"));
-+ }
-+ // External masks use _rowid, not necessarily physical row addresses.
-+ if index.results_are_row_addresses() && self.dataset.manifest.uses_stable_row_ids() {
-+ return Ok(Some("row_id_domain"));
-+ }
-+ let started = Instant::now();
-+ let result = index.search(search.query.as_ref(), metrics).await?;
-+ stats.all_times.insert(
-+ "scalar_segment_search_time".into(),
-+ started.elapsed().as_nanos().min(usize::MAX as u128) as usize,
-+ );
-+ stats
-+ .all_counts
-+ .insert("scalar_segments_searched".into(), 1);
-+ let SearchResult::Exact(rows) = result else {
-+ return Ok(Some("inexact_result"));
-+ };
-+ stats.all_counts.insert(
-+ "scalar_segment_candidate_rows".into(),
-+ rows.len().unwrap_or(0) as usize,
-+ );
-+ // Do not truncate candidates at LIMIT. The reader evaluates the complete
-+ // filter before applying its existing limit/offset operators.
-+ reader.with_row_addr_prefilter(RowAddrMask::from_allowed(rows.selected_rows().clone()));
-+ Ok(None)
-+ }
-+}
-diff --git a/src/scanner.rs b/src/scanner.rs
-index 4ceeb0e..ac3cff6 100644
---- a/src/scanner.rs
-+++ b/src/scanner.rs
-@@ -39,6 +39,7 @@ use crate::fts_query::{
- };
- use crate::helpers;
- use crate::runtime::{RT, block_on};
-+use crate::scalar_segment::PreparedScalarSegment;
- use crate::stream_guard::GuardedReader;
-
- /// Data type tag for query vectors, mirroring the C enum `LanceDataType`.
-@@ -108,6 +109,7 @@ pub struct LanceScanner {
- include_deleted_rows: bool,
- fragment_ids: Option<Vec<u64>>,
- index_segments: Option<Vec<Uuid>>,
-+ scalar_index_segment: Option<Uuid>,
- nearest: Option<NearestQuery>,
- nprobes: NprobesRange,
- approx_mode: Option<LanceApproxMode>,
-@@ -256,6 +258,7 @@ impl LanceScanner {
- include_deleted_rows: false,
- fragment_ids: None,
- index_segments: None,
-+ scalar_index_segment: None,
- nearest: None,
- nprobes: NprobesRange::default(),
- approx_mode: None,
-@@ -470,12 +473,37 @@ impl LanceScanner {
- None
- };
- self.apply_filter(&mut scanner)?;
-+ let scalar_segment = if let Some(segment_uuid) = self.scalar_index_segment {
-+ if self.nearest.is_some()
-+ || self.fts_query.is_some()
-+ || self.fts_context.is_some()
-+ || self.index_segments.is_some()
-+ || self.fts_index_segments.is_some()
-+ {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "scalar_index_segment requires an ordinary scan of live rows".into(),
-+ ));
-+ }
-+ let fragment_ids = self.fragment_ids.as_ref().filter(|ids| !ids.is_empty())
-+ .ok_or_else(|| lance_core::Error::invalid_input_source(
-+ "scalar_index_segment requires explicit nonempty fragment_ids for its read and fallback domain".into(),
-+ ))?;
-+ Some(PreparedScalarSegment {
-+ dataset: Arc::clone(&self.dataset),
-+ segment_uuid,
-+ fragment_ids: fragment_ids.clone(),
-+ callback: self.scan_statistics_callback.clone(),
-+ })
-+ } else {
-+ None
-+ };
- if let Some(callback) = &self.scan_statistics_callback {
- scanner.scan_stats_callback(callback.clone());
- }
- Ok(PreparedScanner {
- scanner,
- distributed_fts,
-+ scalar_segment,
- })
- }
- }
-@@ -490,10 +518,18 @@ struct PreparedFtsExecution {
- struct PreparedScanner {
- scanner: lance::dataset::scanner::Scanner,
- distributed_fts: Option<PreparedFtsExecution>,
-+ scalar_segment: Option<PreparedScalarSegment>,
- }
-
- impl PreparedScanner {
- async fn try_into_stream(self) -> Result<DatasetRecordBatchStream> {
-+ if let Some(scalar_segment) = self.scalar_segment {
-+ return scalar_segment
-+ .configure(self.scanner)
-+ .await?
-+ .try_into_stream()
-+ .await;
-+ }
- let Some(distributed_fts) = self.distributed_fts else {
- return self.scanner.try_into_stream().await;
- };
-@@ -858,6 +894,31 @@ macro_rules! scanner_ffi_try {
- }};
- }
-
-+/// Select one physical scalar index segment. NULL clears the selection.
-+/// Requires explicit fragment_ids and an ordinary live-row scan. See the C header.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_set_scalar_index_segment(
-+ scanner: *mut LanceScanner,
-+ segment_uuid: *const u8,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, {
-+ let scanner = unsafe { scanner.as_mut() }
-+ .ok_or_else(|| lance_core::Error::invalid_input_source("scanner is NULL".into()))?;
-+ scanner.ensure_scan_not_started("scalar_index_segment")?;
-+ let segment = if segment_uuid.is_null() {
-+ None
-+ } else {
-+ Some(
-+ Uuid::from_slice(unsafe { std::slice::from_raw_parts(segment_uuid, 16) })
-+ .map_err(|e| lance_core::Error::invalid_input_source(e.into()))?,
-+ )
-+ };
-+ scanner.scalar_index_segment = segment;
-+ Ok(0)
-+ })
-+}
-+
- // ---------------------------------------------------------------------------
- // Scanner lifecycle + builder
- // ---------------------------------------------------------------------------
-diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index 3b3424b..a8a7eab 100644
---- a/tests/c_api_test.rs
-+++ b/tests/c_api_test.rs
-@@ -12513,3 +12513,252 @@ fn test_add_columns_stream_null_dataset_consumes_stream() {
- assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument);
- assert_stream_consumed(&stream, &drop_count);
- }
-+
-+// Segment scans deliberately use an unprojected nullable key and a residual
-+// predicate so a candidate LIMIT or loss of filter columns changes the answer.
-+fn create_scalar_segment_fixture(
-+ kind: lance_index::IndexType,
-+ stable: bool,
-+) -> (tempfile::TempDir, String, Vec<[u8; 16]>) {
-+ use lance::dataset::WriteParams;
-+ use lance::index::DatasetIndexExt;
-+ use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
-+ let tmp = tempfile::tempdir().unwrap();
-+ let uri = tmp.path().join("segments").to_str().unwrap().to_owned();
-+ let uuids = lance_c::runtime::block_on(async {
-+ let schema = Arc::new(Schema::new(vec![
-+ Field::new("id", DataType::Int32, false),
-+ Field::new("key", DataType::Int32, true),
-+ ]));
-+ let batch = RecordBatch::try_new(
-+ schema.clone(),
-+ vec![
-+ Arc::new(Int32Array::from_iter_values(0..12)),
-+ Arc::new(Int32Array::from(
-+ (0..12)
-+ .map(|id| if id % 4 == 0 { None } else { Some(id % 3) })
-+ .collect::<Vec<_>>(),
-+ )),
-+ ],
-+ )
-+ .unwrap();
-+ let mut ds = Dataset::write(
-+ arrow::record_batch::RecordBatchIterator::new(vec![Ok(batch)], schema),
-+ &uri,
-+ Some(WriteParams {
-+ max_rows_per_file: 4,
-+ enable_stable_row_ids: stable,
-+ ..Default::default()
-+ }),
-+ )
-+ .await
-+ .unwrap();
-+ let params = ScalarIndexParams::for_builtin(if kind == lance_index::IndexType::Bitmap {
-+ BuiltinIndexType::Bitmap
-+ } else {
-+ BuiltinIndexType::BTree
-+ });
-+ let fragments = ds.get_fragments();
-+ assert_eq!(fragments.len(), 3);
-+ let mut segments = Vec::new();
-+ for fragment in fragments.iter().take(2) {
-+ segments.push(
-+ ds.create_index_builder(&["key"], kind, ¶ms)
-+ .name("key_idx".into())
-+ .fragments(vec![fragment.id() as u32])
-+ .execute_uncommitted()
-+ .await
-+ .unwrap(),
-+ );
-+ }
-+ let uuids = segments.iter().map(|s| *s.uuid.as_bytes()).collect();
-+ ds.commit_existing_index_segments("key_idx", "key", segments)
-+ .await
-+ .unwrap();
-+ uuids
-+ });
-+ (tmp, uri, uuids)
-+}
-+
-+fn scalar_segment_ids(
-+ uri: &str,
-+ uuid: &[u8; 16],
-+ fragments: &[u64],
-+ filter: &str,
-+ limit: Option<i64>,
-+ offset: i64,
-+) -> (Vec<i32>, CapturedScanStatistics) {
-+ let uri = c_str(uri);
-+ let filter = c_str(filter);
-+ let id = c_str("id");
-+ let columns = [id.as_ptr(), ptr::null()];
-+ let mut captured = CapturedScanStatistics::default();
-+ let mut ids = Vec::new();
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ assert!(!ds.is_null());
-+ let scanner = lance_scanner_new(ds, columns.as_ptr(), filter.as_ptr());
-+ assert!(!scanner.is_null());
-+ assert_eq!(
-+ lance_scanner_set_fragment_ids(scanner, fragments.as_ptr(), fragments.len()),
-+ 0
-+ );
-+ assert_eq!(
-+ lance_scanner_set_scalar_index_segment(scanner, uuid.as_ptr()),
-+ 0
-+ );
-+ if let Some(limit) = limit {
-+ assert_eq!(lance_scanner_set_limit(scanner, limit), 0);
-+ }
-+ assert_eq!(lance_scanner_set_offset(scanner, offset), 0);
-+ assert_eq!(
-+ lance_scanner_set_statistics_callback(
-+ scanner,
-+ Some(capture_scan_statistics),
-+ (&mut captured as *mut CapturedScanStatistics).cast()
-+ ),
-+ 0
-+ );
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ let rc = lance_scanner_to_arrow_stream(scanner, &mut stream);
-+ assert_eq!(
-+ rc,
-+ 0,
-+ "{}",
-+ if rc != 0 {
-+ take_last_error_message()
-+ } else {
-+ String::new()
-+ }
-+ );
-+ assert_eq!(
-+ lance_scanner_set_scalar_index_segment(scanner, ptr::null()),
-+ -1
-+ );
-+ {
-+ let reader = ArrowArrayStreamReader::from_raw(&mut stream).unwrap();
-+ for batch in reader {
-+ let batch = batch.unwrap();
-+ assert_eq!(batch.num_columns(), 1);
-+ ids.extend(
-+ batch
-+ .column(0)
-+ .as_any()
-+ .downcast_ref::<Int32Array>()
-+ .unwrap()
-+ .values()
-+ .iter()
-+ .copied(),
-+ );
-+ }
-+ }
-+ lance_scanner_close(scanner);
-+ lance_dataset_close(ds);
-+ }
-+ (ids, captured)
-+}
-+
-+#[test]
-+fn test_scalar_segment_scope_residual_limit_and_unindexed_fallback() {
-+ for kind in [
-+ lance_index::IndexType::BTree,
-+ lance_index::IndexType::Bitmap,
-+ ] {
-+ let (_tmp, uri, uuids) = create_scalar_segment_fixture(kind, false);
-+ let (ids, stats) =
-+ scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0);
-+ assert_eq!(ids, vec![2, 3]);
-+ assert_eq!(stats.calls, 1);
-+ assert!(
-+ stats
-+ .metrics
-+ .iter()
-+ .any(|(name, _, value)| name == "scalar_segments_searched" && *value == 1)
-+ );
-+ let (ids, _) =
-+ scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", Some(1), 1);
-+ assert_eq!(
-+ ids,
-+ vec![3],
-+ "offset and limit must apply after residual filtering"
-+ );
-+ let (ids, _) = scalar_segment_ids(&uri, &uuids[1], &[1], "key >= 0 AND id >= 2", None, 0);
-+ assert_eq!(ids, vec![5, 6, 7]);
-+ let (ids, stats) =
-+ scalar_segment_ids(&uri, &uuids[0], &[0, 2], "key >= 0 AND id >= 2", None, 0);
-+ assert_eq!(
-+ ids,
-+ vec![2, 3, 9, 10, 11],
-+ "partial coverage must not omit unindexed rows"
-+ );
-+ assert!(
-+ stats
-+ .metrics
-+ .iter()
-+ .any(|(name, _, _)| name == "scalar_segment_fallback_partial_coverage")
-+ );
-+ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0], "key = 99 OR id = 0", None, 0);
-+ assert_eq!(
-+ ids,
-+ vec![0],
-+ "OR must not use just one branch as candidates"
-+ );
-+ assert!(
-+ stats
-+ .metrics
-+ .iter()
-+ .any(|(name, _, _)| name == "scalar_segment_fallback_no_driver")
-+ );
-+ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0], "key = 99", None, 0);
-+ assert!(ids.is_empty());
-+ }
-+}
-+
-+#[test]
-+fn test_scalar_segment_stable_row_ids_and_deletes() {
-+ use lance::index::DatasetIndexExt;
-+ let (_tmp, uri, uuids) = create_scalar_segment_fixture(lance_index::IndexType::BTree, true);
-+ lance_c::runtime::block_on(async {
-+ let mut ds = Dataset::open(&uri).await.unwrap();
-+ ds.delete("id = 2").await.unwrap();
-+ assert_eq!(ds.load_indices().await.unwrap().len(), 2);
-+ });
-+ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0);
-+ assert_eq!(ids, vec![3]);
-+}
-+
-+#[test]
-+fn test_scalar_segment_requires_explicit_domain_and_checks_uuid() {
-+ let (_tmp, uri, uuids) = create_scalar_segment_fixture(lance_index::IndexType::BTree, false);
-+ let uri = c_str(&uri);
-+ let filter = c_str("key >= 0");
-+ unsafe {
-+ assert_eq!(
-+ lance_scanner_set_scalar_index_segment(ptr::null_mut(), ptr::null()),
-+ -1
-+ );
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ let scanner = lance_scanner_new(ds, ptr::null(), filter.as_ptr());
-+ assert_eq!(
-+ lance_scanner_set_scalar_index_segment(scanner, uuids[0].as_ptr()),
-+ 0
-+ );
-+ let mut batch = ptr::null_mut();
-+ assert_eq!(lance_scanner_next(scanner, &mut batch), -1);
-+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument);
-+ lance_scanner_close(scanner);
-+ let scanner = lance_scanner_new(ds, ptr::null(), filter.as_ptr());
-+ assert_eq!(
-+ lance_scanner_set_fragment_ids(scanner, [0u64].as_ptr(), 1),
-+ 0
-+ );
-+ assert_eq!(
-+ lance_scanner_set_scalar_index_segment(scanner, [0u8; 16].as_ptr()),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_next(scanner, &mut batch), -1);
-+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument);
-+ lance_scanner_close(scanner);
-+ lance_dataset_close(ds);
-+ }
-+}
-
-From 17240674d739293a494c07dfe5aec92a97185363 Mon Sep 17 00:00:00 2001
-From: zhangstar333 <zh...@selectdb.com>
-Date: Tue, 8 Sep 2026 23:17:23 +0800
-Subject: [PATCH 2/5] update
-
----
- docs/scalar-segment-scans.md | 3 ++
- include/lance/lance.h | 4 +--
- src/scalar_segment.rs | 7 +++++
- tests/c_api_test.rs | 60 ++++++++++++++++++++++++++++++++++--
- 4 files changed, 70 insertions(+), 4 deletions(-)
-
-diff --git a/docs/scalar-segment-scans.md b/docs/scalar-segment-scans.md
-index dc7c141..f72dbfc 100644
---- a/docs/scalar-segment-scans.md
-+++ b/docs/scalar-segment-scans.md
-@@ -36,6 +36,9 @@ An unknown UUID, absent fragment or invalid option combination is an error. A
- known segment with incomplete/unknown coverage, no suitable driver, unsupported
- index type, nested key, overlays, fragment reuse, non-exact results or unsupported
- row-ID domain falls back to a non-indexed scan of the entire explicit domain.
-+Legacy (v1) storage also takes this fallback because ordinary scans cannot consume
-+external row masks; it reports `scalar_segment_fallback_legacy_storage` without
-+searching the index.
- I/O and corruption errors are propagated, not converted to empty results or
- successful fallback.
-
-diff --git a/include/lance/lance.h b/include/lance/lance.h
-index cbd8330..ab15247 100644
---- a/include/lance/lance.h
-+++ b/include/lance/lance.h
-@@ -1870,8 +1870,8 @@ int32_t lance_scanner_set_index_segments(
- *
- * BTree/Bitmap searches use a necessary AND-conjunct of the full scanner filter
- * on the selected logical index. All predicates are reapplied during candidate
-- * reads; other scalar indices are disabled. OR/NOT-only filters, overlays,
-- * fragment reuse, unsupported index types / result domains
-+ * reads; other scalar indices are disabled. Legacy storage, OR/NOT-only filters,
-+ * overlays, fragment reuse, unsupported index types / result domains
- * and missing coverage use the same domain without an index. No filter also
- * falls back. LIMIT/OFFSET apply after the complete scanner filter, never to the
- * unfiltered candidate set. Vector/FTS queries are rejected.
-diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs
-index 2ab1297..5e121c9 100644
---- a/src/scalar_segment.rs
-+++ b/src/scalar_segment.rs
-@@ -138,6 +138,13 @@ impl PreparedScalarSegment {
- self.dataset.schema().field_by_id(field_id).ok_or_else(|| {
- invalid("scalar segment key field is absent from the dataset schema")
- })?;
-+ // Match Lance's plain-scan external-mask restriction. Keep the scoped,
-+ // full-filtered reader intact and avoid index work on legacy storage.
-+ if self.dataset.manifest().data_storage_format.lance_file_format()
-+ == lance_file::version::ConcreteFileVersion::V1
-+ {
-+ return Ok(Some("legacy_storage"));
-+ }
- // Keep V1 to flat scalar fields. A dotted name is not sufficient to prove
- // the field path of an evolved or nested schema.
- if !self
-diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index a8a7eab..0cb998b 100644
---- a/tests/c_api_test.rs
-+++ b/tests/c_api_test.rs
-@@ -12519,6 +12519,15 @@ fn test_add_columns_stream_null_dataset_consumes_stream() {
- fn create_scalar_segment_fixture(
- kind: lance_index::IndexType,
- stable: bool,
-+) -> (tempfile::TempDir, String, Vec<[u8; 16]>) {
-+ create_scalar_segment_fixture_with_options(kind, stable, None, &[&[0], &[1]])
-+}
-+
-+fn create_scalar_segment_fixture_with_options(
-+ kind: lance_index::IndexType,
-+ stable: bool,
-+ storage_version: Option<lance_file::version::LanceFileVersion>,
-+ segment_fragments: &[&[u32]],
- ) -> (tempfile::TempDir, String, Vec<[u8; 16]>) {
- use lance::dataset::WriteParams;
- use lance::index::DatasetIndexExt;
-@@ -12548,6 +12557,7 @@ fn create_scalar_segment_fixture(
- Some(WriteParams {
- max_rows_per_file: 4,
- enable_stable_row_ids: stable,
-+ data_storage_version: storage_version,
- ..Default::default()
- }),
- )
-@@ -12561,11 +12571,11 @@ fn create_scalar_segment_fixture(
- let fragments = ds.get_fragments();
- assert_eq!(fragments.len(), 3);
- let mut segments = Vec::new();
-- for fragment in fragments.iter().take(2) {
-+ for fragment_ids in segment_fragments {
- segments.push(
- ds.create_index_builder(&["key"], kind, ¶ms)
- .name("key_idx".into())
-- .fragments(vec![fragment.id() as u32])
-+ .fragments(fragment_ids.to_vec())
- .execute_uncommitted()
- .await
- .unwrap(),
-@@ -12714,6 +12724,52 @@ fn test_scalar_segment_scope_residual_limit_and_unindexed_fallback() {
- }
- }
-
-+#[test]
-+fn test_scalar_segment_legacy_storage_falls_back() {
-+ use lance_file::version::{ConcreteFileVersion, LanceFileVersion};
-+
-+ // Three fragments, with one segment covering 0 and 1. Reading only fragment
-+ // 0 must retain the full predicate and must not leak rows from fragment 1.
-+ let (_tmp, uri, uuids) = create_scalar_segment_fixture_with_options(
-+ lance_index::IndexType::BTree,
-+ false,
-+ Some(LanceFileVersion::Legacy),
-+ &[&[0, 1]],
-+ );
-+ assert_eq!(uuids.len(), 1);
-+ lance_c::runtime::block_on(async {
-+ let ds = Dataset::open(&uri).await.unwrap();
-+ assert_eq!(
-+ ds.manifest().data_storage_format.lance_file_format(),
-+ ConcreteFileVersion::V1
-+ );
-+ });
-+
-+ let (ids, stats) =
-+ scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0);
-+ assert_eq!(ids, vec![2, 3]);
-+ assert_eq!(stats.calls, 1);
-+ assert_eq!(stats.indices_loaded, 0);
-+ assert_eq!(stats.index_comparisons, 0);
-+ assert!(stats.metrics.iter().any(|(name, _, value)| {
-+ name == "scalar_segment_fallback_legacy_storage" && *value == 1
-+ }));
-+ assert!(
-+ !stats
-+ .metrics
-+ .iter()
-+ .any(|(name, _, value)| name == "scalar_segments_searched" && *value != 0)
-+ );
-+
-+ let (ids, _) =
-+ scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", Some(1), 1);
-+ assert_eq!(
-+ ids,
-+ vec![3],
-+ "fallback must retain LIMIT/OFFSET after filtering"
-+ );
-+}
-+
- #[test]
- fn test_scalar_segment_stable_row_ids_and_deletes() {
- use lance::index::DatasetIndexExt;
-
-From f9ce263e4e32540c26afa6ccd18021ca81c090f1 Mon Sep 17 00:00:00 2001
-From: zhangstar333 <zh...@selectdb.com>
-Date: Tue, 8 Sep 2026 23:21:02 +0800
-Subject: [PATCH 3/5] update
-
----
- src/scalar_segment.rs | 6 +++++-
- tests/c_api_test.rs | 6 ++----
- 2 files changed, 7 insertions(+), 5 deletions(-)
-
-diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs
-index 5e121c9..01df58b 100644
---- a/src/scalar_segment.rs
-+++ b/src/scalar_segment.rs
-@@ -140,7 +140,11 @@ impl PreparedScalarSegment {
- })?;
- // Match Lance's plain-scan external-mask restriction. Keep the scoped,
- // full-filtered reader intact and avoid index work on legacy storage.
-- if self.dataset.manifest().data_storage_format.lance_file_format()
-+ if self
-+ .dataset
-+ .manifest()
-+ .data_storage_format
-+ .lance_file_format()
- == lance_file::version::ConcreteFileVersion::V1
- {
- return Ok(Some("legacy_storage"));
-diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index 0cb998b..efd4284 100644
---- a/tests/c_api_test.rs
-+++ b/tests/c_api_test.rs
-@@ -12745,8 +12745,7 @@ fn test_scalar_segment_legacy_storage_falls_back() {
- );
- });
-
-- let (ids, stats) =
-- scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0);
-+ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0);
- assert_eq!(ids, vec![2, 3]);
- assert_eq!(stats.calls, 1);
- assert_eq!(stats.indices_loaded, 0);
-@@ -12761,8 +12760,7 @@ fn test_scalar_segment_legacy_storage_falls_back() {
- .any(|(name, _, value)| name == "scalar_segments_searched" && *value != 0)
- );
-
-- let (ids, _) =
-- scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", Some(1), 1);
-+ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", Some(1), 1);
- assert_eq!(
- ids,
- vec![3],
-
-From 221d8800f56790ab8b48b12f3aaaf9da06359380 Mon Sep 17 00:00:00 2001
-From: zhangstar333 <zh...@selectdb.com>
-Date: Wed, 9 Sep 2026 12:59:08 +0800
-Subject: [PATCH 4/5] add LabelList
-
----
- docs/scalar-segment-scans.md | 12 ++-
- include/lance/lance.h | 8 +-
- include/lance/lance.hpp | 4 +-
- src/scalar_segment.rs | 7 +-
- tests/c_api_test.rs | 190 ++++++++++++++++++++++++++++++++---
- 5 files changed, 200 insertions(+), 21 deletions(-)
-
-diff --git a/docs/scalar-segment-scans.md b/docs/scalar-segment-scans.md
-index f72dbfc..72f6188 100644
---- a/docs/scalar-segment-scans.md
-+++ b/docs/scalar-segment-scans.md
-@@ -1,10 +1,18 @@
- # Scalar index segment scans
-
--An ordinary scanner can use one physical BTree/Bitmap segment to generate
--candidates, then read those candidates with the complete scanner filter. This
-+An ordinary scanner can use one physical BTree, Bitmap, or LabelList segment to
-+generate candidates, then read them with the complete scanner filter. This
- does not run a global search of the other segments of the logical index. It does
- not subdivide a physical segment or make its own index search incremental.
-
-+LabelList supports indexed array membership predicates. Every candidate search
-+must return `SearchResult::Exact`.
-+LabelList query values should match the array element type, for example
-+`array_contains(int32_labels, CAST(42 AS INT))`; a cast on the indexed column
-+can prevent the planner from finding an index driver and cause fallback.
-+`AtMost` and `AtLeast` results still fall back; this mode does not enable FMIndex,
-+NGram, BloomFilter, ZoneMap, or Inverted indices.
-+
- ## Configuring a task
-
- Open a fixed dataset version. Select the physical index UUID from that version's
-diff --git a/include/lance/lance.h b/include/lance/lance.h
-index ab15247..b5c6902 100644
---- a/include/lance/lance.h
-+++ b/include/lance/lance.h
-@@ -1868,9 +1868,11 @@ int32_t lance_scanner_set_index_segments(
- * scan of those fragment_ids. Callers distributing work must assign disjoint
- * fragment domains and separately include any unindexed data they wish to read.
- *
-- * BTree/Bitmap searches use a necessary AND-conjunct of the full scanner filter
-- * on the selected logical index. All predicates are reapplied during candidate
-- * reads; other scalar indices are disabled. Legacy storage, OR/NOT-only filters,
-+ * BTree/Bitmap/LabelList searches use a necessary AND-conjunct of the
-+ * full scanner filter on the selected logical index and require an Exact result.
-+ * AtMost/AtLeast results fall back to a full filtered scan of fragment_ids.
-+ * All predicates are reapplied during candidate reads; other scalar indices
-+ * are disabled. Legacy storage, OR/NOT-only filters,
- * overlays, fragment reuse, unsupported index types / result domains
- * and missing coverage use the same domain without an index. No filter also
- * falls back. LIMIT/OFFSET apply after the complete scanner filter, never to the
-diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp
-index 8e1c6cf..ebb8141 100644
---- a/include/lance/lance.hpp
-+++ b/include/lance/lance.hpp
-@@ -1311,8 +1311,8 @@ class Scanner {
- return *this;
- }
-
-- /// Restrict scalar candidate generation to one segment; fragment_ids is
-- /// required and defines the complete read/fallback domain. See lance.h.
-+ /// Generate exact candidates from one BTree/Bitmap/LabelList segment.
-+ /// fragment_ids is required and defines the complete read/fallback domain. See lance.h.
- Scanner& scalar_index_segment(const std::array<uint8_t, 16>& segment_uuid) {
- if (lance_scanner_set_scalar_index_segment(handle_.get(), segment_uuid.data()) != 0)
- check_error();
-diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs
-index 01df58b..1faf500 100644
---- a/src/scalar_segment.rs
-+++ b/src/scalar_segment.rs
-@@ -204,7 +204,12 @@ impl PreparedScalarSegment {
- .dataset
- .open_scalar_index(&search.column, &self.segment_uuid, metrics)
- .await?;
-- if !matches!(index.index_type(), IndexType::BTree | IndexType::Bitmap) {
-+ // These implementations can return exact candidates. Keep the runtime
-+ // Exact check below: a type alone is not a guarantee for every query.
-+ if !matches!(
-+ index.index_type(),
-+ IndexType::BTree | IndexType::Bitmap | IndexType::LabelList
-+ ) {
- return Ok(Some("index_type"));
- }
- // External masks use _rowid, not necessarily physical row addresses.
-diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index efd4284..3322d42 100644
---- a/tests/c_api_test.rs
-+++ b/tests/c_api_test.rs
-@@ -12528,6 +12528,21 @@ fn create_scalar_segment_fixture_with_options(
- stable: bool,
- storage_version: Option<lance_file::version::LanceFileVersion>,
- segment_fragments: &[&[u32]],
-+) -> (tempfile::TempDir, String, Vec<[u8; 16]>) {
-+ let key = Arc::new(Int32Array::from(
-+ (0..12)
-+ .map(|id| if id % 4 == 0 { None } else { Some(id % 3) })
-+ .collect::<Vec<_>>(),
-+ ));
-+ create_scalar_segment_fixture_from_key(kind, stable, storage_version, segment_fragments, key)
-+}
-+
-+fn create_scalar_segment_fixture_from_key(
-+ kind: lance_index::IndexType,
-+ stable: bool,
-+ storage_version: Option<lance_file::version::LanceFileVersion>,
-+ segment_fragments: &[&[u32]],
-+ key: arrow_array::ArrayRef,
- ) -> (tempfile::TempDir, String, Vec<[u8; 16]>) {
- use lance::dataset::WriteParams;
- use lance::index::DatasetIndexExt;
-@@ -12537,17 +12552,14 @@ fn create_scalar_segment_fixture_with_options(
- let uuids = lance_c::runtime::block_on(async {
- let schema = Arc::new(Schema::new(vec![
- Field::new("id", DataType::Int32, false),
-- Field::new("key", DataType::Int32, true),
-+ Field::new("key", key.data_type().clone(), true),
- ]));
-+ let row_count = key.len();
- let batch = RecordBatch::try_new(
- schema.clone(),
- vec![
-- Arc::new(Int32Array::from_iter_values(0..12)),
-- Arc::new(Int32Array::from(
-- (0..12)
-- .map(|id| if id % 4 == 0 { None } else { Some(id % 3) })
-- .collect::<Vec<_>>(),
-- )),
-+ Arc::new(Int32Array::from_iter_values(0..row_count as i32)),
-+ key,
- ],
- )
- .unwrap();
-@@ -12563,13 +12575,9 @@ fn create_scalar_segment_fixture_with_options(
- )
- .await
- .unwrap();
-- let params = ScalarIndexParams::for_builtin(if kind == lance_index::IndexType::Bitmap {
-- BuiltinIndexType::Bitmap
-- } else {
-- BuiltinIndexType::BTree
-- });
-+ let params = ScalarIndexParams::for_builtin(BuiltinIndexType::try_from(kind).unwrap());
- let fragments = ds.get_fragments();
-- assert_eq!(fragments.len(), 3);
-+ assert_eq!(fragments.len(), row_count.div_ceil(4));
- let mut segments = Vec::new();
- for fragment_ids in segment_fragments {
- segments.push(
-@@ -12724,6 +12732,162 @@ fn test_scalar_segment_scope_residual_limit_and_unindexed_fallback() {
- }
- }
-
-+#[test]
-+fn test_scalar_segment_label_list_exact_candidates() {
-+ use arrow_array::builder::{Int32Builder, ListBuilder};
-+ use lance::index::DatasetIndexExt;
-+ use lance_index::IndexType;
-+
-+ for stable in [false, true] {
-+ let mut lists = ListBuilder::new(Int32Builder::new());
-+ for row in 0..16 {
-+ match row {
-+ 0 | 9 | 13 => lists.append(false),
-+ 1 | 10 | 14 => lists.append(true),
-+ 4 => {
-+ lists.values().append_value(7);
-+ lists.append(true);
-+ }
-+ _ => {
-+ lists.values().append_value(42);
-+ if row == 3 || row == 11 || row == 15 {
-+ lists.values().append_value(7);
-+ }
-+ if row == 6 {
-+ lists.values().append_null();
-+ }
-+ lists.append(true);
-+ }
-+ }
-+ }
-+ // S0 covers fragments 0 and 1, S1 covers 2, and 3 is unindexed.
-+ let (_tmp, uri, uuids) = create_scalar_segment_fixture_from_key(
-+ IndexType::LabelList,
-+ stable,
-+ None,
-+ &[&[0, 1], &[2]],
-+ Arc::new(lists.finish()),
-+ );
-+ let predicate = "array_contains(key, CAST(42 AS INT))";
-+ let filter = format!("{predicate} AND id >= 3");
-+ for (fragments, expected) in [(vec![0, 1], vec![3, 5, 6, 7]), (vec![0], vec![3])] {
-+ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &fragments, &filter, None, 0);
-+ assert_eq!(ids, expected, "stable={stable}");
-+ assert_eq!(stats.calls, 1);
-+ assert!(
-+ stats
-+ .metrics
-+ .iter()
-+ .any(|(name, _, value)| { name == "scalar_segments_searched" && *value == 1 }),
-+ "stable={stable}, metrics={:?}",
-+ stats.metrics
-+ );
-+ assert!(
-+ !stats
-+ .metrics
-+ .iter()
-+ .any(|(name, _, value)| { name == "scalar_segment_fallbacks" && *value != 0 })
-+ );
-+ }
-+ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0, 1], &filter, Some(1), 1);
-+ assert_eq!(ids, vec![5], "limit/offset must follow the residual filter");
-+ let (ids, _) = scalar_segment_ids(&uri, &uuids[1], &[2], &filter, None, 0);
-+ assert_eq!(ids, vec![8, 11]);
-+ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0, 3], &filter, None, 0);
-+ assert_eq!(ids, vec![3, 12, 15]);
-+ assert!(stats.metrics.iter().any(|(name, _, value)| {
-+ name == "scalar_segment_fallback_partial_coverage" && *value == 1
-+ }));
-+ let (ids, stats) = scalar_segment_ids(
-+ &uri,
-+ &uuids[0],
-+ &[0],
-+ &format!("{predicate} OR id = 0"),
-+ None,
-+ 0,
-+ );
-+ assert_eq!(ids, vec![0, 2, 3]);
-+ assert!(stats.metrics.iter().any(|(name, _, value)| {
-+ name == "scalar_segment_fallback_no_driver" && *value == 1
-+ }));
-+
-+ for (predicate, expected) in [
-+ (
-+ "array_has_all(key, [CAST(42 AS INT), CAST(7 AS INT)])",
-+ vec![3],
-+ ),
-+ (
-+ "array_has_any(key, [CAST(42 AS INT), CAST(99 AS INT)])",
-+ vec![2, 3, 5, 6, 7],
-+ ),
-+ ("array_contains(key, CAST(99 AS INT))", vec![]),
-+ ("array_contains(key, CAST(NULL AS INT))", vec![]),
-+ ("array_has_any(key, [])", vec![]),
-+ ] {
-+ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0, 1], predicate, None, 0);
-+ assert_eq!(ids, expected, "{predicate}, stable={stable}");
-+ }
-+ // An untyped integer literal casts this Int32 list to Int64. Such
-+ // a column expression must retain the scan fallback.
-+ let (ids, stats) =
-+ scalar_segment_ids(&uri, &uuids[0], &[0, 1], "array_contains(key, 42)", None, 0);
-+ assert_eq!(ids, vec![2, 3, 5, 6, 7]);
-+ assert!(stats.metrics.iter().any(|(name, _, value)| {
-+ name == "scalar_segment_fallback_no_driver" && *value == 1
-+ }));
-+
-+ lance_c::runtime::block_on(async {
-+ let mut ds = Dataset::open(&uri).await.unwrap();
-+ ds.delete("id = 3").await.unwrap();
-+ assert_eq!(ds.load_indices().await.unwrap().len(), 2);
-+ });
-+ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0, 1], &filter, None, 0);
-+ assert_eq!(ids, vec![5, 6, 7]);
-+ }
-+}
-+
-+#[test]
-+fn test_scalar_segment_text_indices_still_fall_back() {
-+ for kind in [lance_index::IndexType::Fm, lance_index::IndexType::NGram] {
-+ for stable in [false, true] {
-+ let key = Arc::new(StringArray::from(vec![
-+ Some("needle"),
-+ None,
-+ Some(""),
-+ Some("other"),
-+ Some("needle"),
-+ Some("other"),
-+ Some(""),
-+ None,
-+ ]));
-+ let (_tmp, uri, uuids) =
-+ create_scalar_segment_fixture_from_key(kind, stable, None, &[&[0, 1]], key);
-+ for (predicate, expected) in [
-+ ("contains(key, 'needle')", vec![0]),
-+ ("contains(key, '')", vec![0, 2, 3]),
-+ ] {
-+ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0], predicate, None, 0);
-+ assert_eq!(ids, expected, "{kind:?}, stable={stable}, {predicate}");
-+ assert!(
-+ stats.metrics.iter().any(|(name, _, value)| {
-+ name == "scalar_segment_fallbacks" && *value == 1
-+ })
-+ );
-+ if predicate == "contains(key, 'needle')" {
-+ assert!(stats.metrics.iter().any(|(name, _, value)| {
-+ name == "scalar_segment_fallback_index_type" && *value == 1
-+ }));
-+ }
-+ assert!(
-+ !stats.metrics.iter().any(|(name, _, value)| {
-+ name == "scalar_segments_searched" && *value != 0
-+ })
-+ );
-+ }
-+ }
-+ }
-+}
-+
- #[test]
- fn test_scalar_segment_legacy_storage_falls_back() {
- use lance_file::version::{ConcreteFileVersion, LanceFileVersion};
-
-From 30dda06a1cc6bad37d04ed207115409ccef91715 Mon Sep 17 00:00:00 2001
-From: zhangstar333 <zh...@selectdb.com>
-Date: Wed, 9 Sep 2026 13:27:54 +0800
-Subject: [PATCH 5/5] update
-
----
- docs/scalar-segment-scans.md | 55 +++++++++--
- include/lance/lance.h | 16 ++-
- include/lance/lance.hpp | 5 +
- src/scalar_segment.rs | 12 ++-
- src/scanner.rs | 13 ++-
- tests/c_api_test.rs | 187 +++++++++++++++++++++++++++++++++++
- 6 files changed, 272 insertions(+), 16 deletions(-)
-
-diff --git a/docs/scalar-segment-scans.md b/docs/scalar-segment-scans.md
-index 72f6188..efd0498 100644
---- a/docs/scalar-segment-scans.md
-+++ b/docs/scalar-segment-scans.md
-@@ -27,11 +27,19 @@ lance_scanner_set_limit(scanner, 20000);
- /* The scanner-owning thread calls lance_scanner_next as usual. */
- ```
-
-+The segment setter copies the UUID; passing NULL clears it. Final option
-+compatibility and snapshot metadata are checked when preparing the stream, not
-+by the setter. Configure all options before the first `next`, Arrow stream
-+export, or asynchronous scan. A failed preparation also freezes the options;
-+create a new scanner to retry with different settings.
-+
- SQL, Substrait and additional SQL filters keep their existing precedence and AND
- composition. The caller does not supply a separate driver predicate: Lance-C
- uses the typed filter planner and selects a necessary indexed leaf belonging to
- the requested logical index. It only descends through AND, never through OR or
--NOT. It then searches the selected UUID and applies the complete filter while
-+NOT, and chooses the first matching leaf in the planner's expression tree;
-+this is not a selectivity-based choice or a guarantee of SQL text order.
-+It then searches the selected UUID and applies the complete filter while
- reading candidates with automatic scalar-index planning disabled.
-
- Each task's fragment IDs define its result domain, including on fallback. A
-@@ -40,9 +48,16 @@ scan. Unindexed fragments need their own tasks, or an explicit domain including
- them (which causes that task to use fallback). Merely listing indexed segments
- does not include appended, unindexed data automatically.
-
--An unknown UUID, absent fragment or invalid option combination is an error. A
--known segment with incomplete/unknown coverage, no suitable driver, unsupported
--index type, nested key, overlays, fragment reuse, non-exact results or unsupported
-+`use_scalar_index=false` disables segment search regardless of setter order.
-+The scanner validates the selected snapshot UUID and fragment domain, then scans
-+that domain with the full filter and LIMIT/OFFSET without opening the index or
-+generating candidates. It reports `scalar_segment_fallback_disabled`.
-+
-+An unknown UUID, absent fragment, invalid option combination or segment metadata
-+without one valid schema key field is an error. A known segment with
-+incomplete/unknown coverage, no suitable driver, unsupported
-+index type, nested key, overlays, unknown physical row counts, fragment reuse,
-+non-exact results or unsupported
- row-ID domain falls back to a non-indexed scan of the entire explicit domain.
- Legacy (v1) storage also takes this fallback because ordinary scans cannot consume
- external row masks; it reports `scalar_segment_fallback_legacy_storage` without
-@@ -50,8 +65,15 @@ searching the index.
- I/O and corruption errors are propagated, not converted to empty results or
- successful fallback.
-
--The first implementation supports live-row ordinary scans and cannot be combined
--with vector/FTS queries. Physical row-address
-+The first implementation supports live-row ordinary scans and rejects vector/FTS
-+queries and `include_deleted_rows=true` at stream creation, even when
-+`use_scalar_index=false`. An index built after a delete does not contain the
-+tombstoned rows, so even exact segment candidates
-+cannot satisfy a scan that includes deleted rows. To read those rows, clear the
-+segment setting and use an ordinary scan with `with_row_id=true`,
-+`include_deleted_rows=true`, and `use_scalar_index=false`.
-+Fragments removed from the current snapshot are not scanned by this option.
-+Physical row-address
- results on stable-row-ID datasets currently fall back; results already expressed
- in the correct row-ID domain use the candidate path. Deletes and all remaining
- predicates are handled by the ordinary reader. No candidate-count limit is
-@@ -84,6 +106,23 @@ Successful exhaustion merges segment-search metrics into the existing statistics
- callback exactly once. New metrics include `scalar_segments_requested`,
- `scalar_segments_searched`, `scalar_segment_candidate_rows`,
- `scalar_segment_prepare_time`, `scalar_segment_search_time`, and
--`scalar_segment_fallback_*` reasons. `prepare_time` includes search time. Early
--release, cancellation and errors retain the existing callback contract: final
-+`scalar_segment_fallbacks` plus `scalar_segment_fallback_*` reasons.
-+`scalar_segment_prepare_time` includes search time. Metrics describe each
-+successfully exhausted stream, including separately exported streams:
-+
-+- `scalar_segments_requested` is 1 even on fallback.
-+- `scalar_segments_searched` is 1 after a completed index search, including one
-+ whose inexact result causes fallback. A fallback can therefore include index work.
-+- `scalar_segment_candidate_rows` counts TRUE rows in the exact segment result before fragment
-+ restriction, residual filtering, deletion handling and LIMIT/OFFSET. It is not
-+ the output or physical-read row count; a result without a known cardinality is
-+ reported as 0. The reader's mask may also include NULL candidates that the full
-+ filter subsequently discards.
-+- A fallback records one reason, the first eligibility check that fails. Disabled
-+ scalar indices and legacy storage bypass index opening and search after snapshot
-+ validation. Other fallback reasons may be found after opening or searching an index.
-+- Search and candidate metrics may be absent when their stage did not execute;
-+ consumers should treat absent counts as 0.
-+
-+Early release, cancellation and errors retain the existing callback contract: final
- statistics are not guaranteed. Metrics do not establish global task concurrency.
-diff --git a/include/lance/lance.h b/include/lance/lance.h
-index b5c6902..ed5bb6c 100644
---- a/include/lance/lance.h
-+++ b/include/lance/lance.h
-@@ -1015,7 +1015,9 @@ int32_t lance_scanner_set_scan_in_order(LanceScanner* scanner, bool scan_in_orde
- * Configure whether scalar indices may be used to optimize filters.
- *
- * Scalar indices are enabled by default. Disable this to force filter
-- * evaluation without scalar indices. This setting is independent of
-+ * evaluation without scalar indices, including an explicitly selected scalar
-+ * segment (which falls back to a scan of its explicit fragment_ids).
-+ * This setting is independent of
- * `lance_scanner_set_use_index`, which controls vector ANN index usage.
- * Must be set before scanning starts.
- */
-@@ -1051,7 +1053,11 @@ int32_t lance_scanner_with_row_address(LanceScanner* scanner, bool enable);
-
- /**
- * Configure whether deleted rows still present in storage are returned.
-- * Deleted rows have a NULL `_rowid`; callers should also enable row IDs.
-+ * Requires with_row_id=true; deleted rows have a NULL `_rowid`.
-+ * For filtered scans, also set use_scalar_index=false: indices built after a
-+ * deletion may omit tombstoned rows. Incompatible with scalar_index_segment,
-+ * even when scalar indices are disabled.
-+ * Fragments removed from the current snapshot are not scanned.
- * Must be set before scanning starts.
- */
- int32_t lance_scanner_set_include_deleted_rows(
-@@ -1867,16 +1873,20 @@ int32_t lance_scanner_set_index_segments(
- * excluded by fragment_ids; incomplete coverage falls back to a full filtered
- * scan of those fragment_ids. Callers distributing work must assign disjoint
- * fragment domains and separately include any unindexed data they wish to read.
-+ * The segment metadata must identify one key field present in the schema.
- *
- * BTree/Bitmap/LabelList searches use a necessary AND-conjunct of the
- * full scanner filter on the selected logical index and require an Exact result.
-+ * use_scalar_index=false skips segment search and uses the scoped fallback;
-+ * snapshot UUID and fragment validation still applies.
- * AtMost/AtLeast results fall back to a full filtered scan of fragment_ids.
- * All predicates are reapplied during candidate reads; other scalar indices
- * are disabled. Legacy storage, OR/NOT-only filters,
- * overlays, fragment reuse, unsupported index types / result domains
- * and missing coverage use the same domain without an index. No filter also
- * falls back. LIMIT/OFFSET apply after the complete scanner filter, never to the
-- * unfiltered candidate set. Vector/FTS queries are rejected.
-+ * unfiltered candidate set. Vector/FTS queries and include_deleted_rows=true
-+ * are rejected even when use_scalar_index=false; segment mode is live-row-only.
- *
- * UUID bytes are copied. Metadata and final option compatibility are validated
- * when creating the stream. Index corruption or I/O failures remain errors.
-diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp
-index ebb8141..d96cf9f 100644
---- a/include/lance/lance.hpp
-+++ b/include/lance/lance.hpp
-@@ -1270,6 +1270,7 @@ class Scanner {
- }
-
- /// Configure whether scalar indices may be used to optimize filters.
-+ /// False also disables explicit scalar segment search, retaining its fragment domain.
- Scanner& use_scalar_index(bool enable = true) {
- if (lance_scanner_set_use_scalar_index(handle_.get(), enable) != 0)
- check_error();
-@@ -1305,6 +1306,8 @@ class Scanner {
- }
-
- /// Configure whether deleted rows still present in storage are returned.
-+ /// Requires with_row_id(true); use_scalar_index(false) is needed for filtered scans.
-+ /// Incompatible with scalar_index_segment. See lance.h.
- Scanner& include_deleted_rows(bool include_deleted_rows = true) {
- if (lance_scanner_set_include_deleted_rows(handle_.get(), include_deleted_rows) != 0)
- check_error();
-@@ -1313,6 +1316,8 @@ class Scanner {
-
- /// Generate exact candidates from one BTree/Bitmap/LabelList segment.
- /// fragment_ids is required and defines the complete read/fallback domain. See lance.h.
-+ /// Requires live rows only: include_deleted_rows(true) is rejected at stream creation.
-+ /// use_scalar_index(false) selects the scoped fallback without searching the segment.
- Scanner& scalar_index_segment(const std::array<uint8_t, 16>& segment_uuid) {
- if (lance_scanner_set_scalar_index_segment(handle_.get(), segment_uuid.data()) != 0)
- check_error();
-diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs
-index 1faf500..747dac6 100644
---- a/src/scalar_segment.rs
-+++ b/src/scalar_segment.rs
-@@ -27,6 +27,7 @@ pub(crate) struct PreparedScalarSegment {
- pub dataset: Arc<Dataset>,
- pub segment_uuid: Uuid,
- pub fragment_ids: Vec<u64>,
-+ pub use_scalar_index: bool,
- pub callback: Option<ExecutionStatsCallback>,
- }
-
-@@ -138,6 +139,11 @@ impl PreparedScalarSegment {
- self.dataset.schema().field_by_id(field_id).ok_or_else(|| {
- invalid("scalar segment key field is absent from the dataset schema")
- })?;
-+ // Explicitly disabling scalar indices also disables this accelerator.
-+ // Keep snapshot validation above, but do not plan, open or search an index.
-+ if !self.use_scalar_index {
-+ return Ok(Some("disabled"));
-+ }
- // Match Lance's plain-scan external-mask restriction. Keep the scoped,
- // full-filtered reader intact and avoid index work on legacy storage.
- if self
-@@ -149,8 +155,8 @@ impl PreparedScalarSegment {
- {
- return Ok(Some("legacy_storage"));
- }
-- // Keep V1 to flat scalar fields. A dotted name is not sufficient to prove
-- // the field path of an evolved or nested schema.
-+ // Keep this implementation to flat scalar fields. A dotted name cannot
-+ // prove the field path of an evolved or nested schema.
- if !self
- .dataset
- .schema()
-@@ -234,6 +240,8 @@ impl PreparedScalarSegment {
- );
- // Do not truncate candidates at LIMIT. The reader evaluates the complete
- // filter before applying its existing limit/offset operators.
-+ // The raw selected bitmap can overlap NULL rows; the full filter removes
-+ // those as well. The metric above counts semantic TRUE rows, not mask size.
- reader.with_row_addr_prefilter(RowAddrMask::from_allowed(rows.selected_rows().clone()));
- Ok(None)
- }
-diff --git a/src/scanner.rs b/src/scanner.rs
-index ac3cff6..bda554d 100644
---- a/src/scanner.rs
-+++ b/src/scanner.rs
-@@ -479,9 +479,10 @@ impl LanceScanner {
- || self.fts_context.is_some()
- || self.index_segments.is_some()
- || self.fts_index_segments.is_some()
-+ || self.include_deleted_rows
- {
- return Err(lance_core::Error::invalid_input_source(
-- "scalar_index_segment requires an ordinary scan of live rows".into(),
-+ "scalar_index_segment requires an ordinary scan of live rows; vector/FTS queries and include_deleted_rows=true are unsupported".into(),
- ));
- }
- let fragment_ids = self.fragment_ids.as_ref().filter(|ids| !ids.is_empty())
-@@ -492,6 +493,7 @@ impl LanceScanner {
- dataset: Arc::clone(&self.dataset),
- segment_uuid,
- fragment_ids: fragment_ids.clone(),
-+ use_scalar_index: self.use_scalar_index.unwrap_or(true),
- callback: self.scan_statistics_callback.clone(),
- })
- } else {
-@@ -896,6 +898,8 @@ macro_rules! scanner_ffi_try {
-
- /// Select one physical scalar index segment. NULL clears the selection.
- /// Requires explicit fragment_ids and an ordinary live-row scan. See the C header.
-+/// include_deleted_rows=true is rejected when preparing the scan, even if
-+/// use_scalar_index=false selects the scoped non-indexed fallback.
- #[unsafe(no_mangle)]
- pub unsafe extern "C" fn lance_scanner_set_scalar_index_segment(
- scanner: *mut LanceScanner,
-@@ -1239,7 +1243,8 @@ unsafe fn scanner_set_scan_in_order_inner(
- /// Configure whether scalar indices may be used to optimize filters.
- ///
- /// Scalar indices are enabled by default in Lance. Must be set before the scan
--/// starts.
-+/// starts. False also disables explicit scalar segment search while preserving
-+/// the configured fragment domain and snapshot validation.
- #[unsafe(no_mangle)]
- pub unsafe extern "C" fn lance_scanner_set_use_scalar_index(
- scanner: *mut LanceScanner,
-@@ -1381,7 +1386,9 @@ unsafe fn scanner_with_row_address_inner(scanner: *mut LanceScanner, enable: boo
-
- /// Configure whether deleted rows still present in storage are returned.
- ///
--/// Deleted rows have a NULL `_rowid`, so callers should also enable row IDs.
-+/// Requires with_row_id=true; deleted rows have a NULL `_rowid`.
-+/// Filtered scans also need use_scalar_index=false because indices may omit
-+/// tombstoned rows. Incompatible with scalar_index_segment.
- /// Must be set before the scan starts.
- #[unsafe(no_mangle)]
- pub unsafe extern "C" fn lance_scanner_set_include_deleted_rows(
-diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index 3322d42..513f8d0 100644
---- a/tests/c_api_test.rs
-+++ b/tests/c_api_test.rs
-@@ -12945,6 +12945,193 @@ fn test_scalar_segment_stable_row_ids_and_deletes() {
- assert_eq!(ids, vec![3]);
- }
-
-+#[test]
-+fn test_scalar_segment_honors_use_scalar_index_false() {
-+ let (_tmp, uri, uuids) = create_scalar_segment_fixture(lance_index::IndexType::BTree, false);
-+ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0", None, 0);
-+ assert_eq!(ids, vec![1, 2, 3]);
-+ assert!(
-+ stats
-+ .metrics
-+ .iter()
-+ .any(|(name, _, value)| name == "scalar_segments_searched" && *value == 1)
-+ );
-+
-+ let uri = c_str(&uri);
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ assert!(!ds.is_null());
-+ for disable_first in [false, true] {
-+ for (fragments, filter, limit, offset, expected) in [
-+ (vec![0u64], "key >= 0", None, 0, vec![1, 2, 3]),
-+ (vec![0, 2], "key >= 0 AND id >= 2", Some(2), 1, vec![3, 9]),
-+ ] {
-+ let filter = c_str(filter);
-+ let scanner = lance_scanner_new(ds, ptr::null(), filter.as_ptr());
-+ assert!(!scanner.is_null());
-+ assert_eq!(
-+ lance_scanner_set_fragment_ids(scanner, fragments.as_ptr(), fragments.len()),
-+ 0
-+ );
-+ if disable_first {
-+ assert_eq!(lance_scanner_set_use_scalar_index(scanner, false), 0);
-+ }
-+ assert_eq!(
-+ lance_scanner_set_scalar_index_segment(scanner, uuids[0].as_ptr()),
-+ 0
-+ );
-+ if !disable_first {
-+ assert_eq!(lance_scanner_set_use_scalar_index(scanner, false), 0);
-+ }
-+ if let Some(limit) = limit {
-+ assert_eq!(lance_scanner_set_limit(scanner, limit), 0);
-+ }
-+ assert_eq!(lance_scanner_set_offset(scanner, offset), 0);
-+ let mut captured = CapturedScanStatistics::default();
-+ assert_eq!(
-+ lance_scanner_set_statistics_callback(
-+ scanner,
-+ Some(capture_scan_statistics),
-+ (&mut captured as *mut CapturedScanStatistics).cast(),
-+ ),
-+ 0
-+ );
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(lance_scanner_to_arrow_stream(scanner, &mut stream), 0);
-+ let mut ids = Vec::new();
-+ for batch in ArrowArrayStreamReader::from_raw(&mut stream).unwrap() {
-+ let batch = batch.unwrap();
-+ ids.extend_from_slice(
-+ batch
-+ .column_by_name("id")
-+ .unwrap()
-+ .as_any()
-+ .downcast_ref::<Int32Array>()
-+ .unwrap()
-+ .values(),
-+ );
-+ }
-+ assert_eq!(ids, expected);
-+ assert_eq!(captured.calls, 1);
-+ for metric in ["scalar_segments_searched", "scalar_segment_candidate_rows"] {
-+ assert_eq!(
-+ captured
-+ .metrics
-+ .iter()
-+ .filter(|(name, _, _)| name == metric)
-+ .map(|(_, _, value)| *value)
-+ .sum::<u64>(),
-+ 0,
-+ "{metric}"
-+ );
-+ }
-+ assert_eq!(captured.indices_loaded, 0);
-+ assert_eq!(captured.index_comparisons, 0);
-+ assert!(captured.metrics.iter().any(|(name, _, value)| name
-+ == "scalar_segment_fallback_disabled"
-+ && *value == 1));
-+ lance_scanner_close(scanner);
-+ }
-+ }
-+ lance_dataset_close(ds);
-+ }
-+}
-+
-+#[test]
-+fn test_scalar_segment_rejects_include_deleted_rows_after_index_rebuild() {
-+ use lance::index::DatasetIndexExt;
-+ use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
-+
-+ let (_tmp, uri, _) = create_scalar_segment_fixture(lance_index::IndexType::BTree, false);
-+ let uuid = lance_c::runtime::block_on(async {
-+ let mut ds = Dataset::open(&uri).await.unwrap();
-+ ds.delete("id = 2").await.unwrap();
-+ ds.drop_index("key_idx").await.unwrap();
-+ // A segment built after the delete cannot return the tombstoned row,
-+ // even though its search result is Exact for the indexed live rows.
-+ let params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree);
-+ let segment = ds
-+ .create_index_builder(&["key"], lance_index::IndexType::BTree, ¶ms)
-+ .name("key_idx".into())
-+ .fragments(vec![0])
-+ .execute_uncommitted()
-+ .await
-+ .unwrap();
-+ let uuid = *segment.uuid.as_bytes();
-+ ds.commit_existing_index_segments("key_idx", "key", vec![segment])
-+ .await
-+ .unwrap();
-+ uuid
-+ });
-+
-+ let (ids, _) = scalar_segment_ids(&uri, &uuid, &[0], "key >= 0 AND id >= 2", None, 0);
-+ assert_eq!(ids, vec![3], "live-row segment scans remain supported");
-+
-+ let uri = c_str(&uri);
-+ let filter = c_str("key >= 0 AND id >= 2");
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ assert!(!ds.is_null());
-+ // Check both setter orders: compatibility is validated at stream creation.
-+ for segment_first in [None, Some(false), Some(true)] {
-+ let scanner = lance_scanner_new(ds, ptr::null(), filter.as_ptr());
-+ assert!(!scanner.is_null());
-+ assert_eq!(
-+ lance_scanner_set_fragment_ids(scanner, [0u64].as_ptr(), 1),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_with_row_id(scanner, true), 0);
-+ if segment_first.is_none() {
-+ assert_eq!(lance_scanner_set_use_scalar_index(scanner, false), 0);
-+ }
-+ if segment_first == Some(true) {
-+ assert_eq!(
-+ lance_scanner_set_scalar_index_segment(scanner, uuid.as_ptr()),
-+ 0
-+ );
-+ }
-+ assert_eq!(lance_scanner_set_include_deleted_rows(scanner, true), 0);
-+ if segment_first == Some(false) {
-+ assert_eq!(
-+ lance_scanner_set_scalar_index_segment(scanner, uuid.as_ptr()),
-+ 0
-+ );
-+ }
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ let rc = lance_scanner_to_arrow_stream(scanner, &mut stream);
-+ if segment_first.is_some() {
-+ assert_eq!(rc, -1, "segment scans must not silently omit deleted rows");
-+ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument);
-+ assert!(take_last_error_message().contains("include_deleted_rows=true"));
-+ } else {
-+ assert_eq!(rc, 0);
-+ let reader = ArrowArrayStreamReader::from_raw(&mut stream).unwrap();
-+ let mut ids = Vec::new();
-+ for batch in reader {
-+ let batch = batch.unwrap();
-+ ids.extend_from_slice(
-+ batch
-+ .column_by_name("id")
-+ .unwrap()
-+ .as_any()
-+ .downcast_ref::<Int32Array>()
-+ .unwrap()
-+ .values(),
-+ );
-+ }
-+ ids.sort_unstable();
-+ assert_eq!(
-+ ids,
-+ vec![2, 3],
-+ "ordinary scans can still read tombstoned rows"
-+ );
-+ }
-+ lance_scanner_close(scanner);
-+ }
-+ lance_dataset_close(ds);
-+ }
-+}
-+
- #[test]
- fn test_scalar_segment_requires_explicit_domain_and_checks_uuid() {
- let (_tmp, uri, uuids) = create_scalar_segment_fixture(lance_index::IndexType::BTree, false);
diff --git a/thirdparty/patches/lance-c-0.1.9-pr-80.patch b/thirdparty/patches/lance-c-0.1.9-pr-80.patch
deleted file mode 100644
index edc42cad0a1..00000000000
--- a/thirdparty/patches/lance-c-0.1.9-pr-80.patch
+++ /dev/null
@@ -1,225 +0,0 @@
-From 7fcd9c4ff7c03c10bdc9d8a600b3f7b0cf28a9ad Mon Sep 17 00:00:00 2001
-From: zhangstar333 <zh...@selectdb.com>
-Date: Thu, 10 Sep 2026 15:50:53 +0800
-Subject: [PATCH] oss provider error
-
-Doris integration: rebase only Cargo.toml/Cargo.lock hunk context over PR #73.
-All added and removed lines are identical to upstream PR #80 at 7fcd9c4.
-
----
- Cargo.lock | 1 +
- Cargo.toml | 2 +
- src/runtime.rs | 5 ++
- tests/compile_and_run_test.rs | 16 ++++++
- tests/cpp/test_oss_transport.c | 40 +++++++++++++
- tests/static_oss_transport_test.py | 91 ++++++++++++++++++++++++++++++
- 6 files changed, 155 insertions(+)
- create mode 100644 tests/cpp/test_oss_transport.c
- create mode 100644 tests/static_oss_transport_test.py
-
-diff --git a/Cargo.lock b/Cargo.lock
---- a/Cargo.lock
-+++ b/Cargo.lock
-@@ -3970,6 +3970,7 @@
- "libc",
- "log",
- "object_store",
-+ "opendal",
- "pin-project",
- "prost",
- "snafu",
-diff --git a/Cargo.toml b/Cargo.toml
---- a/Cargo.toml
-+++ b/Cargo.toml
-@@ -43,6 +43,8 @@
- log = "0.4"
- libc = "0.2"
- object_store = "0.13.2"
-+# Explicitly install the HTTP transport when embedded in a static C/C++ executable.
-+opendal = { version = "=0.58.2", default-features = false, features = ["http-transport-reqwest"] }
- pin-project = "1.0"
- prost = "0.14"
- snafu = "0.9"
-diff --git a/src/runtime.rs b/src/runtime.rs
-index 0153d3f..3bd8964 100644
---- a/src/runtime.rs
-+++ b/src/runtime.rs
-@@ -8,6 +8,11 @@ use std::sync::LazyLock;
- /// Global multi-threaded Tokio runtime, shared across all FFI calls.
- /// Initialized lazily on first access.
- pub static RT: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
-+ // A native linker can omit OpenDAL's automatic constructor from liblance_c.a.
-+ // Keep initialization reachable from the FFI entry points, before any HTTP I/O.
-+ // Installation is idempotent and preserves an already installed transport.
-+ opendal::install_default();
-+
- tokio::runtime::Builder::new_multi_thread()
- .enable_all()
- .build()
-diff --git a/tests/compile_and_run_test.rs b/tests/compile_and_run_test.rs
-index b419ac9..8566d10 100644
---- a/tests/compile_and_run_test.rs
-+++ b/tests/compile_and_run_test.rs
-@@ -249,3 +249,19 @@ fn test_cpp_compilation_and_execution() {
-
- run_test_binary(&binary, &dataset_uri, &write_uri);
- }
-+
-+/// A fresh C executable must initialize OpenDAL even when archive constructors are omitted.
-+#[cfg(target_os = "linux")]
-+#[test]
-+#[ignore = "requires a C compiler, Python 3, and building the static library"]
-+fn test_static_oss_transport() {
-+ let (shared_library, _) = build_lance_c();
-+ let static_library = shared_library.with_file_name("liblance_c.a");
-+ assert!(static_library.exists(), "static library was not built");
-+ let status = Command::new("python3")
-+ .arg(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/static_oss_transport_test.py"))
-+ .arg(static_library)
-+ .status()
-+ .expect("failed to run the static OSS transport test");
-+ assert!(status.success(), "static OSS HTTP transport test failed");
-+}
-diff --git a/tests/cpp/test_oss_transport.c b/tests/cpp/test_oss_transport.c
-new file mode 100644
-index 0000000..c96ee7e
---- /dev/null
-+++ b/tests/cpp/test_oss_transport.c
-@@ -0,0 +1,40 @@
-+/* SPDX-License-Identifier: Apache-2.0 */
-+/* SPDX-FileCopyrightText: Copyright The Lance Authors */
-+
-+#include "lance/lance.h"
-+#include <stdio.h>
-+#include <string.h>
-+
-+/* The Python harness serves a missing manifest on a local HTTP endpoint. */
-+int main(int argc, char **argv) {
-+ if (argc != 3) return 2;
-+ const char *options[] = {
-+ "oss_endpoint", argv[1],
-+ "oss_region", "cn-test",
-+ "oss_access_key_id", "test-key",
-+ "oss_secret_access_key", "test-secret",
-+ "addressing_style", "path",
-+ NULL
-+ };
-+ LanceSession *session = NULL;
-+ LanceDataset *dataset = NULL;
-+ const char *uri = "oss://test-bucket/missing.lance";
-+ if (strcmp(argv[2], "shared") == 0) {
-+ session = lance_session_new(0, 0);
-+ if (session == NULL) return 3;
-+ dataset = lance_dataset_open_with_session(uri, options, 1, session);
-+ } else {
-+ dataset = lance_dataset_open(uri, options, 1);
-+ }
-+ /* The object does not exist, but the request must reach the HTTP server. */
-+ const char *error = lance_last_error_message();
-+ int failed = dataset != NULL || error == NULL;
-+ if (error != NULL) {
-+ fprintf(stderr, "%s\n", error);
-+ failed |= strstr(error, "default HTTP transport is not installed") != NULL;
-+ lance_free_string(error);
-+ }
-+ lance_dataset_close(dataset);
-+ lance_session_close(session);
-+ return failed ? 1 : 0;
-+}
-diff --git a/tests/static_oss_transport_test.py b/tests/static_oss_transport_test.py
-new file mode 100644
-index 0000000..7d0e87c
---- /dev/null
-+++ b/tests/static_oss_transport_test.py
-@@ -0,0 +1,91 @@
-+# SPDX-License-Identifier: Apache-2.0
-+# SPDX-FileCopyrightText: Copyright The Lance Authors
-+
-+"""Exercise native OSS HTTP initialization from fresh, statically linked C processes.
-+
-+Build lance-c first, then run on Linux:
-+ python3 tests/static_oss_transport_test.py target/release/liblance_c.a
-+
-+No OSS account is needed. A local HTTP server returns 404 for a missing manifest.
-+The assertion is that an HTTP request reaches it, not merely that opening fails.
-+Unlike a Rust test binary, the C executable must pull initialization from the archive.
-+"""
-+
-+import argparse
-+from http.server import BaseHTTPRequestHandler, HTTPServer
-+import os
-+from pathlib import Path
-+import shlex
-+import subprocess
-+import sys
-+import tempfile
-+import threading
-+
-+
-+def main():
-+ parser = argparse.ArgumentParser(description=__doc__)
-+ parser.add_argument("library", type=Path)
-+ args = parser.parse_args()
-+ if not sys.platform.startswith("linux"):
-+ parser.error("this static-link regression test currently supports Linux")
-+ library = args.library.resolve(strict=True)
-+ root = Path(__file__).resolve().parents[1]
-+ requests = []
-+
-+ class Handler(BaseHTTPRequestHandler):
-+ def missing(self):
-+ requests.append((self.command, self.path))
-+ self.send_response(404)
-+ self.send_header("Content-Length", "0")
-+ self.send_header("Connection", "close")
-+ self.end_headers()
-+
-+ do_HEAD = missing
-+ do_GET = missing
-+
-+ def log_message(self, *_args):
-+ pass
-+
-+ with tempfile.TemporaryDirectory(prefix="lance-static-oss-") as directory:
-+ executable = Path(directory) / "test_oss_transport"
-+ # Pass the archive explicitly; -llance_c could silently select the shared library.
-+ # Do not use --whole-archive: ordinary native linking must retain initialization.
-+ subprocess.run(
-+ shlex.split(os.environ.get("CC", "cc"))
-+ + ["-std=c11", "-Wall", "-Wextra", "-Werror", "-Wl,--gc-sections",
-+ "-I", str(root / "include"), str(root / "tests/cpp/test_oss_transport.c"),
-+ str(library), "-lgcc_s", "-lutil", "-lrt", "-lpthread", "-lm", "-ldl",
-+ "-o", str(executable)],
-+ check=True,
-+ )
-+ environment = {
-+ key: value for key, value in os.environ.items()
-+ if not key.startswith(("AWS_", "OSS_", "ALIBABA_CLOUD_"))
-+ and key.lower() not in ("http_proxy", "https_proxy", "all_proxy", "no_proxy")
-+ }
-+ environment["NO_PROXY"] = "127.0.0.1,localhost"
-+ # Each mode starts a new process so an earlier call cannot hide missing initialization.
-+ for mode in ("ordinary", "shared"):
-+ requests.clear()
-+ with HTTPServer(("127.0.0.1", 0), Handler) as server:
-+ thread = threading.Thread(
-+ target=server.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True
-+ )
-+ thread.start()
-+ try:
-+ result = subprocess.run(
-+ [str(executable), f"http://127.0.0.1:{server.server_port}", mode],
-+ env=environment, capture_output=True, text=True, timeout=30,
-+ )
-+ finally:
-+ server.shutdown()
-+ thread.join()
-+ assert result.returncode == 0, f"{mode}: {result.stderr}"
-+ assert any("/_versions/" in path for _, path in requests), (
-+ f"{mode}: no manifest HTTP request reached the server: {result.stderr}"
-+ )
-+ print(f"PASS: {mode} OSS open reached the local HTTP server")
-+
-+
-+if __name__ == "__main__":
-+ main()
diff --git a/thirdparty/patches/lance-c-0.1.9-pr-83.patch b/thirdparty/patches/lance-c-0.1.9-pr-83.patch
deleted file mode 100644
index 86c0c23e227..00000000000
--- a/thirdparty/patches/lance-c-0.1.9-pr-83.patch
+++ /dev/null
@@ -1,1914 +0,0 @@
-From 0a30ee6c5a9d1455ceb36f4745acb79e53a00461 Mon Sep 17 00:00:00 2001
-Subject: [PATCH] feat: support multi-vector queries through C and C++ APIs (#83)
-
-Upstream: https://github.com/lance-format/lance-c/pull/83
-Commit: 0a30ee6c5a9d1455ceb36f4745acb79e53a00461
-
-Adapted to the v0.1.9 community patch chain used by Doris. Resolve context
-conflicts with PR #73 and retain PR #79 scalar_segment fields, execution
-branch, and tests alongside the upstream multi-vector additions. The
-multi-vector implementation and its integration tests are unchanged.
-
-diff --git a/README.md b/README.md
-index d9a5f2b..29c89c4 100644
---- a/README.md
-+++ b/README.md
-@@ -70,6 +70,40 @@ Based on the [liblance RFC](https://github.com/lance-format/lance/discussions/60
- | [x] | Filter pushdown | `lance_scanner_set_substrait_filter()` accepts a serialized Substrait `ExtendedExpression`; `lance_scanner_additional_sql_filter()` adds SQL predicates with AND before scanning starts |
- | [x] | Data-file cache | Optional Foyer memory/disk cache for immutable `data/*.lance` reads |
-
-+## Multi-vector search
-+
-+Use `lance_scanner_nearest_multivector` or the C++ `Scanner::nearest_multivector`
-+method for a `List<FixedSizeList<float16|float32|float64, D>>` column:
-+
-+```cpp
-+const float query[] = {1.0f, 0.0f, 0.0f, 1.0f};
-+auto scanner = dataset.scan();
-+scanner.nearest_multivector("embeddings", query, 2, 2, LANCE_DTYPE_FLOAT32, 10)
-+ .metric(LANCE_METRIC_COSINE)
-+ .prefilter(true);
-+```
-+
-+The copied, row-major matrix is **one query** containing two subvectors. Results
-+rank logical rows by the sum of each query subvector's minimum distance to a
-+stored subvector. Empty or null outer rows do not rank. Inner vectors must be
-+non-nullable; actual stored null or non-finite elements encountered during
-+scoring fail the stream. Float types and dimensions must match the column.
-+Cosine pairs with zero norm have undefined distance and are ignored. A row is
-+excluded if any query subvector has no defined match; a zero-norm query subvector
-+therefore produces no results. Column names use Lance field-path syntax,
-+including nested paths such as `payload.embeddings` and backtick-quoted names.
-+
-+L2 is the default on every fragment. Cosine multi-vector indexes are supported
-+by the pinned Lance version; incompatible metrics use exact search. Indexed
-+candidates are refined against stored values (`refine_factor` defaults to 1).
-+ANN candidate selection remains approximate. Limit and offset apply after
-+restoring distance order, including fragment-scoped searches. Strict row batching
-+is applied after that final result window, preserving full batches except the last.
-+
-+Queries accept at most 128 subvectors. Both `num_vectors * k` and
-+`refine_factor * k` must be at most 100,000 to bound plan expansion and candidate
-+allocation. The existing single-vector API and its defaults are unchanged.
-+
- ## Building
-
- There are four supported entry points; pick whichever matches your toolchain.
-diff --git a/include/lance/lance.h b/include/lance/lance.h
-index a201bfb..e404612 100644
---- a/include/lance/lance.h
-+++ b/include/lance/lance.h
-@@ -1847,6 +1847,21 @@ int32_t lance_scanner_nearest(
- uint32_t k
- );
-
-+/**
-+ * Set one multi-vector query on a List<FixedSizeList<float16|float32|float64>> column.
-+ * Inner vectors must be non-nullable and contain no null elements; the outer list may be nullable.
-+ * query_data contains dimension * num_vectors aligned elements in row-major order.
-+ * Both sizes and k must be positive. At most 128 query subvectors are accepted;
-+ * num_vectors * k and refine_factor * k must each be at most 100000.
-+ * Values are copied before returning. The default metric is L2 on every fragment.
-+ * Scores sum each query vector's minimum distance; refinement defaults to 1.
-+ * Returns 0 on success, -1 on error. Stored invalid elements fail during execution.
-+ */
-+int32_t lance_scanner_nearest_multivector(
-+ LanceScanner* scanner, const char* column, const void* query_data,
-+ size_t dimension, size_t num_vectors, LanceDataType element_type, uint32_t k
-+);
-+
- /**
- * Set both the minimum and maximum vector-index partition-search bounds.
- *
-diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp
-index a231a00..146c6b1 100644
---- a/include/lance/lance.hpp
-+++ b/include/lance/lance.hpp
-@@ -1465,6 +1465,16 @@ public:
- return *this;
- }
-
-+ /// One multi-vector query, copied from dimension * num_vectors row-major elements.
-+ Scanner& nearest_multivector(const std::string& column, const void* query_data,
-+ size_t dimension, size_t num_vectors,
-+ LanceDataType element_type, uint32_t k) {
-+ if (lance_scanner_nearest_multivector(handle_.get(), column.c_str(), query_data,
-+ dimension, num_vectors, element_type, k) != 0)
-+ check_error();
-+ return *this;
-+ }
-+
- /// Replace both minimum and maximum partition-search bounds.
- Scanner& nprobes(uint32_t nprobes) {
- if (lance_scanner_set_nprobes(handle_.get(), nprobes) != 0) check_error();
-diff --git a/src/lib.rs b/src/lib.rs
-index 197da5f..55f6ecf 100644
---- a/src/lib.rs
-+++ b/src/lib.rs
-@@ -39,6 +39,7 @@ mod index;
- mod index_model;
- mod index_segment;
- mod merge_insert;
-+mod multivector;
- mod restore;
- pub mod runtime;
- mod scalar_segment;
-diff --git a/src/multivector.rs b/src/multivector.rs
-new file mode 100644
-index 0000000..c08df28
---- /dev/null
-+++ b/src/multivector.rs
-@@ -0,0 +1,514 @@
-+// SPDX-License-Identifier: Apache-2.0
-+// SPDX-FileCopyrightText: Copyright The Lance Authors
-+
-+//! Correct multi-vector scoring before the pinned Lance plan's candidate limits.
-+
-+use std::collections::HashMap;
-+use std::sync::Arc;
-+
-+use arrow_array::types::{Float16Type, Float32Type, Float64Type};
-+use arrow_array::{
-+ Array, ArrayRef, ArrowPrimitiveType, BooleanArray, FixedSizeListArray, Float32Array, ListArray,
-+ RecordBatch, UInt64Array,
-+};
-+use arrow_schema::{DataType, SchemaRef};
-+use datafusion::error::{DataFusionError, Result};
-+use datafusion::execution::context::TaskContext;
-+use datafusion::physical_plan::{
-+ DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SendableRecordBatchStream,
-+ stream::RecordBatchStreamAdapter,
-+};
-+use futures::{StreamExt, TryStreamExt, stream};
-+use lance::io::exec::KNNVectorDistanceExec;
-+use lance_linalg::distance::{Cosine, DistanceType, Dot, L2};
-+
-+// Lance creates one ANN branch per query vector,
-+// each overfetching 10 * k candidates before scoring; wire bytes alone cannot bound this work.
-+pub(crate) const MAX_QUERY_VECTORS: usize = 128;
-+pub(crate) const MAX_QUERY_VECTOR_CANDIDATES: usize = 100_000;
-+
-+fn invalid(message: impl Into<String>) -> DataFusionError {
-+ DataFusionError::Execution(message.into())
-+}
-+
-+/// Rewrite inside TopK/refinement, before any score can discard a candidate.
-+pub(crate) fn rewrite(plan: Arc<dyn ExecutionPlan>) -> Result<Arc<dyn ExecutionPlan>> {
-+ let children = plan
-+ .children()
-+ .into_iter()
-+ .map(|child| rewrite(child.clone()))
-+ .collect::<Result<Vec<_>>>()?;
-+ let plan = if children.is_empty() {
-+ plan
-+ } else {
-+ plan.with_new_children(children)?
-+ };
-+ let mode = if let Some(exact) = plan.downcast_ref::<KNNVectorDistanceExec>() {
-+ if exact.is_batch {
-+ return Err(invalid(
-+ "expected one logical multi-vector query, not batch queries",
-+ ));
-+ }
-+ Some(Scoring::Exact {
-+ query: exact.query.clone(),
-+ column: exact.column.clone(),
-+ metric: exact.distance_type,
-+ })
-+ // This pinned Lance node is not publicly re-exported, so match its stable plan name.
-+ } else if plan.name() == "MultivectorScoringExec" {
-+ Some(Scoring::Indexed)
-+ } else {
-+ None
-+ };
-+ Ok(match mode {
-+ Some(mode) => Arc::new(MultiVectorScoreExec {
-+ original: plan,
-+ mode,
-+ }),
-+ None => plan,
-+ })
-+}
-+
-+/// Apply the final distance-ordered window without invalidating output batching.
-+pub(crate) fn apply_result_window(
-+ plan: Arc<dyn ExecutionPlan>,
-+ offset: usize,
-+ limit: Option<usize>,
-+) -> Result<Arc<dyn ExecutionPlan>> {
-+ use datafusion::physical_expr::{PhysicalSortExpr, expressions};
-+ use datafusion::physical_plan::{
-+ coalesce_partitions::CoalescePartitionsExec, limit::GlobalLimitExec, sorts::sort::SortExec,
-+ };
-+ if plan
-+ .downcast_ref::<lance_datafusion::exec::StrictBatchSizeExec>()
-+ .is_some()
-+ {
-+ // Offset can split a previously strict batch. Keep Lance's final rechunker
-+ // outside the window, preserving its resolved batch size, including defaults.
-+ let input = apply_result_window(plan.children()[0].clone(), offset, limit)?;
-+ return plan.with_new_children(vec![input]);
-+ }
-+ let sort = PhysicalSortExpr {
-+ expr: expressions::col("_distance", plan.schema().as_ref())?,
-+ options: arrow::compute::SortOptions {
-+ descending: false,
-+ nulls_first: false,
-+ },
-+ };
-+ // Fragment-scoped payload takes can reorder batches. Restore distance order
-+ // before the window; the nearest plan already bounds candidate rows by k.
-+ let sorted = Arc::new(SortExec::new(
-+ [sort].into(),
-+ Arc::new(CoalescePartitionsExec::new(plan)),
-+ ));
-+ Ok(Arc::new(GlobalLimitExec::new(sorted, offset, limit)))
-+}
-+
-+#[derive(Clone, Debug)]
-+enum Scoring {
-+ Exact {
-+ query: ArrayRef,
-+ column: String,
-+ metric: DistanceType,
-+ },
-+ Indexed,
-+}
-+
-+#[derive(Debug)]
-+struct MultiVectorScoreExec {
-+ original: Arc<dyn ExecutionPlan>,
-+ mode: Scoring,
-+}
-+
-+impl DisplayAs for MultiVectorScoreExec {
-+ fn fmt_as(&self, _: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-+ write!(f, "MultiVectorScore: {}", self.original.name())
-+ }
-+}
-+
-+impl ExecutionPlan for MultiVectorScoreExec {
-+ fn name(&self) -> &str {
-+ "MultiVectorScoreExec"
-+ }
-+ fn properties(&self) -> &Arc<PlanProperties> {
-+ self.original.properties()
-+ }
-+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
-+ self.original.children()
-+ }
-+ fn required_input_distribution(&self) -> Vec<datafusion::physical_expr::Distribution> {
-+ self.original.required_input_distribution()
-+ }
-+ fn with_new_children(
-+ self: Arc<Self>,
-+ children: Vec<Arc<dyn ExecutionPlan>>,
-+ ) -> Result<Arc<dyn ExecutionPlan>> {
-+ Ok(Arc::new(Self {
-+ original: self.original.clone().with_new_children(children)?,
-+ mode: self.mode.clone(),
-+ }))
-+ }
-+ fn execute(
-+ &self,
-+ partition: usize,
-+ context: Arc<TaskContext>,
-+ ) -> Result<SendableRecordBatchStream> {
-+ let schema = self.schema();
-+ match &self.mode {
-+ Scoring::Exact {
-+ query,
-+ column,
-+ metric,
-+ } => {
-+ let input = self.children()[0].execute(partition, context)?;
-+ let query = query.clone();
-+ let column = column.clone();
-+ let metric = *metric;
-+ let output_schema = schema.clone();
-+ let output = input
-+ .map(move |batch| {
-+ let query = query.clone();
-+ let column = column.clone();
-+ let schema = output_schema.clone();
-+ async move {
-+ let batch = batch?;
-+ tokio::task::spawn_blocking(move || {
-+ exact_batch(batch, query, &column, metric, schema)
-+ })
-+ .await
-+ .map_err(|e| DataFusionError::External(Box::new(e)))?
-+ }
-+ })
-+ .buffered(lance_core::utils::tokio::get_num_compute_intensive_cpus());
-+ Ok(Box::pin(RecordBatchStreamAdapter::new(schema, output)))
-+ }
-+ Scoring::Indexed => {
-+ let inputs = self
-+ .children()
-+ .into_iter()
-+ .map(|child| child.execute(partition, context.clone()))
-+ .collect::<Result<Vec<_>>>()?;
-+ let output_schema = schema.clone();
-+ let output =
-+ stream::once(async move { indexed_batch(inputs, output_schema).await });
-+ Ok(Box::pin(RecordBatchStreamAdapter::new(schema, output)))
-+ }
-+ }
-+ }
-+}
-+
-+fn row_distance<T: ArrowPrimitiveType>(
-+ query: &dyn Array,
-+ vectors: &FixedSizeListArray,
-+ metric: DistanceType,
-+) -> Result<Option<f32>>
-+where
-+ T::Native: L2 + Cosine + Dot + Into<f64>,
-+{
-+ let q = query
-+ .as_any()
-+ .downcast_ref::<arrow_array::PrimitiveArray<T>>()
-+ .ok_or_else(|| invalid("multi-vector query element type mismatch"))?;
-+ let values = vectors
-+ .values()
-+ .as_any()
-+ .downcast_ref::<arrow_array::PrimitiveArray<T>>()
-+ .ok_or_else(|| invalid("multi-vector stored element type mismatch"))?;
-+ if vectors.null_count() != 0
-+ || values.null_count() != 0
-+ || values
-+ .values()
-+ .iter()
-+ .any(|v| !Into::<f64>::into(*v).is_finite())
-+ {
-+ return Err(invalid(
-+ "multi-vector stored subvectors must contain only finite, non-null elements",
-+ ));
-+ }
-+ let dimension = vectors.value_length() as usize;
-+ let distance = metric.func();
-+ // Subtracting each small distance from 1 rounds it away before TopK. Sum minima
-+ // directly, using f64 only for the accumulator; the base kernels and output remain f32.
-+ let mut score = 0.0f64;
-+ for query_vector in q.values().chunks_exact(dimension) {
-+ let best = values
-+ .values()
-+ .chunks_exact(dimension)
-+ .map(|vector| distance(query_vector, vector))
-+ // Finite zero-norm vectors have undefined cosine distance. Ignore those
-+ // pairs; a query with no defined match masks this row, not the whole scan.
-+ .filter(|distance| !distance.is_nan())
-+ .min_by(f32::total_cmp);
-+ let Some(best) = best else {
-+ return Ok(None);
-+ };
-+ score += best as f64;
-+ }
-+ let score = score as f32;
-+ if !score.is_finite() {
-+ return Err(invalid("multi-vector distance is not finite"));
-+ }
-+ Ok(Some(score))
-+}
-+
-+fn vector_column(batch: &RecordBatch, column: &str) -> Result<ArrayRef> {
-+ if let Some(array) = batch.column_by_name(column) {
-+ return Ok(array.clone());
-+ }
-+ // The planner resolves field paths, including quoted dotted names. Its private
-+ // KNN resolver is not exported, so use the same parser and struct traversal here.
-+ let parts = lance_core::datatypes::parse_field_path(column)
-+ .map_err(|e| invalid(format!("invalid vector column path '{column}': {e}")))?;
-+ let root = parts
-+ .first()
-+ .ok_or_else(|| invalid("empty vector column path"))?;
-+ let mut array = batch
-+ .column_by_name(root)
-+ .cloned()
-+ .ok_or_else(|| invalid(format!("missing vector column '{column}'")))?;
-+ for part in &parts[1..] {
-+ array = array
-+ .as_any()
-+ .downcast_ref::<arrow_array::StructArray>()
-+ .and_then(|parent| parent.column_by_name(part))
-+ .cloned()
-+ .ok_or_else(|| {
-+ invalid(format!(
-+ "missing struct field '{part}' in vector column '{column}'"
-+ ))
-+ })?;
-+ }
-+ Ok(array)
-+}
-+
-+fn exact_batch(
-+ batch: RecordBatch,
-+ query: ArrayRef,
-+ column: &str,
-+ metric: DistanceType,
-+ schema: SchemaRef,
-+) -> Result<RecordBatch> {
-+ if batch.num_rows() == 0 {
-+ return Ok(RecordBatch::new_empty(schema));
-+ }
-+ let array = vector_column(&batch, column)?;
-+ let vectors = array
-+ .as_any()
-+ .downcast_ref::<ListArray>()
-+ .ok_or_else(|| invalid("multi-vector scoring requires a List column"))?;
-+ let row_ids = batch.column_by_name("_rowid");
-+ let mut scores = Vec::with_capacity(batch.num_rows());
-+ for (i, vector) in vectors.iter().enumerate() {
-+ if row_ids.is_some_and(|ids| ids.is_null(i)) {
-+ scores.push(None);
-+ continue;
-+ }
-+ let Some(vector) = vector else {
-+ scores.push(None);
-+ continue;
-+ };
-+ let vector = vector
-+ .as_any()
-+ .downcast_ref::<FixedSizeListArray>()
-+ .ok_or_else(|| invalid("multi-vector row must be a FixedSizeList"))?;
-+ if vector.is_empty() {
-+ scores.push(None);
-+ continue;
-+ }
-+ let score = match query.data_type() {
-+ DataType::Float16 => row_distance::<Float16Type>(query.as_ref(), vector, metric),
-+ DataType::Float32 => row_distance::<Float32Type>(query.as_ref(), vector, metric),
-+ DataType::Float64 => row_distance::<Float64Type>(query.as_ref(), vector, metric),
-+ _ => Err(invalid("unsupported multi-vector element type")),
-+ }?;
-+ scores.push(score);
-+ }
-+ let mask = BooleanArray::from_iter(scores.iter().map(|score| Some(score.is_some())));
-+ let distances: ArrayRef = Arc::new(Float32Array::from(scores));
-+ let columns = schema
-+ .fields()
-+ .iter()
-+ .map(|field| {
-+ if field.name() == "_distance" {
-+ Ok(distances.clone())
-+ } else {
-+ batch
-+ .column_by_name(field.name())
-+ .cloned()
-+ .ok_or_else(|| invalid(format!("missing score output column {}", field.name())))
-+ }
-+ })
-+ .collect::<Result<Vec<_>>>()?;
-+ Ok(arrow::compute::filter_record_batch(
-+ &RecordBatch::try_new(schema, columns)?,
-+ &mask,
-+ )?)
-+}
-+
-+async fn indexed_batch(
-+ inputs: Vec<SendableRecordBatchStream>,
-+ schema: SchemaRef,
-+) -> Result<RecordBatch> {
-+ // A query child may emit several batches. Reduce its entire stream exactly once;
-+ // treating each batch as a query adds spurious missing-query contributions.
-+ let queries = futures::future::try_join_all(inputs.into_iter().map(|mut input| async move {
-+ let mut rows = HashMap::<u64, f32>::new();
-+ let mut maximum: Option<f32> = None;
-+ while let Some(batch) = input.try_next().await? {
-+ let ids = batch
-+ .column_by_name("_rowid")
-+ .and_then(|a| a.as_any().downcast_ref::<UInt64Array>())
-+ .ok_or_else(|| invalid("indexed multi-vector scorer requires row IDs"))?;
-+ let distances = batch
-+ .column_by_name("_distance")
-+ .and_then(|a| a.as_any().downcast_ref::<Float32Array>())
-+ .ok_or_else(|| invalid("indexed multi-vector scorer requires distances"))?;
-+ for i in 0..batch.num_rows() {
-+ let distance = distances.value(i);
-+ if ids.is_null(i) || distances.is_null(i) || !distance.is_finite() {
-+ return Err(invalid(
-+ "indexed multi-vector candidate has invalid row ID or distance",
-+ ));
-+ }
-+ maximum = Some(maximum.map_or(distance, |old| old.max(distance)));
-+ rows.entry(ids.value(i))
-+ .and_modify(|old| *old = old.min(distance))
-+ .or_insert(distance);
-+ }
-+ }
-+ Ok::<_, DataFusionError>((rows, maximum.unwrap_or(1.0)))
-+ }))
-+ .await?;
-+ let mut results = HashMap::<u64, f64>::new();
-+ let mut missed = 0.0f64;
-+ for (rows, maximum) in queries {
-+ for (id, score) in &mut results {
-+ *score += *rows.get(id).unwrap_or(&maximum) as f64;
-+ }
-+ for (id, score) in rows {
-+ results.entry(id).or_insert(score as f64 + missed);
-+ }
-+ missed += maximum as f64;
-+ }
-+ let (ids, scores): (Vec<_>, Vec<_>) = results.into_iter().unzip();
-+ Ok(RecordBatch::try_new(
-+ schema,
-+ vec![
-+ Arc::new(Float32Array::from_iter_values(
-+ scores.into_iter().map(|score| score as f32),
-+ )),
-+ Arc::new(UInt64Array::from(ids)),
-+ ],
-+ )?)
-+}
-+
-+pub(crate) fn validate_query(values: &dyn Array) -> Result<()> {
-+ fn finite<T: ArrowPrimitiveType>(values: &dyn Array) -> bool
-+ where
-+ T::Native: Into<f64>,
-+ {
-+ values
-+ .as_any()
-+ .downcast_ref::<arrow_array::PrimitiveArray<T>>()
-+ .is_some_and(|array| {
-+ array.null_count() == 0
-+ && array
-+ .values()
-+ .iter()
-+ .all(|v| Into::<f64>::into(*v).is_finite())
-+ })
-+ }
-+ let valid = match values.data_type() {
-+ DataType::Float16 => finite::<Float16Type>(values),
-+ DataType::Float32 => finite::<Float32Type>(values),
-+ DataType::Float64 => finite::<Float64Type>(values),
-+ _ => false,
-+ };
-+ if valid {
-+ Ok(())
-+ } else {
-+ Err(invalid(
-+ "multi-vector query must contain only finite, non-null elements",
-+ ))
-+ }
-+}
-+
-+#[cfg(test)]
-+mod tests {
-+ use super::*;
-+ use arrow_schema::{Field, Schema};
-+ use lance_datafusion::exec::{
-+ LanceExecutionOptions, OneShotExec, StrictBatchSizeExec, execute_plan,
-+ };
-+
-+ #[test]
-+ fn result_window_preserves_strict_batching_and_distance_order() {
-+ crate::runtime::block_on(async {
-+ for size in [2, 3] {
-+ for (offset, limit) in [
-+ (1, Some(4)),
-+ (1, Some(3)),
-+ (5, Some(4)),
-+ (6, Some(4)),
-+ (1, None),
-+ ] {
-+ let schema = Arc::new(Schema::new(vec![Field::new(
-+ "_distance",
-+ DataType::Float32,
-+ false,
-+ )]));
-+ let batches = [[5., 0.], [4., 1.], [3., 2.]]
-+ .into_iter()
-+ .map(|values| {
-+ RecordBatch::try_new(
-+ schema.clone(),
-+ vec![Arc::new(Float32Array::from(values.to_vec()))],
-+ )
-+ .map_err(DataFusionError::from)
-+ })
-+ .collect::<Vec<_>>();
-+ let input = Arc::new(OneShotExec::new(Box::pin(
-+ RecordBatchStreamAdapter::new(schema, stream::iter(batches)),
-+ )));
-+ let plan = Arc::new(StrictBatchSizeExec::new(input, size));
-+ let plan = apply_result_window(plan, offset, limit).unwrap();
-+ let batches: Vec<_> = execute_plan(
-+ plan,
-+ LanceExecutionOptions {
-+ batch_size: Some(2),
-+ ..Default::default()
-+ },
-+ )
-+ .unwrap()
-+ .try_collect()
-+ .await
-+ .unwrap();
-+ let actual: Vec<_> = batches
-+ .iter()
-+ .flat_map(|b| {
-+ b.column(0)
-+ .as_any()
-+ .downcast_ref::<Float32Array>()
-+ .unwrap()
-+ .values()
-+ .to_vec()
-+ })
-+ .collect();
-+ let expected: Vec<_> = (0..6)
-+ .skip(offset)
-+ .take(limit.unwrap_or(6))
-+ .map(|i| i as f32)
-+ .collect();
-+ assert_eq!(actual, expected);
-+ assert_eq!(
-+ batches
-+ .iter()
-+ .map(RecordBatch::num_rows)
-+ .collect::<Vec<_>>(),
-+ expected.chunks(size).map(<[f32]>::len).collect::<Vec<_>>()
-+ );
-+ }
-+ }
-+ });
-+ }
-+}
-diff --git a/src/scanner.rs b/src/scanner.rs
-index bda554d..f4b3f36 100644
---- a/src/scanner.rs
-+++ b/src/scanner.rs
-@@ -363,8 +363,18 @@ impl LanceScanner {
- if let Some(cols) = &self.columns {
- scanner.project(cols)?;
- }
-+ let multi_vector = self.nearest.as_ref().is_some_and(|query| {
-+ matches!(
-+ query.query.data_type(),
-+ arrow_schema::DataType::FixedSizeList(_, _)
-+ )
-+ });
- if self.limit.is_some() || self.offset.is_some() {
- scanner.limit(self.limit, self.offset)?;
-+ if multi_vector {
-+ // Retain Lance's window validation, but defer truncation until the final sort.
-+ scanner.limit(None, None)?;
-+ }
- }
- if let Some(bs) = self.batch_size {
- scanner.batch_size(bs);
-@@ -440,7 +450,27 @@ impl LanceScanner {
- if let Some(query_parallelism) = self.query_parallelism {
- scanner.query_parallelism(query_parallelism);
- }
-- if let Some(rf) = self.refine_factor {
-+ if multi_vector {
-+ if matches!(
-+ self.metric_override,
-+ Some(crate::index::LanceMetricType::Hamming)
-+ ) {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "multi-vector queries support only l2, cosine, and dot metrics".into(),
-+ ));
-+ }
-+ let refine = self.refine_factor.unwrap_or(1);
-+ if refine == 0
-+ || n.k as usize
-+ > crate::multivector::MAX_QUERY_VECTOR_CANDIDATES / refine as usize
-+ {
-+ return Err(lance_core::Error::invalid_input_source(
-+ "multi-vector refined candidate count must be in 1..=100000".into(),
-+ ));
-+ }
-+ // Validate actual stored values and refine candidate scores before TopK.
-+ scanner.refine(refine);
-+ } else if let Some(rf) = self.refine_factor {
- scanner.refine(rf);
- }
- if let Some(ef) = self.ef {
-@@ -448,6 +478,9 @@ impl LanceScanner {
- }
- if let Some(m) = self.metric_override {
- scanner.distance_metric(m.to_distance());
-+ } else if multi_vector {
-+ // Resolve the same default on indexed and uncovered fragments.
-+ scanner.distance_metric(lance_linalg::distance::DistanceType::L2);
- }
- if let Some(ui) = self.use_index {
- scanner.use_index(ui);
-@@ -506,6 +539,12 @@ impl LanceScanner {
- scanner,
- distributed_fts,
- scalar_segment,
-+ multi_vector_window: multi_vector.then_some((
-+ self.offset.unwrap_or(0) as usize,
-+ self.limit.map(|n| n as usize),
-+ )),
-+ batch_size: self.batch_size,
-+ scan_statistics_callback: self.scan_statistics_callback.clone(),
- })
- }
- }
-@@ -521,6 +560,9 @@ struct PreparedScanner {
- scanner: lance::dataset::scanner::Scanner,
- distributed_fts: Option<PreparedFtsExecution>,
- scalar_segment: Option<PreparedScalarSegment>,
-+ multi_vector_window: Option<(usize, Option<usize>)>,
-+ batch_size: Option<usize>,
-+ scan_statistics_callback: Option<ExecutionStatsCallback>,
- }
-
- impl PreparedScanner {
-@@ -532,6 +574,19 @@ impl PreparedScanner {
- .try_into_stream()
- .await;
- }
-+ if let Some((offset, limit)) = self.multi_vector_window {
-+ let plan = crate::multivector::rewrite(self.scanner.create_plan().await?)?;
-+ let plan = crate::multivector::apply_result_window(plan, offset, limit)?;
-+ let stream = lance_datafusion::exec::execute_plan(
-+ plan,
-+ lance_datafusion::exec::LanceExecutionOptions {
-+ batch_size: self.batch_size,
-+ execution_stats_callback: self.scan_statistics_callback,
-+ ..Default::default()
-+ },
-+ )?;
-+ return Ok(DatasetRecordBatchStream::new(stream));
-+ }
- let Some(distributed_fts) = self.distributed_fts else {
- return self.scanner.try_into_stream().await;
- };
-@@ -2744,6 +2799,21 @@ unsafe fn scanner_nearest_inner(
- }
- let column_str = unsafe { helpers::parse_c_string(column)? }.unwrap();
-
-+ let query = unsafe { decode_query_values(query_data, query_len, element_type)? };
-+
-+ s.nearest = Some(NearestQuery {
-+ column: column_str.to_string(),
-+ query,
-+ k,
-+ });
-+ Ok(0)
-+}
-+
-+unsafe fn decode_query_values(
-+ query_data: *const c_void,
-+ query_len: usize,
-+ element_type: i32,
-+) -> Result<arrow_array::ArrayRef> {
- let dtype = match element_type {
- 0 => LanceDataType::Float32,
- 1 => LanceDataType::Float16,
-@@ -2782,9 +2852,112 @@ unsafe fn scanner_nearest_inner(
- }
- };
-
-+ Ok(query)
-+}
-+
-+/// Set one multi-vector query, supplied as a row-major matrix of floating-point values.
-+/// The caller must supply dimension * num_vectors aligned elements matching the column type.
-+#[unsafe(no_mangle)]
-+pub unsafe extern "C" fn lance_scanner_nearest_multivector(
-+ scanner: *mut LanceScanner,
-+ column: *const c_char,
-+ query_data: *const c_void,
-+ dimension: usize,
-+ num_vectors: usize,
-+ element_type: i32,
-+ k: u32,
-+) -> i32 {
-+ scanner_poison_check!(scanner, -1);
-+ scanner_ffi_try!(scanner, unsafe {
-+ nearest_multivector_inner(
-+ scanner,
-+ column,
-+ query_data,
-+ dimension,
-+ num_vectors,
-+ element_type,
-+ k,
-+ )
-+ },)
-+}
-+
-+unsafe fn nearest_multivector_inner(
-+ scanner: *mut LanceScanner,
-+ column: *const c_char,
-+ query_data: *const c_void,
-+ dimension: usize,
-+ num_vectors: usize,
-+ element_type: i32,
-+ k: u32,
-+) -> Result<i32> {
-+ use arrow_schema::{DataType, Field};
-+ let invalid = |message: &str| lance_core::Error::invalid_input_source(message.into());
-+ if scanner.is_null() || column.is_null() || query_data.is_null() {
-+ return Err(invalid("scanner, column, and query_data must not be NULL"));
-+ }
-+ if dimension == 0 || dimension > i32::MAX as usize || num_vectors == 0 || k == 0 {
-+ return Err(invalid(
-+ "dimension, num_vectors, and k must be positive; dimension must fit int32",
-+ ));
-+ }
-+ if num_vectors > crate::multivector::MAX_QUERY_VECTORS
-+ || num_vectors > crate::multivector::MAX_QUERY_VECTOR_CANDIDATES / k as usize
-+ {
-+ return Err(invalid(
-+ "multi-vector query exceeds 128 subvectors or 100000 subvector-candidates",
-+ ));
-+ }
-+ let (data_type, width) = match element_type {
-+ 0 => (DataType::Float32, 4),
-+ 1 => (DataType::Float16, 2),
-+ 2 => (DataType::Float64, 8),
-+ _ => {
-+ return Err(invalid(
-+ "multi-vector queries require float16, float32, or float64",
-+ ));
-+ }
-+ };
-+ let count = dimension
-+ .checked_mul(num_vectors)
-+ .filter(|count| *count <= isize::MAX as usize / width)
-+ .ok_or_else(|| invalid("query matrix byte size overflows"))?;
-+ let s = unsafe { &mut *scanner };
-+ if s.fts_query.is_some() || s.fts_context.is_some() {
-+ return Err(invalid(
-+ "nearest and full-text search are mutually exclusive",
-+ ));
-+ }
-+ let column = unsafe { helpers::parse_c_string(column)? }.unwrap();
-+ let field = s
-+ .dataset
-+ .schema()
-+ .field(column)
-+ .ok_or_else(|| invalid("multi-vector column does not exist"))?;
-+ match field.data_type() {
-+ DataType::List(child) if !child.is_nullable() => match child.data_type() {
-+ DataType::FixedSizeList(element, dim)
-+ if *dim == dimension as i32 && *element.data_type() == data_type => {}
-+ _ => return Err(invalid("multi-vector dimension/type mismatch")),
-+ },
-+ _ => {
-+ return Err(invalid(
-+ "multi-vector column must be List of non-nullable FixedSizeList",
-+ ));
-+ }
-+ }
-+ // A primitive array is interpreted as one vector by Lance. Preserve matrix shape even
-+ // for a single subvector. Lance does not preserve element nullability in its schema.
-+ let values = unsafe { decode_query_values(query_data, count, element_type)? };
-+ crate::multivector::validate_query(values.as_ref())?;
-+ let query = arrow_array::FixedSizeListArray::try_new(
-+ Arc::new(Field::new("item", data_type, false)),
-+ dimension as i32,
-+ values,
-+ None,
-+ )?;
- s.nearest = Some(NearestQuery {
-- column: column_str.to_string(),
-- query,
-+ column: column.to_string(),
-+ query: Arc::new(query),
- k,
- });
- Ok(0)
-diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index 04699da..24bef9f 100644
---- a/tests/c_api_test.rs
-+++ b/tests/c_api_test.rs
-@@ -13388,3 +13388,28 @@ fn test_scalar_segment_requires_explicit_domain_and_checks_uuid() {
- lance_dataset_close(ds);
- }
- }
-+
-+#[test]
-+fn test_multivector_nearest_rejects_null_handle() {
-+ let column = c_str("vectors");
-+ let query = [1.0f32, 0.0];
-+ let status = unsafe {
-+ lance_scanner_nearest_multivector(
-+ ptr::null_mut(),
-+ column.as_ptr(),
-+ query.as_ptr().cast(),
-+ 2,
-+ 1,
-+ 0,
-+ 1,
-+ )
-+ };
-+ assert_eq!(status, -1);
-+ let error = lance_last_error_message();
-+ assert!(!error.is_null());
-+ let message = unsafe { std::ffi::CStr::from_ptr(error) }
-+ .to_string_lossy()
-+ .into_owned();
-+ unsafe { lance_free_string(error) };
-+ assert!(message.contains("NULL"));
-+}
-diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp
-index fc6fc82..11badc6 100644
---- a/tests/cpp/test_cpp_api.cpp
-+++ b/tests/cpp/test_cpp_api.cpp
-@@ -427,6 +427,20 @@ static void test_nearest_smoke(const std::string& uri) {
- PASS();
- }
-
-+static void test_multivector_rejects_flat_column(const std::string& uri) {
-+ TEST(test_multivector_rejects_flat_column);
-+ auto scanner = lance::Dataset::open(uri).scan();
-+ const float query[8] = {1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
-+ bool caught = false;
-+ try {
-+ scanner.nearest_multivector("embedding", query, 8, 1, LANCE_DTYPE_FLOAT32, 1);
-+ } catch (const lance::Error&) {
-+ caught = true;
-+ }
-+ assert(caught);
-+ PASS();
-+}
-+
- static void test_index_segments_smoke(const std::string& /*uri*/) {
- TEST(test_index_segments_smoke);
-
-@@ -971,6 +985,7 @@ int main(int argc, char** argv) {
- test_error_exception(uri);
- test_index_lifecycle(uri);
- test_nearest_smoke(uri);
-+ test_multivector_rejects_flat_column(uri);
- test_index_segments_smoke(uri);
- test_index_segment_builder(uri);
- test_vector_models_and_reusable_segments(uri);
-diff --git a/tests/multivector_test.rs b/tests/multivector_test.rs
-new file mode 100644
-index 0000000..f02e850
---- /dev/null
-+++ b/tests/multivector_test.rs
-@@ -0,0 +1,965 @@
-+// SPDX-License-Identifier: Apache-2.0
-+// SPDX-FileCopyrightText: Copyright The Lance Authors
-+
-+use std::ffi::{CString, c_void};
-+use std::ptr;
-+use std::sync::Arc;
-+
-+use arrow::buffer::{NullBuffer, OffsetBuffer};
-+use arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
-+use arrow::record_batch::RecordBatchIterator;
-+use arrow_array::{Array, FixedSizeListArray, Float32Array, Int32Array, ListArray, RecordBatch};
-+use arrow_schema::{DataType, Field, Schema};
-+use lance::Dataset;
-+use lance_c::*;
-+
-+fn fixture() -> (tempfile::TempDir, CString) {
-+ let dir = tempfile::tempdir().unwrap();
-+ let path = dir.path().join("vectors.lance");
-+ let element = Arc::new(Field::new("item", DataType::Float32, false));
-+ let vectors = FixedSizeListArray::try_new(
-+ element,
-+ 2,
-+ Arc::new(Float32Array::from(vec![
-+ 1., 0., 0., 1., 2., 0., 3., 0., 0., 3.,
-+ ])),
-+ None,
-+ )
-+ .unwrap();
-+ let child = Arc::new(Field::new("item", vectors.data_type().clone(), false));
-+ let rows = ListArray::try_new(
-+ child,
-+ OffsetBuffer::new(vec![0i32, 2, 3, 5, 5, 5].into()),
-+ Arc::new(vectors),
-+ Some(NullBuffer::from(vec![true, true, true, true, false])),
-+ )
-+ .unwrap();
-+ let schema = Arc::new(Schema::new(vec![
-+ Field::new("id", DataType::Int32, false),
-+ Field::new("vectors", rows.data_type().clone(), true),
-+ ]));
-+ let batch = RecordBatch::try_new(
-+ schema.clone(),
-+ vec![
-+ Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])),
-+ Arc::new(rows),
-+ ],
-+ )
-+ .unwrap();
-+ lance_c::runtime::block_on(async {
-+ Dataset::write(
-+ RecordBatchIterator::new(vec![Ok(batch)], schema),
-+ path.to_str().unwrap(),
-+ None,
-+ )
-+ .await
-+ .unwrap();
-+ });
-+ (dir, CString::new(path.to_str().unwrap()).unwrap())
-+}
-+
-+unsafe fn collect(scanner: *mut LanceScanner) -> Vec<(i32, f32)> {
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ let status = unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) };
-+ if status != 0 {
-+ panic!(
-+ "{}",
-+ unsafe { std::ffi::CStr::from_ptr(lance_last_error_message()) }.to_string_lossy()
-+ );
-+ }
-+ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap();
-+ reader
-+ .flat_map(|batch| {
-+ let batch = batch.unwrap();
-+ let ids = batch
-+ .column_by_name("id")
-+ .unwrap()
-+ .as_any()
-+ .downcast_ref::<Int32Array>()
-+ .unwrap();
-+ let distances = batch
-+ .column_by_name("_distance")
-+ .unwrap()
-+ .as_any()
-+ .downcast_ref::<Float32Array>()
-+ .unwrap();
-+ (0..batch.num_rows())
-+ .map(|i| (ids.value(i), distances.value(i)))
-+ .collect::<Vec<_>>()
-+ })
-+ .collect()
-+}
-+
-+#[test]
-+fn multivector_search_scores_logical_rows_and_excludes_empty_and_null_rows() {
-+ let (_dir, uri) = fixture();
-+ let column = CString::new("vectors").unwrap();
-+ let q = [1.0f32, 0., 0., 1.];
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ assert!(!ds.is_null());
-+ for count in [1, 2] {
-+ let scan = lance_scanner_new(ds, ptr::null(), ptr::null());
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ column.as_ptr(),
-+ q.as_ptr() as *const c_void,
-+ 2,
-+ count,
-+ 0,
-+ 10
-+ ),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_set_use_index(scan, false), 0);
-+ assert_eq!(lance_scanner_set_prefilter(scan, true), 0);
-+ let rows = collect(scan);
-+ let expected = if count == 2 {
-+ vec![(1, 0.), (2, 6.), (3, 8.)]
-+ } else {
-+ vec![(1, 0.), (2, 1.), (3, 4.)]
-+ };
-+ assert_eq!(rows, expected);
-+ lance_scanner_close(scan);
-+ }
-+ lance_dataset_close(ds);
-+ }
-+}
-+
-+#[test]
-+fn multivector_rejects_invalid_shape_and_preserves_previous_query() {
-+ let (_dir, uri) = fixture();
-+ let column = CString::new("vectors").unwrap();
-+ let q = [1.0f32, 0., 0., 1.];
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ let scan = lance_scanner_new(ds, ptr::null(), ptr::null());
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ column.as_ptr(),
-+ q.as_ptr() as *const c_void,
-+ 2,
-+ 2,
-+ 0,
-+ 10
-+ ),
-+ 0
-+ );
-+ for (dim, count, dtype, k) in [
-+ (0, 2, 0, 10),
-+ (2, 0, 0, 10),
-+ (3, 1, 0, 10),
-+ (2, 2, 3, 10),
-+ (2, 2, 4, 10),
-+ (2, 2, 1, 10),
-+ (2, 2, 0, 0),
-+ (usize::MAX, 2, 0, 10),
-+ (2, usize::MAX, 0, 10),
-+ ] {
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ column.as_ptr(),
-+ q.as_ptr() as *const c_void,
-+ dim,
-+ count,
-+ dtype,
-+ k
-+ ),
-+ -1
-+ );
-+ }
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(scan, column.as_ptr(), ptr::null(), 2, 2, 0, 10),
-+ -1
-+ );
-+ assert_eq!(lance_scanner_set_use_index(scan, false), 0);
-+ for (limit, offset) in [(-1, 0), (10, -1)] {
-+ assert_eq!(lance_scanner_set_limit(scan, limit), 0);
-+ assert_eq!(lance_scanner_set_offset(scan, offset), 0);
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(lance_scanner_to_arrow_stream(scan, &mut stream), -1);
-+ }
-+ assert_eq!(lance_scanner_set_limit(scan, 10), 0);
-+ assert_eq!(lance_scanner_set_offset(scan, 0), 0);
-+ assert_eq!(collect(scan), vec![(1, 0.), (2, 6.), (3, 8.)]);
-+ lance_scanner_close(scan);
-+ lance_dataset_close(ds);
-+ }
-+}
-+
-+#[test]
-+fn fragment_scoped_indexed_search_orders_candidates_before_offset() {
-+ use lance::index::DatasetIndexExt;
-+ use lance::index::vector::VectorIndexParams;
-+ use lance_index::IndexType;
-+ use lance_linalg::distance::MetricType;
-+
-+ let (_dir, uri) = fixture();
-+ lance_c::runtime::block_on(async {
-+ let mut ds = Dataset::open(uri.to_str().unwrap()).await.unwrap();
-+ let batch = ds.scan().try_into_batch().await.unwrap();
-+ ds.create_index(
-+ &["vectors"],
-+ IndexType::Vector,
-+ None,
-+ &VectorIndexParams::ivf_flat(1, MetricType::Cosine),
-+ false,
-+ )
-+ .await
-+ .unwrap();
-+ ds.append(
-+ RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()),
-+ None,
-+ )
-+ .await
-+ .unwrap();
-+ });
-+ let column = CString::new("vectors").unwrap();
-+ let filter = CString::new("id >= 2").unwrap();
-+ let q = [1.0f32, 0., 0., 1.];
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ for _ in 0..10 {
-+ let scan = lance_scanner_new(ds, ptr::null(), filter.as_ptr());
-+ assert_eq!(
-+ lance_scanner_set_fragment_ids(scan, [0u64, 1].as_ptr(), 2),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_set_prefilter(scan, true), 0);
-+ assert_eq!(lance_scanner_set_batch_size(scan, 1), 0);
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ column.as_ptr(),
-+ q.as_ptr() as *const c_void,
-+ 2,
-+ 2,
-+ 0,
-+ 4
-+ ),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_set_metric(scan, 1), 0);
-+ assert_eq!(lance_scanner_set_refine_factor(scan, 1), 0);
-+ assert_eq!(lance_scanner_set_offset(scan, 1), 0);
-+ assert_eq!(lance_scanner_set_limit(scan, 2), 0);
-+ assert_eq!(collect(scan), vec![(3, 0.), (2, 1.)]);
-+ lance_scanner_close(scan);
-+ }
-+ lance_dataset_close(ds);
-+ }
-+}
-+
-+fn custom_fixture(
-+ rows: Vec<Vec<Option<f32>>>,
-+ dim: i32,
-+ indexed: bool,
-+) -> (tempfile::TempDir, CString) {
-+ custom_fixture_at(rows, dim, indexed, "vectors")
-+}
-+
-+fn custom_fixture_at(
-+ rows: Vec<Vec<Option<f32>>>,
-+ dim: i32,
-+ indexed: bool,
-+ column: &str,
-+) -> (tempfile::TempDir, CString) {
-+ use lance::index::DatasetIndexExt;
-+ use lance::index::vector::VectorIndexParams;
-+ use lance_index::IndexType;
-+ use lance_linalg::distance::MetricType;
-+ let dir = tempfile::tempdir().unwrap();
-+ let path = dir.path().join("vectors.lance");
-+ let mut offsets = vec![0i32];
-+ let mut values = Vec::new();
-+ for row in &rows {
-+ values.extend_from_slice(row);
-+ offsets.push(values.len() as i32 / dim);
-+ }
-+ let vectors = FixedSizeListArray::try_new(
-+ Arc::new(Field::new("item", DataType::Float32, true)),
-+ dim,
-+ Arc::new(Float32Array::from(values)),
-+ None,
-+ )
-+ .unwrap();
-+ let rows_array = ListArray::try_new(
-+ Arc::new(Field::new("item", vectors.data_type().clone(), false)),
-+ OffsetBuffer::new(offsets.into()),
-+ Arc::new(vectors),
-+ None,
-+ )
-+ .unwrap();
-+ let parts = lance_core::datatypes::parse_field_path(column).unwrap();
-+ let mut array: arrow_array::ArrayRef = Arc::new(rows_array);
-+ for name in parts[1..].iter().rev() {
-+ let field = Arc::new(Field::new(name, array.data_type().clone(), true));
-+ let labels: arrow_array::ArrayRef = Arc::new(Int32Array::from(vec![7; rows.len()]));
-+ array = Arc::new(arrow_array::StructArray::from(vec![
-+ (field, array),
-+ (
-+ Arc::new(Field::new("label", DataType::Int32, false)),
-+ labels,
-+ ),
-+ ]));
-+ }
-+ let schema = Arc::new(Schema::new(vec![
-+ Field::new("id", DataType::Int32, false),
-+ Field::new(&parts[0], array.data_type().clone(), true),
-+ ]));
-+ let batch = RecordBatch::try_new(
-+ schema.clone(),
-+ vec![
-+ Arc::new(Int32Array::from_iter_values(1..=rows.len() as i32)),
-+ array,
-+ ],
-+ )
-+ .unwrap();
-+ lance_c::runtime::block_on(async {
-+ let mut ds = Dataset::write(
-+ RecordBatchIterator::new(vec![Ok(batch)], schema),
-+ path.to_str().unwrap(),
-+ None,
-+ )
-+ .await
-+ .unwrap();
-+ if indexed {
-+ ds.create_index(
-+ &[column],
-+ IndexType::Vector,
-+ None,
-+ &VectorIndexParams::ivf_flat(1, MetricType::Cosine),
-+ false,
-+ )
-+ .await
-+ .unwrap();
-+ }
-+ });
-+ (dir, CString::new(path.to_str().unwrap()).unwrap())
-+}
-+
-+#[test]
-+fn exact_top_one_preserves_small_distances_before_truncation() {
-+ let (_dir, uri) = custom_fixture(vec![vec![Some(0.00015)], vec![Some(0.0001)]], 1, false);
-+ let column = CString::new("vectors").unwrap();
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ for count in [1, 2] {
-+ for batch_size in [1, 1024] {
-+ let scan = lance_scanner_new(ds, ptr::null(), ptr::null());
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ column.as_ptr(),
-+ [0.0f32, 0.0].as_ptr().cast(),
-+ 1,
-+ count,
-+ 0,
-+ 1
-+ ),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_set_use_index(scan, false), 0);
-+ assert_eq!(lance_scanner_set_batch_size(scan, batch_size), 0);
-+ let rows = collect(scan);
-+ assert_eq!(rows.len(), 1);
-+ assert_eq!(rows[0].0, 2);
-+ assert!((rows[0].1 - count as f32 * 1e-8).abs() < 1e-14, "{rows:?}");
-+ lance_scanner_close(scan);
-+ }
-+ }
-+ lance_dataset_close(ds);
-+ }
-+}
-+
-+#[test]
-+fn indexed_top_one_is_independent_of_child_batch_boundaries() {
-+ let rows = vec![
-+ vec![Some(1.), Some(0.)],
-+ vec![Some(0.), Some(1.)],
-+ vec![Some(1.), Some(1.)],
-+ vec![Some(2.), Some(1.)],
-+ vec![Some(1.), Some(2.)],
-+ ];
-+ let (_dir, uri) = custom_fixture(rows, 2, true);
-+ let column = CString::new("vectors").unwrap();
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ for batch_size in [1, 2, 1024] {
-+ let scan = lance_scanner_new(ds, ptr::null(), ptr::null());
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ column.as_ptr(),
-+ [1.0f32, 0., 0., 1.].as_ptr().cast(),
-+ 2,
-+ 2,
-+ 0,
-+ 1
-+ ),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_set_metric(scan, 1), 0);
-+ assert_eq!(lance_scanner_set_refine_factor(scan, 1), 0);
-+ assert_eq!(lance_scanner_set_batch_size(scan, batch_size), 0);
-+ let result = collect(scan);
-+ assert_eq!(result[0].0, 3, "batch_size={batch_size}");
-+ assert!((result[0].1 - (2.0 - 2.0f32.sqrt())).abs() < 1e-6);
-+ lance_scanner_close(scan);
-+ }
-+ lance_dataset_close(ds);
-+ }
-+}
-+
-+#[test]
-+fn actual_null_and_nonfinite_stored_elements_fail_search() {
-+ let column = CString::new("vectors").unwrap();
-+ for value in [
-+ None,
-+ Some(f32::NAN),
-+ Some(f32::INFINITY),
-+ Some(f32::NEG_INFINITY),
-+ ] {
-+ let (_dir, uri) = custom_fixture(vec![vec![value, Some(0.)]], 2, false);
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ let scan = lance_scanner_new(ds, ptr::null(), ptr::null());
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ column.as_ptr(),
-+ [0.0f32, 0.].as_ptr().cast(),
-+ 2,
-+ 1,
-+ 0,
-+ 1
-+ ),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_set_use_index(scan, false), 0);
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(lance_scanner_to_arrow_stream(scan, &mut stream), 0);
-+ let mut reader = ArrowArrayStreamReader::from_raw(&mut stream).unwrap();
-+ let result = reader.next();
-+ assert!(
-+ matches!(result, Some(Err(_))),
-+ "value={value:?}: {result:?}"
-+ );
-+ drop(reader);
-+ lance_scanner_close(scan);
-+ lance_dataset_close(ds);
-+ }
-+ }
-+}
-+
-+#[test]
-+fn rejects_excessive_multivector_plan_width_and_candidate_work() {
-+ let (_dir, uri) = fixture();
-+ let column = CString::new("vectors").unwrap();
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ let scan = lance_scanner_new(ds, ptr::null(), ptr::null());
-+ let q = [0.0f32; 258];
-+ for (count, k) in [(129, 1), (128, 782), (1, 100001)] {
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ column.as_ptr(),
-+ q.as_ptr().cast(),
-+ 2,
-+ count,
-+ 0,
-+ k
-+ ),
-+ -1
-+ );
-+ }
-+ lance_scanner_close(scan);
-+ lance_dataset_close(ds);
-+ }
-+}
-+
-+#[test]
-+fn omitted_metric_is_l2_on_indexed_and_unindexed_fragments() {
-+ let (_dir, uri) = custom_fixture(vec![vec![Some(2.), Some(0.)]], 2, true);
-+ lance_c::runtime::block_on(async {
-+ let mut ds = Dataset::open(uri.to_str().unwrap()).await.unwrap();
-+ let old = ds.scan().try_into_batch().await.unwrap();
-+ let vectors = FixedSizeListArray::try_new(
-+ Arc::new(Field::new("item", DataType::Float32, true)),
-+ 2,
-+ Arc::new(Float32Array::from(vec![1.0, 0.1])),
-+ None,
-+ )
-+ .unwrap();
-+ let lists = ListArray::try_new(
-+ Arc::new(Field::new("item", vectors.data_type().clone(), false)),
-+ OffsetBuffer::new(vec![0i32, 1].into()),
-+ Arc::new(vectors),
-+ None,
-+ )
-+ .unwrap();
-+ let batch = RecordBatch::try_new(
-+ old.schema(),
-+ vec![Arc::new(Int32Array::from(vec![2])), Arc::new(lists)],
-+ )
-+ .unwrap();
-+ ds.append(
-+ RecordBatchIterator::new(vec![Ok(batch)], old.schema()),
-+ None,
-+ )
-+ .await
-+ .unwrap();
-+ });
-+ let column = CString::new("vectors").unwrap();
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ for (fragment, id, score) in [(0u64, 1, 1.0f32), (1u64, 2, 0.01f32)] {
-+ let scan = lance_scanner_new(ds, ptr::null(), ptr::null());
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ column.as_ptr(),
-+ [1.0f32, 0.].as_ptr().cast(),
-+ 2,
-+ 1,
-+ 0,
-+ 1
-+ ),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_set_prefilter(scan, true), 0);
-+ assert_eq!(lance_scanner_set_fragment_ids(scan, &fragment, 1), 0);
-+ let rows = collect(scan);
-+ assert_eq!(rows[0].0, id);
-+ assert!(
-+ (rows[0].1 - score).abs() < 1e-6,
-+ "fragment={fragment}: {rows:?}"
-+ );
-+ lance_scanner_close(scan);
-+ }
-+ lance_dataset_close(ds);
-+ }
-+}
-+
-+#[test]
-+fn query_values_and_refinement_are_validated_before_execution() {
-+ let (_dir, uri) = fixture();
-+ let column = CString::new("vectors").unwrap();
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ let scan = lance_scanner_new(ds, ptr::null(), ptr::null());
-+ for value in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] {
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ column.as_ptr(),
-+ [value, 0.0f32].as_ptr().cast(),
-+ 2,
-+ 1,
-+ 0,
-+ 1
-+ ),
-+ -1
-+ );
-+ }
-+ let query = [1.0f32; 256];
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ column.as_ptr(),
-+ query.as_ptr().cast(),
-+ 2,
-+ 128,
-+ 0,
-+ 781
-+ ),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_set_metric(scan, 3), 0);
-+ let mut invalid_metric_stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(
-+ lance_scanner_to_arrow_stream(scan, &mut invalid_metric_stream),
-+ -1
-+ );
-+ assert_eq!(lance_scanner_set_metric(scan, 0), 0);
-+ assert_eq!(lance_scanner_set_refine_factor(scan, u32::MAX), 0);
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(lance_scanner_to_arrow_stream(scan, &mut stream), -1);
-+ lance_scanner_close(scan);
-+ lance_dataset_close(ds);
-+ }
-+}
-+
-+#[test]
-+fn float16_and_float64_queries_support_all_distance_metrics() {
-+ let (dir, source) = fixture();
-+ for (code, dtype) in [(1, DataType::Float16), (2, DataType::Float64)] {
-+ let path = dir.path().join(format!("typed_{code}.lance"));
-+ lance_c::runtime::block_on(async {
-+ let source = Dataset::open(source.to_str().unwrap()).await.unwrap();
-+ let batch = source.scan().try_into_batch().await.unwrap();
-+ let vector_type = DataType::List(Arc::new(Field::new(
-+ "item",
-+ DataType::FixedSizeList(Arc::new(Field::new("item", dtype, true)), 2),
-+ false,
-+ )));
-+ let vectors =
-+ arrow::compute::cast(batch.column_by_name("vectors").unwrap(), &vector_type)
-+ .unwrap();
-+ let schema = Arc::new(Schema::new(vec![
-+ Field::new("id", DataType::Int32, false),
-+ Field::new("vectors", vector_type, true),
-+ ]));
-+ let batch =
-+ RecordBatch::try_new(schema.clone(), vec![batch.column(0).clone(), vectors])
-+ .unwrap();
-+ Dataset::write(
-+ RecordBatchIterator::new(vec![Ok(batch)], schema),
-+ path.to_str().unwrap(),
-+ None,
-+ )
-+ .await
-+ .unwrap();
-+ });
-+ let uri = CString::new(path.to_str().unwrap()).unwrap();
-+ let column = CString::new("vectors").unwrap();
-+ let f16_query = [1.0f32, 0., 0., 1.].map(half::f16::from_f32);
-+ let f64_query = [1.0f64, 0., 0., 1.];
-+ let query = if code == 1 {
-+ f16_query.as_ptr().cast()
-+ } else {
-+ f64_query.as_ptr().cast()
-+ };
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ for (metric, expected) in [(0, [0., 6., 8.]), (1, [0., 1., 0.]), (2, [0., 0., -4.])] {
-+ let scanner = lance_scanner_new(ds, ptr::null(), ptr::null());
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scanner,
-+ column.as_ptr(),
-+ query,
-+ 2,
-+ 2,
-+ code,
-+ 10
-+ ),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_set_metric(scanner, metric), 0);
-+ assert_eq!(lance_scanner_set_use_index(scanner, false), 0);
-+ let mut rows = collect(scanner);
-+ rows.sort_by_key(|row| row.0);
-+ assert_eq!(
-+ rows,
-+ vec![(1, expected[0]), (2, expected[1]), (3, expected[2])]
-+ );
-+ lance_scanner_close(scanner);
-+ }
-+ lance_dataset_close(ds);
-+ }
-+ }
-+}
-+
-+#[test]
-+fn cosine_zero_norm_rows_do_not_abort_exact_or_refined_search() {
-+ let rows = vec![
-+ vec![Some(0.), Some(0.)],
-+ vec![Some(1.), Some(0.)],
-+ vec![Some(0.), Some(0.), Some(0.), Some(1.)],
-+ ];
-+ for indexed in [false, true] {
-+ let (_dir, uri) = custom_fixture(rows.clone(), 2, indexed);
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ for query in [[1.0f32, 0., 0., 1.], [0., 0., 0., 1.]] {
-+ for count in [1, 2] {
-+ let scan = lance_scanner_new(ds, ptr::null(), ptr::null());
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ c"vectors".as_ptr(),
-+ query.as_ptr().cast(),
-+ 2,
-+ count,
-+ 0,
-+ 3
-+ ),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_set_use_index(scan, indexed), 0);
-+ assert_eq!(lance_scanner_set_metric(scan, 1), 0);
-+ assert_eq!(lance_scanner_set_refine_factor(scan, 2), 0);
-+ let mut actual = collect(scan);
-+ actual.sort_by_key(|row| row.0);
-+ let expected = if query[0] == 0. {
-+ vec![]
-+ } else if count == 1 {
-+ vec![(2, 0.), (3, 1.)]
-+ } else {
-+ vec![(2, 1.), (3, 1.)]
-+ };
-+ assert_eq!(
-+ actual, expected,
-+ "indexed={indexed}, query={query:?}, count={count}"
-+ );
-+ lance_scanner_close(scan);
-+ }
-+ }
-+ lance_dataset_close(ds);
-+ }
-+ }
-+}
-+
-+#[test]
-+fn nested_and_quoted_columns_support_exact_and_refined_search() {
-+ for column in [
-+ "payload.vectors",
-+ "payload.`vectors.with.dot`",
-+ "payload.inner.`vectors.with.dot`",
-+ ] {
-+ let rows = vec![
-+ vec![Some(1.), Some(0.), Some(0.), Some(1.)],
-+ vec![Some(2.), Some(0.)],
-+ vec![Some(1.), Some(1.)],
-+ ];
-+ for indexed in [false, true] {
-+ let (_dir, uri) = custom_fixture_at(rows.clone(), 2, indexed, column);
-+ let column = CString::new(column).unwrap();
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ let scan = lance_scanner_new(ds, ptr::null(), ptr::null());
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ column.as_ptr(),
-+ [1.0f32, 0., 0., 1.].as_ptr().cast(),
-+ 2,
-+ 2,
-+ 0,
-+ 3
-+ ),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_set_use_index(scan, indexed), 0);
-+ assert_eq!(lance_scanner_set_metric(scan, 1), 0);
-+ assert_eq!(lance_scanner_set_refine_factor(scan, 2), 0);
-+ let actual = collect(scan);
-+ assert_eq!(actual.len(), 3);
-+ assert_eq!(
-+ actual.iter().map(|row| row.0).collect::<Vec<_>>(),
-+ vec![1, 3, 2]
-+ );
-+ assert_eq!(actual[0].1, 0.);
-+ assert!((actual[1].1 - (2. - 2.0f32.sqrt())).abs() < 1e-6);
-+ assert_eq!(actual[2].1, 1.);
-+ lance_scanner_close(scan);
-+ lance_dataset_close(ds);
-+ }
-+ }
-+ }
-+}
-+
-+#[test]
-+fn nested_projections_preserve_schema_and_values_in_exact_indexed_and_hybrid_search() {
-+ for column in ["payload.vectors", "payload.`vectors.with.dot`"] {
-+ for indexed in [false, true] {
-+ for appended in [false, true] {
-+ let (_dir, uri) = custom_fixture_at(
-+ vec![vec![Some(1.), Some(0.)], vec![Some(0.), Some(1.)]],
-+ 2,
-+ indexed,
-+ column,
-+ );
-+ if appended {
-+ lance_c::runtime::block_on(async {
-+ let mut ds = Dataset::open(uri.to_str().unwrap()).await.unwrap();
-+ let batch = ds.scan().try_into_batch().await.unwrap();
-+ ds.append(
-+ RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()),
-+ None,
-+ )
-+ .await
-+ .unwrap();
-+ });
-+ }
-+ for projection in [
-+ vec![],
-+ vec!["id"],
-+ vec!["payload.label"],
-+ vec!["id", column],
-+ ] {
-+ let expected = lance_c::runtime::block_on(async {
-+ let ds = Dataset::open(uri.to_str().unwrap()).await.unwrap();
-+ let mut scanner = ds.scan();
-+ scanner.scan_in_order(true);
-+ if !projection.is_empty() {
-+ scanner.project(&projection).unwrap();
-+ }
-+ scanner.try_into_batch().await.unwrap()
-+ });
-+ // The appended fragment repeats the same two values; distance order groups
-+ // both exact matches before the orthogonal rows, regardless of tie order.
-+ let indices = arrow_array::UInt32Array::from(if appended {
-+ vec![0, 2, 1, 3]
-+ } else {
-+ vec![0, 1]
-+ });
-+ let expected = RecordBatch::try_new(
-+ expected.schema(),
-+ expected
-+ .columns()
-+ .iter()
-+ .map(|array| {
-+ arrow::compute::take(array.as_ref(), &indices, None).unwrap()
-+ })
-+ .collect(),
-+ )
-+ .unwrap();
-+ unsafe {
-+ let names: Vec<_> = projection
-+ .iter()
-+ .map(|name| CString::new(*name).unwrap())
-+ .collect();
-+ let mut columns: Vec<_> = names.iter().map(|name| name.as_ptr()).collect();
-+ columns.push(ptr::null());
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ let scan = lance_scanner_new(
-+ ds,
-+ if projection.is_empty() {
-+ ptr::null()
-+ } else {
-+ columns.as_ptr()
-+ },
-+ ptr::null(),
-+ );
-+ let column = CString::new(column).unwrap();
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ column.as_ptr(),
-+ [1.0f32, 0.].as_ptr().cast(),
-+ 2,
-+ 1,
-+ 0,
-+ 10
-+ ),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_set_use_index(scan, indexed), 0);
-+ assert_eq!(lance_scanner_set_metric(scan, 1), 0);
-+ assert_eq!(lance_scanner_set_batch_size(scan, 1), 0);
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(lance_scanner_to_arrow_stream(scan, &mut stream), 0);
-+ let batches = ArrowArrayStreamReader::from_raw(&mut stream)
-+ .unwrap()
-+ .collect::<Result<Vec<_>, _>>()
-+ .unwrap();
-+ lance_scanner_close(scan);
-+ lance_dataset_close(ds);
-+ let actual =
-+ arrow::compute::concat_batches(&batches[0].schema(), &batches).unwrap();
-+ let distances = actual
-+ .column_by_name("_distance")
-+ .unwrap()
-+ .as_any()
-+ .downcast_ref::<Float32Array>()
-+ .unwrap();
-+ assert_eq!(
-+ distances.values().as_ref(),
-+ if appended {
-+ &[0., 0., 1., 1.][..]
-+ } else {
-+ &[0., 1.][..]
-+ }
-+ );
-+ let positions: Vec<_> = actual
-+ .schema()
-+ .fields()
-+ .iter()
-+ .enumerate()
-+ .filter_map(|(i, field)| (field.name() != "_distance").then_some(i))
-+ .collect();
-+ assert_eq!(
-+ actual.project(&positions).unwrap(),
-+ expected,
-+ "column={column:?} indexed={indexed} appended={appended} projection={projection:?}"
-+ );
-+ }
-+ }
-+ }
-+ }
-+ }
-+}
-+
-+#[test]
-+fn strict_batches_apply_after_multivector_offset_and_limit() {
-+ let (_dir, uri) = custom_fixture(
-+ (1..=6).map(|i| vec![Some(i as f32), Some(0.)]).collect(),
-+ 2,
-+ false,
-+ );
-+ unsafe {
-+ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0);
-+ for batch_size in [Some(2), None] {
-+ for (offset, limit) in [(1, 4), (1, 3), (5, 4), (6, 4), (0, 6)] {
-+ let scan = lance_scanner_new(ds, ptr::null(), ptr::null());
-+ assert_eq!(
-+ lance_scanner_nearest_multivector(
-+ scan,
-+ c"vectors".as_ptr(),
-+ [1.0f32, 0.].as_ptr().cast(),
-+ 2,
-+ 1,
-+ 0,
-+ 6
-+ ),
-+ 0
-+ );
-+ assert_eq!(lance_scanner_set_use_index(scan, false), 0);
-+ if let Some(size) = batch_size {
-+ assert_eq!(lance_scanner_set_batch_size(scan, size), 0);
-+ }
-+ assert_eq!(lance_scanner_set_strict_batch_size(scan, true), 0);
-+ assert_eq!(lance_scanner_set_offset(scan, offset), 0);
-+ assert_eq!(lance_scanner_set_limit(scan, limit), 0);
-+ let mut stream = FFI_ArrowArrayStream::empty();
-+ assert_eq!(lance_scanner_to_arrow_stream(scan, &mut stream), 0);
-+ let batches = ArrowArrayStreamReader::from_raw(&mut stream)
-+ .unwrap()
-+ .collect::<Result<Vec<_>, _>>()
-+ .unwrap();
-+ lance_scanner_close(scan);
-+ let sizes: Vec<_> = batches.iter().map(RecordBatch::num_rows).collect();
-+ let ids: Vec<_> = batches
-+ .iter()
-+ .flat_map(|b| {
-+ b.column_by_name("id")
-+ .unwrap()
-+ .as_any()
-+ .downcast_ref::<Int32Array>()
-+ .unwrap()
-+ .values()
-+ .to_vec()
-+ })
-+ .collect();
-+ let expected_ids: Vec<_> =
-+ (1..=6).skip(offset as usize).take(limit as usize).collect();
-+ let expected_sizes: Vec<_> = expected_ids
-+ .chunks(batch_size.unwrap_or(8192) as usize)
-+ .map(<[i32]>::len)
-+ .collect();
-+ assert_eq!(ids, expected_ids);
-+ assert_eq!(
-+ sizes, expected_sizes,
-+ "batch_size={batch_size:?} offset={offset} limit={limit}"
-+ );
-+ }
-+ }
-+ lance_dataset_close(ds);
-+ }
-+}
diff --git a/thirdparty/patches/lance-c-0.1.9-prefilter-fts.patch b/thirdparty/patches/lance-c-0.1.9-prefilter-fts.patch
deleted file mode 100644
index 890093886fb..00000000000
--- a/thirdparty/patches/lance-c-0.1.9-prefilter-fts.patch
+++ /dev/null
@@ -1,148 +0,0 @@
-Subject: [PATCH] Include FTS prefilter metrics and full-snapshot test coverage
-
-Upstream: https://github.com/lance-format/lance/pull/9460
-Commit: f202fe41ac18323ca0cd7bc6efd8fd30b8722116
-
-Backport the applicable review updates from Lance PR #9471 to v11.
-Keep the C API and other dependencies unchanged, and upgrade previously
-patched source caches with the same revision as fresh builds.
-
-diff --git a/Cargo.toml b/Cargo.toml
---- a/Cargo.toml
-+++ b/Cargo.toml
-@@ -20,10 +20,10 @@
- [dependencies]
--lance = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] }
--lance-core = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
--lance-file = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
--lance-index = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
--lance-io = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
--lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
--lance-table = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
--lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] }
-+lance = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116", features = ["substrait"] }
-+lance-core = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" }
-+lance-file = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" }
-+lance-index = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" }
-+lance-io = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" }
-+lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" }
-+lance-table = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" }
-+lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116", features = ["substrait"] }
- datafusion = { version = "54.0.0", default-features = false }
-@@ -53,5 +53,5 @@
- [dev-dependencies]
--lance = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] }
--lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
--lance-file = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
-+lance = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116", features = ["substrait"] }
-+lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" }
-+lance-file = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" }
- tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
-diff --git a/Cargo.lock b/Cargo.lock
---- a/Cargo.lock
-+++ b/Cargo.lock
-@@ -2608,3 +2608,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -3820,3 +3820,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -3892,3 +3892,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -3914,3 +3914,3 @@
- version = "58.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -3928,3 +3928,3 @@
- version = "58.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -3938,3 +3938,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -3984,3 +3984,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -4022,3 +4022,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -4054,3 +4054,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -4072,3 +4072,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -4082,3 +4082,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -4116,3 +4116,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -4148,3 +4148,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -4163,3 +4163,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -4231,3 +4231,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -4254,3 +4254,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -4294,3 +4294,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -4309,3 +4309,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -4336,3 +4336,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -4351,3 +4351,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
-@@ -4390,3 +4390,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
-+source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116"
- dependencies = [
diff --git a/thirdparty/patches/lance-c-0.1.9-prefilter.patch b/thirdparty/patches/lance-c-0.1.9-prefilter.patch
deleted file mode 100644
index c3df2f5d262..00000000000
--- a/thirdparty/patches/lance-c-0.1.9-prefilter.patch
+++ /dev/null
@@ -1,147 +0,0 @@
-Subject: [PATCH] Use Lance full-snapshot prefilter optimization and loader metrics
-
-Upstream: https://github.com/lance-format/lance/pull/9460
-Commit: f75f3343b5e125c42da1bd7acc8d8217cd5660a6
-
-Pin the tested Lance v11 change without upgrading the release or changing
-the C API. Keep all Lance crates on the same revision.
-
-diff --git a/Cargo.toml b/Cargo.toml
---- a/Cargo.toml
-+++ b/Cargo.toml
-@@ -20,10 +20,10 @@
- [dependencies]
--lance = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] }
--lance-core = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
--lance-file = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
--lance-index = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
--lance-io = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
--lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
--lance-table = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
--lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] }
-+lance = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] }
-+lance-core = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
-+lance-file = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
-+lance-index = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
-+lance-io = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
-+lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
-+lance-table = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
-+lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] }
- datafusion = { version = "54.0.0", default-features = false }
-@@ -53,5 +53,5 @@
- [dev-dependencies]
--lance = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] }
--lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
--lance-file = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-+lance = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] }
-+lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
-+lance-file = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
- tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
-diff --git a/Cargo.lock b/Cargo.lock
---- a/Cargo.lock
-+++ b/Cargo.lock
-@@ -2608,3 +2608,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
-+source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
- dependencies = [
-@@ -3820,3 +3820,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
-+source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
- dependencies = [
-@@ -3892,3 +3892,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
-+source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
- dependencies = [
-@@ -3914,3 +3914,3 @@
- version = "58.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
-+source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
- dependencies = [
-@@ -3928,3 +3928,3 @@
- version = "58.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
-+source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
- dependencies = [
-@@ -3938,3 +3938,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
-+source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
- dependencies = [
-@@ -3984,3 +3984,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
-+source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
- dependencies = [
-@@ -4022,3 +4022,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
-+source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
- dependencies = [
-@@ -4054,3 +4054,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
-+source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
- dependencies = [
-@@ -4072,3 +4072,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
-+source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
- dependencies = [
-@@ -4082,3 +4082,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
-+source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
- dependencies = [
-@@ -4116,3 +4116,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
-+source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
- dependencies = [
-@@ -4148,3 +4148,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
-+source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
- dependencies = [
-@@ -4163,3 +4163,3 @@
- version = "11.0.0"
--source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
... 1565 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 34/35: [fix](arrow-flight) Preserve nested logical type metadata (#68614)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1bbc51ef3c5a891c5b7de9e55d7731e2ba1ec85d
Author: Gabriel <li...@selectdb.com>
AuthorDate: Thu Oct 8 10:46:33 2026 +0800
[fix](arrow-flight) Preserve nested logical type metadata (#68614)
### What problem does this PR solve?
Flight SQL attaches `doris_type` metadata to top-level LARGEINT/IP
fields, but ARRAY items, MAP keys/values and STRUCT children lose it. A
nested LARGEINT therefore arrives as an Arrow string that clients cannot
distinguish from ordinary text. JSON and VARIANT also lack the marker at
the top level.
For example, in `SELECT named_struct('number', CAST(17 AS LARGEINT),
'text', '17')`, both children currently appear as unannotated strings.
This change annotates only the LARGEINT child, allowing clients to
restore its logical type safely.
Related PR: #67530 (closed while awaiting type-mapping refactoring).
This change targets the current `branch-4.1` schema and converter
implementation.
Move schema construction behind the abstract `ArrowSchemaConvertor`
interface. Doris, Flight, legacy Flight, Parquet, Iceberg and Paimon own
their conversion policies; recursive Doris type conversion no longer
accepts protocol or metadata flags. Flight selects its compatibility
implementation once during result-sink preparation. Iceberg retains
table field IDs and schema metadata, and Paimon consumes the
writer-provided IPC schema so nested nullability and physical layouts
remain authoritative.
Build nested fields through the existing metadata helper and add
JSON/VARIANT to that helper. Preserve Arrow storage types, child names,
nullability and encoded values; in particular, MAP keys remain
non-nullable and LARGEINT retains its string encoding for the full
signed 128-bit range. Metadata is carried at every nesting depth and
survives Arrow IPC serialization. No session setting is required.
Rolling upgrades use an optional result-sink capability flag. Old FEs
omit it, so upgraded BEs return the exact legacy metadata layout for
both schema fetches and result streams. Upgraded FEs enable the flag and
fill missing markers from the planned Doris output types before
comparing and advertising schemas. Physical types, names, nullability,
dictionaries, conflicting markers and unrelated metadata still
participate in strict schema equality. Mixed old/new BE endpoints
produce the same advertised metadata in either order. An old BE's stream
can still lack the markers advertised by the upgraded FE.
PyArrow does not automatically interpret custom field metadata.
Applications can use `doris_type=LARGEINT` to convert the corresponding
values to integers without guessing from their text.
### Release note
Preserve Doris logical type metadata for LARGEINT, IPV4, IPV6, JSON and
VARIANT inside Arrow arrays, maps and structs. JSON and VARIANT now also
include the marker at the top level.
### Check List (For Author)
- Test:
- BE ASAN: all 84 tests passed with `bash run-be-ut.sh --run
--filter='*MetadataTest*:ArrowSchemaConvertorTest.*:ArrowBlockConvertorTest.*:ArrowSchemaUtilTest.*:DataTypeSerDeArrowTest.*:VParquetWriterTest.*:ArrowFlightTimestampTest.*'
-j 48`. Coverage includes metadata compatibility, independent converter
policies, table schemas and field IDs, Arrow SerDe, timestamp bindings,
and Parquet writer round trips.
- The new polymorphic schema-converter test verifies that nested
timestamp and LARGEINT metadata policies stay independent across Doris,
Flight and legacy Flight instances.
- Before the fix, 8 of the 9 new tests failed on missing metadata; the
native-type negative control passed.
- Coverage includes all five logical types at the root and in
ARRAY/STRUCT/MAP, LARGEINT map keys, deep nesting, names/nullability,
signed 128-bit extremes, NULL values, and Arrow IPC round trips.
- FE: 13 tests passed (5 new compatibility tests and 8 existing Arrow
schema helper tests). Tests cover mixed-version schema fetches in both
orders, all-old/all-new endpoints, strict rejection of physical and
unrelated metadata differences, and result-sink capability negotiation.
The pre-fix FE fails the mixed-version and all-old metadata cases.
- Legacy BE schema tests cover all five logical types, LARGEINT map
keys, and IPC schema round trips. Populated LARGEINT batches also run
through the legacy schema path to verify values, NULLs, and IPC remain
compatible.
- FE Checkstyle, clang-format 16 and `git diff --check` passed.
- The optional ADBC integration script has five independent cases for
LARGEINT, IPV4, IPV6, JSON and VARIANT. JSON/VARIANT use ARRAY_REPEAT to
support nested values in the default mode.
- A temporary FE analyzer test checks all five exact sample queries and
asserts ARRAY<JSON>/ARRAY<VARIANT> result types. The original combined
query fails analysis; the updated queries pass.
- Python syntax/import checks passed; all five live ADBC tests were
skipped because no cluster URI was provided. No live-cluster result is
claimed.
- Behavior changed: additional Arrow field metadata only; physical types
and values remain compatible.
- Documentation: describe nested metadata and client conversion in the
Python Flight SQL sample README.
---
.../exec/operator/memory_scratch_sink_operator.cpp | 3 +-
be/src/exec/operator/result_sink_operator.cpp | 23 +-
be/src/exec/operator/result_sink_operator.h | 3 +
.../aggregate/aggregate_function_python_udaf.cpp | 4 +-
be/src/format/arrow/arrow_block_convertor.cpp | 15 +-
be/src/format/arrow/arrow_block_convertor.h | 9 +-
be/src/format/arrow/arrow_row_batch.cpp | 147 ++++++----
be/src/format/arrow/arrow_row_batch.h | 71 ++++-
.../parquet/parquet_arrow_block_convertor.cpp | 18 +-
.../format/parquet/parquet_arrow_block_convertor.h | 20 ++
be/src/format/table/iceberg/arrow_schema_util.cpp | 46 ++--
be/src/format/table/iceberg/arrow_schema_util.h | 20 +-
.../iceberg/iceberg_arrow_block_convertor.cpp | 12 +-
.../table/paimon/paimon_arrow_block_convertor.cpp | 16 +-
.../table/paimon/paimon_arrow_block_convertor.h | 11 +
be/src/udf/python/python_udf_meta.cpp | 3 +-
.../core/data_type/common_data_type_serder_test.h | 4 +-
.../data_type_serde/data_type_serde_arrow_test.cpp | 31 ++-
.../format/arrow/arrow_flight_timestamp_test.cpp | 21 +-
be/test/format/arrow/arrow_row_batch_test.cpp | 305 +++++++++++++++++++++
.../table/iceberg/arrow_schema_util_test.cpp | 16 +-
.../java/org/apache/doris/planner/ResultSink.java | 3 +
.../arrowflight/FlightSqlConnectProcessor.java | 9 +-
.../service/arrowflight/FlightSqlSchemaHelper.java | 45 +++
.../doris/planner/ResultSinkMetadataTest.java | 38 +++
.../FlightSqlConnectProcessorSchemaTest.java | 167 +++++++++++
gensrc/thrift/DataSinks.thrift | 2 +
samples/arrow-flight-sql/python/README.md | 22 ++
.../python/test_nested_type_metadata.py | 108 ++++++++
29 files changed, 1034 insertions(+), 158 deletions(-)
diff --git a/be/src/exec/operator/memory_scratch_sink_operator.cpp b/be/src/exec/operator/memory_scratch_sink_operator.cpp
index 7b3bf3eed0b..3520491ebb2 100644
--- a/be/src/exec/operator/memory_scratch_sink_operator.cpp
+++ b/be/src/exec/operator/memory_scratch_sink_operator.cpp
@@ -103,8 +103,7 @@ Status MemoryScratchSinkOperatorX::sink_impl(RuntimeState* state, Block* input_b
RETURN_IF_ERROR(VExprContext::get_output_block_after_execute_exprs(
local_state._output_vexpr_ctxs, *input_block, &block));
}
- DorisArrowBlockConvertor converter(block, state->timezone(), _timezone_obj,
- /*datetime_naive=*/true);
+ ArrowFlightArrowBlockConvertor converter(block, state->timezone(), _timezone_obj);
{
SCOPED_TIMER(local_state._get_arrow_schema_timer);
// After expr executed, use recaculated schema as final schema
diff --git a/be/src/exec/operator/result_sink_operator.cpp b/be/src/exec/operator/result_sink_operator.cpp
index 24c9f6ad18e..88e21fafd1a 100644
--- a/be/src/exec/operator/result_sink_operator.cpp
+++ b/be/src/exec/operator/result_sink_operator.cpp
@@ -57,9 +57,8 @@ Status ResultSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info)
} else {
std::shared_ptr<arrow::Schema> arrow_schema;
if (p._sink_type == TResultSinkType::ARROW_FLIGHT_PROTOCOL) {
- RETURN_IF_ERROR(get_arrow_schema_from_expr_ctxs(_output_vexpr_ctxs, &arrow_schema,
- state->timezone(),
- /*datetime_naive=*/true));
+ RETURN_IF_ERROR(p._arrow_schema_convertor->get_arrow_schema_from_expr_ctxs(
+ _output_vexpr_ctxs, &arrow_schema));
}
VLOG_DEBUG << "create sender in INIT with instance id " << fragment_instance_id;
RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(
@@ -103,6 +102,8 @@ ResultSinkOperatorX::ResultSinkOperatorX(int operator_id, int node_id,
_sink_type(!sink.__isset.type || sink.type == TResultSinkType::MYSQL_PROTOCOL
? TResultSinkType::MYSQL_PROTOCOL
: sink.type),
+ _enable_arrow_type_metadata(sink.__isset.enable_arrow_type_metadata &&
+ sink.enable_arrow_type_metadata),
_result_sink_buffer_size_rows(_sink_type == TResultSinkType::ARROW_FLIGHT_PROTOCOL
? config::arrow_flight_result_sink_buffer_size_rows
: RESULT_SINK_BUFFER_SIZE),
@@ -120,12 +121,22 @@ Status ResultSinkOperatorX::prepare(RuntimeState* state) {
// Prepare the exprs to run.
RETURN_IF_ERROR(VExpr::prepare(_output_vexpr_ctxs, state, _row_desc));
+ if (_sink_type == TResultSinkType::ARROW_FLIGHT_PROTOCOL) {
+ // Capability negotiation selects one immutable policy for schema fetches and batches.
+ if (_enable_arrow_type_metadata) {
+ _arrow_schema_convertor =
+ std::make_shared<ArrowFlightSchemaConvertor>(state->timezone());
+ } else {
+ _arrow_schema_convertor =
+ std::make_shared<LegacyArrowFlightSchemaConvertor>(state->timezone());
+ }
+ }
+
if (state->query_options().enable_parallel_result_sink) {
std::shared_ptr<arrow::Schema> arrow_schema;
if (_sink_type == TResultSinkType::ARROW_FLIGHT_PROTOCOL) {
- RETURN_IF_ERROR(get_arrow_schema_from_expr_ctxs(_output_vexpr_ctxs, &arrow_schema,
- state->timezone(),
- /*datetime_naive=*/true));
+ RETURN_IF_ERROR(_arrow_schema_convertor->get_arrow_schema_from_expr_ctxs(
+ _output_vexpr_ctxs, &arrow_schema));
}
VLOG_DEBUG << "create sender in prepare with query id " << state->query_id();
RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(
diff --git a/be/src/exec/operator/result_sink_operator.h b/be/src/exec/operator/result_sink_operator.h
index 84c86f1127e..173a5b52e82 100644
--- a/be/src/exec/operator/result_sink_operator.h
+++ b/be/src/exec/operator/result_sink_operator.h
@@ -25,6 +25,7 @@
#include "runtime/result_block_buffer.h"
namespace doris {
+class DorisArrowSchemaConvertor;
#include "common/compile_check_begin.h"
class ResultBlockBufferBase;
@@ -167,6 +168,8 @@ private:
Status _second_phase_fetch_data(RuntimeState* state, Block* final_block);
const TResultSinkType::type _sink_type;
+ const bool _enable_arrow_type_metadata;
+ std::shared_ptr<const DorisArrowSchemaConvertor> _arrow_schema_convertor;
const int _result_sink_buffer_size_rows;
// set file options when sink type is FILE
std::unique_ptr<ResultFileOptions> _file_opts = nullptr;
diff --git a/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp b/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp
index 495daee2018..2e2fbd266fb 100644
--- a/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp
@@ -245,10 +245,10 @@ void AggregatePythonUDAF::create(AggregateDataPtr __restrict place) const {
std::call_once(_schema_init_flag, [this]() {
std::vector<std::shared_ptr<arrow::Field>> fields;
- std::string timezone = TimezoneUtils::default_time_zone;
+ const DorisArrowSchemaConvertor schema_convertor(TimezoneUtils::default_time_zone);
for (size_t i = 0; i < argument_types.size(); ++i) {
std::shared_ptr<arrow::DataType> arrow_type;
- Status st = convert_to_arrow_type(argument_types[i], &arrow_type, timezone);
+ Status st = schema_convertor.convert_to_arrow_type(argument_types[i], &arrow_type);
if (!st.ok()) {
throw doris::Exception(ErrorCode::INTERNAL_ERROR,
"Failed to convert argument type {} to Arrow type: {}", i,
diff --git a/be/src/format/arrow/arrow_block_convertor.cpp b/be/src/format/arrow/arrow_block_convertor.cpp
index 6b2ed5a71e8..2148d0934af 100644
--- a/be/src/format/arrow/arrow_block_convertor.cpp
+++ b/be/src/format/arrow/arrow_block_convertor.cpp
@@ -443,7 +443,8 @@ Status ArrowBlockConvertor::write_plain_arrow_column(const std::shared_ptr<const
int64_t start, int64_t end,
const cctz::time_zone& ctz) const {
std::shared_ptr<arrow::DataType> plain_arrow_type;
- RETURN_IF_ERROR(convert_to_arrow_type(type, &plain_arrow_type, ctz.name()));
+ RETURN_IF_ERROR(
+ DorisArrowSchemaConvertor(ctz.name()).convert_to_arrow_type(type, &plain_arrow_type));
const auto storage_type = extension_storage_type(field->type());
// This is an exact binding check selected by the target converter, not a recovery path. A
// mismatch returns without invoking SerDe, and a SerDe error is never retried elsewhere.
@@ -501,8 +502,16 @@ Status DorisArrowBlockConvertor::init() {
if (_arrow_schema == nullptr) {
// cctz names fixed offsets as "Fixed/UTC+HH:MM:SS", which is not the Arrow
// protocol label. Keep the declared name so Python metadata and batches agree.
- RETURN_IF_ERROR(get_arrow_schema_from_block(_header, &_arrow_schema, _timezone_name,
- _datetime_naive));
+ RETURN_IF_ERROR(DorisArrowSchemaConvertor(_header, _timezone_name)
+ .get_arrow_schema(&_arrow_schema));
+ }
+ return ArrowBlockConvertor::init();
+}
+
+Status ArrowFlightArrowBlockConvertor::init() {
+ if (_arrow_schema == nullptr) {
+ RETURN_IF_ERROR(ArrowFlightSchemaConvertor(_header, _timezone_name)
+ .get_arrow_schema(&_arrow_schema));
}
return ArrowBlockConvertor::init();
}
diff --git a/be/src/format/arrow/arrow_block_convertor.h b/be/src/format/arrow/arrow_block_convertor.h
index c923507786d..e0b58d76838 100644
--- a/be/src/format/arrow/arrow_block_convertor.h
+++ b/be/src/format/arrow/arrow_block_convertor.h
@@ -89,11 +89,10 @@ class DorisArrowBlockConvertor : public ArrowBlockConvertor {
public:
using ArrowBlockConvertor::ArrowBlockConvertor;
DorisArrowBlockConvertor(const Block& header, std::string timezone_name,
- const cctz::time_zone& timezone, bool datetime_naive = false)
+ const cctz::time_zone& timezone)
: ArrowBlockConvertor(nullptr, timezone),
_header(header.clone_empty()),
- _timezone_name(std::move(timezone_name)),
- _datetime_naive(datetime_naive) {}
+ _timezone_name(std::move(timezone_name)) {}
Status init() override;
Status convert_from_arrow(const std::shared_ptr<arrow::RecordBatch>& batch,
@@ -106,16 +105,14 @@ protected:
arrow::ArrayBuilder* array_builder, int64_t start, int64_t end,
const cctz::time_zone& ctz) const override;
-private:
Block _header;
std::string _timezone_name;
- bool _datetime_naive = false;
};
class ArrowFlightArrowBlockConvertor final : public DorisArrowBlockConvertor {
public:
using DorisArrowBlockConvertor::DorisArrowBlockConvertor;
-
+ Status init() override;
Status convert_to_arrow(const Block& block, arrow::MemoryPool* pool,
std::shared_ptr<arrow::RecordBatch>* result, size_t start_row = 0,
size_t end_row = 0) const override;
diff --git a/be/src/format/arrow/arrow_row_batch.cpp b/be/src/format/arrow/arrow_row_batch.cpp
index f291cf33ac3..a958843e519 100644
--- a/be/src/format/arrow/arrow_row_batch.cpp
+++ b/be/src/format/arrow/arrow_row_batch.cpp
@@ -48,9 +48,8 @@
namespace doris {
-Status convert_to_arrow_type(const DataTypePtr& origin_type,
- std::shared_ptr<arrow::DataType>* result, const std::string& timezone,
- bool datetime_naive) {
+Status DorisArrowSchemaConvertor::convert_to_arrow_type(
+ const DataTypePtr& origin_type, std::shared_ptr<arrow::DataType>* result) const {
auto type = get_serialized_type(origin_type);
switch (type->get_primitive_type()) {
case TYPE_NULL:
@@ -107,18 +106,8 @@ Status convert_to_arrow_type(const DataTypePtr& origin_type,
} else {
time_unit = arrow::TimeUnit::SECOND;
}
- // Doris DATETIMEV2 represents a wall-clock value without a timezone. Arrow Flight
- // exposes it as a timezone-naive timestamp so clients do not interpret it as an instant.
- // This option only changes the DATETIMEV2 output schema. TIMESTAMPTZ remains timezone-aware,
- // and Arrow-to-Doris conversions are unaffected.
- if (type->get_primitive_type() == TYPE_DATETIMEV2 && datetime_naive) {
- *result = std::make_shared<arrow::TimestampType>(time_unit);
- } else {
- // Arrow clients resolve timezone metadata as an IANA name; use the canonical UTC
- // name instead of the ISO-8601 "Z" alias without changing the encoded instant.
- *result = std::make_shared<arrow::TimestampType>(time_unit,
- timezone == "Z" ? "UTC" : timezone);
- }
+ *result = std::make_shared<arrow::TimestampType>(
+ time_unit, timestamp_timezone(type->get_primitive_type()));
break;
}
case TYPE_DECIMALV2:
@@ -136,20 +125,23 @@ Status convert_to_arrow_type(const DataTypePtr& origin_type,
case TYPE_ARRAY: {
const auto* type_arr = assert_cast<const DataTypeArray*>(remove_nullable(type).get());
std::shared_ptr<arrow::DataType> item_type;
- RETURN_IF_ERROR(convert_to_arrow_type(type_arr->get_nested_type(), &item_type, timezone,
- datetime_naive));
- *result = std::make_shared<arrow::ListType>(item_type);
+ RETURN_IF_ERROR(convert_to_arrow_type(type_arr->get_nested_type(), &item_type));
+ // Arrow stores metadata on fields, so implicit child fields lose the Doris logical type.
+ *result = std::make_shared<arrow::ListType>(make_child_field(
+ "item", item_type, true, type_arr->get_nested_type()->get_primitive_type()));
break;
}
case TYPE_MAP: {
const auto* type_map = assert_cast<const DataTypeMap*>(remove_nullable(type).get());
std::shared_ptr<arrow::DataType> key_type;
std::shared_ptr<arrow::DataType> val_type;
- RETURN_IF_ERROR(convert_to_arrow_type(type_map->get_key_type(), &key_type, timezone,
- datetime_naive));
- RETURN_IF_ERROR(convert_to_arrow_type(type_map->get_value_type(), &val_type, timezone,
- datetime_naive));
- *result = std::make_shared<arrow::MapType>(key_type, val_type);
+ RETURN_IF_ERROR(convert_to_arrow_type(type_map->get_key_type(), &key_type));
+ RETURN_IF_ERROR(convert_to_arrow_type(type_map->get_value_type(), &val_type));
+ auto key_field = make_child_field("key", key_type, false,
+ type_map->get_key_type()->get_primitive_type());
+ auto value_field = make_child_field("value", val_type, true,
+ type_map->get_value_type()->get_primitive_type());
+ *result = std::make_shared<arrow::MapType>(key_field, value_field);
break;
}
case TYPE_STRUCT: {
@@ -157,11 +149,10 @@ Status convert_to_arrow_type(const DataTypePtr& origin_type,
std::vector<std::shared_ptr<arrow::Field>> fields;
for (size_t i = 0; i < type_struct->get_elements().size(); i++) {
std::shared_ptr<arrow::DataType> field_type;
- RETURN_IF_ERROR(convert_to_arrow_type(type_struct->get_element(i), &field_type,
- timezone, datetime_naive));
- fields.push_back(
- std::make_shared<arrow::Field>(type_struct->get_element_name(i), field_type,
- type_struct->get_element(i)->is_nullable()));
+ RETURN_IF_ERROR(convert_to_arrow_type(type_struct->get_element(i), &field_type));
+ fields.push_back(make_child_field(type_struct->get_element_name(i), field_type,
+ type_struct->get_element(i)->is_nullable(),
+ type_struct->get_element(i)->get_primitive_type()));
}
*result = std::make_shared<arrow::StructType>(fields);
break;
@@ -187,56 +178,102 @@ Status convert_to_arrow_type(const DataTypePtr& origin_type,
return Status::OK();
}
-// Helper function to create an Arrow Field with type metadata if applicable, such as IP types
+// Logical types sharing Arrow storage need the same marker at the root and every child field.
std::shared_ptr<arrow::Field> create_arrow_field_with_metadata(
const std::string& field_name, const std::shared_ptr<arrow::DataType>& arrow_type,
bool is_nullable, PrimitiveType primitive_type) {
- if (primitive_type == PrimitiveType::TYPE_IPV4) {
- auto metadata = arrow::KeyValueMetadata::Make({"doris_type"}, {"IPV4"});
- return std::make_shared<arrow::Field>(field_name, arrow_type, is_nullable, metadata);
- } else if (primitive_type == PrimitiveType::TYPE_IPV6) {
- auto metadata = arrow::KeyValueMetadata::Make({"doris_type"}, {"IPV6"});
- return std::make_shared<arrow::Field>(field_name, arrow_type, is_nullable, metadata);
- } else if (primitive_type == PrimitiveType::TYPE_LARGEINT) {
- auto metadata = arrow::KeyValueMetadata::Make({"doris_type"}, {"LARGEINT"});
- return std::make_shared<arrow::Field>(field_name, arrow_type, is_nullable, metadata);
- } else {
+ const char* type_name;
+ switch (primitive_type) {
+ case TYPE_IPV4:
+ type_name = "IPV4";
+ break;
+ case TYPE_IPV6:
+ type_name = "IPV6";
+ break;
+ case TYPE_LARGEINT:
+ type_name = "LARGEINT";
+ break;
+ case TYPE_JSONB:
+ type_name = "JSON";
+ break;
+ case TYPE_VARIANT:
+ type_name = "VARIANT";
+ break;
+ default:
return std::make_shared<arrow::Field>(field_name, arrow_type, is_nullable);
}
+ auto metadata = arrow::KeyValueMetadata::Make({"doris_type"}, {type_name});
+ return std::make_shared<arrow::Field>(field_name, arrow_type, is_nullable, metadata);
}
-Status get_arrow_schema_from_block(const Block& block, std::shared_ptr<arrow::Schema>* result,
- const std::string& timezone, bool datetime_naive) {
+Status DorisArrowSchemaConvertor::get_arrow_schema_from_block(
+ const Block& block, std::shared_ptr<arrow::Schema>* result) const {
std::vector<std::shared_ptr<arrow::Field>> fields;
for (const auto& type_and_name : block) {
std::shared_ptr<arrow::DataType> arrow_type;
- RETURN_IF_ERROR(
- convert_to_arrow_type(type_and_name.type, &arrow_type, timezone, datetime_naive));
- auto field = create_arrow_field_with_metadata(type_and_name.name, arrow_type,
- type_and_name.type->is_nullable(),
- type_and_name.type->get_primitive_type());
+ RETURN_IF_ERROR(convert_to_arrow_type(type_and_name.type, &arrow_type));
+ auto field = make_field(type_and_name.name, arrow_type, type_and_name.type->is_nullable(),
+ type_and_name.type->get_primitive_type());
fields.push_back(field);
}
*result = arrow::schema(std::move(fields));
return Status::OK();
}
-Status get_arrow_schema_from_expr_ctxs(const VExprContextSPtrs& output_vexpr_ctxs,
- std::shared_ptr<arrow::Schema>* result,
- const std::string& timezone, bool datetime_naive) {
+Status DorisArrowSchemaConvertor::get_arrow_schema(std::shared_ptr<arrow::Schema>* result) const {
+ return get_arrow_schema_from_block(_header, result);
+}
+
+std::string DorisArrowSchemaConvertor::timestamp_timezone(PrimitiveType) const {
+ // Arrow clients expect a timezone name rather than the ISO-8601 UTC alias.
+ return _timezone == "Z" ? "UTC" : _timezone;
+}
+
+std::string ArrowFlightSchemaConvertor::timestamp_timezone(PrimitiveType type) const {
+ // DATETIMEV2 is wall-clock time; TIMESTAMPTZ must still describe an instant.
+ return type == TYPE_DATETIMEV2 ? "" : DorisArrowSchemaConvertor::timestamp_timezone(type);
+}
+
+std::shared_ptr<arrow::Field> DorisArrowSchemaConvertor::make_field(
+ const std::string& name, const std::shared_ptr<arrow::DataType>& type, bool nullable,
+ PrimitiveType primitive) const {
+ return create_arrow_field_with_metadata(name, type, nullable, primitive);
+}
+
+std::shared_ptr<arrow::Field> DorisArrowSchemaConvertor::make_child_field(
+ const std::string& name, const std::shared_ptr<arrow::DataType>& type, bool nullable,
+ PrimitiveType primitive) const {
+ return create_arrow_field_with_metadata(name, type, nullable, primitive);
+}
+
+std::shared_ptr<arrow::Field> LegacyArrowFlightSchemaConvertor::make_field(
+ const std::string& name, const std::shared_ptr<arrow::DataType>& type, bool nullable,
+ PrimitiveType primitive) const {
+ if (primitive == TYPE_JSONB || primitive == TYPE_VARIANT) {
+ return arrow::field(name, type, nullable);
+ }
+ return DorisArrowSchemaConvertor::make_field(name, type, nullable, primitive);
+}
+
+std::shared_ptr<arrow::Field> LegacyArrowFlightSchemaConvertor::make_child_field(
+ const std::string& name, const std::shared_ptr<arrow::DataType>& type, bool nullable,
+ PrimitiveType) const {
+ return arrow::field(name, type, nullable);
+}
+
+Status DorisArrowSchemaConvertor::get_arrow_schema_from_expr_ctxs(
+ const VExprContextSPtrs& output_vexpr_ctxs, std::shared_ptr<arrow::Schema>* result) const {
std::vector<std::shared_ptr<arrow::Field>> fields;
for (int i = 0; i < output_vexpr_ctxs.size(); i++) {
std::shared_ptr<arrow::DataType> arrow_type;
auto root_expr = output_vexpr_ctxs.at(i)->root();
- RETURN_IF_ERROR(convert_to_arrow_type(root_expr->data_type(), &arrow_type, timezone,
- datetime_naive));
+ RETURN_IF_ERROR(convert_to_arrow_type(root_expr->data_type(), &arrow_type));
auto field_name = root_expr->is_slot_ref() && !root_expr->expr_label().empty()
? root_expr->expr_label()
: fmt::format("{}_{}", root_expr->data_type()->get_name(), i);
- auto field =
- create_arrow_field_with_metadata(field_name, arrow_type, root_expr->is_nullable(),
- root_expr->data_type()->get_primitive_type());
- fields.push_back(field);
+ auto field = make_field(field_name, arrow_type, root_expr->is_nullable(),
+ root_expr->data_type()->get_primitive_type());
+ fields.push_back(std::move(field));
}
*result = arrow::schema(std::move(fields));
return Status::OK();
diff --git a/be/src/format/arrow/arrow_row_batch.h b/be/src/format/arrow/arrow_row_batch.h
index d5ba5cb0ed0..60b192716fe 100644
--- a/be/src/format/arrow/arrow_row_batch.h
+++ b/be/src/format/arrow/arrow_row_batch.h
@@ -19,6 +19,7 @@
#include <memory>
#include <string>
+#include <utility>
#include "common/status.h"
#include "core/block/block.h"
@@ -43,23 +44,69 @@ constexpr size_t MAX_ARROW_UTF8 = (1ULL << 31); // 2G
class RowDescriptor;
-// datetime_naive only controls how Doris DATETIMEV2 is represented in the output Arrow schema.
-// When enabled, DATETIMEV2 is mapped to a timestamp without a timezone to preserve its wall-clock
-// semantics. TIMESTAMPTZ remains timezone-aware, and Arrow-to-Doris conversions are unaffected.
-Status convert_to_arrow_type(const DataTypePtr& type, std::shared_ptr<arrow::DataType>* result,
- const std::string& timezone, bool datetime_naive = false);
+// Each protocol owns its schema source and conversion rules. Table formats retain their
+// authoritative schemas instead of reconstructing field IDs or physical layouts from Doris types.
+class ArrowSchemaConvertor {
+public:
+ virtual ~ArrowSchemaConvertor() = default;
+ virtual Status get_arrow_schema(std::shared_ptr<arrow::Schema>* result) const = 0;
+};
+
+class DorisArrowSchemaConvertor : public ArrowSchemaConvertor {
+public:
+ explicit DorisArrowSchemaConvertor(std::string timezone) : _timezone(std::move(timezone)) {}
+ DorisArrowSchemaConvertor(const Block& header, std::string timezone)
+ : _header(header.clone_empty()), _timezone(std::move(timezone)) {}
+
+ Status get_arrow_schema(std::shared_ptr<arrow::Schema>* result) const override;
+ Status get_arrow_schema_from_block(const Block& block,
+ std::shared_ptr<arrow::Schema>* result) const;
+ Status get_arrow_schema_from_expr_ctxs(const VExprContextSPtrs& output_vexpr_ctxs,
+ std::shared_ptr<arrow::Schema>* result) const;
+ Status convert_to_arrow_type(const DataTypePtr& type,
+ std::shared_ptr<arrow::DataType>* result) const;
+
+protected:
+ virtual std::string timestamp_timezone(PrimitiveType type) const;
+ virtual std::shared_ptr<arrow::Field> make_field(const std::string& name,
+ const std::shared_ptr<arrow::DataType>& type,
+ bool nullable, PrimitiveType primitive) const;
+ virtual std::shared_ptr<arrow::Field> make_child_field(
+ const std::string& name, const std::shared_ptr<arrow::DataType>& type, bool nullable,
+ PrimitiveType primitive) const;
+
+private:
+ Block _header;
+ const std::string _timezone;
+};
+
+class ArrowFlightSchemaConvertor : public DorisArrowSchemaConvertor {
+public:
+ using DorisArrowSchemaConvertor::DorisArrowSchemaConvertor;
+
+protected:
+ std::string timestamp_timezone(PrimitiveType type) const override;
+};
+
+// Old FEs require the pre-capability metadata layout, including metadata-free nested fields.
+class LegacyArrowFlightSchemaConvertor final : public ArrowFlightSchemaConvertor {
+public:
+ using ArrowFlightSchemaConvertor::ArrowFlightSchemaConvertor;
+
+protected:
+ std::shared_ptr<arrow::Field> make_field(const std::string& name,
+ const std::shared_ptr<arrow::DataType>& type,
+ bool nullable, PrimitiveType primitive) const override;
+ std::shared_ptr<arrow::Field> make_child_field(const std::string& name,
+ const std::shared_ptr<arrow::DataType>& type,
+ bool nullable,
+ PrimitiveType primitive) const override;
+};
std::shared_ptr<arrow::Field> create_arrow_field_with_metadata(
const std::string& field_name, const std::shared_ptr<arrow::DataType>& arrow_type,
bool is_nullable, PrimitiveType primitive_type);
-Status get_arrow_schema_from_block(const Block& block, std::shared_ptr<arrow::Schema>* result,
- const std::string& timezone, bool datetime_naive = false);
-
-Status get_arrow_schema_from_expr_ctxs(const VExprContextSPtrs& output_vexpr_ctxs,
- std::shared_ptr<arrow::Schema>* result,
- const std::string& timezone, bool datetime_naive = false);
-
Status serialize_record_batch(const arrow::RecordBatch& record_batch, std::string* result);
Status serialize_arrow_schema(std::shared_ptr<arrow::Schema>* schema, std::string* result);
diff --git a/be/src/format/parquet/parquet_arrow_block_convertor.cpp b/be/src/format/parquet/parquet_arrow_block_convertor.cpp
index cd8b933195a..156ee634ac5 100644
--- a/be/src/format/parquet/parquet_arrow_block_convertor.cpp
+++ b/be/src/format/parquet/parquet_arrow_block_convertor.cpp
@@ -24,7 +24,13 @@
namespace doris {
#include "common/compile_check_begin.h"
-Status ParquetArrowBlockConvertor::init() {
+std::string ParquetArrowSchemaConvertor::timestamp_timezone(PrimitiveType type) const {
+ return type == TYPE_DATETIMEV2 && !_enable_int96_timestamps
+ ? ""
+ : DorisArrowSchemaConvertor::timestamp_timezone(type);
+}
+
+Status ParquetArrowSchemaConvertor::get_arrow_schema(std::shared_ptr<arrow::Schema>* result) const {
if (_types.size() != _names.size()) {
return Status::InvalidArgument("Parquet column names and types must have the same size");
}
@@ -34,14 +40,18 @@ Status ParquetArrowBlockConvertor::init() {
// INT96 normalization and schema construction must use the same instance's timezone.
for (size_t i = 0; i < _types.size(); ++i) {
std::shared_ptr<arrow::DataType> type;
- RETURN_IF_ERROR(
- convert_to_arrow_type(_types[i], &type, _timezone_name, !_enable_int96_timestamps));
+ RETURN_IF_ERROR(convert_to_arrow_type(_types[i], &type));
fields.emplace_back(arrow::field(_names[i], type, _types[i]->is_nullable()));
}
- _arrow_schema = arrow::schema(std::move(fields));
+ *result = arrow::schema(std::move(fields));
return Status::OK();
}
+Status ParquetArrowBlockConvertor::init() {
+ return ParquetArrowSchemaConvertor(_types, _names, _timezone_name, _enable_int96_timestamps)
+ .get_arrow_schema(&_arrow_schema);
+}
+
Status ParquetArrowBlockConvertor::write_column(const DataTypePtr& type, const DataTypeSerDe& serde,
const IColumn& column, const NullMap* null_map,
const std::shared_ptr<arrow::Field>& field,
diff --git a/be/src/format/parquet/parquet_arrow_block_convertor.h b/be/src/format/parquet/parquet_arrow_block_convertor.h
index 24a490a565b..28744817f9c 100644
--- a/be/src/format/parquet/parquet_arrow_block_convertor.h
+++ b/be/src/format/parquet/parquet_arrow_block_convertor.h
@@ -18,9 +18,29 @@
#pragma once
#include "format/arrow/arrow_block_convertor.h"
+#include "format/arrow/arrow_row_batch.h"
namespace doris {
+class ParquetArrowSchemaConvertor final : public DorisArrowSchemaConvertor {
+public:
+ ParquetArrowSchemaConvertor(DataTypes types, std::vector<std::string> names,
+ std::string timezone, bool enable_int96_timestamps)
+ : DorisArrowSchemaConvertor(std::move(timezone)),
+ _types(std::move(types)),
+ _names(std::move(names)),
+ _enable_int96_timestamps(enable_int96_timestamps) {}
+ Status get_arrow_schema(std::shared_ptr<arrow::Schema>* result) const override;
+
+protected:
+ std::string timestamp_timezone(PrimitiveType type) const override;
+
+private:
+ const DataTypes _types;
+ const std::vector<std::string> _names;
+ const bool _enable_int96_timestamps;
+};
+
class ParquetArrowBlockConvertor : public ArrowBlockConvertor {
public:
ParquetArrowBlockConvertor(DataTypes types, std::vector<std::string> names,
diff --git a/be/src/format/table/iceberg/arrow_schema_util.cpp b/be/src/format/table/iceberg/arrow_schema_util.cpp
index 84dc29f594d..848ce74f903 100644
--- a/be/src/format/table/iceberg/arrow_schema_util.cpp
+++ b/be/src/format/table/iceberg/arrow_schema_util.cpp
@@ -24,24 +24,36 @@
namespace doris::iceberg {
#include "common/compile_check_begin.h"
-const char* ArrowSchemaUtil::PARQUET_FIELD_ID = "PARQUET:field_id";
-const char* ArrowSchemaUtil::ORIGINAL_TYPE = "originalType";
-const char* ArrowSchemaUtil::MAP_TYPE_VALUE = "mapType";
-const char* ArrowSchemaUtil::UUID_TYPE_VALUE = "uuid";
-
-Status ArrowSchemaUtil::convert(const Schema* schema, const std::string& timezone,
- std::vector<std::shared_ptr<arrow::Field>>& fields) {
- for (const auto& column : schema->columns()) {
+const char* IcebergArrowSchemaConvertor::PARQUET_FIELD_ID = "PARQUET:field_id";
+const char* IcebergArrowSchemaConvertor::ORIGINAL_TYPE = "originalType";
+const char* IcebergArrowSchemaConvertor::MAP_TYPE_VALUE = "mapType";
+const char* IcebergArrowSchemaConvertor::UUID_TYPE_VALUE = "uuid";
+
+Status IcebergArrowSchemaConvertor::get_arrow_schema(std::shared_ptr<arrow::Schema>* result) const {
+ // Field IDs, Variant storage and timestamp bindings all come from the pinned table schema.
+ std::vector<std::shared_ptr<arrow::Field>> fields;
+ RETURN_IF_ERROR(convert_fields(fields));
+ auto schema = arrow::schema(std::move(fields));
+ if (!_schema_json.empty()) {
+ schema = schema->WithMetadata(
+ arrow::KeyValueMetadata::Make({"iceberg.schema"}, {_schema_json}));
+ }
+ *result = std::move(schema);
+ return Status::OK();
+}
+
+Status IcebergArrowSchemaConvertor::convert_fields(
+ std::vector<std::shared_ptr<arrow::Field>>& fields) const {
+ for (const auto& column : _schema.columns()) {
std::shared_ptr<arrow::Field> arrow_field;
- RETURN_IF_ERROR(convert_to(column, &arrow_field, timezone));
+ RETURN_IF_ERROR(convert_to_arrow_field(column, &arrow_field));
fields.push_back(arrow_field);
}
return Status::OK();
}
-Status ArrowSchemaUtil::convert_to(const iceberg::NestedField& field,
- std::shared_ptr<arrow::Field>* arrow_field,
- const std::string& timezone) {
+Status IcebergArrowSchemaConvertor::convert_to_arrow_field(
+ const iceberg::NestedField& field, std::shared_ptr<arrow::Field>* arrow_field) const {
std::shared_ptr<arrow::DataType> arrow_type;
std::unordered_map<std::string, std::string> metadata;
metadata[PARQUET_FIELD_ID] = std::to_string(field.field_id());
@@ -73,7 +85,7 @@ Status ArrowSchemaUtil::convert_to(const iceberg::NestedField& field,
case iceberg::TypeID::TIMESTAMP: {
iceberg::TimestampType* t_type = static_cast<iceberg::TimestampType*>(field.field_type());
- std::string real_tz = t_type->should_adjust_to_utc() ? timezone : "";
+ std::string real_tz = t_type->should_adjust_to_utc() ? _timezone : "";
arrow_type = std::make_shared<arrow::TimestampType>(arrow::TimeUnit::MICRO, real_tz);
break;
}
@@ -115,7 +127,7 @@ Status ArrowSchemaUtil::convert_to(const iceberg::NestedField& field,
StructType* st = field.field_type()->as_struct_type();
for (const auto& column : st->fields()) {
std::shared_ptr<arrow::Field> element_field;
- RETURN_IF_ERROR(convert_to(column, &element_field, timezone));
+ RETURN_IF_ERROR(convert_to_arrow_field(column, &element_field));
element_fields.push_back(element_field);
}
arrow_type = arrow::struct_(element_fields);
@@ -125,7 +137,7 @@ Status ArrowSchemaUtil::convert_to(const iceberg::NestedField& field,
case iceberg::TypeID::LIST: {
std::shared_ptr<arrow::Field> item_field;
ListType* list_type = field.field_type()->as_list_type();
- RETURN_IF_ERROR(convert_to(list_type->element_field(), &item_field, timezone));
+ RETURN_IF_ERROR(convert_to_arrow_field(list_type->element_field(), &item_field));
arrow_type = arrow::list(item_field);
break;
}
@@ -134,8 +146,8 @@ Status ArrowSchemaUtil::convert_to(const iceberg::NestedField& field,
std::shared_ptr<arrow::Field> key_field;
std::shared_ptr<arrow::Field> value_field;
MapType* map_type = field.field_type()->as_map_type();
- RETURN_IF_ERROR(convert_to(map_type->key_field(), &key_field, timezone));
- RETURN_IF_ERROR(convert_to(map_type->value_field(), &value_field, timezone));
+ RETURN_IF_ERROR(convert_to_arrow_field(map_type->key_field(), &key_field));
+ RETURN_IF_ERROR(convert_to_arrow_field(map_type->value_field(), &value_field));
metadata[ORIGINAL_TYPE] = MAP_TYPE_VALUE;
arrow_type = std::make_shared<arrow::MapType>(key_field, value_field);
break;
diff --git a/be/src/format/table/iceberg/arrow_schema_util.h b/be/src/format/table/iceberg/arrow_schema_util.h
index 2942edacbaa..9d0f6d022c7 100644
--- a/be/src/format/table/iceberg/arrow_schema_util.h
+++ b/be/src/format/table/iceberg/arrow_schema_util.h
@@ -19,15 +19,21 @@
#include <arrow/type.h>
+#include "format/arrow/arrow_row_batch.h"
#include "format/table/iceberg/schema.h"
namespace doris::iceberg {
#include "common/compile_check_begin.h"
-class ArrowSchemaUtil {
+class IcebergArrowSchemaConvertor final : public ArrowSchemaConvertor {
public:
- static Status convert(const Schema* schema, const std::string& timezone,
- std::vector<std::shared_ptr<arrow::Field>>& fields);
+ IcebergArrowSchemaConvertor(const Schema& schema, std::string timezone,
+ std::string schema_json = {})
+ : _schema(schema),
+ _timezone(std::move(timezone)),
+ _schema_json(std::move(schema_json)) {}
+ Status get_arrow_schema(std::shared_ptr<arrow::Schema>* result) const override;
+ Status convert_fields(std::vector<std::shared_ptr<arrow::Field>>& fields) const;
private:
static const char* PARQUET_FIELD_ID;
@@ -35,9 +41,11 @@ private:
static const char* MAP_TYPE_VALUE;
static const char* UUID_TYPE_VALUE;
- static Status convert_to(const iceberg::NestedField& field,
- std::shared_ptr<arrow::Field>* arrow_field,
- const std::string& timezone);
+ Status convert_to_arrow_field(const iceberg::NestedField& field,
+ std::shared_ptr<arrow::Field>* arrow_field) const;
+ const Schema& _schema;
+ const std::string _timezone;
+ const std::string _schema_json;
};
#include "common/compile_check_end.h"
diff --git a/be/src/format/table/iceberg/iceberg_arrow_block_convertor.cpp b/be/src/format/table/iceberg/iceberg_arrow_block_convertor.cpp
index f5260b76e6d..0cbcebd2ecb 100644
--- a/be/src/format/table/iceberg/iceberg_arrow_block_convertor.cpp
+++ b/be/src/format/table/iceberg/iceberg_arrow_block_convertor.cpp
@@ -30,16 +30,8 @@ Status IcebergArrowBlockConvertor::init() {
if (_schema == nullptr) {
return ArrowBlockConvertor::init();
}
- // Field IDs, Variant storage and timestamp bindings must share the same target schema.
- std::vector<std::shared_ptr<arrow::Field>> fields;
- // Arrow consumers need the declared label, not cctz's internal fixed-offset name.
- RETURN_IF_ERROR(ArrowSchemaUtil::convert(_schema, _timezone_name, fields));
- _arrow_schema = arrow::schema(std::move(fields));
- if (!_schema_json.empty()) {
- _arrow_schema = _arrow_schema->WithMetadata(
- arrow::KeyValueMetadata::Make({"iceberg.schema"}, {_schema_json}));
- }
- return Status::OK();
+ return IcebergArrowSchemaConvertor(*_schema, _timezone_name, _schema_json)
+ .get_arrow_schema(&_arrow_schema);
}
Status IcebergArrowBlockConvertor::write_column(const std::shared_ptr<const IDataType>& type,
diff --git a/be/src/format/table/paimon/paimon_arrow_block_convertor.cpp b/be/src/format/table/paimon/paimon_arrow_block_convertor.cpp
index 9404d622938..3baa787cd40 100644
--- a/be/src/format/table/paimon/paimon_arrow_block_convertor.cpp
+++ b/be/src/format/table/paimon/paimon_arrow_block_convertor.cpp
@@ -25,10 +25,7 @@
namespace doris::paimon {
#include "common/compile_check_begin.h"
-Status PaimonArrowBlockConvertor::init() {
- if (_arrow_schema != nullptr) {
- return Status::OK();
- }
+Status PaimonArrowSchemaConvertor::get_arrow_schema(std::shared_ptr<arrow::Schema>* result) const {
// Decode the pinned table schema here; rebuilding it from Doris types would lose
// nested nullability, timestamp precision and Paimon's physical Variant layout.
auto input = std::make_shared<arrow::io::BufferReader>(
@@ -38,7 +35,16 @@ Status PaimonArrowBlockConvertor::init() {
return Status::InvalidArgument("Failed to deserialize Paimon Arrow schema: {}",
reader.status().ToString());
}
- _arrow_schema = reader.ValueOrDie()->schema();
+ *result = reader.ValueOrDie()->schema();
+ return Status::OK();
+}
+
+Status PaimonArrowBlockConvertor::init() {
+ if (_arrow_schema != nullptr) {
+ return Status::OK();
+ }
+ RETURN_IF_ERROR(
+ PaimonArrowSchemaConvertor(_serialized_schema).get_arrow_schema(&_arrow_schema));
_serialized_schema.clear();
return Status::OK();
}
diff --git a/be/src/format/table/paimon/paimon_arrow_block_convertor.h b/be/src/format/table/paimon/paimon_arrow_block_convertor.h
index caa6501e0a6..8dd159dca3f 100644
--- a/be/src/format/table/paimon/paimon_arrow_block_convertor.h
+++ b/be/src/format/table/paimon/paimon_arrow_block_convertor.h
@@ -18,9 +18,20 @@
#pragma once
#include "format/arrow/arrow_block_convertor.h"
+#include "format/arrow/arrow_row_batch.h"
namespace doris::paimon {
+class PaimonArrowSchemaConvertor final : public ArrowSchemaConvertor {
+public:
+ explicit PaimonArrowSchemaConvertor(std::string serialized_schema)
+ : _serialized_schema(std::move(serialized_schema)) {}
+ Status get_arrow_schema(std::shared_ptr<arrow::Schema>* result) const override;
+
+private:
+ const std::string _serialized_schema;
+};
+
// Paimon reads use native Parquet/ORC or the JNI scanner, which apply the table's
// timestamp and nested-type semantics. This adapter implements the Arrow write protocol only;
// convert_from_arrow deliberately inherits NotSupported instead of a generic SerDe fallback.
diff --git a/be/src/udf/python/python_udf_meta.cpp b/be/src/udf/python/python_udf_meta.cpp
index cbf885094a2..e0cd98ba01c 100644
--- a/be/src/udf/python/python_udf_meta.cpp
+++ b/be/src/udf/python/python_udf_meta.cpp
@@ -34,9 +34,10 @@ namespace doris {
Status PythonUDFMeta::convert_types_to_schema(const DataTypes& types, const std::string& timezone,
std::shared_ptr<arrow::Schema>* schema) {
arrow::SchemaBuilder builder;
+ const DorisArrowSchemaConvertor schema_convertor(timezone);
for (size_t i = 0; i < types.size(); ++i) {
std::shared_ptr<arrow::DataType> arrow_type;
- RETURN_IF_ERROR(convert_to_arrow_type(types[i], &arrow_type, timezone));
+ RETURN_IF_ERROR(schema_convertor.convert_to_arrow_type(types[i], &arrow_type));
std::shared_ptr<arrow::Field> field = std::make_shared<arrow::Field>(
"arg" + std::to_string(i), arrow_type, types[i]->is_nullable());
RETURN_DORIS_STATUS_IF_ERROR(builder.AddField(field));
diff --git a/be/test/core/data_type/common_data_type_serder_test.h b/be/test/core/data_type/common_data_type_serder_test.h
index ad8667f1454..1760e0d6c39 100644
--- a/be/test/core/data_type/common_data_type_serder_test.h
+++ b/be/test/core/data_type/common_data_type_serder_test.h
@@ -362,7 +362,9 @@ public:
static std::shared_ptr<arrow::RecordBatch> serialize_arrow(
const std::shared_ptr<Block>& block) {
std::shared_ptr<arrow::Schema> block_arrow_schema;
- EXPECT_EQ(get_arrow_schema_from_block(*block, &block_arrow_schema, "UTC"), Status::OK());
+ EXPECT_EQ(DorisArrowSchemaConvertor("UTC").get_arrow_schema_from_block(*block,
+ &block_arrow_schema),
+ Status::OK());
std::cout << "schema: " << block_arrow_schema->ToString(true) << std::endl;
// convert block to arrow
std::shared_ptr<arrow::RecordBatch> result;
diff --git a/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp b/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp
index fed7e4c7be3..d83b2ff8f8e 100644
--- a/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp
+++ b/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp
@@ -550,7 +550,8 @@ void block_converter_test(std::vector<PrimitiveType> cols, int row_num, bool is_
std::shared_ptr<arrow::RecordBatch> record_batch;
std::shared_ptr<arrow::Schema> schema;
Status status = Status::OK();
- status = get_arrow_schema_from_block(*source_block, &schema, TimezoneUtils::default_time_zone);
+ status = DorisArrowSchemaConvertor(TimezoneUtils::default_time_zone)
+ .get_arrow_schema_from_block(*source_block, &schema);
ASSERT_TRUE(status.ok() && schema);
cctz::time_zone default_timezone;
ASSERT_TRUE(
@@ -587,7 +588,8 @@ TEST(DataTypeSerDeArrowTest, DataTypeCollectionSerDeTest) {
TEST(DataTypeSerDeArrowTest, ArrowBlockConvertorReusesBothDirectionsAndValidatesSlices) {
auto source = create_test_block({TYPE_INT, TYPE_STRING, TYPE_VARBINARY}, 4, true);
std::shared_ptr<arrow::Schema> schema;
- ASSERT_TRUE(get_arrow_schema_from_block(*source, &schema, "UTC").ok());
+ ASSERT_TRUE(
+ DorisArrowSchemaConvertor("UTC").get_arrow_schema_from_block(*source, &schema).ok());
ArrowFlightArrowBlockConvertor convertor(schema, cctz::utc_time_zone());
std::shared_ptr<arrow::RecordBatch> batch;
ASSERT_TRUE(convertor.convert_to_arrow(*source, arrow::default_memory_pool(), &batch).ok());
@@ -607,7 +609,7 @@ void expect_target_converter_matches_plain(const std::vector<PrimitiveType>& typ
ArrowConvertorFactory target_converter) {
auto block = create_test_block(types, 4, false);
std::shared_ptr<arrow::Schema> schema;
- ASSERT_TRUE(get_arrow_schema_from_block(*block, &schema, "UTC").ok());
+ ASSERT_TRUE(DorisArrowSchemaConvertor("UTC").get_arrow_schema_from_block(*block, &schema).ok());
std::shared_ptr<arrow::RecordBatch> plain_batch;
ASSERT_TRUE(
@@ -651,7 +653,7 @@ TEST(DataTypeSerDeArrowTest, PaimonCommonScalarTypesUseDeclaredConverter) {
TEST(DataTypeSerDeArrowTest, PlainArrowWritesAggregateStateBinaryTypes) {
auto block = create_test_block({TYPE_HLL, TYPE_BITMAP, TYPE_QUANTILE_STATE}, 3, false);
std::shared_ptr<arrow::Schema> schema;
- ASSERT_TRUE(get_arrow_schema_from_block(*block, &schema, "UTC").ok());
+ ASSERT_TRUE(DorisArrowSchemaConvertor("UTC").get_arrow_schema_from_block(*block, &schema).ok());
std::shared_ptr<arrow::RecordBatch> batch;
Status status = convert_to_arrow_batch_for_test(
*block, schema, arrow::default_memory_pool(), &batch, cctz::utc_time_zone(), 0,
@@ -668,7 +670,7 @@ TEST(DataTypeSerDeArrowTest, PlainArrowWritesAggregateStateBinaryTypes) {
TEST(DataTypeSerDeArrowTest, PlainArrowWritesTimeV2) {
auto block = create_test_block({TYPE_TIMEV2}, 3, false);
std::shared_ptr<arrow::Schema> schema;
- ASSERT_TRUE(get_arrow_schema_from_block(*block, &schema, "UTC").ok());
+ ASSERT_TRUE(DorisArrowSchemaConvertor("UTC").get_arrow_schema_from_block(*block, &schema).ok());
ASSERT_EQ(arrow::Type::DOUBLE, schema->field(0)->type()->id());
std::shared_ptr<arrow::RecordBatch> batch;
@@ -1428,27 +1430,31 @@ TEST(DataTypeSerDeArrowTest, ConvertDateTimeV2ToNaiveArrowType) {
const auto datetime_type = std::make_shared<DataTypeDateTimeV2>(6);
std::shared_ptr<arrow::DataType> arrow_type;
- auto status = convert_to_arrow_type(datetime_type, &arrow_type, "Asia/Shanghai");
+ auto status = DorisArrowSchemaConvertor("Asia/Shanghai")
+ .convert_to_arrow_type(datetime_type, &arrow_type);
ASSERT_TRUE(status.ok()) << status;
auto timestamp_type = std::static_pointer_cast<arrow::TimestampType>(arrow_type);
EXPECT_EQ(arrow::TimeUnit::MICRO, timestamp_type->unit());
EXPECT_EQ("Asia/Shanghai", timestamp_type->timezone());
- status = convert_to_arrow_type(datetime_type, &arrow_type, "Asia/Shanghai", true);
+ status = ArrowFlightSchemaConvertor("Asia/Shanghai")
+ .convert_to_arrow_type(datetime_type, &arrow_type);
ASSERT_TRUE(status.ok()) << status;
timestamp_type = std::static_pointer_cast<arrow::TimestampType>(arrow_type);
EXPECT_EQ(arrow::TimeUnit::MICRO, timestamp_type->unit());
EXPECT_TRUE(timestamp_type->timezone().empty());
const auto timestamptz_type = std::make_shared<DataTypeTimeStampTz>(6);
- status = convert_to_arrow_type(timestamptz_type, &arrow_type, "Asia/Shanghai", true);
+ status = ArrowFlightSchemaConvertor("Asia/Shanghai")
+ .convert_to_arrow_type(timestamptz_type, &arrow_type);
ASSERT_TRUE(status.ok()) << status;
timestamp_type = std::static_pointer_cast<arrow::TimestampType>(arrow_type);
EXPECT_EQ(arrow::TimeUnit::MICRO, timestamp_type->unit());
EXPECT_EQ("Asia/Shanghai", timestamp_type->timezone());
const auto array_type = std::make_shared<DataTypeArray>(datetime_type);
- status = convert_to_arrow_type(array_type, &arrow_type, "Asia/Shanghai", true);
+ status = ArrowFlightSchemaConvertor("Asia/Shanghai")
+ .convert_to_arrow_type(array_type, &arrow_type);
ASSERT_TRUE(status.ok()) << status;
const auto list_type = std::static_pointer_cast<arrow::ListType>(arrow_type);
timestamp_type = std::static_pointer_cast<arrow::TimestampType>(list_type->value_type());
@@ -1459,7 +1465,8 @@ TEST(DataTypeSerDeArrowTest, CanonicalizeUtcTimezoneForArrow) {
const auto timestamptz_type = std::make_shared<DataTypeTimeStampTz>(6);
std::shared_ptr<arrow::DataType> arrow_type;
- auto status = convert_to_arrow_type(timestamptz_type, &arrow_type, "Z", true);
+ auto status =
+ ArrowFlightSchemaConvertor("Z").convert_to_arrow_type(timestamptz_type, &arrow_type);
ASSERT_TRUE(status.ok()) << status;
const auto timestamp_type = std::static_pointer_cast<arrow::TimestampType>(arrow_type);
EXPECT_EQ("UTC", timestamp_type->timezone());
@@ -1544,7 +1551,9 @@ TEST(DataTypeSerDeArrowTest, NestedDateTimeV2PlainArrowAcceptsNaiveSchema) {
block.insert(ColumnWithTypeAndName(struct_column->get_ptr(), struct_type, "event_struct"));
std::shared_ptr<arrow::Schema> naive_schema;
- ASSERT_TRUE(get_arrow_schema_from_block(block, &naive_schema, "Asia/Shanghai", true).ok());
+ ASSERT_TRUE(ArrowFlightSchemaConvertor("Asia/Shanghai")
+ .get_arrow_schema_from_block(block, &naive_schema)
+ .ok());
std::shared_ptr<arrow::RecordBatch> naive_batch;
const auto status = convert_to_arrow_batch_for_test(
block, naive_schema, arrow::default_memory_pool(), &naive_batch,
diff --git a/be/test/format/arrow/arrow_flight_timestamp_test.cpp b/be/test/format/arrow/arrow_flight_timestamp_test.cpp
index ebcafc04a91..1d278f75503 100644
--- a/be/test/format/arrow/arrow_flight_timestamp_test.cpp
+++ b/be/test/format/arrow/arrow_flight_timestamp_test.cpp
@@ -26,6 +26,7 @@
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_struct.h"
#include "format/arrow/arrow_block_convertor.h"
+#include "format/arrow/arrow_row_batch.h"
#include "util/timezone_utils.h"
namespace doris {
@@ -59,7 +60,7 @@ TEST_F(ArrowFlightTimestampTest, RejectsOutOfRangeInEveryUnitWithoutPublishingBa
make_datetime(10000, 1, 1, 0, 0, 0, 0)}) {
SCOPED_TRACE(scale);
auto block = timestamp_block(scale, {value});
- ArrowFlightArrowBlockConvertor flight(block, "UTC", cctz::utc_time_zone(), true);
+ ArrowFlightArrowBlockConvertor flight(block, "UTC", cctz::utc_time_zone());
ASSERT_TRUE(flight.init().ok());
const ArrowBlockConvertor& converter = flight;
std::shared_ptr<arrow::RecordBatch> batch;
@@ -72,7 +73,7 @@ TEST_F(ArrowFlightTimestampTest, RejectsOutOfRangeInEveryUnitWithoutPublishingBa
EXPECT_EQ(nullptr, batch);
// Other Arrow consumers retain their existing date semantics.
- DorisArrowBlockConvertor ordinary(block, "UTC", cctz::utc_time_zone(), true);
+ DorisArrowBlockConvertor ordinary(flight.arrow_schema(), cctz::utc_time_zone());
ASSERT_TRUE(ordinary.init().ok());
ASSERT_TRUE(
ordinary.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
@@ -88,7 +89,7 @@ TEST_F(ArrowFlightTimestampTest, PreservesCalendarBoundariesAndPreEpochFractions
auto block = timestamp_block(scale, {make_datetime(1, 1, 1, 0, 0, 0, 0),
make_datetime(9999, 12, 31, 23, 59, 59, fraction),
make_datetime(1969, 12, 31, 23, 59, 59, fraction)});
- ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone(), true);
+ ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone());
ASSERT_TRUE(converter.init().ok());
std::shared_ptr<arrow::RecordBatch> batch;
ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
@@ -101,7 +102,7 @@ TEST_F(ArrowFlightTimestampTest, PreservesCalendarBoundariesAndPreEpochFractions
TEST_F(ArrowFlightTimestampTest, ChecksSlicesAndSubsequentBatches) {
auto block = timestamp_block(6, {make_datetime(2024, 1, 1, 0, 0, 0, 0), DateTime {}});
- ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone(), true);
+ ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone());
ASSERT_TRUE(converter.init().ok());
std::shared_ptr<arrow::RecordBatch> batch;
ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch, 0, 1).ok());
@@ -138,7 +139,7 @@ TEST_F(ArrowFlightTimestampTest, RejectsNestedTimestampValuesAndMapKeys) {
column->insert_default();
column->insert(fields[i]);
Block block {{std::move(column), types[i], "nested"}};
- ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone(), true);
+ ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone());
ASSERT_TRUE(converter.init().ok());
std::shared_ptr<arrow::RecordBatch> batch;
const auto status =
@@ -161,7 +162,11 @@ TEST_F(ArrowFlightTimestampTest, ChecksBothUtcAndZonedCalendarBounds) {
cctz::time_zone timezone;
ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone(zone, timezone));
auto block = timestamp_block(6, {value});
- ArrowFlightArrowBlockConvertor converter(block, zone, timezone);
+ // Flight-generated DATETIMEV2 schemas are naive; keep an explicit zoned schema
+ // here so this test still checks both UTC and local calendar bounds.
+ std::shared_ptr<arrow::Schema> schema;
+ ASSERT_TRUE(DorisArrowSchemaConvertor(block, zone).get_arrow_schema(&schema).ok());
+ ArrowFlightArrowBlockConvertor converter(schema, timezone);
ASSERT_TRUE(converter.init().ok());
std::shared_ptr<arrow::RecordBatch> batch;
const auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
@@ -179,7 +184,7 @@ TEST_F(ArrowFlightTimestampTest, NaiveBoundsDoNotDependOnSessionTimezone) {
ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("+08:00", timezone));
auto block = timestamp_block(6, {make_datetime(1, 1, 1, 0, 0, 0, 0),
make_datetime(9999, 12, 31, 23, 59, 59, 999999)});
- ArrowFlightArrowBlockConvertor converter(block, "+08:00", timezone, true);
+ ArrowFlightArrowBlockConvertor converter(block, "+08:00", timezone);
ASSERT_TRUE(converter.init().ok());
std::shared_ptr<arrow::RecordBatch> batch;
ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
@@ -207,7 +212,7 @@ TEST_F(ArrowFlightTimestampTest, IgnoresTimestampsMaskedByNullParents) {
nulls->insert_value(1);
Block block {{ColumnNullable::create(std::move(data), std::move(nulls)),
make_nullable(types[i]), "masked"}};
- ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone(), true);
+ ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone());
ASSERT_TRUE(converter.init().ok());
std::shared_ptr<arrow::RecordBatch> batch;
const auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
diff --git a/be/test/format/arrow/arrow_row_batch_test.cpp b/be/test/format/arrow/arrow_row_batch_test.cpp
new file mode 100644
index 00000000000..8b720c675d8
--- /dev/null
+++ b/be/test/format/arrow/arrow_row_batch_test.cpp
@@ -0,0 +1,305 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/arrow/arrow_row_batch.h"
+
+#include <arrow/api.h>
+#include <arrow/io/api.h>
+#include <arrow/ipc/api.h>
+#include <gtest/gtest.h>
+
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_factory.hpp"
+#include "core/data_type/data_type_map.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
+#include "exprs/vexpr_context.h"
+#include "exprs/vslot_ref.h"
+#include "format/arrow/arrow_block_convertor.h"
+
+namespace doris {
+namespace {
+
+void expect_logical_type(const std::shared_ptr<arrow::Field>& field, const std::string& name) {
+ ASSERT_NE(nullptr, field->metadata()) << field->ToString();
+ auto value = field->metadata()->Get("doris_type");
+ ASSERT_TRUE(value.ok()) << value.status();
+ EXPECT_EQ(name, *value);
+}
+
+void expect_legacy_batch(const Block& block, const std::shared_ptr<arrow::RecordBatch>& extended) {
+ VExprContextSPtrs expressions;
+ for (size_t i = 0; i < block.columns(); ++i) {
+ const auto& column = block.get_by_position(i);
+ expressions.push_back(VExprContext::create_shared(
+ VSlotRef::create_shared(i, i, -1, column.type, column.name)));
+ }
+ std::shared_ptr<arrow::Schema> schema;
+ ASSERT_TRUE(LegacyArrowFlightSchemaConvertor("UTC")
+ .get_arrow_schema_from_expr_ctxs(expressions, &schema)
+ .ok());
+ // Synthetic slot references have no expression labels; compare batches using the same names.
+ schema = schema->WithNames(extended->schema()->field_names()).ValueOrDie();
+ std::shared_ptr<arrow::RecordBatch> batch;
+ auto status = ArrowFlightArrowBlockConvertor(schema, cctz::utc_time_zone())
+ .convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ EXPECT_TRUE(extended->Equals(*batch, false));
+ EXPECT_FALSE(extended->schema()->Equals(*schema, true));
+ std::string serialized;
+ ASSERT_TRUE(serialize_record_batch(*batch, &serialized).ok());
+ auto source = std::make_shared<arrow::io::BufferReader>(arrow::Buffer::FromString(serialized));
+ auto reader = arrow::ipc::RecordBatchStreamReader::Open(source).ValueOrDie();
+ ASSERT_TRUE(reader->ReadNext(&batch).ok());
+ EXPECT_TRUE(schema->Equals(*batch->schema(), true));
+ EXPECT_TRUE(extended->Equals(*batch, false));
+}
+
+TEST(ArrowSchemaConvertorTest, ProtocolsKeepNestedTimestampAndMetadataPoliciesSeparate) {
+ const auto datetime =
+ DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, false, 0, 6);
+ const auto largeint = std::make_shared<DataTypeInt128>();
+ const auto nested = std::make_shared<DataTypeStruct>(DataTypes {datetime, largeint},
+ Strings {"time", "number"});
+ Block block;
+ block.insert({nested->create_column(), nested, "value"});
+ std::vector<std::unique_ptr<ArrowSchemaConvertor>> convertors;
+ convertors.push_back(std::make_unique<DorisArrowSchemaConvertor>(block, "Asia/Shanghai"));
+ convertors.push_back(std::make_unique<ArrowFlightSchemaConvertor>(block, "Asia/Shanghai"));
+ convertors.push_back(
+ std::make_unique<LegacyArrowFlightSchemaConvertor>(block, "Asia/Shanghai"));
+ for (size_t i = 0; i < convertors.size(); ++i) {
+ std::shared_ptr<arrow::Schema> schema;
+ ASSERT_TRUE(convertors[i]->get_arrow_schema(&schema).ok());
+ const auto& fields = schema->field(0)->type()->fields();
+ const auto& timestamp = static_cast<const arrow::TimestampType&>(*fields[0]->type());
+ EXPECT_EQ(i == 0 ? "Asia/Shanghai" : "", timestamp.timezone());
+ if (i == 2) {
+ EXPECT_EQ(nullptr, fields[1]->metadata());
+ } else {
+ expect_logical_type(fields[1], "LARGEINT");
+ }
+ }
+}
+
+class ArrowLogicalTypeMetadataTest
+ : public testing::TestWithParam<std::pair<PrimitiveType, const char*>> {};
+
+TEST_P(ArrowLogicalTypeMetadataTest, PreservesTopLevelAndNestedFields) {
+ const auto [primitive, name] = GetParam();
+ auto type = make_nullable(DataTypeFactory::instance().create_data_type(primitive, false));
+ auto string_type = std::make_shared<DataTypeString>();
+ DataTypes types {type, std::make_shared<DataTypeArray>(type),
+ std::make_shared<DataTypeStruct>(DataTypes {type, string_type},
+ Strings {"typed", "text"}),
+ std::make_shared<DataTypeMap>(string_type, type)};
+ Block block;
+ for (size_t i = 0; i < types.size(); ++i) {
+ block.insert({types[i]->create_column(), types[i], std::to_string(i)});
+ }
+ std::shared_ptr<arrow::Schema> schema;
+ ASSERT_TRUE(ArrowFlightSchemaConvertor("UTC").get_arrow_schema_from_block(block, &schema).ok());
+ expect_logical_type(schema->field(0), name);
+ expect_logical_type(schema->field(1)->type()->field(0), name);
+ expect_logical_type(schema->field(2)->type()->field(0), name);
+ EXPECT_EQ(nullptr, schema->field(2)->type()->field(1)->metadata());
+ const auto& map = static_cast<const arrow::MapType&>(*schema->field(3)->type());
+ expect_logical_type(map.item_field(), name);
+ EXPECT_FALSE(map.key_field()->nullable());
+ EXPECT_TRUE(map.item_field()->nullable());
+ EXPECT_EQ(nullptr, map.key_field()->metadata());
+}
+
+TEST_P(ArrowLogicalTypeMetadataTest, OldFeReceivesLegacySchema) {
+ const auto [primitive, name] = GetParam();
+ auto type = make_nullable(DataTypeFactory::instance().create_data_type(primitive, false));
+ auto string_type = std::make_shared<DataTypeString>();
+ DataTypes types {
+ type, std::make_shared<DataTypeArray>(type),
+ std::make_shared<DataTypeStruct>(DataTypes {type}, Strings {"typed"}),
+ std::make_shared<DataTypeMap>(primitive == TYPE_LARGEINT ? type : string_type, type)};
+ VExprContextSPtrs expressions;
+ for (size_t i = 0; i < types.size(); ++i) {
+ expressions.push_back(VExprContext::create_shared(
+ VSlotRef::create_shared(i, i, -1, types[i], std::to_string(i))));
+ }
+ std::shared_ptr<arrow::Schema> legacy;
+ std::shared_ptr<arrow::Schema> extended;
+ ASSERT_TRUE(LegacyArrowFlightSchemaConvertor("UTC")
+ .get_arrow_schema_from_expr_ctxs(expressions, &legacy)
+ .ok());
+ ASSERT_TRUE(ArrowFlightSchemaConvertor("UTC")
+ .get_arrow_schema_from_expr_ctxs(expressions, &extended)
+ .ok());
+ std::shared_ptr<arrow::DataType> storage;
+ ASSERT_TRUE(ArrowFlightSchemaConvertor("UTC").convert_to_arrow_type(type, &storage).ok());
+ auto scalar = arrow::field(legacy->field(0)->name(), storage, type->is_nullable());
+ if (primitive != TYPE_JSONB && primitive != TYPE_VARIANT) {
+ scalar = scalar->WithMetadata(arrow::key_value_metadata({"doris_type"}, {name}));
+ }
+ auto expected = arrow::schema(
+ {scalar,
+ arrow::field(legacy->field(1)->name(), arrow::list(storage), types[1]->is_nullable()),
+ arrow::field(legacy->field(2)->name(),
+ arrow::struct_({arrow::field("typed", storage, true)}),
+ types[2]->is_nullable()),
+ arrow::field(legacy->field(3)->name(), arrow::map(arrow::utf8(), storage),
+ types[3]->is_nullable())});
+ EXPECT_TRUE(expected->Equals(*legacy, true));
+ EXPECT_TRUE(legacy->Equals(*extended, false));
+ EXPECT_FALSE(legacy->Equals(*extended, true));
+ expect_logical_type(extended->field(0), name);
+ expect_logical_type(extended->field(1)->type()->field(0), name);
+ std::string serialized;
+ ASSERT_TRUE(serialize_arrow_schema(&legacy, &serialized).ok());
+ auto source = std::make_shared<arrow::io::BufferReader>(arrow::Buffer::FromString(serialized));
+ auto reader = arrow::ipc::RecordBatchStreamReader::Open(source).ValueOrDie();
+ EXPECT_TRUE(expected->Equals(*reader->schema(), true));
+}
+
+INSTANTIATE_TEST_SUITE_P(LogicalTypes, ArrowLogicalTypeMetadataTest,
+ testing::Values(std::make_pair(TYPE_LARGEINT, "LARGEINT"),
+ std::make_pair(TYPE_IPV4, "IPV4"),
+ std::make_pair(TYPE_IPV6, "IPV6"),
+ std::make_pair(TYPE_JSONB, "JSON"),
+ std::make_pair(TYPE_VARIANT, "VARIANT")));
+
+TEST(ArrowRowBatchMetadataTest, PreservesMapKeysAndDeepNestingThroughIpc) {
+ auto integer = std::make_shared<DataTypeInt128>();
+ auto structure = std::make_shared<DataTypeStruct>(DataTypes {integer}, Strings {"number"});
+ auto array = std::make_shared<DataTypeArray>(structure);
+ auto type = std::make_shared<DataTypeMap>(integer, array);
+ Block block;
+ block.insert({type->create_column(), type, "m"});
+ std::shared_ptr<arrow::Schema> schema;
+ ASSERT_TRUE(DorisArrowSchemaConvertor("UTC").get_arrow_schema_from_block(block, &schema).ok());
+ std::string serialized;
+ ASSERT_TRUE(serialize_arrow_schema(&schema, &serialized).ok());
+ auto source = std::make_shared<arrow::io::BufferReader>(arrow::Buffer::FromString(serialized));
+ auto reader = arrow::ipc::RecordBatchStreamReader::Open(source).ValueOrDie();
+ EXPECT_TRUE(schema->Equals(*reader->schema(), true));
+ const auto& map = static_cast<const arrow::MapType&>(*reader->schema()->field(0)->type());
+ expect_logical_type(map.key_field(), "LARGEINT");
+ EXPECT_EQ("key", map.key_field()->name());
+ EXPECT_FALSE(map.key_field()->nullable());
+ EXPECT_EQ("value", map.item_field()->name());
+ auto item = map.item_type()->field(0);
+ EXPECT_EQ("item", item->name());
+ EXPECT_TRUE(item->nullable());
+ expect_logical_type(item->type()->field(0), "LARGEINT");
+ EXPECT_FALSE(item->type()->field(0)->nullable());
+ auto physical =
+ arrow::map(arrow::utf8(),
+ arrow::list(arrow::struct_({arrow::field("number", arrow::utf8(), false)})));
+ EXPECT_TRUE(map.Equals(physical));
+ EXPECT_FALSE(map.Equals(physical, true));
+}
+
+TEST(ArrowRowBatchMetadataTest, KeepsNativeTypesUnannotated) {
+ auto type = std::make_shared<DataTypeStruct>(
+ DataTypes {std::make_shared<DataTypeInt32>(), std::make_shared<DataTypeString>()},
+ Strings {"number", "text"});
+ std::shared_ptr<arrow::DataType> arrow_type;
+ ASSERT_TRUE(DorisArrowSchemaConvertor("UTC").convert_to_arrow_type(type, &arrow_type).ok());
+ for (const auto& field : arrow_type->fields()) {
+ EXPECT_EQ(nullptr, field->metadata());
+ EXPECT_FALSE(field->nullable());
+ }
+}
+
+TEST(ArrowRowBatchMetadataTest, PreservesLargeintExtremesAndNullsInRecordBatches) {
+ auto integer = make_nullable(std::make_shared<DataTypeInt128>());
+ auto array_type = std::make_shared<DataTypeArray>(integer);
+ auto column = array_type->create_column();
+ column->insert(Field::create_field<TYPE_ARRAY>(
+ Array {Field::create_field<TYPE_LARGEINT>(MAX_INT128),
+ Field::create_field<TYPE_LARGEINT>(MIN_INT128), Field()}));
+ column->insert(Field::create_field<TYPE_ARRAY>(Array {}));
+ Block block;
+ block.insert({std::move(column), array_type, "numbers"});
+ std::shared_ptr<arrow::Schema> schema;
+ ASSERT_TRUE(ArrowFlightSchemaConvertor("UTC").get_arrow_schema_from_block(block, &schema).ok());
+ ArrowFlightArrowBlockConvertor converter(schema, cctz::utc_time_zone());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ std::string serialized;
+ ASSERT_TRUE(serialize_record_batch(*batch, &serialized).ok());
+ auto source = std::make_shared<arrow::io::BufferReader>(arrow::Buffer::FromString(serialized));
+ auto reader = arrow::ipc::RecordBatchStreamReader::Open(source).ValueOrDie();
+ ASSERT_TRUE(reader->ReadNext(&batch).ok());
+ EXPECT_TRUE(batch->schema()->Equals(*schema, true));
+ expect_logical_type(batch->schema()->field(0)->type()->field(0), "LARGEINT");
+ const auto& lists = static_cast<const arrow::ListArray&>(*batch->column(0));
+ EXPECT_EQ(0, lists.value_length(1));
+ const auto& values = static_cast<const arrow::StringArray&>(*lists.values());
+ EXPECT_EQ("170141183460469231731687303715884105727", values.GetString(0));
+ EXPECT_EQ("-170141183460469231731687303715884105728", values.GetString(1));
+ EXPECT_TRUE(values.IsNull(2));
+ expect_legacy_batch(block, batch);
+}
+
+TEST(ArrowRowBatchMetadataTest, PreservesNestedMapMetadataAndValuesInRecordBatches) {
+ auto integer = make_nullable(std::make_shared<DataTypeInt128>());
+ auto map_type = make_nullable(std::make_shared<DataTypeMap>(integer, integer));
+ auto type = make_nullable(std::make_shared<DataTypeStruct>(
+ DataTypes {map_type, std::make_shared<DataTypeString>()}, Strings {"mapping", "text"}));
+ Map map;
+ map.push_back(Field::create_field<TYPE_ARRAY>(
+ Array {Field::create_field<TYPE_LARGEINT>(MAX_INT128)}));
+ map.push_back(Field::create_field<TYPE_ARRAY>(
+ Array {Field::create_field<TYPE_LARGEINT>(MIN_INT128)}));
+ auto column = type->create_column();
+ column->insert(Field::create_field<TYPE_STRUCT>(
+ Struct {Field::create_field<TYPE_MAP>(map), Field::create_field<TYPE_STRING>("17")}));
+ column->insert_default();
+ Block block;
+ block.insert({std::move(column), type, "nested"});
+ std::shared_ptr<arrow::Schema> schema;
+ ASSERT_TRUE(ArrowFlightSchemaConvertor("UTC").get_arrow_schema_from_block(block, &schema).ok());
+ ArrowFlightArrowBlockConvertor converter(schema, cctz::utc_time_zone());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ std::string serialized;
+ ASSERT_TRUE(serialize_record_batch(*batch, &serialized).ok());
+ auto source = std::make_shared<arrow::io::BufferReader>(arrow::Buffer::FromString(serialized));
+ auto reader = arrow::ipc::RecordBatchStreamReader::Open(source).ValueOrDie();
+ ASSERT_TRUE(reader->ReadNext(&batch).ok());
+ const auto& structure = static_cast<const arrow::StructArray&>(*batch->column(0));
+ EXPECT_TRUE(structure.IsNull(1));
+ const auto& maps = static_cast<const arrow::MapArray&>(*structure.field(0));
+ const auto& arrow_map_type = static_cast<const arrow::MapType&>(*maps.type());
+ expect_logical_type(arrow_map_type.key_field(), "LARGEINT");
+ expect_logical_type(arrow_map_type.item_field(), "LARGEINT");
+ EXPECT_FALSE(arrow_map_type.key_field()->nullable());
+ EXPECT_EQ("170141183460469231731687303715884105727",
+ static_cast<const arrow::StringArray&>(*maps.keys()).GetString(0));
+ EXPECT_EQ("-170141183460469231731687303715884105728",
+ static_cast<const arrow::StringArray&>(*maps.items()).GetString(0));
+ EXPECT_EQ("17", static_cast<const arrow::StringArray&>(*structure.field(1)).GetString(0));
+ EXPECT_EQ(nullptr, structure.type()->field(1)->metadata());
+ expect_legacy_batch(block, batch);
+}
+
+} // namespace
+} // namespace doris
diff --git a/be/test/format/table/iceberg/arrow_schema_util_test.cpp b/be/test/format/table/iceberg/arrow_schema_util_test.cpp
index d8241812639..063302a3252 100644
--- a/be/test/format/table/iceberg/arrow_schema_util_test.cpp
+++ b/be/test/format/table/iceberg/arrow_schema_util_test.cpp
@@ -56,7 +56,7 @@ TEST(ArrowSchemaUtilTest, test_simple_field) {
std::vector<std::shared_ptr<arrow::Field>> fields;
Status st;
- st = ArrowSchemaUtil::convert(&schema, "utc", fields);
+ st = IcebergArrowSchemaConvertor(schema, "utc").convert_fields(fields);
EXPECT_TRUE(st.ok());
EXPECT_EQ(2, fields.size());
EXPECT_EQ("field1", fields[0]->name());
@@ -114,7 +114,7 @@ TEST(ArrowSchemaUtilTest, test_stuct_field) {
std::vector<std::shared_ptr<arrow::Field>> fields;
Status st;
- st = ArrowSchemaUtil::convert(schema.get(), "utc", fields);
+ st = IcebergArrowSchemaConvertor(*schema, "utc").convert_fields(fields);
EXPECT_TRUE(st.ok());
EXPECT_EQ(1, fields.size());
EXPECT_EQ("st_col", fields[0]->name());
@@ -166,7 +166,7 @@ TEST(ArrowSchemaUtilTest, test_map_field) {
std::vector<std::shared_ptr<arrow::Field>> fields;
Status st;
- st = ArrowSchemaUtil::convert(schema.get(), "utc", fields);
+ st = IcebergArrowSchemaConvertor(*schema, "utc").convert_fields(fields);
EXPECT_TRUE(st.ok());
EXPECT_EQ(1, fields.size());
EXPECT_EQ("map_col", fields[0]->name());
@@ -213,7 +213,7 @@ TEST(ArrowSchemaUtilTest, test_list_field) {
std::vector<std::shared_ptr<arrow::Field>> fields;
Status st;
- st = ArrowSchemaUtil::convert(schema.get(), "utc", fields);
+ st = IcebergArrowSchemaConvertor(*schema, "utc").convert_fields(fields);
EXPECT_TRUE(st.ok());
EXPECT_EQ(1, fields.size());
EXPECT_EQ("list_col", fields[0]->name());
@@ -237,7 +237,7 @@ TEST(ArrowSchemaUtilTest, test_binary_field_types) {
Schema schema(1, std::move(nested_fields));
std::vector<std::shared_ptr<arrow::Field>> fields;
- Status st = ArrowSchemaUtil::convert(&schema, "utc", fields);
+ Status st = IcebergArrowSchemaConvertor(schema, "utc").convert_fields(fields);
ASSERT_TRUE(st.ok()) << st;
ASSERT_EQ(4, fields.size());
@@ -260,7 +260,7 @@ TEST(ArrowSchemaUtilTest, test_variant_field) {
Schema schema(1, std::move(nested_fields));
std::vector<std::shared_ptr<arrow::Field>> fields;
- Status st = ArrowSchemaUtil::convert(&schema, "utc", fields);
+ Status st = IcebergArrowSchemaConvertor(schema, "utc").convert_fields(fields);
ASSERT_TRUE(st.ok()) << st;
ASSERT_EQ(1, fields.size());
EXPECT_EQ("21", fields[0]->metadata()->Get(pfid).ValueUnsafe());
@@ -324,7 +324,7 @@ TEST(ArrowSchemaUtilTest, test_nested_variant_fields) {
std::unique_ptr<Schema> schema = SchemaParser::from_json(schema_json);
std::vector<std::shared_ptr<arrow::Field>> fields;
- Status status = ArrowSchemaUtil::convert(schema.get(), "UTC", fields);
+ Status status = IcebergArrowSchemaConvertor(*schema, "UTC").convert_fields(fields);
ASSERT_TRUE(status.ok()) << status;
ASSERT_EQ(3, fields.size());
@@ -387,7 +387,7 @@ TEST(ArrowSchemaUtilTest, test_parquet_filed_id) {
Schema schema(1, std::move(nested_fields));
std::vector<std::shared_ptr<arrow::Field>> fields;
- st = ArrowSchemaUtil::convert(&schema, "utc", fields);
+ st = IcebergArrowSchemaConvertor(schema, "utc").convert_fields(fields);
auto arrow_schema = arrow::schema(fields);
// create arrow table
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ResultSink.java
index 25e72ed7598..b7d7dd4d876 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ResultSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ResultSink.java
@@ -73,6 +73,9 @@ public class ResultSink extends DataSink {
tResultSink.setFetchOption(fetchOption);
}
tResultSink.setType(resultSinkType);
+ if (resultSinkType == TResultSinkType.ARROW_FLIGHT_PROTOCOL) {
+ tResultSink.setEnableArrowTypeMetadata(true);
+ }
result.setResultSink(tResultSink);
return result;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java
index 2296986c535..5942424750c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java
@@ -40,6 +40,7 @@ import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowStreamReader;
+import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -158,7 +159,13 @@ public class FlightSqlConnectProcessor extends ConnectProcessor implements AutoC
String.format("Schema size %s' is not equal to arrow field size %s, queryId: %s.",
fieldVectors.size(), resultOutputExprs.size(), DebugUtil.printId(tid)));
}
- schema = root.getSchema();
+ List<Field> fields = new ArrayList<>();
+ for (int i = 0; i < resultOutputExprs.size(); i++) {
+ fields.add(FlightSqlSchemaHelper.withDorisTypeMetadata(
+ root.getSchema().getFields().get(i), resultOutputExprs.get(i).getType()));
+ }
+ // Compare and advertise the same complete metadata regardless of BE upgrade order.
+ schema = new Schema(fields, root.getSchema().getCustomMetadata());
if (arrowSchema == null) {
arrowSchema = schema;
} else if (!arrowSchema.equals(schema)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java
index 9d91153ebd6..6af37014132 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java
@@ -17,8 +17,12 @@
package org.apache.doris.service.arrowflight;
+import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MapType;
import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.StructType;
+import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.service.ExecuteEnv;
@@ -165,6 +169,47 @@ public class FlightSqlSchemaHelper {
}
}
+ static Field withDorisTypeMetadata(Field field, Type type) {
+ List<Field> children = new ArrayList<>(field.getChildren());
+ if (type.isArrayType()) {
+ children.set(0, withDorisTypeMetadata(children.get(0), ((ArrayType) type).getItemType()));
+ } else if (type.isMapType()) {
+ Field entries = children.get(0);
+ List<Field> pair = new ArrayList<>(entries.getChildren());
+ pair.set(0, withDorisTypeMetadata(pair.get(0), ((MapType) type).getKeyType()));
+ pair.set(1, withDorisTypeMetadata(pair.get(1), ((MapType) type).getValueType()));
+ children.set(0, new Field(entries.getName(), entries.getFieldType(), pair));
+ } else if (type.isStructType()) {
+ StructType struct = (StructType) type;
+ for (int i = 0; i < children.size(); i++) {
+ children.set(i, withDorisTypeMetadata(children.get(i), struct.getFields().get(i).getType()));
+ }
+ }
+ String marker = null;
+ switch (type.getPrimitiveType()) {
+ case LARGEINT:
+ case IPV4:
+ case IPV6:
+ case VARIANT:
+ marker = type.getPrimitiveType().name();
+ break;
+ case JSONB:
+ marker = "JSON";
+ break;
+ default:
+ break;
+ }
+ FieldType fieldType = field.getFieldType();
+ if (marker != null && !field.getMetadata().containsKey("doris_type")) {
+ Map<String, String> metadata = new HashMap<>(field.getMetadata());
+ metadata.put("doris_type", marker);
+ fieldType = new FieldType(field.isNullable(), field.getType(), field.getDictionary(), metadata);
+ }
+ // Old BEs omit these markers. Fill only missing ones from the planned Doris type;
+ // preserve conflicting markers and all physical properties for the strict schema comparison.
+ return new Field(field.getName(), fieldType, children);
+ }
+
private static ArrowType columnDescToArrowType(final TColumnDesc desc) {
PrimitiveType primitiveType = PrimitiveType.fromThrift(desc.getColumnType());
Integer precision = desc.isSetColumnPrecision() ? desc.getColumnPrecision() : null;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/ResultSinkMetadataTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/ResultSinkMetadataTest.java
new file mode 100644
index 00000000000..d41c1062155
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/ResultSinkMetadataTest.java
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.planner;
+
+import org.apache.doris.thrift.TResultSink;
+import org.apache.doris.thrift.TResultSinkType;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class ResultSinkMetadataTest {
+ @Test
+ void onlyUpgradedFlightPlannerOptsIntoExtendedMetadata() {
+ Assertions.assertFalse(new TResultSink().isEnableArrowTypeMetadata());
+ TResultSink flight = new ResultSink(new PlanNodeId(0), TResultSinkType.ARROW_FLIGHT_PROTOCOL)
+ .toThrift().getResultSink();
+ Assertions.assertTrue(flight.isSetEnableArrowTypeMetadata());
+ Assertions.assertTrue(flight.isEnableArrowTypeMetadata());
+ TResultSink mysql = new ResultSink(new PlanNodeId(0), TResultSinkType.MYSQL_PROTOCOL)
+ .toThrift().getResultSink();
+ Assertions.assertFalse(mysql.isSetEnableArrowTypeMetadata());
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessorSchemaTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessorSchemaTest.java
new file mode 100644
index 00000000000..b7f6725478b
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessorSchemaTest.java
@@ -0,0 +1,167 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service.arrowflight;
+
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.StructField;
+import org.apache.doris.catalog.StructType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.proto.InternalService.PFetchArrowFlightSchemaResult;
+import org.apache.doris.proto.Types.PStatus;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.rpc.BackendServiceProxy;
+import org.apache.doris.service.arrowflight.results.FlightSqlEndpointsLocation;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TUniqueId;
+
+import com.google.protobuf.ByteString;
+import org.apache.arrow.vector.ipc.WriteChannel;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.channels.Channels;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicInteger;
+
+class FlightSqlConnectProcessorSchemaTest {
+ private static Field field(String name, ArrowType type, boolean nullable, String marker, Field... children) {
+ Map<String, String> metadata = marker == null ? Collections.emptyMap()
+ : Collections.singletonMap("doris_type", marker);
+ return new Field(name, new FieldType(nullable, type, null, metadata), Arrays.asList(children));
+ }
+
+ private static Schema schema(Field... fields) {
+ return new Schema(Arrays.asList(fields), Collections.singletonMap("schema_key", "schema_value"));
+ }
+
+ private static Schema nestedSchema(boolean annotated) {
+ return schema(
+ field("a", new ArrowType.List(), true, null,
+ field("item", new ArrowType.Utf8(), true, annotated ? "LARGEINT" : null)),
+ field("m", new ArrowType.Map(false), true, null,
+ field("entries", new ArrowType.Struct(), false, null,
+ field("key", new ArrowType.Utf8(), false, annotated ? "LARGEINT" : null),
+ field("value", new ArrowType.Struct(), true, null,
+ field("ip4", new ArrowType.Int(32, true), true, annotated ? "IPV4" : null),
+ field("ip6", new ArrowType.Utf8(), true, annotated ? "IPV6" : null),
+ field("json", new ArrowType.Utf8(), true, annotated ? "JSON" : null),
+ field("variant", new ArrowType.Utf8(), true, annotated ? "VARIANT" : null),
+ field("text", new ArrowType.Utf8(), true, null)))),
+ field("json", new ArrowType.Utf8(), true, annotated ? "JSON" : null),
+ field("variant", new ArrowType.Utf8(), true, annotated ? "VARIANT" : null));
+ }
+
+ private static List<Type> resultTypes() {
+ return Arrays.asList(new ArrayType(Type.LARGEINT),
+ new MapType(Type.LARGEINT, new StructType(
+ new StructField("ip4", Type.IPV4), new StructField("ip6", Type.IPV6),
+ new StructField("json", Type.JSONB), new StructField("variant", Type.VARIANT),
+ new StructField("text", Type.STRING))), Type.JSONB, Type.VARIANT);
+ }
+
+ private static Schema fetch(List<Type> types, Schema... schemas) throws Exception {
+ List<FlightSqlEndpointsLocation> endpoints = new ArrayList<>();
+ List<CompletableFuture<PFetchArrowFlightSchemaResult>> responses = new ArrayList<>();
+ for (int i = 0; i < schemas.length; i++) {
+ ArrayList<Expr> exprs = new ArrayList<>();
+ for (Type type : types) {
+ Expr expr = Mockito.mock(Expr.class);
+ Mockito.when(expr.getType()).thenReturn(type);
+ exprs.add(expr);
+ }
+ endpoints.add(new FlightSqlEndpointsLocation(new TUniqueId(1, i),
+ new TNetworkAddress("localhost", 10000 + i),
+ new TNetworkAddress("localhost", 11000 + i), exprs));
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ MessageSerializer.serialize(new WriteChannel(Channels.newChannel(bytes)), schemas[i]);
+ responses.add(CompletableFuture.completedFuture(PFetchArrowFlightSchemaResult.newBuilder()
+ .setStatus(PStatus.newBuilder().setStatusCode(0))
+ .setSchema(ByteString.copyFrom(bytes.toByteArray())).build()));
+ }
+ ConnectContext context = Mockito.mock(ConnectContext.class);
+ Mockito.when(context.getFlightSqlEndpointsLocations()).thenReturn(endpoints);
+ BackendServiceProxy proxy = Mockito.mock(BackendServiceProxy.class);
+ AtomicInteger next = new AtomicInteger();
+ Mockito.when(proxy.fetchArrowFlightSchema(Mockito.any(), Mockito.any()))
+ .thenAnswer(invocation -> responses.get(next.getAndIncrement()));
+ try (MockedStatic<BackendServiceProxy> singleton = Mockito.mockStatic(BackendServiceProxy.class);
+ FlightSqlConnectProcessor processor = new FlightSqlConnectProcessor(context)) {
+ singleton.when(BackendServiceProxy::getInstance).thenReturn(proxy);
+ processor.fetchArrowFlightSchema(1000);
+ return processor.getArrowSchema();
+ }
+ }
+
+ @Test
+ void mixedVersionsAdvertiseCompleteMetadataInEitherOrder() throws Exception {
+ Schema oldSchema = nestedSchema(false);
+ Schema newSchema = nestedSchema(true);
+ Assertions.assertEquals(newSchema, fetch(resultTypes(), oldSchema, newSchema));
+ Assertions.assertEquals(newSchema, fetch(resultTypes(), newSchema, oldSchema));
+ }
+
+ @Test
+ void oldOnlyAndNewOnlyAdvertiseTheSameSchema() throws Exception {
+ Assertions.assertEquals(nestedSchema(true), fetch(resultTypes(), nestedSchema(false)));
+ Assertions.assertEquals(nestedSchema(true), fetch(resultTypes(), nestedSchema(true), nestedSchema(true)));
+ }
+
+ private static void assertSchemaMismatch(Schema first, Schema second) {
+ RuntimeException failure = Assertions.assertThrows(RuntimeException.class,
+ () -> fetch(Collections.singletonList(Type.JSONB), first, second));
+ Assertions.assertTrue(failure.getCause().getMessage()
+ .startsWith("The schema returned by results BE is different"));
+ }
+
+ @Test
+ void stillRejectsPhysicalDifferences() {
+ Schema expected = schema(field("value", new ArrowType.Utf8(), true, "JSON"));
+ for (Field incompatible : Arrays.asList(
+ field("value", new ArrowType.Int(32, true), true, "JSON"),
+ field("renamed", new ArrowType.Utf8(), true, "JSON"),
+ field("value", new ArrowType.Utf8(), false, "JSON"))) {
+ assertSchemaMismatch(expected, schema(incompatible));
+ }
+ }
+
+ @Test
+ void stillRejectsConflictingOrUnrelatedMetadata() {
+ Schema expected = schema(field("value", new ArrowType.Utf8(), true, "JSON"));
+ Schema wrongMarker = schema(field("value", new ArrowType.Utf8(), true, "VARIANT"));
+ Field extraMetadata = new Field("value", new FieldType(true, new ArrowType.Utf8(), null,
+ Collections.singletonMap("other_key", "other_value")), Collections.emptyList());
+ assertSchemaMismatch(expected, wrongMarker);
+ assertSchemaMismatch(expected, schema(extraMetadata));
+ assertSchemaMismatch(expected, new Schema(expected.getFields()));
+ }
+}
diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift
index c873e90529c..321bdf94469 100644
--- a/gensrc/thrift/DataSinks.thrift
+++ b/gensrc/thrift/DataSinks.thrift
@@ -232,6 +232,8 @@ struct TResultSink {
1: optional TResultSinkType type;
2: optional TResultFileSinkOptions file_options; // deprecated
3: optional TFetchOption fetch_option;
+ // Absent for old FEs, whose Flight schema comparison requires legacy field metadata.
+ 4: optional bool enable_arrow_type_metadata = false;
}
struct TResultFileSink {
diff --git a/samples/arrow-flight-sql/python/README.md b/samples/arrow-flight-sql/python/README.md
index e729054b813..7a7586940a1 100644
--- a/samples/arrow-flight-sql/python/README.md
+++ b/samples/arrow-flight-sql/python/README.md
@@ -35,6 +35,28 @@ under the License.
Section 6.1 of https://github.com/apache/doris/issues/25514 is the performance test
results of the doris arrow flight sql using python.
+## Logical type metadata
+
+Some Doris types share an Arrow storage type with ordinary strings or integers.
+The `doris_type` field metadata identifies `LARGEINT`, `IPV4`, `IPV6`, `JSON`, and
+`VARIANT`, including fields nested in arrays, maps, and structs. Consumers should
+inspect each nested Arrow field instead of inferring a type from its value.
+
+LARGEINT retains its Arrow string encoding, including the full signed 128-bit
+range. PyArrow does not automatically convert custom metadata into Python types;
+a client can use `doris_type=LARGEINT` to safely convert that field's non-NULL
+values with `int(value)`, while leaving ordinary STRING fields unchanged.
+
+During rolling upgrades, queries planned by older FEs retain the legacy metadata
+layout. Upgraded FEs advertise complete logical type metadata in the FlightInfo
+schema even when some BEs are older. An older BE's DoGet stream may still omit
+these markers; use the FlightInfo schema as the logical type reference until the
+upgrade finishes. No session setting is required.
+
+Run `python test_nested_type_metadata.py` with `DORIS_FLIGHT_SQL_URI` and optional
+`DORIS_USER` / `DORIS_PASSWORD` to verify the metadata and values against a cluster.
+The tests execute only read-only queries.
+
# Notes
For more details, refer to [Python Usage] in the document https://doris.apache.org/zh-CN/docs/dev/db-connect/arrow-flight-sql-connect
diff --git a/samples/arrow-flight-sql/python/test_nested_type_metadata.py b/samples/arrow-flight-sql/python/test_nested_type_metadata.py
new file mode 100644
index 00000000000..f918cf5702c
--- /dev/null
+++ b/samples/arrow-flight-sql/python/test_nested_type_metadata.py
@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Run with DORIS_FLIGHT_SQL_URI and optional DORIS_USER/DORIS_PASSWORD.
+
+LARGEINT retains its string storage encoding. PyArrow does not interpret custom
+field metadata automatically; applications can use doris_type to distinguish
+these strings from text and convert them to Python integers without losing range.
+"""
+
+import os
+import unittest
+
+import adbc_driver_flightsql.dbapi as flight_sql
+import adbc_driver_manager
+import pyarrow as pa
+
+
+@unittest.skipUnless(os.getenv("DORIS_FLIGHT_SQL_URI"), "Set DORIS_FLIGHT_SQL_URI to run against Doris")
+class NestedTypeMetadataTest(unittest.TestCase):
+ def setUp(self):
+ self.connection = flight_sql.connect(
+ uri=os.environ["DORIS_FLIGHT_SQL_URI"],
+ db_kwargs={
+ adbc_driver_manager.DatabaseOptions.USERNAME.value: os.getenv("DORIS_USER", "root"),
+ adbc_driver_manager.DatabaseOptions.PASSWORD.value: os.getenv("DORIS_PASSWORD", ""),
+ },
+ )
+ self.addCleanup(self.connection.close)
+ self.cursor = self.connection.cursor()
+ self.addCleanup(self.cursor.close)
+
+ def assert_logical_type(self, field, name):
+ self.assertEqual((field.metadata or {}).get(b"doris_type"), name.encode())
+
+ def query(self, sql):
+ self.cursor.execute(sql)
+ table = self.cursor.fetch_arrow_table()
+ table.validate(full=True)
+ return table
+
+ def test_largeint_metadata_and_full_range(self):
+ table = self.query("""
+ SELECT CAST('170141183460469231731687303715884105727' AS LARGEINT) AS scalar_value,
+ named_struct('number', CAST(17 AS LARGEINT), 'text', '17') AS struct_value,
+ array(CAST('-170141183460469231731687303715884105728' AS LARGEINT),
+ CAST(NULL AS LARGEINT)) AS array_value,
+ map(CAST(17 AS LARGEINT), CAST(19 AS LARGEINT)) AS map_value
+ """)
+ self.assert_logical_type(table.schema.field("scalar_value"), "LARGEINT")
+ structure = table.schema.field("struct_value").type
+ self.assert_logical_type(structure.field("number"), "LARGEINT")
+ self.assertNotIn(b"doris_type", structure.field("text").metadata or {})
+ self.assert_logical_type(table.schema.field("array_value").type.value_field, "LARGEINT")
+ mapping = table.schema.field("map_value").type
+ self.assert_logical_type(mapping.key_field, "LARGEINT")
+ self.assert_logical_type(mapping.item_field, "LARGEINT")
+ self.assertFalse(mapping.key_field.nullable)
+ self.assertTrue(pa.types.is_string(table.schema.field("scalar_value").type))
+ row = table.to_pylist()[0]
+ self.assertEqual(row["scalar_value"], str(2**127 - 1))
+ self.assertEqual(row["array_value"], [str(-(2**127)), None])
+ self.assertEqual(row["struct_value"], {"number": "17", "text": "17"})
+ self.assertEqual(row["map_value"], [("17", "19")])
+
+ def test_ipv4_metadata(self):
+ table = self.query("""
+ SELECT array(CAST('192.0.2.1' AS IPV4)) AS ip4
+ """)
+ self.assert_logical_type(table.schema.field("ip4").type.value_field, "IPV4")
+
+ def test_ipv6_metadata(self):
+ table = self.query("""
+ SELECT named_struct('address', CAST('2001:db8::1' AS IPV6)) AS ip6
+ """)
+ self.assert_logical_type(table.schema.field("ip6").type.field("address"), "IPV6")
+
+ def test_json_metadata(self):
+ # ARRAY/MAP constructors reject JSON; ARRAY_REPEAT preserves its element type.
+ table = self.query("""
+ SELECT array_repeat(CAST('{"n":1}' AS JSON), 1) AS json_value
+ """)
+ self.assert_logical_type(table.schema.field("json_value").type.value_field, "JSON")
+
+ def test_variant_metadata(self):
+ # ARRAY rejects legacy VARIANT; ARRAY_REPEAT also supports the default mode.
+ table = self.query("""
+ SELECT array_repeat(CAST('{"n":1}' AS VARIANT), 1) AS variant_value
+ """)
+ self.assert_logical_type(table.schema.field("variant_value").type.value_field, "VARIANT")
+
+
+if __name__ == "__main__":
+ unittest.main()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 29/35: branch-4.1: [fix](storage) Do not cache the failure of loading segment rows in BetaRowset #68675 (#68683)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0cd51660c4b42c16e0abb841b95eda88df17a518
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Thu Oct 8 09:31:45 2026 +0800
branch-4.1: [fix](storage) Do not cache the failure of loading segment rows in BetaRowset #68675 (#68683)
Cherry-picked from #68675
Co-authored-by: Xin Liao <li...@selectdb.com>
---
be/src/storage/rowset/beta_rowset.cpp | 17 +++++++--
be/src/storage/rowset/beta_rowset.h | 5 ++-
be/test/storage/rowset/beta_rowset_test.cpp | 54 +++++++++++++++++++++++++++++
3 files changed, 73 insertions(+), 3 deletions(-)
diff --git a/be/src/storage/rowset/beta_rowset.cpp b/be/src/storage/rowset/beta_rowset.cpp
index 637b0aa6fb1..5a9ac59375b 100644
--- a/be/src/storage/rowset/beta_rowset.cpp
+++ b/be/src/storage/rowset/beta_rowset.cpp
@@ -115,7 +115,9 @@ Status BetaRowset::get_segment_num_rows(std::vector<uint32_t>* segment_rows,
// So here `ROWSET_UNLOADING` is allowed.
DCHECK_NE(_rowset_state_machine.rowset_state(), ROWSET_UNLOADED);
#endif
- RETURN_IF_ERROR(_load_segment_rows_once.call([this, enable_segment_cache, read_stats, io_ctx] {
+ auto load_segment_rows = [this, enable_segment_cache, read_stats, io_ctx]() -> Status {
+ // Start from scratch, a previous failed attempt may have left partial rows.
+ _segments_rows.clear();
auto segment_count = num_segments();
if (segment_count == 0) {
return Status::OK();
@@ -165,7 +167,18 @@ Status BetaRowset::get_segment_num_rows(std::vector<uint32_t>* segment_rows,
auto self = std::dynamic_pointer_cast<BetaRowset>(shared_from_this());
return load_segment_rows_from_footer(self, &_segments_rows, enable_segment_cache,
read_stats, io_ctx);
- }));
+ };
+
+ // Only remember a successful load. A failure (e.g. a transient S3 SlowDown) must not be
+ // cached, otherwise every later caller of this long-lived rowset gets the same stale error.
+ if (!_segment_rows_loaded.load(std::memory_order_acquire)) {
+ std::lock_guard lock(_segment_rows_mutex);
+ if (!_segment_rows_loaded.load(std::memory_order_relaxed)) {
+ RETURN_IF_ERROR(load_segment_rows());
+ // `_segments_rows` is never modified once loaded, so it can be read without lock.
+ _segment_rows_loaded.store(true, std::memory_order_release);
+ }
+ }
segment_rows->assign(_segments_rows.cbegin(), _segments_rows.cend());
return Status::OK();
}
diff --git a/be/src/storage/rowset/beta_rowset.h b/be/src/storage/rowset/beta_rowset.h
index 8a2c13efe92..07182f34ede 100644
--- a/be/src/storage/rowset/beta_rowset.h
+++ b/be/src/storage/rowset/beta_rowset.h
@@ -20,8 +20,10 @@
#include <stddef.h>
+#include <atomic>
#include <cstdint>
#include <memory>
+#include <mutex>
#include <string>
#include <vector>
@@ -114,7 +116,8 @@ private:
friend class RowsetFactory;
friend class BetaRowsetReader;
- DorisCallOnce<Status> _load_segment_rows_once;
+ std::mutex _segment_rows_mutex;
+ std::atomic<bool> _segment_rows_loaded {false};
std::vector<uint32_t> _segments_rows;
};
diff --git a/be/test/storage/rowset/beta_rowset_test.cpp b/be/test/storage/rowset/beta_rowset_test.cpp
index 07d135702e8..e9c2cda99a4 100644
--- a/be/test/storage/rowset/beta_rowset_test.cpp
+++ b/be/test/storage/rowset/beta_rowset_test.cpp
@@ -555,6 +555,60 @@ TEST_F(BetaRowsetTest, GetSegmentNumRowsCorruptedMeta) {
sp->clear_trace();
}
+TEST_F(BetaRowsetTest, GetSegmentNumRowsRetryAfterFailure) {
+ // A failed load must not be cached. The rowset lives as long as its tablet version, so a
+ // cached transient error (e.g. S3 SlowDown) would fail every later caller, such as compaction.
+ auto tablet_schema = std::make_shared<TabletSchema>();
+ create_tablet_schema(tablet_schema);
+
+ auto rowset_meta = std::make_shared<RowsetMeta>();
+ init_rs_meta(rowset_meta, 1, 1);
+ // Use a dedicated rowset id so that no segment of other tests can be hit in segment cache.
+ RowsetId rowset_id;
+ rowset_id.init(540099);
+ rowset_meta->set_rowset_id(rowset_id);
+ rowset_meta->set_num_segments(2);
+ // No segment rows in meta and no segment files, so loading from segment footer fails.
+
+ auto rowset = std::make_shared<BetaRowset>(tablet_schema, rowset_meta, "");
+
+ auto sp = SyncPoint::get_instance();
+ int meta_path_count = 0;
+ int footer_path_count = 0;
+
+ sp->set_call_back("BetaRowset::get_segment_num_rows:use_segment_rows_from_meta",
+ [&](auto&& args) { meta_path_count++; });
+
+ sp->set_call_back("BetaRowset::get_segment_num_rows:load_from_segment_footer",
+ [&](auto&& args) { footer_path_count++; });
+
+ sp->enable_processing();
+
+ std::vector<uint32_t> segment_rows;
+ Status st = rowset->get_segment_num_rows(&segment_rows, false, &_stats);
+ ASSERT_FALSE(st.ok());
+ ASSERT_EQ(footer_path_count, 1);
+
+ // The failure is not cached, so the next call loads again and succeeds.
+ rowset_meta->set_num_segment_rows({100, 200});
+ st = rowset->get_segment_num_rows(&segment_rows, false, &_stats);
+ ASSERT_TRUE(st.ok()) << st;
+ ASSERT_EQ(segment_rows, (std::vector<uint32_t> {100, 200}));
+ ASSERT_EQ(meta_path_count, 1);
+
+ // The success is cached, so the following call does not load again.
+ std::vector<uint32_t> segment_rows_2;
+ st = rowset->get_segment_num_rows(&segment_rows_2, false, &_stats);
+ ASSERT_TRUE(st.ok()) << st;
+ ASSERT_EQ(segment_rows_2, (std::vector<uint32_t> {100, 200}));
+ ASSERT_EQ(meta_path_count, 1);
+ ASSERT_EQ(footer_path_count, 1);
+
+ sp->clear_all_call_backs();
+ sp->disable_processing();
+ sp->clear_trace();
+}
+
TEST_F(BetaRowsetTest, GetNumSegmentRowsAPI) {
// Test the simple get_num_segment_rows API (without loading)
auto tablet_schema = std::make_shared<TabletSchema>();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 19/35: branch-4.1: [fix](cloud) Invalidate version caches on visible commit retries (#67813) (#68650)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7606fcebd4e4ce230e8e9b4e7f89f5d6ef64fdd9
Author: Luwei <81...@qq.com>
AuthorDate: Wed Sep 30 20:55:45 2026 +0800
branch-4.1: [fix](cloud) Invalidate version caches on visible commit retries (#67813) (#68650)
### What problem does this PR solve?
Issue Number: #67099
Related PR: #67813
Problem Summary: A retry after a lost cloud commit response can return a
VISIBLE transaction without version results. Invalidate the affected
table and partition version caches, and notify other FEs so subsequent
reads fetch current versions from the MetaService. This cherry-picks the
merged #67813 commit (`3dbe4d3ca53307f73db96eda23d4efc9f02963c5`) to
`branch-4.1`.
The conflict resolution keeps the branch-4.1 partition cache API, which
has no commit-TSO argument, and adapts the tests to its Java 8 and JUnit
4 setup. It preserves the source PR's cache invalidation epochs and
batch publication locks.
### Release note
Prevent stale reads and false commit failures after cloud commit
retries.
### Check List (For Author)
- Test
- [x] Unit Test: 51 passed (`CloudGlobalTransactionMgrTest`,
`CloudPartitionTest`, `OlapTableTest`, `VersionHelperTest`).
- [x] FE Checkstyle: 0 violations; `git diff --check` passed.
- [ ] Live cluster / SQL regression: not run.
- Behavior changed:
- [x] Yes. VISIBLE retries without versions invalidate local and peer
caches; subsequent reads fetch fresh versions without changing the
successful commit outcome.
- Does this need documentation?
- [x] No.
---
.../java/org/apache/doris/catalog/OlapTable.java | 20 +-
.../cloud/catalog/CloudFEVersionSynchronizer.java | 59 +++
.../apache/doris/cloud/catalog/CloudPartition.java | 43 ++-
.../transaction/CloudGlobalTransactionMgr.java | 11 +
.../transaction/CloudGlobalTransactionMgrTest.java | 400 ++++++++++++++++++++-
5 files changed, 520 insertions(+), 13 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index e327aa5e883..1f5a54657d0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -242,6 +242,10 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc
private volatile long lastTableVersionCachedTimeMs = 0;
private volatile long cachedTableVersion = -1;
+ // Commit notifications cannot clear invalidation: their versions may precede the missing commit result.
+ private final AtomicLong tableVersionCacheEpoch = new AtomicLong();
+ private final AtomicLong refreshedTableVersionCacheEpoch = new AtomicLong();
+
private ReadWriteLock versionLock = Config.isCloudMode() ? new ReentrantReadWriteLock(true) : null;
public OlapTable() {
@@ -3369,7 +3373,7 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc
@VisibleForTesting
protected boolean isCachedTableVersionExpired() {
// -1 means no cache yet, need to fetch from MS
- if (cachedTableVersion == -1) {
+ if (cachedTableVersion == -1 || tableVersionCacheEpoch.get() != refreshedTableVersionCacheEpoch.get()) {
return true;
}
ConnectContext ctx = ConnectContext.get();
@@ -3383,13 +3387,18 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc
public boolean isCachedTableVersionExpired(long expirationMs) {
// -1 means no cache yet, need to fetch from MS
- if (cachedTableVersion == -1 || expirationMs <= 0) {
+ if (cachedTableVersion == -1 || expirationMs <= 0
+ || tableVersionCacheEpoch.get() != refreshedTableVersionCacheEpoch.get()) {
return true;
}
return System.currentTimeMillis() - lastTableVersionCachedTimeMs > expirationMs;
}
- public void setCachedTableVersion(long version) {
+ public void invalidateCachedTableVersion() {
+ tableVersionCacheEpoch.incrementAndGet();
+ }
+
+ public synchronized void setCachedTableVersion(long version) {
if (version >= cachedTableVersion) {
cachedTableVersion = version;
lastTableVersionCachedTimeMs = System.currentTimeMillis();
@@ -3410,6 +3419,7 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc
return getCachedTableVersion();
}
+ long cacheEpoch = tableVersionCacheEpoch.get();
// get version rpc
Cloud.GetVersionRequest request = Cloud.GetVersionRequest.newBuilder()
.setRequestIp(FrontendOptions.getLocalHostAddressCached())
@@ -3436,6 +3446,7 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc
}
// update cache
setCachedTableVersion(version);
+ refreshedTableVersionCacheEpoch.accumulateAndGet(cacheEpoch, Math::max);
return version;
} catch (RpcException e) {
LOG.warn("get version from meta service failed", e);
@@ -3500,9 +3511,11 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc
private static List<Long> getVisibleVersionInBatchFromMs(List<OlapTable> tables) {
List<Long> dbIds = new ArrayList<>(tables.size());
List<Long> tableIds = new ArrayList<>(tables.size());
+ List<Long> cacheEpochs = new ArrayList<>(tables.size());
for (OlapTable table : tables) {
dbIds.add(table.getDatabase().getId());
tableIds.add(table.getId());
+ cacheEpochs.add(table.tableVersionCacheEpoch.get());
}
List<Long> versions = getVisibleVersionFromMeta(dbIds, tableIds);
@@ -3511,6 +3524,7 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc
Preconditions.checkState(tables.size() == versions.size());
for (int i = 0; i < tables.size(); i++) {
tables.get(i).setCachedTableVersion(versions.get(i));
+ tables.get(i).refreshedTableVersionCacheEpoch.accumulateAndGet(cacheEpochs.get(i), Math::max);
}
return versions;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudFEVersionSynchronizer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudFEVersionSynchronizer.java
index f843b8b58e7..d88348b704a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudFEVersionSynchronizer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudFEVersionSynchronizer.java
@@ -49,6 +49,9 @@ import java.util.stream.Collectors;
public class CloudFEVersionSynchronizer {
private static final Logger LOG = LogManager.getLogger(CloudFEVersionSynchronizer.class);
+ // Reuse the version RPC for whole-table invalidation when a commit response has no versions.
+ // Older FEs ignore this value and continue relying on their periodic version synchronization.
+ private static final long INVALIDATE_VERSION_CACHE = -1;
private static final ExecutorService SYNC_VERSION_THREAD_POOL = Executors.newFixedThreadPool(
Config.cloud_sync_version_task_threads_num,
@@ -57,6 +60,54 @@ public class CloudFEVersionSynchronizer {
public CloudFEVersionSynchronizer() {
}
+ public void invalidateVersionCaches(long dbId, List<Long> tableIds) {
+ Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
+ if (db == null) {
+ return; // The database may have been dropped since the original commit.
+ }
+ List<OlapTable> tables = new ArrayList<>();
+ for (long tableId : tableIds) {
+ Table table = db.getTableNullable(tableId);
+ if (table != null && table.isManagedTable()) {
+ tables.add((OlapTable) table);
+ }
+ }
+ invalidateVersionCaches(tables);
+ List<Pair<OlapTable, Long>> tableVersions = tables.stream()
+ .map(table -> Pair.of(table, INVALIDATE_VERSION_CACHE)).collect(Collectors.toList());
+ pushVersionAsync(dbId, tableVersions, Collections.emptyMap());
+ }
+
+ private void invalidateVersionCaches(List<OlapTable> tables) {
+ tables.sort(Comparator.comparingLong(OlapTable::getId));
+ for (OlapTable table : tables) {
+ table.readLock();
+ }
+ try {
+ for (OlapTable table : tables) {
+ table.versionWriteLock();
+ }
+ try {
+ for (OlapTable table : tables) {
+ table.invalidateCachedTableVersion();
+ for (Partition partition : table.getAllPartitions()) {
+ ((CloudPartition) partition).invalidateCachedVisibleVersion();
+ }
+ }
+ } finally {
+ for (int i = tables.size() - 1; i >= 0; i--) {
+ tables.get(i).versionWriteUnlock();
+ }
+ }
+ } finally {
+ for (int i = tables.size() - 1; i >= 0; i--) {
+ tables.get(i).readUnlock();
+ }
+ }
+ LOG.info("Invalidated cloud version caches for tables {}",
+ tables.stream().map(OlapTable::getId).collect(Collectors.toList()));
+ }
+
// master FE send sync version rpc to other FEs
public void pushVersionAsync(long dbId, OlapTable table, long version) {
if (!Config.cloud_enable_version_syncer) {
@@ -153,17 +204,25 @@ public class CloudFEVersionSynchronizer {
}
// only update table version
if (request.getPartitionVersionInfos().isEmpty()) {
+ List<OlapTable> invalidatedTables = new ArrayList<>();
request.getTableVersionInfos().forEach(tableVersionInfo -> {
Table table = db.getTableNullable(tableVersionInfo.getTableId());
if (table == null || !table.isManagedTable()) {
return;
}
OlapTable olapTable = (OlapTable) table;
+ if (tableVersionInfo.getVersion() == INVALIDATE_VERSION_CACHE) {
+ invalidatedTables.add(olapTable);
+ return;
+ }
olapTable.setCachedTableVersion(tableVersionInfo.getVersion());
if (LOG.isDebugEnabled()) {
LOG.debug("Update tableId: {}, version: {}", olapTable.getId(), tableVersionInfo.getVersion());
}
});
+ if (!invalidatedTables.isEmpty()) {
+ invalidateVersionCaches(invalidatedTables);
+ }
return;
}
// update partition and table version
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java
index cfb9fabe907..eb1193602ad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java
@@ -48,6 +48,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
@@ -66,6 +67,10 @@ public class CloudPartition extends Partition {
// This value is set when get the version from meta-service, 0 means version is not cached yet
private volatile long lastVersionCachedTimeMs = 0;
+ // Only an MS read started after invalidation may make this cache valid again.
+ private final AtomicLong versionCacheEpoch = new AtomicLong();
+ private final AtomicLong refreshedVersionCacheEpoch = new AtomicLong();
+
private ReentrantLock lock = new ReentrantLock(true);
public CloudPartition(long id, String name, MaterializedIndex baseIndex,
@@ -123,9 +128,13 @@ public class CloudPartition extends Partition {
return super.getVisibleVersion();
}
+ public void invalidateCachedVisibleVersion() {
+ versionCacheEpoch.incrementAndGet();
+ }
+
@VisibleForTesting
protected boolean isCachedVersionExpired() {
- if (lastVersionCachedTimeMs == 0) {
+ if (lastVersionCachedTimeMs == 0 || versionCacheEpoch.get() != refreshedVersionCacheEpoch.get()) {
return true;
}
ConnectContext ctx = ConnectContext.get();
@@ -151,6 +160,7 @@ public class CloudPartition extends Partition {
}
private long getVisibleVersionFromMs(boolean waitForPendingTxns) {
+ long cacheEpoch = versionCacheEpoch.get();
if (LOG.isDebugEnabled()) {
LOG.debug("getVisibleVersionFromMs use CloudPartition {}, waitForPendingTxns: {}",
super.getName(), waitForPendingTxns);
@@ -182,6 +192,7 @@ public class CloudPartition extends Partition {
LOG.debug("get version from meta service, version: {}, partition: {}", version, super.getId());
}
setCachedVisibleVersion(version, mTime);
+ refreshedVersionCacheEpoch.accumulateAndGet(cacheEpoch, Math::max);
return version;
} catch (RpcException e) {
throw new RuntimeException("get version from meta service failed");
@@ -242,10 +253,12 @@ public class CloudPartition extends Partition {
List<Long> tableIds = new ArrayList<>();
List<Long> partitionIds = new ArrayList<>();
List<Long> versionUpdateTimesMs = new ArrayList<>();
+ List<Long> cacheEpochs = new ArrayList<>();
for (CloudPartition partition : partitions) {
dbIds.add(partition.getDbId());
tableIds.add(partition.getTableId());
partitionIds.add(partition.getId());
+ cacheEpochs.add(partition.versionCacheEpoch.get());
}
List<Long> versions = getSnapshotVisibleVersion(
@@ -253,14 +266,26 @@ public class CloudPartition extends Partition {
// Cache visible version, see hasData() for details.
int size = versions.size();
- for (int i = 0; i < size; ++i) {
- Long version = versions.get(i);
- if (version > Partition.PARTITION_INIT_VERSION) {
- // For compatibility, the existing partitions may not have mtime
- long mTime = versions.size() == versionUpdateTimesMs.size() ? versionUpdateTimesMs.get(i) : 0;
- partitions.get(i).setCachedVisibleVersion(versions.get(i), mTime);
- } else { // No data has been written to this partition
- partitions.get(i).setCachedVisibleVersion(Partition.PARTITION_INIT_VERSION, System.currentTimeMillis());
+ List<OlapTable> tables = getTables(partitions);
+ for (OlapTable table : tables) {
+ table.versionWriteLock();
+ }
+ try {
+ for (int i = 0; i < size; ++i) {
+ Long version = versions.get(i);
+ if (version > Partition.PARTITION_INIT_VERSION) {
+ // For compatibility, the existing partitions may not have mtime
+ long mTime = versions.size() == versionUpdateTimesMs.size() ? versionUpdateTimesMs.get(i) : 0;
+ partitions.get(i).setCachedVisibleVersion(versions.get(i), mTime);
+ } else { // No data has been written to this partition
+ partitions.get(i).setCachedVisibleVersion(Partition.PARTITION_INIT_VERSION,
+ System.currentTimeMillis());
+ }
+ partitions.get(i).refreshedVersionCacheEpoch.accumulateAndGet(cacheEpochs.get(i), Math::max);
+ }
+ } finally {
+ for (int i = tables.size() - 1; i >= 0; i--) {
+ tables.get(i).versionWriteUnlock();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
index 1c49b3c11b2..9f8a133491e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
@@ -478,6 +478,10 @@ public class CloudGlobalTransactionMgr implements GlobalTransactionMgrIface {
+ "] is already aborted. abort reason: " + transactionState.getReason());
} else if (transactionState.getTransactionStatus() == TransactionStatus.COMMITTED
|| transactionState.getTransactionStatus() == TransactionStatus.VISIBLE) {
+ if (transactionState.getTransactionStatus() == TransactionStatus.VISIBLE) {
+ ((CloudEnv) Env.getCurrentEnv()).getCloudFEVersionSynchronizer()
+ .invalidateVersionCaches(dbId, transactionState.getTableIdList());
+ }
LOG.info("txn={}, status={} not need to calculate delete bitmap again, just return ",
transactionId,
transactionState.getTransactionStatus().toString());
@@ -582,6 +586,13 @@ public class CloudGlobalTransactionMgr implements GlobalTransactionMgrIface {
long txnId = commitTxnResponse.getTxnInfo().getTxnId();
int totalPartitionNum = commitTxnResponse.getPartitionIdsList().size();
if (totalPartitionNum == 0 && commitTxnResponse.getTableStatsList().isEmpty()) {
+ // A visible retry can contain only txn_info. Invalidate locally without another commit-time RPC;
+ // subsequent reads fetch fresh versions, including the table version used to validate SQL caches.
+ if (commitTxnResponse.getTxnInfo().getStatus() == TxnStatusPB.TXN_STATUS_VISIBLE
+ && !(commitTxnResponse.getIsLazyCommit() && commitTxnResponse.getIsLazyCommitIncomplete())) {
+ ((CloudEnv) Env.getCurrentEnv()).getCloudFEVersionSynchronizer()
+ .invalidateVersionCaches(dbId, commitTxnResponse.getTxnInfo().getTableIdsList());
+ }
return Collections.emptyMap();
}
Env env = Env.getCurrentEnv();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java
index 1daaaf51de8..490851d52db 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java
@@ -21,7 +21,15 @@ import org.apache.doris.catalog.CatalogTestUtil;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.FakeEditLog;
import org.apache.doris.catalog.FakeEnv;
+import org.apache.doris.catalog.KeysType;
+import org.apache.doris.catalog.MaterializedIndex;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.PartitionInfo;
+import org.apache.doris.catalog.RandomDistributionInfo;
import org.apache.doris.catalog.Table;
+import org.apache.doris.cloud.catalog.CloudEnv;
+import org.apache.doris.cloud.catalog.CloudFEVersionSynchronizer;
+import org.apache.doris.cloud.catalog.CloudPartition;
import org.apache.doris.cloud.proto.Cloud;
import org.apache.doris.cloud.proto.Cloud.AbortTxnResponse;
import org.apache.doris.cloud.proto.Cloud.BeginTxnResponse;
@@ -31,29 +39,47 @@ import org.apache.doris.cloud.proto.Cloud.GetCurrentMaxTxnResponse;
import org.apache.doris.cloud.proto.Cloud.MetaServiceCode;
import org.apache.doris.cloud.proto.Cloud.TxnInfoPB;
import org.apache.doris.cloud.rpc.MetaServiceProxy;
+import org.apache.doris.cloud.rpc.VersionHelper;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ClientPool;
import org.apache.doris.common.Config;
import org.apache.doris.common.DuplicatedRequestException;
import org.apache.doris.common.FeMetaVersion;
+import org.apache.doris.common.GenericPool;
import org.apache.doris.common.LabelAlreadyUsedException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.QuotaExceedException;
import org.apache.doris.common.UserException;
+import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.load.routineload.RLTaskTxnCommitAttachment;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.rpc.RpcException;
+import org.apache.doris.service.FrontendServiceImpl;
+import org.apache.doris.system.Frontend;
+import org.apache.doris.system.SystemInfoService.HostInfo;
+import org.apache.doris.thrift.FrontendService;
+import org.apache.doris.thrift.TCloudVersionInfo;
+import org.apache.doris.thrift.TFrontendSyncCloudVersionRequest;
+import org.apache.doris.thrift.TStatus;
+import org.apache.doris.thrift.TStatusCode;
import org.apache.doris.thrift.TTabletCommitInfo;
import org.apache.doris.transaction.BeginTransactionException;
import org.apache.doris.transaction.GlobalTransactionMgrIface;
import org.apache.doris.transaction.TabletCommitInfo;
import org.apache.doris.transaction.TransactionState;
+import org.apache.doris.transaction.TransactionStatus;
import org.apache.doris.transaction.TxnStateChangeCallback;
import com.google.common.collect.Lists;
import mockit.Mock;
import mockit.MockUp;
+import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.Assertions;
+import org.mockito.AdditionalAnswers;
import org.mockito.ArgumentCaptor;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
@@ -62,8 +88,15 @@ import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
public class CloudGlobalTransactionMgrTest {
@@ -83,11 +116,21 @@ public class CloudGlobalTransactionMgrTest {
Config.meta_service_endpoint = "127.0.0.1:20121";
fakeEditLog = new FakeEditLog();
fakeEnv = new FakeEnv();
- masterEnv = CatalogTestUtil.createTestCatalog();
+ Env catalog = CatalogTestUtil.createTestCatalog();
+ masterEnv = Mockito.mock(CloudEnv.class, AdditionalAnswers.delegatesTo(catalog));
+ Mockito.doReturn(new CloudFEVersionSynchronizer()).when((CloudEnv) masterEnv).getCloudFEVersionSynchronizer();
+ // Env.getCurrentGlobalTransactionMgr reads the field directly rather than calling the delegated getter.
+ Deencapsulation.setField(masterEnv, "globalTransactionMgr", catalog.getGlobalTransactionMgr());
+ FakeEnv.setEnv(masterEnv);
FakeEnv.setMetaVersion(FeMetaVersion.VERSION_CURRENT);
masterTransMgr = masterEnv.getGlobalTransactionMgr();
}
+ @After
+ public void tearDown() {
+ ConnectContext.remove();
+ }
+
@Test
public void testBeginTransaction() throws LabelAlreadyUsedException, AnalysisException,
BeginTransactionException, DuplicatedRequestException, QuotaExceedException, MetaNotFoundException {
@@ -554,6 +597,361 @@ public class CloudGlobalTransactionMgrTest {
Assert.assertEquals(1000, result);
}
+ @Test
+ public void testVisibleRetryInvalidatesTableAndPartitionVersions() throws Exception {
+ useVersionCaches();
+ CloudPartition first = addCloudPartition(1000);
+ CloudPartition second = addCloudPartition(2000);
+ OlapTable firstTable = getCloudTable(first);
+ OlapTable secondTable = getCloudTable(second);
+ CommitTxnResponse response = visibleRetry(Lists.newArrayList(first.getTableId(), second.getTableId()));
+ try (MockedStatic<VersionHelper> versions = mockVersionHelper()) {
+ versions.when(() -> VersionHelper.getVersionFromMeta(Mockito.any())).thenReturn(partitionVersion(4));
+ masterTransMgr.afterCommitTxnResp(response, null, Lists.newArrayList());
+ versions.verifyNoInteractions();
+ Assertions.assertEquals(2, first.getCachedVisibleVersion());
+ Assertions.assertEquals(2, firstTable.getCachedTableVersion());
+ Assertions.assertEquals(4, first.getVisibleVersion());
+ Assertions.assertEquals(Lists.newArrayList(4L), CloudPartition.getSnapshotVisibleVersion(Lists.newArrayList(second)));
+ Assertions.assertEquals(4, firstTable.getVisibleVersion());
+ Assertions.assertEquals(Lists.newArrayList(4L), OlapTable.getVisibleVersionInBatch(Lists.newArrayList(secondTable)));
+ Assertions.assertEquals(4, first.getCachedVisibleVersion());
+ Assertions.assertEquals(4, second.getCachedVisibleVersion());
+ Assertions.assertEquals(40, first.getVisibleVersionTime());
+ // Current versions must not become the original transaction's BE promotion outcome.
+ Assertions.assertEquals(0, response.getVersionsCount());
+ Assertions.assertEquals(4, first.getVisibleVersion());
+ Assertions.assertEquals(Lists.newArrayList(4L), CloudPartition.getSnapshotVisibleVersion(Lists.newArrayList(second)));
+ Assertions.assertEquals(4, firstTable.getVisibleVersion());
+ Assertions.assertEquals(Lists.newArrayList(4L), OlapTable.getVisibleVersionInBatch(Lists.newArrayList(secondTable)));
+ versions.verify(() -> VersionHelper.getVersionFromMeta(Mockito.any()), Mockito.times(4));
+ }
+ }
+
+ @Test
+ public void testIncompleteCommitDoesNotRefreshPartitionVersions() throws Exception {
+ CloudPartition partition = addCloudPartition(1000);
+ CommitTxnResponse response = visibleRetry(Lists.newArrayList(partition.getTableId()));
+ try (MockedStatic<VersionHelper> versions = mockVersionHelper()) {
+ masterTransMgr.afterCommitTxnResp(response.toBuilder().setTxnInfo(response.getTxnInfo().toBuilder()
+ .setStatus(Cloud.TxnStatusPB.TXN_STATUS_COMMITTED)).build(), null, Lists.newArrayList());
+ masterTransMgr.afterCommitTxnResp(response.toBuilder().setIsLazyCommit(true)
+ .setIsLazyCommitIncomplete(true).build(), null, Lists.newArrayList());
+ Assertions.assertEquals(2, partition.getCachedVisibleVersion());
+ versions.verifyNoInteractions();
+ }
+ }
+
+ @Test
+ public void testVisibleRetrySkipsDroppedTable() throws Exception {
+ try (MockedStatic<VersionHelper> versions = mockVersionHelper()) {
+ masterTransMgr.afterCommitTxnResp(visibleRetry(Lists.newArrayList(1000L)), null, Lists.newArrayList());
+ versions.verifyNoInteractions();
+ }
+ }
+
+ @Test
+ public void testMowVisiblePrecheckInvalidatesVersions() throws Exception {
+ useVersionCaches();
+ CloudPartition partition = addCloudPartition(1000);
+ TxnInfoPB txnInfo = visibleRetry(Lists.newArrayList(partition.getTableId())).getTxnInfo();
+ MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class);
+ Mockito.when(proxy.getTxn(Mockito.any())).thenReturn(Cloud.GetTxnResponse.newBuilder()
+ .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK))
+ .setTxnInfo(txnInfo).build());
+ try (MockedStatic<MetaServiceProxy> proxyMock = Mockito.mockStatic(MetaServiceProxy.class);
+ MockedStatic<VersionHelper> versions = mockVersionHelper()) {
+ proxyMock.when(MetaServiceProxy::getInstance).thenReturn(proxy);
+ versions.when(() -> VersionHelper.getVersionFromMeta(Mockito.any())).thenReturn(partitionVersion(4));
+ Method check = CloudGlobalTransactionMgr.class.getDeclaredMethod("checkTransactionStateBeforeCommit",
+ long.class, long.class);
+ check.setAccessible(true);
+ Assertions.assertEquals(false, check.invoke(masterTransMgr, txnInfo.getDbId(), txnInfo.getTxnId()));
+ versions.verifyNoInteractions();
+ Assertions.assertEquals(4, partition.getVisibleVersion());
+ Assertions.assertEquals(4, getCloudTable(partition).getVisibleVersion());
+ Assertions.assertEquals(4, partition.getCachedVisibleVersion());
+ }
+ }
+
+ @Test
+ public void testRefreshFailurePreservesCommitCallbacks() throws Exception {
+ useVersionCaches();
+ CloudPartition partition = addCloudPartition(1000);
+ CommitTxnResponse retry = visibleRetry(Lists.newArrayList(partition.getTableId()));
+ CommitTxnResponse response = retry.toBuilder()
+ .setTxnInfo(retry.getTxnInfo().toBuilder().setListenerId(42)).build();
+ TxnStateChangeCallback callback = Mockito.mock(TxnStateChangeCallback.class);
+ Mockito.when(callback.getId()).thenReturn(42L);
+ masterTransMgr.getCallbackFactory().addCallback(callback);
+ Mockito.clearInvocations(callback);
+ MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class);
+ Mockito.when(proxy.commitTxn(Mockito.any())).thenReturn(response);
+ Table table = masterEnv.getInternalCatalog().getDbOrMetaException(CatalogTestUtil.testDbId1)
+ .getTableOrMetaException(partition.getTableId());
+ try (MockedStatic<MetaServiceProxy> proxyMock = Mockito.mockStatic(MetaServiceProxy.class);
+ MockedStatic<VersionHelper> versions = mockVersionHelper()) {
+ proxyMock.when(MetaServiceProxy::getInstance).thenReturn(proxy);
+ versions.when(() -> VersionHelper.getVersionFromMeta(Mockito.any()))
+ .thenThrow(new RpcException("MS", "unavailable"));
+ Assertions.assertDoesNotThrow(() -> masterTransMgr.commitTransactionWithoutLock(CatalogTestUtil.testDbId1,
+ Lists.newArrayList(table), response.getTxnInfo().getTxnId(), null, null));
+ Assertions.assertEquals(2, partition.getCachedVisibleVersion());
+ versions.verifyNoInteractions();
+ Mockito.verify(callback).afterCommitted(Mockito.argThat(state ->
+ state.getTransactionStatus() == TransactionStatus.VISIBLE), Mockito.eq(true));
+ Mockito.verify(callback).afterVisible(Mockito.argThat(state ->
+ state.getTransactionStatus() == TransactionStatus.VISIBLE), Mockito.eq(true));
+ // A version outage fails the following read, not the already committed transaction.
+ Assertions.assertThrows(RuntimeException.class, partition::getVisibleVersion);
+ Assertions.assertThrows(RpcException.class, ((OlapTable) table)::getVisibleVersion);
+ versions.when(() -> VersionHelper.getVersionFromMeta(Mockito.any())).thenReturn(partitionVersion(4));
+ Assertions.assertEquals(4, partition.getVisibleVersion());
+ Assertions.assertEquals(4, ((OlapTable) table).getVisibleVersion());
+ Mockito.verifyNoMoreInteractions(callback);
+ }
+ }
+
+ @Test
+ public void testSinglePartitionReadCannotClearConcurrentInvalidation() throws Exception {
+ checkReadCannotClearConcurrentInvalidation(false, false);
+ }
+
+ @Test
+ public void testBatchPartitionReadCannotClearConcurrentInvalidation() throws Exception {
+ checkReadCannotClearConcurrentInvalidation(false, true);
+ }
+
+ @Test
+ public void testSingleTableReadCannotClearConcurrentInvalidation() throws Exception {
+ checkReadCannotClearConcurrentInvalidation(true, false);
+ }
+
+ @Test
+ public void testBatchTableReadCannotClearConcurrentInvalidation() throws Exception {
+ checkReadCannotClearConcurrentInvalidation(true, true);
+ }
+
+ private void checkReadCannotClearConcurrentInvalidation(boolean tableVersion, boolean batch) throws Exception {
+ useVersionCaches();
+ CloudPartition partition = addCloudPartition(1000);
+ OlapTable table = getCloudTable(partition);
+ ConnectContext.get().getSessionVariable().cloudPartitionVersionCacheTtlMs = 0;
+ ConnectContext.get().getSessionVariable().cloudTableVersionCacheTtlMs = 0;
+ try (MockedStatic<VersionHelper> versions = mockVersionHelper()) {
+ versions.when(() -> VersionHelper.getVersionFromMeta(Mockito.any())).thenAnswer(invocation -> {
+ // The MS snapshot predates the commit, but its reply arrives after cache invalidation.
+ masterTransMgr.afterCommitTxnResp(visibleRetry(Lists.newArrayList(table.getId())), null, Lists.newArrayList());
+ useVersionCaches();
+ return partitionVersion(2);
+ });
+ if (tableVersion) {
+ Assertions.assertEquals(2L, batch
+ ? OlapTable.getVisibleVersionInBatch(Lists.newArrayList(table)).get(0) : table.getVisibleVersion());
+ } else {
+ Assertions.assertEquals(2L, batch
+ ? CloudPartition.getSnapshotVisibleVersion(Lists.newArrayList(partition)).get(0)
+ : partition.getVisibleVersion());
+ }
+ // A delayed ordinary commit notification cannot repair an unknown version either.
+ table.setCachedTableVersion(3);
+ partition.setCachedVisibleVersion(3, 30);
+ versions.when(() -> VersionHelper.getVersionFromMeta(Mockito.any())).thenReturn(partitionVersion(4));
+ if (tableVersion) {
+ Assertions.assertEquals(4, table.getVisibleVersion());
+ Assertions.assertEquals(4, table.getVisibleVersion());
+ } else {
+ Assertions.assertEquals(4, partition.getVisibleVersion());
+ Assertions.assertEquals(4, partition.getVisibleVersion());
+ }
+ versions.verify(() -> VersionHelper.getVersionFromMeta(Mockito.any()), Mockito.times(2));
+ }
+ }
+
+ @Test
+ public void testInvalidationWaitsForVersionSnapshotReaders() throws Exception {
+ CloudPartition partition = addCloudPartition(1000);
+ OlapTable table = getCloudTable(partition);
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ CountDownLatch started = new CountDownLatch(1);
+ table.versionReadLock();
+ CompletableFuture<Void> invalidation;
+ try {
+ invalidation = CompletableFuture.runAsync(() -> {
+ started.countDown();
+ ((CloudEnv) masterEnv).getCloudFEVersionSynchronizer()
+ .invalidateVersionCaches(CatalogTestUtil.testDbId1, Lists.newArrayList(table.getId()));
+ }, executor);
+ Assertions.assertTrue(started.await(5, TimeUnit.SECONDS));
+ Assertions.assertThrows(TimeoutException.class, () -> invalidation.get(200, TimeUnit.MILLISECONDS));
+ } finally {
+ table.versionReadUnlock();
+ executor.shutdown();
+ }
+ invalidation.get(5, TimeUnit.SECONDS);
+ Assertions.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
+ }
+
+ @Test
+ public void testBatchRefreshInstallsVersionsUnderWriteLock() throws Exception {
+ CloudPartition first = addCloudPartition(1000);
+ OlapTable table = getCloudTable(first);
+ CloudPartition second = Mockito.spy(new CloudPartition(1002, "p2", new MaterializedIndex(),
+ new RandomDistributionInfo(1), CatalogTestUtil.testDbId1, table.getId()));
+ table.addPartition(second);
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ AtomicReference<CompletableFuture<Void>> snapshotReader = new AtomicReference<>();
+ try (MockedStatic<VersionHelper> versions = mockVersionHelper()) {
+ versions.when(() -> VersionHelper.getVersionFromMeta(Mockito.any())).thenReturn(
+ partitionVersion(4).toBuilder().addVersions(4).addVersionUpdateTimeMs(40).build());
+ Mockito.doAnswer(invocation -> {
+ CountDownLatch started = new CountDownLatch(1);
+ CompletableFuture<Void> reader = CompletableFuture.runAsync(() -> {
+ started.countDown();
+ table.versionReadLock();
+ try {
+ Assertions.assertEquals(4, first.getCachedVisibleVersion());
+ Assertions.assertEquals(4, second.getCachedVisibleVersion());
+ } finally {
+ table.versionReadUnlock();
+ }
+ }, executor);
+ snapshotReader.set(reader);
+ Assertions.assertTrue(started.await(5, TimeUnit.SECONDS));
+ Assertions.assertThrows(TimeoutException.class, () -> reader.get(200, TimeUnit.MILLISECONDS));
+ return invocation.callRealMethod();
+ }).when(second).setCachedVisibleVersion(4, 40);
+ CloudPartition.getSnapshotVisibleVersionFromMs(Lists.newArrayList(first, second), false);
+ snapshotReader.get().get(5, TimeUnit.SECONDS);
+ } finally {
+ executor.shutdown();
+ Assertions.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ public void testFollowerInvalidatesTableAndAllPartitions() throws Exception {
+ useVersionCaches();
+ CloudPartition partition = addCloudPartition(1000);
+ OlapTable table = getCloudTable(partition);
+ CloudPartition second = new CloudPartition(1002, "p2", new MaterializedIndex(),
+ new RandomDistributionInfo(1), CatalogTestUtil.testDbId1, table.getId());
+ second.setCachedVisibleVersion(2, 20);
+ table.addPartition(second);
+ TFrontendSyncCloudVersionRequest request = new TFrontendSyncCloudVersionRequest()
+ .setDbId(CatalogTestUtil.testDbId1).setPartitionVersionInfos(Lists.newArrayList())
+ .setTableVersionInfos(Lists.newArrayList(new TCloudVersionInfo().setTableId(table.getId()).setVersion(-1)));
+ Mockito.doReturn(false).when(masterEnv).isMaster();
+ FrontendServiceImpl service = new FrontendServiceImpl(null);
+ try (MockedStatic<VersionHelper> versions = mockVersionHelper()) {
+ Assertions.assertEquals(TStatusCode.OK, service.syncCloudVersion(request).getStatusCode());
+ versions.verifyNoInteractions();
+ // Even a delayed regular version push must leave these caches invalid.
+ service.syncCloudVersion(new TFrontendSyncCloudVersionRequest()
+ .setDbId(CatalogTestUtil.testDbId1).setPartitionVersionInfos(Lists.newArrayList())
+ .setTableVersionInfos(Lists.newArrayList(new TCloudVersionInfo().setTableId(table.getId()).setVersion(3))));
+ versions.when(() -> VersionHelper.getVersionFromMeta(Mockito.any())).thenReturn(partitionVersion(4));
+ Assertions.assertEquals(4, table.getVisibleVersion());
+ Assertions.assertEquals(4, partition.getVisibleVersion());
+ Assertions.assertEquals(4, second.getVisibleVersion());
+ versions.verify(() -> VersionHelper.getVersionFromMeta(Mockito.any()), Mockito.times(3));
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testInvalidationIsPushedThroughVersionRpc() throws Exception {
+ CloudPartition partition = addCloudPartition(1000);
+ boolean syncEnabled = Config.cloud_enable_version_syncer;
+ GenericPool<FrontendService.Client> originalPool = ClientPool.frontendVersionPool;
+ GenericPool<FrontendService.Client> pool = Mockito.mock(GenericPool.class);
+ FrontendService.Client client = Mockito.mock(FrontendService.Client.class);
+ Mockito.when(pool.borrowObject(Mockito.any())).thenReturn(client);
+ CompletableFuture<TFrontendSyncCloudVersionRequest> sent = new CompletableFuture<>();
+ CountDownLatch returned = new CountDownLatch(1);
+ Mockito.when(client.syncCloudVersion(Mockito.any())).thenAnswer(invocation -> {
+ sent.complete(invocation.getArgument(0));
+ return new TStatus(TStatusCode.OK);
+ });
+ Mockito.doAnswer(invocation -> {
+ returned.countDown();
+ return null;
+ }).when(pool).returnObject(Mockito.any(), Mockito.eq(client));
+ CloudEnv sender = (CloudEnv) masterEnv;
+ Frontend follower = Mockito.mock(Frontend.class);
+ Mockito.when(follower.isAlive()).thenReturn(true);
+ Mockito.when(follower.getHost()).thenReturn("127.0.0.2");
+ Mockito.when(follower.getRpcPort()).thenReturn(9020);
+ Mockito.doReturn(Lists.newArrayList(follower)).when(sender).getFrontends(null);
+ Mockito.doReturn(new HostInfo("127.0.0.1", 9010)).when(sender).getSelfNode();
+ try {
+ FakeEnv.setEnv(sender);
+ Config.cloud_enable_version_syncer = true;
+ ClientPool.frontendVersionPool = pool;
+ masterTransMgr.afterCommitTxnResp(visibleRetry(Lists.newArrayList(partition.getTableId())), null, Lists.newArrayList());
+ TFrontendSyncCloudVersionRequest request = sent.get(5, TimeUnit.SECONDS);
+ Assertions.assertTrue(returned.await(5, TimeUnit.SECONDS));
+ // Check the actual Thrift payload as well as the sender's Java objects.
+ TFrontendSyncCloudVersionRequest decoded = new TFrontendSyncCloudVersionRequest();
+ new org.apache.thrift.TDeserializer().deserialize(decoded,
+ new org.apache.thrift.TSerializer().serialize(request));
+ Assertions.assertEquals(CatalogTestUtil.testDbId1, decoded.getDbId());
+ Assertions.assertTrue(decoded.getPartitionVersionInfos().isEmpty());
+ Assertions.assertEquals(1, decoded.getTableVersionInfosSize());
+ Assertions.assertEquals(partition.getTableId(), decoded.getTableVersionInfos().get(0).getTableId());
+ Assertions.assertEquals(-1, decoded.getTableVersionInfos().get(0).getVersion());
+ } finally {
+ FakeEnv.setEnv(masterEnv);
+ ClientPool.frontendVersionPool = originalPool;
+ Config.cloud_enable_version_syncer = syncEnabled;
+ }
+ }
+
+ private CloudPartition addCloudPartition(long tableId) throws Exception {
+ OlapTable table = new OlapTable(tableId, "version_cache_" + tableId, Lists.newArrayList(), KeysType.DUP_KEYS,
+ new PartitionInfo(), new RandomDistributionInfo(1));
+ CloudPartition partition = new CloudPartition(tableId + 1, "p1", new MaterializedIndex(),
+ new RandomDistributionInfo(1), CatalogTestUtil.testDbId1, tableId);
+ partition.setCachedVisibleVersion(2, 20);
+ table.addPartition(partition);
+ table.setCachedTableVersion(2);
+ masterEnv.getInternalCatalog().getDbOrMetaException(CatalogTestUtil.testDbId1).registerTable(table);
+ return partition;
+ }
+
+ private MockedStatic<VersionHelper> mockVersionHelper() {
+ MockedStatic<VersionHelper> versions = Mockito.mockStatic(VersionHelper.class);
+ // Batch reads pass an explicit retry limit; share the response stub with single-version reads.
+ versions.when(() -> VersionHelper.getVersionFromMeta(Mockito.any(), Mockito.anyInt()))
+ .thenAnswer(invocation -> VersionHelper.getVersionFromMeta(invocation.getArgument(0)));
+ return versions;
+ }
+
+ private CommitTxnResponse visibleRetry(List<Long> tableIds) {
+ return CommitTxnResponse.newBuilder()
+ .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK))
+ .setTxnInfo(buildTxnInfo(100).toBuilder().clearTableIds().addAllTableIds(tableIds)
+ .setStatus(Cloud.TxnStatusPB.TXN_STATUS_VISIBLE)).build();
+ }
+
+ private Cloud.GetVersionResponse partitionVersion(long version) {
+ return Cloud.GetVersionResponse.newBuilder().setVersion(version).addVersions(version)
+ .addVersionUpdateTimeMs(version * 10).build();
+ }
+
+ private OlapTable getCloudTable(CloudPartition partition) throws Exception {
+ return (OlapTable) masterEnv.getInternalCatalog().getDbOrMetaException(CatalogTestUtil.testDbId1)
+ .getTableOrMetaException(partition.getTableId());
+ }
+
+ private void useVersionCaches() {
+ ConnectContext context = new ConnectContext();
+ context.setSessionVariable(new SessionVariable());
+ context.getSessionVariable().cloudPartitionVersionCacheTtlMs = Long.MAX_VALUE;
+ context.getSessionVariable().cloudTableVersionCacheTtlMs = Long.MAX_VALUE;
+ context.setThreadLocalInfo();
+ }
+
private TxnInfoPB buildTxnInfo(long transactionId) {
return TxnInfoPB.newBuilder()
.setDbId(CatalogTestUtil.testDbId1)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 28/35: [improvement](lance) Expose query parallelism and complete row fetch profiles (branch-4.1) (#68690)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5320dddf5dc6d96b476f111f4484be83c30828a5
Author: Gabriel <li...@selectdb.com>
AuthorDate: Thu Oct 8 09:25:47 2026 +0800
[improvement](lance) Expose query parallelism and complete row fetch profiles (branch-4.1) (#68690)
### What problem does this PR solve?
Lance vector searches cannot configure the SDK's query parallelism from
Doris, and second-phase row-ID fetch profiles omit available read
counters. Materialization also nests two timers for the same `ExecTime`
counter, counting the synchronous fetch wait twice.
This PR targets `branch-4.1` and:
- Adds optional `vector_search("query_parallelism" = "4", ...)` support
through FE validation, Thrift, and the existing Lance-C API. Accepts
`-1`, `0`, and positive integers; omitting it preserves the SDK default.
EXPLAIN shows an explicit setting.
- Carries row-ID fetch call counts, returned row counts, and per-dataset
Foyer cache/origin logical read bytes through the fetch RPC into each
`RowIDFetcher` profile, preserving their units. Statistics are captured
after the reader closes.
- Removes the nested materialization execution timer; the operator
framework continues to count the fetch wait once.
The read-byte counters exclude metadata/index IO and block-alignment
amplification. The current take API does not expose physical request
counts or a separate decode timer; zero Foyer counters on bypassed paths
do not mean zero physical IO. These boundaries are documented in
`docs/lance-ann-profile.md`.
No third-party dependency or patch changes are required.
### Validation
- 43 focused FE tests passed, including option bounds, omitted defaults,
Thrift round trips, and EXPLAIN propagation.
- FE Checkstyle passed with zero violations.
- Clang-format 16 checks passed for every changed C++ source/header.
- C++ syntax checks passed for all changed translation units and both
affected BE test files using regenerated Thrift definitions.
- Groovy compilation passed for the changed regression suite.
- Added BE tests for parameter validation, result preservation,
row-fetch counters, RPC profile propagation, and avoiding duplicate
execution timing.
- Added external regression coverage comparing indexed results across
parallelism settings and rejecting invalid values. Full BE UT and
external regression execution remain for CI; local C++ checks were
syntax-only.
### Release note
Add configurable Lance vector query parallelism and row-ID fetch read
counters, and correct materialization execution-time accounting.
### Check List (For Author)
- Test
- [x] Regression test
- [x] Unit Test
- Behavior changed:
- [x] Yes. Explicit query parallelism is now accepted; existing defaults
are preserved. Materialization profiles no longer double-count fetch
execution time.
- Does this need documentation?
- [x] Yes. Updated `docs/lance-ann-profile.md` in this PR.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
be/src/exec/operator/materialization_opertor.cpp | 6 ++-
be/src/exec/rowid_fetcher.cpp | 21 ++++++++
be/src/exec/rowid_fetcher.h | 2 +
be/src/format_v2/table/lance_reader.cpp | 13 +++++
.../operator/materialization_shared_state_test.cpp | 49 ++++++++++++++++++
be/test/format_v2/table/lance_reader_test.cpp | 60 ++++++++++++++++++++++
docs/lance-ann-profile.md | 36 +++++++++++++
.../datasource/lance/source/LanceScanNode.java | 5 ++
.../VectorSearchTableValuedFunction.java | 11 +++-
.../datasource/lance/source/LanceScanNodeTest.java | 6 ++-
.../VectorSearchTableValuedFunctionTest.java | 32 ++++++++++++
gensrc/thrift/PlanNodes.thrift | 2 +
.../lance/test_lance_vector_search.groovy | 20 ++++++++
13 files changed, 259 insertions(+), 4 deletions(-)
diff --git a/be/src/exec/operator/materialization_opertor.cpp b/be/src/exec/operator/materialization_opertor.cpp
index 47f65cd8c5f..16c2150ebf3 100644
--- a/be/src/exec/operator/materialization_opertor.cpp
+++ b/be/src/exec/operator/materialization_opertor.cpp
@@ -481,6 +481,9 @@ void MaterializationSharedState::_update_profile_info(int64_t backend_id,
update_profile_info_key(RowIdStorageReader::LanceRowIdTakeReadTimeProfile, false);
update_profile_info_key(RowIdStorageReader::LanceArrowToDorisBlockTimeProfile, false);
update_profile_info_key(RowIdStorageReader::LanceRowIdFetchTotalTimeProfile, false);
+ for (const auto& [name, unit] : RowIdStorageReader::LanceFetchCountersProfile) {
+ update_profile_info_key(name, false);
+ }
}
Status MaterializationSharedState::create_muiltget_result(const Columns& columns, bool child_eos,
@@ -674,7 +677,8 @@ Status MaterializationOperator::pull(RuntimeState* state, Block* output_block, b
Status MaterializationOperator::push(RuntimeState* state, Block* in_block, bool eos) const {
auto& local_state = get_local_state(state);
- SCOPED_TIMER(local_state.exec_time_counter());
+ // StatefulOperatorX::get_block_impl already times push() with this counter.
+ // Nesting the same timer counts the synchronous row-fetch RPC wait twice.
if (!local_state._materialization_state.rpc_struct_inited) {
RETURN_IF_ERROR(local_state._materialization_state.init_multi_requests(
_materialization_node, state));
diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp
index 8c3b0023831..92f8de5c2e6 100644
--- a/be/src/exec/rowid_fetcher.cpp
+++ b/be/src/exec/rowid_fetcher.cpp
@@ -755,6 +755,12 @@ const std::string RowIdStorageReader::LanceRowIdTakeReadTimeProfile = "LanceRowI
const std::string RowIdStorageReader::LanceArrowToDorisBlockTimeProfile =
"LanceArrowToDorisBlockTime";
const std::string RowIdStorageReader::LanceRowIdFetchTotalTimeProfile = "LanceRowIdFetchTotalTime";
+const std::map<std::string, TUnit::type> RowIdStorageReader::LanceFetchCountersProfile = {
+ {"LanceRowIdFetchRows", TUnit::UNIT},
+ {"LanceRowIdFetchCalls", TUnit::UNIT},
+ {"LanceDataCacheBytesReadFromCache", TUnit::BYTES},
+ {"LanceDataCacheBytesReadFromRemote", TUnit::BYTES},
+};
const std::string RowIdStorageReader::TopNLazyMaterializationSecondPhaseLocalIOCount =
"TopNLazyMaterializationSecondPhaseLocalIOCount";
const std::string RowIdStorageReader::TopNLazyMaterializationSecondPhaseLocalIOBytes =
@@ -889,6 +895,13 @@ Status RowIdStorageReader::read_lance_rows_by_row_ids(
collect_lance_fetch_time(LanceRowIdTakeReadTimeProfile);
collect_lance_fetch_time(LanceArrowToDorisBlockTimeProfile);
collect_lance_fetch_time(LanceRowIdFetchTotalTimeProfile);
+ // close() publishes dataset-handle cache statistics; collect after it and preserve the
+ // units across the RPC instead of formatting byte/count values as nanoseconds.
+ for (const auto& [name, unit] : LanceFetchCountersProfile) {
+ if (const auto* counter = runtime_profile->get_counter(name); counter != nullptr) {
+ fetch_statistics->lance_fetch_counters.emplace(name, counter->value());
+ }
+ }
return Status::OK();
}
@@ -1203,6 +1216,7 @@ Status RowIdStorageReader::read_batch_external_row(
format_to(file_read_times_buffer, "[");
std::map<std::string, int64_t> lance_fetch_times_ns;
+ std::map<std::string, int64_t> lance_fetch_counters;
size_t idx = 0;
for (const auto& [_, scan_info] : scan_rows) {
format_to(file_read_lines_buffer, "{}, ", scan_info.first.size());
@@ -1213,6 +1227,9 @@ Status RowIdStorageReader::read_batch_external_row(
for (const auto& [time_name, time_value] : fetch_statistics[idx].lance_fetch_times_ns) {
lance_fetch_times_ns[time_name] += time_value;
}
+ for (const auto& [name, value] : fetch_statistics[idx].lance_fetch_counters) {
+ lance_fetch_counters[name] += value;
+ }
idx++;
}
@@ -1232,6 +1249,10 @@ Status RowIdStorageReader::read_batch_external_row(
fmt::to_string(file_read_bytes_buffer));
runtime_profile->add_info_string(FileScannerV2::FileReadTimeProfile,
fmt::to_string(file_read_times_buffer));
+ for (const auto& [name, value] : lance_fetch_counters) {
+ runtime_profile->add_info_string(
+ name, PrettyPrinter::print(value, LanceFetchCountersProfile.at(name)));
+ }
for (const auto& [time_name, time_value] : lance_fetch_times_ns) {
runtime_profile->add_info_string(time_name,
PrettyPrinter::print(time_value, TUnit::TIME_NS));
diff --git a/be/src/exec/rowid_fetcher.h b/be/src/exec/rowid_fetcher.h
index ab8019e8f32..fed8e5ccd52 100644
--- a/be/src/exec/rowid_fetcher.h
+++ b/be/src/exec/rowid_fetcher.h
@@ -108,6 +108,7 @@ public:
static const std::string LanceRowIdTakeReadTimeProfile;
static const std::string LanceArrowToDorisBlockTimeProfile;
static const std::string LanceRowIdFetchTotalTimeProfile;
+ static const std::map<std::string, TUnit::type> LanceFetchCountersProfile;
static const std::string TopNLazyMaterializationSecondPhaseLocalIOCount;
static const std::string TopNLazyMaterializationSecondPhaseLocalIOBytes;
static const std::string TopNLazyMaterializationSecondPhaseRemoteIOCount;
@@ -183,6 +184,7 @@ private:
int64_t init_reader_ms = 0;
int64_t get_block_ms = 0;
std::map<std::string, int64_t> lance_fetch_times_ns;
+ std::map<std::string, int64_t> lance_fetch_counters;
std::string file_read_bytes;
std::string file_read_times;
};
diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp
index 52cf45533e5..b9e5054a2a6 100644
--- a/be/src/format_v2/table/lance_reader.cpp
+++ b/be/src/format_v2/table/lance_reader.cpp
@@ -255,6 +255,10 @@ Status LanceTableReader::read_by_row_ids(const TFileRangeDesc& range,
_row_id_fetch_total_time = ADD_CHILD_TIMER_WITH_LEVEL(
_scanner_profile, "LanceRowIdFetchTotalTime", LANCE_READER_PROFILE, 1);
}
+ auto* fetch_calls = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceRowIdFetchCalls",
+ TUnit::UNIT, LANCE_READER_PROFILE, 1);
+ auto* fetch_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceRowIdFetchRows",
+ TUnit::UNIT, LANCE_READER_PROFILE, 1);
SCOPED_TIMER(_row_id_fetch_total_time);
// Phase-two row fetch does not execute FTS, so a reader created only for take_rows must not
@@ -271,6 +275,7 @@ Status LanceTableReader::read_by_row_ids(const TFileRangeDesc& range,
int32_t take_rows_status = 0;
{
SCOPED_TIMER(_row_id_take_read_time);
+ COUNTER_UPDATE(fetch_calls, 1);
take_rows_status = lance_dataset_take_rows(_dataset, row_ids.data(), row_ids.size(),
columns.data(), &stream);
}
@@ -313,6 +318,7 @@ Status LanceTableReader::read_by_row_ids(const TFileRangeDesc& range,
record_batch, block, _global_rowid_context, &rows));
}
fetched_rows += rows;
+ COUNTER_UPDATE(fetch_rows, rows);
}
if (fetched_rows != row_ids.size()) {
return Status::InternalError("Lance row-id fetch returned {} rows for {} requested row ids",
@@ -541,6 +547,9 @@ Status LanceTableReader::_validate_external_search_request() const {
if (_search_kind == SearchKind::VECTOR && request.__isset.vector_search_options) {
const auto& options = request.vector_search_options;
+ if (options.__isset.query_parallelism && options.query_parallelism < -1) {
+ return Status::InvalidArgument("Lance query_parallelism must be -1, 0, or positive");
+ }
if (options.__isset.nprobes && options.nprobes <= 0) {
return Status::InvalidArgument("Lance nprobes must be positive");
}
@@ -1076,6 +1085,10 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner,
}
if (request.__isset.vector_search_options) {
const auto& options = request.vector_search_options;
+ if (options.__isset.query_parallelism &&
+ lance_scanner_set_query_parallelism(scanner, options.query_parallelism) != 0) {
+ return lance_error("set Lance vector query parallelism");
+ }
if (options.__isset.nprobes &&
lance_scanner_set_nprobes(scanner, static_cast<uint32_t>(options.nprobes)) != 0) {
return lance_error("set Lance vector nprobes");
diff --git a/be/test/exec/operator/materialization_shared_state_test.cpp b/be/test/exec/operator/materialization_shared_state_test.cpp
index 95ec24d0879..b59ba2540ad 100644
--- a/be/test/exec/operator/materialization_shared_state_test.cpp
+++ b/be/test/exec/operator/materialization_shared_state_test.cpp
@@ -27,6 +27,7 @@
#include "exec/operator/materialization_opertor.h"
#include "exec/pipeline/dependency.h"
#include "runtime/runtime_profile.h"
+#include "testutil/mock/mock_runtime_state.h"
namespace doris {
@@ -43,6 +44,10 @@ void set_lance_fetch_profile(PMultiGetBlockV2* response_block, int64_t scale) {
profile.add_info_string("LanceRowIdTakeReadTime", std::to_string(2 * scale) + "ns");
profile.add_info_string("LanceArrowToDorisBlockTime", std::to_string(3 * scale) + "ns");
profile.add_info_string("LanceRowIdFetchTotalTime", std::to_string(4 * scale) + "ns");
+ profile.add_info_string("LanceRowIdFetchRows", std::to_string(5 * scale));
+ profile.add_info_string("LanceRowIdFetchCalls", "1");
+ profile.add_info_string("LanceDataCacheBytesReadFromCache", "64.00 KB");
+ profile.add_info_string("LanceDataCacheBytesReadFromRemote", "32.00 KB");
profile.add_info_string("ScannersRunningTime", "0ms");
profile.add_info_string("InitReaderAvgTime", "0ms");
profile.add_info_string("GetBlockAvgTime", "0ms");
@@ -54,6 +59,46 @@ void set_lance_fetch_profile(PMultiGetBlockV2* response_block, int64_t scale) {
} // namespace
+TEST(MaterializationOperatorTimingTest, PushDoesNotDuplicateFrameworkExecTimer) {
+ ObjectPool pool;
+ MockRuntimeState state;
+ // Even an empty-block timing test needs a real tuple: OperatorXBase builds
+ // a RowDescriptor from the plan and requires non-empty, resolvable tuple IDs.
+ TTupleDescriptor tuple;
+ tuple.__set_id(0);
+ TDescriptorTable thrift_desc;
+ thrift_desc.__set_tupleDescriptors({tuple});
+ DescriptorTbl* desc_tbl = nullptr;
+ ASSERT_TRUE(DescriptorTbl::create(&pool, thrift_desc, &desc_tbl).ok());
+ state.set_desc_tbl(desc_tbl);
+
+ TPlanNode node;
+ node.__set_node_id(0);
+ node.__set_node_type(TPlanNodeType::MATERIALIZATION_NODE);
+ node.__set_row_tuples({0});
+ node.__set_nullable_tuples({false});
+ MaterializationOperator op(&pool, node, 0, state.desc_tbl());
+ RuntimeProfile profile("materialization_timing");
+ auto local = MaterializationLocalState::create_unique(&state, &op);
+ LocalStateInfo info {.parent_profile = &profile,
+ .scan_ranges = {},
+ .shared_state = nullptr,
+ .shared_state_map = {},
+ .task_idx = 0};
+ ASSERT_TRUE(local->init(&state, info).ok());
+ local->_materialization_state.rpc_struct_inited = true;
+ auto* timer = local->exec_time_counter();
+ state.resize_op_id_to_local_state(-1);
+ state.emplace_local_state(0, std::move(local));
+ Block block;
+ const auto before = timer->value();
+ // The framework owns ExecTime. Direct push calls must not contribute a second sample.
+ for (int i = 0; i < 100; ++i) {
+ ASSERT_TRUE(op.push(&state, &block, false).ok());
+ }
+ EXPECT_EQ(before, timer->value());
+}
+
class MaterializationSharedStateTest : public testing::Test {
protected:
void SetUp() override {
@@ -316,6 +361,10 @@ TEST_F(MaterializationSharedStateTest, TestMergeMultiResponse) {
EXPECT_EQ("20ns, ", fmt::to_string(backend1_info.at("LanceRowIdTakeReadTime")));
EXPECT_EQ("30ns, ", fmt::to_string(backend1_info.at("LanceArrowToDorisBlockTime")));
EXPECT_EQ("40ns, ", fmt::to_string(backend1_info.at("LanceRowIdFetchTotalTime")));
+ EXPECT_EQ("50, ", fmt::to_string(backend1_info.at("LanceRowIdFetchRows")));
+ EXPECT_EQ("1, ", fmt::to_string(backend1_info.at("LanceRowIdFetchCalls")));
+ EXPECT_EQ("64.00 KB, ", fmt::to_string(backend1_info.at("LanceDataCacheBytesReadFromCache")));
+ EXPECT_EQ("32.00 KB, ", fmt::to_string(backend1_info.at("LanceDataCacheBytesReadFromRemote")));
const auto& backend2_info = _shared_state->backend_profile_info_string.at(_backend_id2);
EXPECT_EQ("1ns, ", fmt::to_string(backend2_info.at("LanceDatasetOpenTime")));
EXPECT_EQ("2ns, ", fmt::to_string(backend2_info.at("LanceRowIdTakeReadTime")));
diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp
index 93531688634..e952d8da483 100644
--- a/be/test/format_v2/table/lance_reader_test.cpp
+++ b/be/test/format_v2/table/lance_reader_test.cpp
@@ -1226,6 +1226,47 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorRejectsActualNullAndNonFiniteE
}
}
+TEST(LanceTableReaderVectorSearchTest, RejectsInvalidQueryParallelismBeforeOpeningDataset) {
+ TQueryGlobals globals;
+ RuntimeState state(globals);
+ RuntimeProfile profile("lance_invalid_query_parallelism");
+ auto params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 0);
+ params.lance_scan_params.external_search_request.vector_search_options.__set_query_parallelism(
+ -2);
+ const Columns columns {projected_column("_distance", TYPE_FLOAT, true)};
+ LanceTableReader reader;
+ const auto status = init_reader(&reader, columns, &state, &profile, ¶ms);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(std::string::npos, status.to_string().find("query_parallelism"));
+}
+
+TEST(LanceTableReaderVectorSearchTest, QueryParallelismPreservesResults) {
+ const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/all_types.lance";
+ LanceFixtureInfo fixture;
+ ASSERT_TRUE(get_fixture_info(uri, &fixture).ok());
+ const Columns columns {projected_column("row_id", TYPE_BIGINT, false),
+ projected_column("_distance", TYPE_FLOAT, true)};
+ TQueryGlobals globals;
+ RuntimeState state(globals);
+ for (const int parallelism : {-1, 0, 1, 4}) {
+ SCOPED_TRACE(parallelism);
+ auto params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 1);
+ params.lance_scan_params.external_search_request.vector_search_options
+ .__set_query_parallelism(parallelism);
+ RuntimeProfile profile("lance_query_parallelism");
+ LanceTableReader reader;
+ ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok());
+ ASSERT_TRUE(prepare_fixture(&reader, uri, fixture, fixture.fragment_ids).ok());
+ Block block;
+ add_output_columns(&block, columns);
+ const auto rows = read_vector_search_rows(&reader, &block);
+ ASSERT_EQ(2U, rows.size());
+ EXPECT_EQ(2, rows[0].first);
+ EXPECT_EQ(4, rows[1].first);
+ ASSERT_TRUE(reader.close().ok());
+ }
+}
+
TEST(LanceTableReaderVectorSearchTest, SearchesWholeSnapshotWithOffsetAndDistance) {
const std::filesystem::path dataset_uri =
"./be/test/format_v2/table/lance/data/all_types.lance";
@@ -1431,6 +1472,25 @@ TEST(LanceTableReaderVectorSearchTest, ReturnsStableGlobalRowIdsAndFetchesPayloa
EXPECT_EQ("extra", label_values.get_data_at(0).to_string());
EXPECT_EQ("unit-x", label_values.get_data_at(1).to_string());
EXPECT_EQ("extra", label_values.get_data_at(2).to_string());
+ ASSERT_NE(nullptr, fetch_profile.get_counter("LanceRowIdFetchRows"));
+ EXPECT_EQ(3, fetch_profile.get_counter("LanceRowIdFetchRows")->value());
+ ASSERT_NE(nullptr, fetch_profile.get_counter("LanceRowIdFetchCalls"));
+ EXPECT_EQ(1, fetch_profile.get_counter("LanceRowIdFetchCalls")->value());
+ Block second_block;
+ add_output_columns(&second_block, payload_columns);
+ ASSERT_TRUE(payload_reader
+ .read_by_row_ids(make_lance_range(dataset_uri, fixture.version,
+ fixture.fragment_ids),
+ fetch_row_ids, &second_block)
+ .ok());
+ EXPECT_EQ(6, fetch_profile.get_counter("LanceRowIdFetchRows")->value());
+ EXPECT_EQ(2, fetch_profile.get_counter("LanceRowIdFetchCalls")->value());
+ ASSERT_TRUE(payload_reader
+ .read_by_row_ids(make_lance_range(dataset_uri, fixture.version,
+ fixture.fragment_ids),
+ {}, &second_block)
+ .ok());
+ EXPECT_EQ(2, fetch_profile.get_counter("LanceRowIdFetchCalls")->value());
EXPECT_TRUE(payload_reader.close().ok());
}
diff --git a/docs/lance-ann-profile.md b/docs/lance-ann-profile.md
index 51fdc15d095..6a2a8bc92aa 100644
--- a/docs/lance-ann-profile.md
+++ b/docs/lance-ann-profile.md
@@ -65,3 +65,39 @@ queries, inspect partition load together with execution bytes, requests, and
partition cache misses. Use repeated queries and the operator-level elapsed
times to assess latency; cumulative parallel stage times alone are not a critical
path trace.
+
+## Query parallelism
+
+`vector_search` accepts an optional `"query_parallelism"` integer:
+
+- `0` (also the default when omitted): let Lance choose the parallelism.
+- `-1`: use the available Lance CPU parallelism.
+- Positive values: request that many concurrent partition searches, capped by
+ Lance's compute pool and execution-plan limits.
+
+For example, add `"query_parallelism" = "4"` alongside `"nprobes" = "64"`.
+This controls concurrency inside a Lance search, independently of Doris scan
+instances. Increasing it can increase intermediate candidates and memory usage;
+measure both single-query latency and concurrent throughput. EXPLAIN displays an
+explicit setting as `lanceQueryParallelism`.
+
+## Second-phase row-ID fetch
+
+Each `RowIDFetcher: BackendId:...` profile also reports:
+
+| Counter | Scope |
+| --- | --- |
+| `LanceRowIdFetchCalls` | Non-empty dataset `take_rows` calls. |
+| `LanceRowIdFetchRows` | Rows converted from successful returned batches, including duplicate requested row IDs. |
+| `LanceDataCacheBytesReadFromCache` | Logical data-file bytes served by Foyer for the dataset handles used by this fetch. |
+| `LanceDataCacheBytesReadFromRemote` | Logical data-file bytes served through the Foyer origin path for those handles. |
+
+Byte counts are collected after closing the reader and summed across dataset
+handles in the fetch RPC. They exclude block-alignment amplification, metadata,
+and index reads. With Foyer disabled or for paths outside its data-file wrapper,
+zero values do not imply zero physical IO. The current take API does not expose
+physical request counts or a separate decode timer; scanner-plan IO counters
+must not be substituted for them.
+
+`MATERIALIZATION_OPERATOR.ExecTime` includes its synchronous fetch wait once.
+`MaxRpcTime` is nested within that execution time, not an additional duration.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java
index a483ef52aa2..3f9505b87b9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java
@@ -354,6 +354,11 @@ public class LanceScanNode extends FileQueryScanNode {
.append(scanPlan.vectorIndexStatus).append("\n");
result.append(prefix).append("lanceVectorColumn=")
.append(vector.getColumn()).append("\n");
+ if (externalSearchRequest.isSetVectorSearchOptions()
+ && externalSearchRequest.getVectorSearchOptions().isSetQueryParallelism()) {
+ result.append(prefix).append("lanceQueryParallelism=")
+ .append(externalSearchRequest.getVectorSearchOptions().getQueryParallelism()).append("\n");
+ }
result.append(prefix).append("lanceMetric=")
.append(vector.isSetMetric()
? VectorSearchTableValuedFunction.metricName(vector.getMetric()) : "default")
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunction.java
index c394adbe099..93f7b2f4d56 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunction.java
@@ -60,13 +60,14 @@ public class VectorSearchTableValuedFunction extends LanceExternalSearchTableVal
private static final String ARROW_EXTENSION_NAME = "ARROW:extension:name";
private static final String QUERY_VECTOR = "query_vector";
private static final String METRIC = "metric";
+ private static final String QUERY_PARALLELISM = "query_parallelism";
private static final String NPROBES = "nprobes";
private static final String REFINE_FACTOR = "refine_factor";
private static final String EF = "ef";
private static final String USE_INDEX = "use_index";
private static final Set<String> PROPERTIES = ImmutableSet.of(
TABLE, COLUMN, QUERY_VECTOR, TOP_K, OFFSET, METRIC, FILTER,
- NPROBES, REFINE_FACTOR, EF, USE_INDEX);
+ NPROBES, REFINE_FACTOR, EF, USE_INDEX, QUERY_PARALLELISM);
public VectorSearchTableValuedFunction(Map<String, String> properties)
throws AnalysisException {
@@ -125,10 +126,16 @@ public class VectorSearchTableValuedFunction extends LanceExternalSearchTableVal
common, vectorFieldId, searchRequest, DISTANCE_COLUMN, "vector search");
}
- private static TVectorSearchOptions buildVectorSearchOptions(
+ @VisibleForTesting
+ static TVectorSearchOptions buildVectorSearchOptions(
Map<String, String> params, boolean useIndex) throws AnalysisException {
TVectorSearchOptions options = new TVectorSearchOptions();
boolean configured = false;
+ if (params.containsKey(QUERY_PARALLELISM)) {
+ options.setQueryParallelism((int) parseLong(
+ params.get(QUERY_PARALLELISM), QUERY_PARALLELISM, -1, Integer.MAX_VALUE));
+ configured = true;
+ }
if (params.containsKey(NPROBES)) {
options.setNprobes(parsePositiveInt(params.get(NPROBES), NPROBES));
configured = true;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java
index 6131a2fcb2c..b68ca1f437c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java
@@ -563,10 +563,14 @@ public class LanceScanNodeTest {
new LanceIndexSegmentInfo(secondSegment, "vector_idx",
Collections.singletonList(9), Arrays.asList(3L, 4L),
IndexType.VECTOR, "L2")));
- LanceScanNode node = newSearchNode(metadata, vectorSearchRequest(5, 0));
+ TExternalSearchRequest request = vectorSearchRequest(5, 0);
+ request.setVectorSearchOptions(new TVectorSearchOptions().setQueryParallelism(4));
+ LanceScanNode node = newSearchNode(metadata, request);
List<Split> splits = node.getSplits(3);
+ Assert.assertTrue(node.getNodeExplainString("", TExplainLevel.NORMAL)
+ .contains("lanceQueryParallelism=4"));
Assert.assertTrue(node.getNodeExplainString("", TExplainLevel.NORMAL)
.contains("lanceVectorIndexStatus=USED"));
diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java
index 1fe7caefa33..c8c3da2c6c0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java
@@ -22,18 +22,50 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.datasource.lance.metadata.LanceTableAccess;
import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
+import org.apache.doris.thrift.TVectorSearchOptions;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TSerializer;
import org.junit.Assert;
import org.junit.Test;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Map;
public class VectorSearchTableValuedFunctionTest {
+ private TVectorSearchOptions parseOptions(Map<String, String> params) throws Exception {
+ return VectorSearchTableValuedFunction.buildVectorSearchOptions(params, true);
+ }
+
+ @Test
+ public void testQueryParallelismOptions() throws Exception {
+ Assert.assertNull(parseOptions(Collections.emptyMap()));
+ Assert.assertFalse(parseOptions(Collections.singletonMap("nprobes", "4")).isSetQueryParallelism());
+ for (String value : new String[] {"-1", "0", "1", "4", "2147483647"}) {
+ TVectorSearchOptions options = parseOptions(Collections.singletonMap("query_parallelism", value));
+ Assert.assertNotNull("query_parallelism must be serialized", options);
+ TVectorSearchOptions decoded = new TVectorSearchOptions();
+ new TDeserializer().deserialize(decoded, new TSerializer().serialize(options));
+ Assert.assertTrue(decoded.isSetQueryParallelism());
+ Assert.assertEquals(Integer.parseInt(value), decoded.getQueryParallelism());
+ Assert.assertFalse(decoded.isSetNprobes());
+ }
+ }
+
+ @Test
+ public void testRejectInvalidQueryParallelism() {
+ for (String value : new String[] {"-2", "2147483648", "1.5", "abc", ""}) {
+ AnalysisException error = Assert.assertThrows(AnalysisException.class,
+ () -> parseOptions(Collections.singletonMap("query_parallelism", value)));
+ Assert.assertTrue(error.getMessage(), error.getMessage().contains("query_parallelism"));
+ }
+ }
+
@Test
public void testParseQuotedMultiLevelNamespace() throws AnalysisException {
TableName tableName = VectorSearchTableValuedFunction.parseTableName(
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 3e626e6f0ad..6e4fa8fd77b 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -552,6 +552,8 @@ struct TVectorSearchOptions {
2: optional i32 refine_factor
3: optional i32 ef
4: optional bool use_index
+ // Lance: -1 uses available CPU parallelism, 0 selects automatically, positive values cap it.
+ 5: optional i32 query_parallelism
}
// The active union field identifies the logical search kind. A future hybrid field can contain both
diff --git a/regression-test/suites/external_table_p0/lance/test_lance_vector_search.groovy b/regression-test/suites/external_table_p0/lance/test_lance_vector_search.groovy
index 7215db90265..d90acafef7b 100644
--- a/regression-test/suites/external_table_p0/lance/test_lance_vector_search.groovy
+++ b/regression-test/suites/external_table_p0/lance/test_lance_vector_search.groovy
@@ -148,6 +148,26 @@ suite("test_lance_vector_search", "p0,external") {
ORDER BY _distance, row_id
"""
+ def baselineRows = sql "SELECT row_id, label, _distance FROM ${indexedTopFive} ORDER BY _distance, row_id"
+ [-1, 0, 1, 2, 4].each { parallelism ->
+ String parallelSearch = indexedTopFive.substring(0, indexedTopFive.length() - 1) +
+ ", \"query_parallelism\"=\"${parallelism}\")"
+ explain {
+ sql("SELECT row_id, label, _distance FROM ${parallelSearch}")
+ contains "lanceQueryParallelism=${parallelism}"
+ }
+ assertEquals(baselineRows,
+ sql("SELECT row_id, label, _distance FROM ${parallelSearch} ORDER BY _distance, row_id"))
+ }
+ ["-2", "2147483648", "1.5", "invalid"].each { invalid ->
+ String invalidSearch = indexedTopFive.substring(0, indexedTopFive.length() - 1) +
+ ", \"query_parallelism\"=\"${invalid}\")"
+ test {
+ sql "SELECT row_id FROM ${invalidSearch}"
+ exception "query_parallelism"
+ }
+ }
+
// Disable the index explicitly for the exact flat baseline.
qt_flat_l2_topk """
SELECT row_id, label, _distance
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 23/35: branch-4.1: [fix](be) Restore BOOLEAN support in linear_histogram #68478 (#68676)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d30e392db32b802b577bfdb6238f7eaf18bad6b6
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Wed Sep 30 21:58:54 2026 +0800
branch-4.1: [fix](be) Restore BOOLEAN support in linear_histogram #68478 (#68676)
Cherry-picked from #68478
Co-authored-by: HappenLee <ha...@selectdb.com>
---
.../aggregate_function_linear_histogram.cpp | 7 +--
.../exprs/aggregate/agg_linear_histogram_test.cpp | 31 ++++++++++-
.../test_linear_histogram_boolean.out | 17 ++++++
.../test_linear_histogram_boolean.groovy | 63 ++++++++++++++++++++++
4 files changed, 114 insertions(+), 4 deletions(-)
diff --git a/be/src/exprs/aggregate/aggregate_function_linear_histogram.cpp b/be/src/exprs/aggregate/aggregate_function_linear_histogram.cpp
index 3fcf8a9e288..da004bc34cb 100644
--- a/be/src/exprs/aggregate/aggregate_function_linear_histogram.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_linear_histogram.cpp
@@ -35,9 +35,10 @@ AggregateFunctionPtr create_aggregate_function_linear_histogram(const std::strin
const DataTypePtr& result_type,
const bool result_is_nullable,
const AggregateFunctionAttr& attr) {
- using creator = creator_with_type_list<TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT,
- TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE, TYPE_DECIMAL32,
- TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256>;
+ using creator =
+ creator_with_type_list<TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT,
+ TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE, TYPE_DECIMAL32,
+ TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256>;
bool has_offset = (argument_types.size() == 3);
if (has_offset) {
return creator::create<HistogramWithInputParam, AggregateFunctionLinearHistogramData>(
diff --git a/be/test/exprs/aggregate/agg_linear_histogram_test.cpp b/be/test/exprs/aggregate/agg_linear_histogram_test.cpp
index 456868b0b52..5064150d660 100644
--- a/be/test/exprs/aggregate/agg_linear_histogram_test.cpp
+++ b/be/test/exprs/aggregate/agg_linear_histogram_test.cpp
@@ -206,7 +206,7 @@ public:
AggregateFunctionSimpleFactory factory = AggregateFunctionSimpleFactory::instance();
auto agg_function = factory.get("linear_histogram", data_types, nullptr, false, -1,
{.column_names = {""}});
- EXPECT_NE(agg_function, nullptr);
+ ASSERT_NE(agg_function, nullptr);
std::unique_ptr<char[]> memory(new char[agg_function->size_of_data()]);
AggregateDataPtr place = memory.get();
@@ -256,6 +256,28 @@ public:
EXPECT_EQ(result2, expect_empty_result);
}
+ if (input_rows == 2 && interval == 1 && offset == 0) {
+ EXPECT_EQ(result1,
+ "{\"num_buckets\":2,\"buckets\":["
+ "{\"lower\":0.0,\"upper\":1.0,\"count\":2,\"acc_count\":2},"
+ "{\"lower\":1.0,\"upper\":2.0,\"count\":2,\"acc_count\":4}]}");
+ EXPECT_EQ(result2,
+ "{\"num_buckets\":2,\"buckets\":["
+ "{\"lower\":0.0,\"upper\":1.0,\"count\":1,\"acc_count\":1},"
+ "{\"lower\":1.0,\"upper\":2.0,\"count\":1,\"acc_count\":2}]}");
+ }
+
+ if (input_rows == 2 && interval == 1 && offset == 0.5) {
+ EXPECT_EQ(result1,
+ "{\"num_buckets\":2,\"buckets\":["
+ "{\"lower\":-0.5,\"upper\":0.5,\"count\":2,\"acc_count\":2},"
+ "{\"lower\":0.5,\"upper\":1.5,\"count\":2,\"acc_count\":4}]}");
+ EXPECT_EQ(result2,
+ "{\"num_buckets\":2,\"buckets\":["
+ "{\"lower\":-0.5,\"upper\":0.5,\"count\":1,\"acc_count\":1},"
+ "{\"lower\":0.5,\"upper\":1.5,\"count\":1,\"acc_count\":2}]}");
+ }
+
// test with data
if (input_rows == 100 && interval == 10 && offset == 0) {
std::string expect_result1 = s1;
@@ -290,6 +312,13 @@ private:
Arena _agg_arena_pool;
};
+TEST_F(AggLinearHistogramTest, test_boolean) {
+ test_agg_linear_histogram<DataTypeBool>(0, 1, 0);
+ test_agg_linear_histogram<DataTypeBool>(0, 1, 0.5);
+ test_agg_linear_histogram<DataTypeBool>(2, 1, 0);
+ test_agg_linear_histogram<DataTypeBool>(2, 1, 0.5);
+}
+
TEST_F(AggLinearHistogramTest, test_empty) {
test_agg_linear_histogram<DataTypeInt8>(0, 10, 0);
test_agg_linear_histogram<DataTypeInt16>(0, 10, 0);
diff --git a/regression-test/data/query_p0/sql_functions/aggregate_functions/test_linear_histogram_boolean.out b/regression-test/data/query_p0/sql_functions/aggregate_functions/test_linear_histogram_boolean.out
new file mode 100644
index 00000000000..b005daccc5e
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/aggregate_functions/test_linear_histogram_boolean.out
@@ -0,0 +1,17 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !nullable --
+{"num_buckets":2,"buckets":[{"lower":0.0,"upper":1.0,"count":1,"acc_count":1},{"lower":1.0,"upper":2.0,"count":2,"acc_count":3}]} {"num_buckets":2,"buckets":[{"lower":-0.5,"upper":0.5,"count":1,"acc_count":1},{"lower":0.5,"upper":1.5,"count":2,"acc_count":3}]}
+
+-- !not_nullable --
+{"num_buckets":2,"buckets":[{"lower":0.0,"upper":1.0,"count":2,"acc_count":2},{"lower":1.0,"upper":2.0,"count":2,"acc_count":4}]} {"num_buckets":2,"buckets":[{"lower":-0.5,"upper":0.5,"count":2,"acc_count":2},{"lower":0.5,"upper":1.5,"count":2,"acc_count":4}]}
+
+-- !grouped --
+0 {"num_buckets":1,"buckets":[{"lower":0.0,"upper":1.0,"count":1,"acc_count":1}]} {"num_buckets":1,"buckets":[{"lower":-0.5,"upper":0.5,"count":1,"acc_count":1}]}
+1 {"num_buckets":1,"buckets":[{"lower":1.0,"upper":2.0,"count":2,"acc_count":2}]} {"num_buckets":1,"buckets":[{"lower":0.5,"upper":1.5,"count":2,"acc_count":2}]}
+
+-- !all_null --
+{"num_buckets":0,"buckets":[]} {"num_buckets":0,"buckets":[]}
+
+-- !empty --
+{"num_buckets":0,"buckets":[]} {"num_buckets":0,"buckets":[]}
+
diff --git a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_linear_histogram_boolean.groovy b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_linear_histogram_boolean.groovy
new file mode 100644
index 00000000000..99a936b33e4
--- /dev/null
+++ b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_linear_histogram_boolean.groovy
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_linear_histogram_boolean") {
+ sql "DROP TABLE IF EXISTS linear_histogram_boolean"
+ sql """
+ CREATE TABLE linear_histogram_boolean (
+ id INT NOT NULL,
+ b BOOLEAN NULL,
+ non_null_b BOOLEAN NOT NULL
+ )
+ DISTRIBUTED BY HASH(id) BUCKETS 3
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO linear_histogram_boolean VALUES
+ (1, TRUE, TRUE), (2, FALSE, FALSE), (3, TRUE, TRUE), (4, NULL, FALSE)
+ """
+
+ order_qt_nullable """
+ SELECT linear_histogram(b, 1.0), linear_histogram(b, 1.0, 0.5)
+ FROM linear_histogram_boolean
+ """
+ order_qt_not_nullable """
+ SELECT linear_histogram(non_null_b, 1.0), linear_histogram(non_null_b, 1.0, 0.5)
+ FROM linear_histogram_boolean
+ """
+ order_qt_grouped """
+ SELECT id % 2, linear_histogram(b, 1.0), linear_histogram(b, 1.0, 0.5)
+ FROM linear_histogram_boolean GROUP BY id % 2
+ """
+ order_qt_all_null """
+ SELECT linear_histogram(b, 1.0), linear_histogram(b, 1.0, 0.5)
+ FROM linear_histogram_boolean WHERE b IS NULL
+ """
+ order_qt_empty """
+ SELECT linear_histogram(b, 1.0), linear_histogram(b, 1.0, 0.5)
+ FROM linear_histogram_boolean WHERE id < 0
+ """
+
+ test {
+ sql "SELECT linear_histogram(non_null_b, 0.0) FROM linear_histogram_boolean"
+ exception "interval should be larger than 0"
+ }
+ test {
+ sql "SELECT linear_histogram(non_null_b, 1.0, 1.0) FROM linear_histogram_boolean"
+ exception "offset should be in [0, interval)"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 13/35: [fix](arrow) Reject out-of-range Flight timestamps (#68604)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 20a412ebe479b4e3e60558c9a279f8ed54dac193
Author: Gabriel <li...@selectdb.com>
AuthorDate: Tue Sep 29 21:57:29 2026 +0800
[fix](arrow) Reject out-of-range Flight timestamps (#68604)
### What problem does this PR solve?
Flight SQL can publish out-of-range timestamps from legacy INT96 date
materialization. For example, a non-null zero DATETIME becomes
`timestamp[us]` with value `-62169984000000000`: Arrow validation
succeeds, but PyArrow scalar conversion raises `OverflowError`.
Validate timestamps in `ArrowFlightArrowBlockConvertor` before
publishing each batch. Check the actual timestamp unit and both
UTC/local calendar bounds for zoned values. Recursively check arrays,
map keys/values, and structs while skipping values masked by a NULL
parent. Reject unsupported timestamps with an error identifying the
field path and block row. Other Arrow consumers and Parquet scan
compatibility remain unchanged.
Related PR: #68596 adds separate UTF-8 validation. This PR is
independent of that change; both checks must be retained when
integrating the shared Flight conversion entry point.
### Testing
- Added seven BE tests covering zero dates, years 0/10000,
seconds/milliseconds/microseconds, valid calendar boundaries, pre-epoch
fractions, timezone offsets, slices/subsequent batches, nested
timestamps, NULL parents, and other consumers.
- Before the fix: four new test groups failed because malformed
timestamps were published; three valid/NULL groups passed.
- After the fix: all 36 tests in
`ArrowFlightTimestampTest.*:DataTypeSerDeArrowTest.*` passed under ASAN.
Recompiled the changed converter and new tests against existing ASAN BE
libraries, with C++ access control enabled. Test dates are constructed
through the public unchecked setter so invalid calendar values reach the
validator.
- Added a Flight JDBC regression suite and a separate HDFS INT96 suite:
four existing group4 fixtures must be read successfully with microsecond
truncation at the upper calendar boundary; the zero-year fixture must
return a range error. Both suites preserve the configured Flight
credentials and avoid the TLS-dependent `connect` shortcut. The HDFS
suite discovers the live FE Flight endpoint through MySQL because
external clusters can override the configured port. The HDFS suite
requires `enableHiveTest` and the external test environment.
- Fixed the nested NULL assertion that caused the P0/Cloud P0 Groovy
`getAt()` failure by consuming the JDBC array directly.
- Clang-format 16 passed for all affected C++ files. Groovy 4.0.19
executed both scripts against mock JDBC connections in 12 checks
covering TLS on/off, successful assertions, missing expected errors, and
unrelated errors. These are script-level checks; the live external
validation below additionally exercises real SQL and HDFS data.
### Live external regression validation
- Used the CI-built FE/BE binaries for `8d1158d3f5` in an isolated local
cluster and rebuilt the regression framework from this branch. This
follow-up changes only test scripts and expected results.
- Before the update: all four affected suites failed locally,
reproducing the stale Flight port and zero-year timestamp failures.
- After the update: all four suites passed, with zero failures and zero
skips: `test_flight_int96_timestamp_range`,
`test_remote_doris_all_types_select`, `test_remote_doris_statistics`,
and `test_remote_doris_table_stats`.
- The configured Flight port intentionally remained incorrect. The INT96
suite discovered the running FE endpoint, consumed all four valid HDFS
fixtures, and asserted the expected error for the invalid fixture.
- Remote Doris success fixtures now use year 0001 as the minimum
timestamp, with matching golden results. A separate assertion preserves
coverage of zero-year rejection through a Remote Doris catalog.
### Release note
Flight SQL now returns a server-side error for timestamps outside the
supported 0001–9999 calendar range instead of publishing values that
fail client conversion. Valid timestamps and NULLs retain their existing
values.
### Check List (For Author)
- Test
- [x] Regression test
- [x] Unit Test
- Behavior changed:
- [x] Yes: reject out-of-range Flight timestamps before publishing a
batch.
- Does this need documentation?
- [x] No.
---
be/src/format/arrow/arrow_block_convertor.cpp | 137 +++++++++++++
.../format/arrow/arrow_flight_timestamp_test.cpp | 220 +++++++++++++++++++++
.../test_remote_doris_all_types_select.out | 4 +-
.../remote_doris/test_remote_doris_statistics.out | 2 +-
.../test_flight_int96_timestamp_range.groovy | 83 ++++++++
.../test_flight_timestamp_range.groovy | 81 ++++++++
.../test_remote_doris_all_types_select.groovy | 15 +-
.../test_remote_doris_statistics.groovy | 3 +-
.../test_remote_doris_table_stats.groovy | 3 +-
9 files changed, 541 insertions(+), 7 deletions(-)
diff --git a/be/src/format/arrow/arrow_block_convertor.cpp b/be/src/format/arrow/arrow_block_convertor.cpp
index b62ca98cde9..6b2ed5a71e8 100644
--- a/be/src/format/arrow/arrow_block_convertor.cpp
+++ b/be/src/format/arrow/arrow_block_convertor.cpp
@@ -17,6 +17,8 @@
#include "format/arrow/arrow_block_convertor.h"
+#include <arrow/array/array_nested.h>
+#include <arrow/array/array_primitive.h>
#include <arrow/array/builder_base.h>
#include <arrow/array/builder_binary.h>
#include <arrow/array/builder_decimal.h>
@@ -34,6 +36,7 @@
#include <cctz/time_zone.h>
#include <glog/logging.h>
+#include <algorithm>
#include <array>
#include <cstring>
#include <ctime>
@@ -63,6 +66,134 @@ namespace doris {
namespace {
+class FlightTimestampValidator {
+public:
+ Status init(std::shared_ptr<arrow::Array> array, std::string path,
+ const cctz::time_zone& timezone) {
+ _array = std::move(array);
+ _path = std::move(path);
+ switch (_array->type_id()) {
+ case arrow::Type::TIMESTAMP: {
+ const auto& type = static_cast<const arrow::TimestampType&>(*_array->type());
+ int64_t units_per_second = 1;
+ switch (type.unit()) {
+ case arrow::TimeUnit::SECOND:
+ units_per_second = 1;
+ break;
+ case arrow::TimeUnit::MILLI:
+ units_per_second = 1000;
+ break;
+ case arrow::TimeUnit::MICRO:
+ units_per_second = 1000000;
+ break;
+ case arrow::TimeUnit::NANO:
+ units_per_second = 1000000000;
+ break;
+ }
+ int64_t min_seconds = -62135596800LL;
+ int64_t end_seconds = 253402300800LL;
+ if (!type.timezone().empty()) {
+ // The ordinary Arrow writer has already checked the schema/timezone binding.
+ const auto local_start = cctz::convert(cctz::civil_second(1, 1, 1), timezone);
+ const auto local_end = cctz::convert(cctz::civil_second(10000, 1, 1), timezone);
+ // Python first constructs UTC and then applies the Arrow timezone. Both calendar
+ // representations must fit, including offsets at the first and last supported day.
+ min_seconds =
+ std::max<int64_t>(min_seconds, local_start.time_since_epoch().count());
+ end_seconds = std::min<int64_t>(end_seconds, local_end.time_since_epoch().count());
+ }
+ // Nanosecond bounds do not fit int64_t, even though every encoded value does.
+ _min = static_cast<__int128>(min_seconds) * units_per_second;
+ _end = static_cast<__int128>(end_seconds) * units_per_second;
+ break;
+ }
+ case arrow::Type::LIST: {
+ const auto& list = static_cast<const arrow::ListArray&>(*_array);
+ RETURN_IF_ERROR(add_child(list.values(), _path + "[]", timezone));
+ break;
+ }
+ case arrow::Type::MAP: {
+ const auto& map = static_cast<const arrow::MapArray&>(*_array);
+ RETURN_IF_ERROR(add_child(map.keys(), _path + ".key", timezone));
+ RETURN_IF_ERROR(add_child(map.items(), _path + ".value", timezone));
+ break;
+ }
+ case arrow::Type::STRUCT: {
+ const auto& structure = static_cast<const arrow::StructArray&>(*_array);
+ for (int i = 0; i < structure.num_fields(); ++i) {
+ RETURN_IF_ERROR(add_child(structure.field(i),
+ _path + "." + _array->type()->field(i)->name(),
+ timezone));
+ }
+ break;
+ }
+ default:
+ break;
+ }
+ return Status::OK();
+ }
+
+ Status validate(int64_t start, int64_t end, size_t block_start, int64_t parent_row = -1) const {
+ if (!has_timestamps()) {
+ return Status::OK();
+ }
+ for (int64_t row = start; row < end; ++row) {
+ // Children of a NULL struct/list/map are not observable, even if their physical
+ // buffers contain default zero dates. Never validate those masked values.
+ if (_array->IsNull(row)) {
+ continue;
+ }
+ const int64_t output_row = parent_row < 0 ? row : parent_row;
+ if (_array->type_id() == arrow::Type::TIMESTAMP) {
+ const int64_t value = static_cast<const arrow::TimestampArray&>(*_array).Value(row);
+ if (value < _min || value >= _end) {
+ return Status::InvalidArgument(
+ "Arrow Flight timestamp in column '{}' at row {} is outside the "
+ "supported 0001-9999 range (type: {})",
+ _path, block_start + output_row + 1, _array->type()->ToString());
+ }
+ continue;
+ }
+ int64_t child_start = row;
+ int64_t child_end = row + 1;
+ if (_array->type_id() == arrow::Type::LIST) {
+ const auto& list = static_cast<const arrow::ListArray&>(*_array);
+ child_start = list.value_offset(row);
+ child_end = child_start + list.value_length(row);
+ } else if (_array->type_id() == arrow::Type::MAP) {
+ const auto& map = static_cast<const arrow::MapArray&>(*_array);
+ child_start = map.value_offset(row);
+ child_end = child_start + map.value_length(row);
+ }
+ for (const auto& child : _children) {
+ RETURN_IF_ERROR(child.validate(child_start, child_end, block_start, output_row));
+ }
+ }
+ return Status::OK();
+ }
+
+private:
+ bool has_timestamps() const {
+ return _array->type_id() == arrow::Type::TIMESTAMP || !_children.empty();
+ }
+
+ Status add_child(std::shared_ptr<arrow::Array> array, std::string path,
+ const cctz::time_zone& timezone) {
+ FlightTimestampValidator child;
+ RETURN_IF_ERROR(child.init(std::move(array), std::move(path), timezone));
+ if (child.has_timestamps()) {
+ _children.push_back(std::move(child));
+ }
+ return Status::OK();
+ }
+
+ std::shared_ptr<arrow::Array> _array;
+ std::string _path;
+ std::vector<FlightTimestampValidator> _children;
+ __int128 _min = 0;
+ __int128 _end = 0;
+};
+
int hex_value(char c) {
if (c >= '0' && c <= '9') {
return c - '0';
@@ -355,6 +486,12 @@ Status ArrowFlightArrowBlockConvertor::convert_to_arrow(const Block& block, arro
i + 1, batch->schema()->field(i)->name(),
status.ToString());
}
+ // Arrow accepts the full int64 timestamp domain; ValidateFull cannot enforce the calendar
+ // range required by Flight clients. Check the encoded values before publishing the batch.
+ FlightTimestampValidator validator;
+ RETURN_IF_ERROR(
+ validator.init(batch->column(i), batch->schema()->field(i)->name(), _timezone));
+ RETURN_IF_ERROR(validator.validate(0, batch->num_rows(), start_row));
}
*result = std::move(batch);
return Status::OK();
diff --git a/be/test/format/arrow/arrow_flight_timestamp_test.cpp b/be/test/format/arrow/arrow_flight_timestamp_test.cpp
new file mode 100644
index 00000000000..ebcafc04a91
--- /dev/null
+++ b/be/test/format/arrow/arrow_flight_timestamp_test.cpp
@@ -0,0 +1,220 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/api.h>
+#include <gtest/gtest.h>
+
+#include "core/column/column_nullable.h"
+#include "core/column/column_vector.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_date_or_datetime_v2.h"
+#include "core/data_type/data_type_map.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_struct.h"
+#include "format/arrow/arrow_block_convertor.h"
+#include "util/timezone_utils.h"
+
+namespace doris {
+namespace {
+using DateTime = DateV2Value<DateTimeV2ValueType>;
+
+DateTime make_datetime(uint16_t year, uint8_t month, uint8_t day, uint8_t hour, uint8_t minute,
+ uint16_t second, uint32_t microsecond) {
+ DateTime value;
+ // Invalid calendar values must reach the converter to exercise its range validation.
+ value.unchecked_set_time(year, month, day, hour, minute, second, microsecond);
+ return value;
+}
+
+Block timestamp_block(int scale, const std::vector<DateTime>& values) {
+ auto column = ColumnDateTimeV2::create();
+ for (const auto& value : values) {
+ column->insert_value(value);
+ }
+ return Block {{std::move(column), std::make_shared<DataTypeDateTimeV2>(scale), "event_time"}};
+}
+
+class ArrowFlightTimestampTest : public testing::Test {
+protected:
+ static void SetUpTestSuite() { TimezoneUtils::load_timezones_to_cache(); }
+};
+
+TEST_F(ArrowFlightTimestampTest, RejectsOutOfRangeInEveryUnitWithoutPublishingBatch) {
+ for (int scale : {0, 3, 6}) {
+ for (const auto& value : {DateTime {}, make_datetime(0, 12, 31, 0, 0, 0, 0),
+ make_datetime(10000, 1, 1, 0, 0, 0, 0)}) {
+ SCOPED_TRACE(scale);
+ auto block = timestamp_block(scale, {value});
+ ArrowFlightArrowBlockConvertor flight(block, "UTC", cctz::utc_time_zone(), true);
+ ASSERT_TRUE(flight.init().ok());
+ const ArrowBlockConvertor& converter = flight;
+ std::shared_ptr<arrow::RecordBatch> batch;
+ const auto status =
+ converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()) << status;
+ EXPECT_NE(std::string::npos, status.to_string().find("event_time"));
+ EXPECT_NE(std::string::npos, status.to_string().find("row 1"));
+ EXPECT_NE(std::string::npos, status.to_string().find("0001-9999"));
+ EXPECT_EQ(nullptr, batch);
+
+ // Other Arrow consumers retain their existing date semantics.
+ DorisArrowBlockConvertor ordinary(block, "UTC", cctz::utc_time_zone(), true);
+ ASSERT_TRUE(ordinary.init().ok());
+ ASSERT_TRUE(
+ ordinary.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
+ EXPECT_TRUE(batch->ValidateFull().ok());
+ }
+ }
+}
+
+TEST_F(ArrowFlightTimestampTest, PreservesCalendarBoundariesAndPreEpochFractions) {
+ for (int scale : {0, 3, 6}) {
+ const int64_t factor = scale == 0 ? 1 : scale == 3 ? 1000 : 1000000;
+ const uint32_t fraction = scale == 0 ? 0 : scale == 3 ? 999000 : 999999;
+ auto block = timestamp_block(scale, {make_datetime(1, 1, 1, 0, 0, 0, 0),
+ make_datetime(9999, 12, 31, 23, 59, 59, fraction),
+ make_datetime(1969, 12, 31, 23, 59, 59, fraction)});
+ ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone(), true);
+ ASSERT_TRUE(converter.init().ok());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
+ const auto& values = static_cast<const arrow::TimestampArray&>(*batch->column(0));
+ EXPECT_EQ(-62135596800LL * factor, values.Value(0));
+ EXPECT_EQ(253402300800LL * factor - 1, values.Value(1));
+ EXPECT_EQ(-1, values.Value(2));
+ }
+}
+
+TEST_F(ArrowFlightTimestampTest, ChecksSlicesAndSubsequentBatches) {
+ auto block = timestamp_block(6, {make_datetime(2024, 1, 1, 0, 0, 0, 0), DateTime {}});
+ ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone(), true);
+ ASSERT_TRUE(converter.init().ok());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch, 0, 1).ok());
+ auto previous = batch;
+ const auto status =
+ converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch, 1, 2);
+ EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code());
+ EXPECT_NE(std::string::npos, status.to_string().find("row 2"));
+ EXPECT_EQ(previous, batch);
+}
+
+TEST_F(ArrowFlightTimestampTest, RejectsNestedTimestampValuesAndMapKeys) {
+ auto datetime = make_nullable(std::make_shared<DataTypeDateTimeV2>(6));
+ const auto invalid = Field::create_field<TYPE_DATETIMEV2>(DateTime {});
+ const auto valid = Field::create_field<TYPE_DATETIMEV2>(make_datetime(2024, 1, 1, 0, 0, 0, 0));
+ DataTypes types {std::make_shared<DataTypeArray>(datetime),
+ std::make_shared<DataTypeStruct>(DataTypes {datetime}, Strings {"child"}),
+ std::make_shared<DataTypeMap>(datetime, datetime),
+ std::make_shared<DataTypeMap>(datetime, datetime),
+ std::make_shared<DataTypeArray>(std::make_shared<DataTypeStruct>(
+ DataTypes {datetime}, Strings {"child"}))};
+ FieldVector fields {
+ Field::create_field<TYPE_ARRAY>(Array {valid, invalid}),
+ Field::create_field<TYPE_STRUCT>(Struct {invalid}),
+ Field::create_field<TYPE_MAP>(Map {Field::create_field<TYPE_ARRAY>(Array {valid}),
+ Field::create_field<TYPE_ARRAY>(Array {invalid})}),
+ Field::create_field<TYPE_MAP>(Map {Field::create_field<TYPE_ARRAY>(Array {invalid}),
+ Field::create_field<TYPE_ARRAY>(Array {valid})}),
+ Field::create_field<TYPE_ARRAY>(
+ Array {Field::create_field<TYPE_STRUCT>(Struct {invalid})})};
+ for (size_t i = 0; i < types.size(); ++i) {
+ SCOPED_TRACE(types[i]->get_name());
+ auto column = types[i]->create_column();
+ column->insert_default();
+ column->insert(fields[i]);
+ Block block {{std::move(column), types[i], "nested"}};
+ ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone(), true);
+ ASSERT_TRUE(converter.init().ok());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ const auto status =
+ converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch, 1, 2);
+ EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()) << status;
+ EXPECT_NE(std::string::npos, status.to_string().find("nested"));
+ EXPECT_NE(std::string::npos, status.to_string().find("row 2"));
+ EXPECT_EQ(nullptr, batch);
+ }
+}
+
+TEST_F(ArrowFlightTimestampTest, ChecksBothUtcAndZonedCalendarBounds) {
+ const std::vector<std::pair<std::string, DateTime>> cases {
+ {"+08:00", make_datetime(1, 1, 1, 0, 0, 0, 0)},
+ {"-08:00", make_datetime(9999, 12, 31, 23, 0, 0, 0)},
+ {"-08:00", make_datetime(0, 12, 31, 23, 0, 0, 0)},
+ {"+08:00", make_datetime(10000, 1, 1, 1, 0, 0, 0)}};
+ for (const auto& [zone, value] : cases) {
+ SCOPED_TRACE(zone);
+ cctz::time_zone timezone;
+ ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone(zone, timezone));
+ auto block = timestamp_block(6, {value});
+ ArrowFlightArrowBlockConvertor converter(block, zone, timezone);
+ ASSERT_TRUE(converter.init().ok());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ const auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ EXPECT_EQ(ErrorCode::INVALID_ARGUMENT, status.code()) << status;
+ EXPECT_EQ(nullptr, batch);
+
+ auto valid = timestamp_block(
+ 6, {make_datetime(1, 1, 2, 0, 0, 0, 0), make_datetime(9999, 12, 30, 23, 0, 0, 0)});
+ ASSERT_TRUE(converter.convert_to_arrow(valid, arrow::default_memory_pool(), &batch).ok());
+ }
+}
+
+TEST_F(ArrowFlightTimestampTest, NaiveBoundsDoNotDependOnSessionTimezone) {
+ cctz::time_zone timezone;
+ ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("+08:00", timezone));
+ auto block = timestamp_block(6, {make_datetime(1, 1, 1, 0, 0, 0, 0),
+ make_datetime(9999, 12, 31, 23, 59, 59, 999999)});
+ ArrowFlightArrowBlockConvertor converter(block, "+08:00", timezone, true);
+ ASSERT_TRUE(converter.init().ok());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
+ const auto& values = static_cast<const arrow::TimestampArray&>(*batch->column(0));
+ EXPECT_EQ(-62135596800000000LL, values.Value(0));
+ EXPECT_EQ(253402300799999999LL, values.Value(1));
+}
+
+TEST_F(ArrowFlightTimestampTest, IgnoresTimestampsMaskedByNullParents) {
+ auto datetime = std::make_shared<DataTypeDateTimeV2>(6);
+ const auto invalid = Field::create_field<TYPE_DATETIMEV2>(DateTime {});
+ DataTypes types {
+ datetime, std::make_shared<DataTypeStruct>(DataTypes {datetime}, Strings {"child"}),
+ std::make_shared<DataTypeArray>(datetime),
+ std::make_shared<DataTypeMap>(make_nullable(datetime), make_nullable(datetime))};
+ FieldVector fields {
+ invalid, Field::create_field<TYPE_STRUCT>(Struct {invalid}),
+ Field::create_field<TYPE_ARRAY>(Array {invalid}),
+ Field::create_field<TYPE_MAP>(Map {Field::create_field<TYPE_ARRAY>(Array {invalid}),
+ Field::create_field<TYPE_ARRAY>(Array {invalid})})};
+ for (size_t i = 0; i < types.size(); ++i) {
+ auto data = types[i]->create_column();
+ data->insert(fields[i]);
+ auto nulls = ColumnUInt8::create();
+ nulls->insert_value(1);
+ Block block {{ColumnNullable::create(std::move(data), std::move(nulls)),
+ make_nullable(types[i]), "masked"}};
+ ArrowFlightArrowBlockConvertor converter(block, "UTC", cctz::utc_time_zone(), true);
+ ASSERT_TRUE(converter.init().ok());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ const auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_TRUE(batch->column(0)->IsNull(0));
+ }
+}
+
+} // namespace
+} // namespace doris
diff --git a/regression-test/data/external_table_p0/remote_doris/test_remote_doris_all_types_select.out b/regression-test/data/external_table_p0/remote_doris/test_remote_doris_all_types_select.out
index a8823bb027e..70fb18a6da0 100644
--- a/regression-test/data/external_table_p0/remote_doris/test_remote_doris_all_types_select.out
+++ b/regression-test/data/external_table_p0/remote_doris/test_remote_doris_all_types_select.out
@@ -1,12 +1,12 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql --
-2025-05-18T01:00 true -128 -32768 -2147483648 -9223372036854775808 -1234567890123456790 -123.456 -123456.789 -123457 -123456789012346 -1234567890123456789012345678 1970-01-01 0000-01-01T00:00 A Hello Hello, Doris! ["apple", "banana", "orange"] {"Emily":101, "age":25} {"f1":11, "f2":3.14, "f3":"Emily"} {"k1":"v31","k2":300,"k3":[123,456],"k4":[],"k5":{"i1":"iv1"}}
+2025-05-18T01:00 true -128 -32768 -2147483648 -9223372036854775808 -1234567890123456790 -123.456 -123456.789 -123457 -123456789012346 -1234567890123456789012345678 1970-01-01 0001-01-01T00:00 A Hello Hello, Doris! ["apple", "banana", "orange"] {"Emily":101, "age":25} {"f1":11, "f2":3.14, "f3":"Emily"} {"k1":"v31","k2":300,"k3":[123,456],"k4":[],"k5":{"i1":"iv1"}}
2025-05-18T02:00 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N
2025-05-18T03:00 false 127 32767 2147483647 9223372036854775807 1234567890123456789 123.456 123456.789 123457 123456789012346 1234567890123456789012345678 9999-12-31 9999-12-31T23:59:59 [] {} {"f1":11, "f2":3.14, "f3":"Emily"} {}
2025-05-18T04:00 true 0 0 0 0 0 0.0 0 0 0 0 2023-10-01 2023-10-01T12:34:56 A Hello Hello, Doris! ["apple", "banana", "orange"] {"Emily":101, "age":25} {"f1":11, "f2":3.14, "f3":"Emily"} []
-- !sql --
-2025-05-18T01:00 [1] [-128] [-32768] [-2147483648] [-9223372036854775808] [-1234567890123456790] [-123.456] [-123456.789] [-123457] [-123456789012346] [-1234567890123456789012345678] ["0000-01-01"] ["0000-01-01 00:00:00"] ["A"] ["Hello"] ["Hello, Doris!"]
+2025-05-18T01:00 [1] [-128] [-32768] [-2147483648] [-9223372036854775808] [-1234567890123456790] [-123.456] [-123456.789] [-123457] [-123456789012346] [-1234567890123456789012345678] ["0000-01-01"] ["0001-01-01 00:00:00"] ["A"] ["Hello"] ["Hello, Doris!"]
2025-05-18T02:00 [null] [null] [null] [null] [null] [null] [null] [null] [null] [null] [null] [null] [null] [null] [null] [null]
2025-05-18T03:00 [0] [127] [32767] [2147483647] [9223372036854775807] [1234567890123456789] [123.456] [123456.789] [123457] [123456789012346] [1234567890123456789012345678] ["9999-12-31"] ["9999-12-31 23:59:59"] [""] [""] [""]
2025-05-18T04:00 [1] [0] [0] [0] [0] [0] [0] [0] [0] [0] [0] ["2023-10-01"] ["2023-10-01 12:34:56"] ["A"] ["Hello"] ["Hello, Doris!"]
diff --git a/regression-test/data/external_table_p0/remote_doris/test_remote_doris_statistics.out b/regression-test/data/external_table_p0/remote_doris/test_remote_doris_statistics.out
index eecba5cbcd6..aebd716491e 100644
--- a/regression-test/data/external_table_p0/remote_doris/test_remote_doris_statistics.out
+++ b/regression-test/data/external_table_p0/remote_doris/test_remote_doris_statistics.out
@@ -4,7 +4,7 @@ c_bigint 4 3 1 -9223372036854775808 9223372036854775807 32
c_boolean 4 2 1 0 1 4
c_char 4 2 1 A 2
c_date 4 3 1 1970-01-01 9999-12-31 16
-c_datetime 4 3 1 0000-01-01 00:00:00 9999-12-31 23:59:59 32
+c_datetime 4 3 1 0001-01-01 00:00:00 9999-12-31 23:59:59 32
c_decimal18 4 3 1 -123456789012346 123456789012346 32
c_decimal32 4 3 1 -1234567890123456789012345678 1234567890123456789012345678 64
c_decimal9 4 3 1 -123457 123457 16
diff --git a/regression-test/suites/arrow_flight_sql_p0/test_flight_int96_timestamp_range.groovy b/regression-test/suites/arrow_flight_sql_p0/test_flight_int96_timestamp_range.groovy
new file mode 100644
index 00000000000..7cdc5f1bbfe
--- /dev/null
+++ b/regression-test/suites/arrow_flight_sql_p0/test_flight_int96_timestamp_range.groovy
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.sql.DriverManager
+import java.sql.Types
+import java.time.LocalDateTime
+
+suite("test_flight_int96_timestamp_range", "arrow_flight_sql,external,hive,tvf,external_docker") {
+ if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableHiveTest"))) {
+ return
+ }
+ def host = context.config.otherConfigs.get("externalEnvIp")
+ def port = context.config.otherConfigs.get("hive2HdfsPort")
+ // External clusters can override the configured Flight port; use the running FE's endpoint.
+ def frontend = jdbc_sql_return_maparray("SHOW FRONTENDS").find {
+ it.IsMaster.toString().toBoolean() && it.Alive.toString().toBoolean()
+ }
+ assertNotNull(frontend)
+ assertTrue(frontend.ArrowFlightSqlPort.toString().toInteger() > 0)
+ def flightUrl = "jdbc:arrow-flight-sql://${frontend.Host}:${frontend.ArrowFlightSqlPort}" +
+ "/catalog=${context.dbName}?useServerPrepStmts=false&useSSL=false&useEncryption=false"
+ Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver")
+ def query = { String file ->
+ """SELECT * FROM HDFS(
+ "uri" = "hdfs://${host}:${port}/user/doris/tvf_data/test_hdfs_parquet/group4/${file}",
+ "hadoop.username" = "doris", "format" = "parquet") LIMIT 10"""
+ }
+ // Keep the configured Flight identity without the TLS-dependent connect() wrapper.
+ DriverManager.getConnection(flightUrl, context.config.otherConfigs.get("extArrowFlightSqlUser"),
+ context.config.otherConfigs.get("extArrowFlightSqlPassword")).withCloseable { flight ->
+ flight.createStatement().withCloseable { statement ->
+ // Nanosecond fractions in these fixtures truncate to valid DATETIME(6) values.
+ for (def file : ["part-00000-570d8e52-652d-4892-8bdc-7fa5466ffa69.c000.snappy.parquet",
+ "part-00000-b945dfb5-9982-4f86-b903-dabef99caba1.c000.snappy.parquet",
+ "part-00000-721700d2-26d7-42a3-a8f9-b6601628ccd4.c000.snappy.parquet",
+ "part-00000-afeef968-a917-4d51-a652-e5a4214df453.c000.snappy.parquet"]) {
+ logger.info("Read INT96 Flight fixture: ${file}")
+ statement.executeQuery(query(file)).withCloseable { rows ->
+ assertTrue(rows.next())
+ def metadata = rows.getMetaData()
+ for (int column = 1; column <= metadata.getColumnCount(); ++column) {
+ if (metadata.getColumnType(column) == Types.TIMESTAMP) {
+ assertNotNull(rows.getObject(column, LocalDateTime.class))
+ }
+ }
+ assertEquals(LocalDateTime.parse("9999-12-31T23:59:59.999999"),
+ rows.getObject(metadata.getColumnCount(), LocalDateTime.class))
+ assertFalse(rows.next())
+ }
+ }
+ }
+ // Consume the result on the same discovered endpoint so zero-year values must fail over Flight.
+ try {
+ flight.createStatement().withCloseable { statement ->
+ statement.executeQuery(query("int96_timestamps_nanos_outside_day_range.parquet"))
+ .withCloseable { rows ->
+ while (rows.next()) {
+ for (int column = 1; column <= rows.getMetaData().getColumnCount(); ++column) {
+ rows.getObject(column)
+ }
+ }
+ }
+ }
+ assertTrue(false, "Expected an out-of-range Flight timestamp error")
+ } catch (Exception error) {
+ assertTrue(error.toString().contains("outside the supported 0001-9999 range"), error.toString())
+ }
+ }
+}
diff --git a/regression-test/suites/arrow_flight_sql_p0/test_flight_timestamp_range.groovy b/regression-test/suites/arrow_flight_sql_p0/test_flight_timestamp_range.groovy
new file mode 100644
index 00000000000..c494bd84684
--- /dev/null
+++ b/regression-test/suites/arrow_flight_sql_p0/test_flight_timestamp_range.groovy
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.time.LocalDateTime
+
+suite("test_flight_timestamp_range", "arrow_flight_sql") {
+ // Reuse the configured Flight connection so TLS settings cannot skip the assertions or change credentials.
+ def flight = context.getArrowFlightSqlConnection()
+ def input = "${context.dbName}.flight_timestamp_input"
+ def expectRangeError = { String query ->
+ try {
+ arrow_flight_sql(query)
+ assertTrue(false, "Expected an out-of-range Flight timestamp error")
+ } catch (Exception error) {
+ assertTrue(error.toString().contains("outside the supported 0001-9999 range"), error.toString())
+ }
+ }
+ arrow_flight_sql "DROP TABLE IF EXISTS ${input}"
+ arrow_flight_sql """CREATE TABLE ${input} (id INT, value STRING)
+ DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")"""
+ try {
+ arrow_flight_sql """INSERT INTO ${input} VALUES
+ (1, '0001-01-01 00:00:00.000000'),
+ (2, '9999-12-31 23:59:59.999999'),
+ (3, '1969-12-31 23:59:59.999999'),
+ (4, NULL), (5, '0000-12-31 00:00:00.000000')"""
+ // Check that the source value reaches the date conversion instead of becoming SQL NULL.
+ assertEquals([['0000-12-31 00:00:00.000000']],
+ arrow_flight_sql("SELECT CAST(CAST(value AS DATETIME(6)) AS STRING) FROM ${input} WHERE id = 5"))
+ for (def scale : [0, 3, 6]) {
+ expectRangeError("SELECT CAST(value AS DATETIME(${scale})) AS event_time FROM ${input} WHERE id = 5")
+ }
+ expectRangeError("SELECT CAST('0000-12-31 00:00:00' AS DATETIME(6)) AS event_time")
+ for (def expression : ["array(CAST(value AS DATETIME(6)))",
+ "named_struct('child', CAST(value AS DATETIME(6)))",
+ "map('key', CAST(value AS DATETIME(6)))",
+ "map(CAST(value AS DATETIME(6)), 'value')"]) {
+ expectRangeError("SELECT ${expression} AS nested FROM ${input} WHERE id = 5")
+ }
+ expectRangeError("SELECT CAST(value AS DATETIME(6)) AS event_time FROM ${input} ORDER BY id")
+ flight.createStatement().withCloseable { statement ->
+ statement.executeQuery("SELECT CAST(value AS DATETIME(6)) FROM ${input} WHERE id <= 4 ORDER BY id")
+ .withCloseable { rows ->
+ // Typed access preserves proleptic calendar dates and microseconds.
+ for (def expected : ["0001-01-01T00:00:00", "9999-12-31T23:59:59.999999",
+ "1969-12-31T23:59:59.999999"]) {
+ assertTrue(rows.next())
+ assertEquals(LocalDateTime.parse(expected), rows.getObject(1, LocalDateTime.class))
+ }
+ assertTrue(rows.next())
+ assertNull(rows.getObject(1, LocalDateTime.class))
+ assertFalse(rows.next())
+ }
+ statement.executeQuery("SELECT array(CAST(value AS DATETIME(6))) FROM ${input} WHERE id = 4")
+ .withCloseable { rows ->
+ assertTrue(rows.next())
+ def values = rows.getArray(1).getArray()
+ assertEquals(1, values.length)
+ assertNull(values[0])
+ assertFalse(rows.next())
+ }
+ }
+ } finally {
+ arrow_flight_sql "DROP TABLE IF EXISTS ${input}"
+ }
+}
diff --git a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_all_types_select.groovy b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_all_types_select.groovy
index 31530d7bfc1..c6919d62b57 100644
--- a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_all_types_select.groovy
+++ b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_all_types_select.groovy
@@ -67,8 +67,9 @@ suite("test_remote_doris_all_types_select", "p0,external,doris,external_docker,e
);
"""
+ // Successful Flight reads require timestamps within the supported 0001-9999 range.
sql """
- INSERT INTO `test_remote_doris_all_types_select_db`.`test_remote_doris_all_types_select_t` values('2025-05-18 01:00:00.000', true, -128, -32768, -2147483648, -9223372036854775808, -1234567890123456790, -123.456, -123456.789, -123457, -123456789012346, -1234567890123456789012345678, '1970-01-01', '0000-01-01 00:00:00', 'A', 'Hello', 'Hello, Doris!', '["apple", "banana", "orange"]', {"Emily":101,"age":25} , {11, 3.14, "Emily"}, '{"k1":"v31", "k2": 300, "k3": [123, 456], "k4": [], " [...]
+ INSERT INTO `test_remote_doris_all_types_select_db`.`test_remote_doris_all_types_select_t` values('2025-05-18 01:00:00.000', true, -128, -32768, -2147483648, -9223372036854775808, -1234567890123456790, -123.456, -123456.789, -123457, -123456789012346, -1234567890123456789012345678, '1970-01-01', '0001-01-01 00:00:00', 'A', 'Hello', 'Hello, Doris!', '["apple", "banana", "orange"]', {"Emily":101,"age":25} , {11, 3.14, "Emily"}, '{"k1":"v31", "k2": 300, "k3": [123, 456], "k4": [], " [...]
"""
sql """
INSERT INTO `test_remote_doris_all_types_select_db`.`test_remote_doris_all_types_select_t` values('2025-05-18 02:00:00.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)
@@ -108,7 +109,7 @@ suite("test_remote_doris_all_types_select", "p0,external,doris,external_docker,e
"""
sql """
- INSERT INTO `test_remote_doris_all_types_select_db`.`test_remote_doris_all_types_select_t2` values('2025-05-18 01:00:00.000', [true], [-128], [-32768], [-2147483648], [-9223372036854775808], [-1234567890123456790], [-123.456], [-123456.789], [-123457], [-123456789012346], [-1234567890123456789012345678], ['0000-01-01'], ['0000-01-01 00:00:00'], ['A'], ['Hello'], ['Hello, Doris!'])
+ INSERT INTO `test_remote_doris_all_types_select_db`.`test_remote_doris_all_types_select_t2` values('2025-05-18 01:00:00.000', [true], [-128], [-32768], [-2147483648], [-9223372036854775808], [-1234567890123456790], [-123.456], [-123456.789], [-123457], [-123456789012346], [-1234567890123456789012345678], ['0000-01-01'], ['0001-01-01 00:00:00'], ['A'], ['Hello'], ['Hello, Doris!'])
"""
sql """
INSERT INTO `test_remote_doris_all_types_select_db`.`test_remote_doris_all_types_select_t2` values('2025-05-18 02:00:00.000', [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL])
@@ -172,6 +173,16 @@ suite("test_remote_doris_all_types_select", "p0,external,doris,external_docker,e
select * from `test_remote_doris_all_types_select_catalog`.`test_remote_doris_all_types_select_db`.`test_remote_doris_all_types_select_t3` order by id
"""
+ // Keep zero-year rejection covered separately from successful all-type round trips.
+ sql """INSERT INTO test_remote_doris_all_types_select_db.test_remote_doris_all_types_select_t3
+ (id, datetime_0) VALUES ('2025-05-18 02:00:00', '0000-01-01 00:00:00')"""
+ test {
+ sql """SELECT datetime_0 FROM test_remote_doris_all_types_select_catalog
+ .test_remote_doris_all_types_select_db.test_remote_doris_all_types_select_t3
+ WHERE id = '2025-05-18 02:00:00'"""
+ exception "outside the supported 0001-9999 range"
+ }
+
sql """ DROP DATABASE IF EXISTS test_remote_doris_all_types_select_db """
sql """ DROP CATALOG IF EXISTS `test_remote_doris_all_types_select_catalog` """
}
diff --git a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_statistics.groovy b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_statistics.groovy
index 223c294d811..210946f397d 100644
--- a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_statistics.groovy
+++ b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_statistics.groovy
@@ -82,8 +82,9 @@ suite("test_remote_doris_statistics", "p0,external,doris,external_docker,externa
);
"""
+ // Successful Flight reads require timestamps within the supported 0001-9999 range.
sql """
- INSERT INTO `test_remote_doris_statistics_db`.`test_remote_doris_statistics_t1` values('2025-05-18 01:00:00.000', true, -128, -32768, -2147483648, -9223372036854775808, -1234567890123456790, -123.456, -123456.789, -123457, -123456789012346, -1234567890123456789012345678, '1970-01-01', '0000-01-01 00:00:00', 'A', 'Hello', 'Hello, Doris!', '["apple", "banana", "orange"]', {"Emily":101,"age":25} , {11, 3.14, "Emily"})
+ INSERT INTO `test_remote_doris_statistics_db`.`test_remote_doris_statistics_t1` values('2025-05-18 01:00:00.000', true, -128, -32768, -2147483648, -9223372036854775808, -1234567890123456790, -123.456, -123456.789, -123457, -123456789012346, -1234567890123456789012345678, '1970-01-01', '0001-01-01 00:00:00', 'A', 'Hello', 'Hello, Doris!', '["apple", "banana", "orange"]', {"Emily":101,"age":25} , {11, 3.14, "Emily"})
"""
sql """
INSERT INTO `test_remote_doris_statistics_db`.`test_remote_doris_statistics_t1` values('2025-05-18 02:00:00.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)
diff --git a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_table_stats.groovy b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_table_stats.groovy
index e6de44e005a..9486e888a6d 100644
--- a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_table_stats.groovy
+++ b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_table_stats.groovy
@@ -82,8 +82,9 @@ suite("test_remote_doris_table_stats", "p0,external,doris,external_docker,extern
);
"""
+ // Successful Flight reads require timestamps within the supported 0001-9999 range.
sql """
- INSERT INTO `test_remote_doris_table_stats_db`.`test_remote_doris_table_stats_t1` values('2025-05-18 01:00:00.000', true, -128, -32768, -2147483648, -9223372036854775808, -1234567890123456790, -123.456, -123456.789, -123457, -123456789012346, -1234567890123456789012345678, '1970-01-01', '0000-01-01 00:00:00', 'A', 'Hello', 'Hello, Doris!', '["apple", "banana", "orange"]', {"Emily":101,"age":25} , {11, 3.14, "Emily"})
+ INSERT INTO `test_remote_doris_table_stats_db`.`test_remote_doris_table_stats_t1` values('2025-05-18 01:00:00.000', true, -128, -32768, -2147483648, -9223372036854775808, -1234567890123456790, -123.456, -123456.789, -123457, -123456789012346, -1234567890123456789012345678, '1970-01-01', '0001-01-01 00:00:00', 'A', 'Hello', 'Hello, Doris!', '["apple", "banana", "orange"]', {"Emily":101,"age":25} , {11, 3.14, "Emily"})
"""
sql """
INSERT INTO `test_remote_doris_table_stats_db`.`test_remote_doris_table_stats_t1` values('2025-05-18 02:00:00.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 30/35: branch-4.1: [fix](array-func) Handle outer-null containers with hidden nested data (#68022) (#68696)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 26004ed0c6dde60e705f7a3c7726e825c94212c6
Author: linrrarity <li...@selectdb.com>
AuthorDate: Thu Oct 8 09:32:38 2026 +0800
branch-4.1: [fix](array-func) Handle outer-null containers with hidden nested data (#68022) (#68696)
pick: https://github.com/apache/doris/pull/68022
---
.../exprs/aggregate/aggregate_function_foreach.h | 181 +++++-
be/src/exprs/aggregate/aggregate_function_null.h | 114 ++++
.../array/function_array_enumerate_uniq.cpp | 146 +++--
.../exprs/function/array/function_array_sortby.cpp | 35 +-
.../exprs/function/array/function_array_split.cpp | 127 +++--
be/src/exprs/function/array/function_array_zip.cpp | 175 ++++--
.../lambda_function/varray_filter_function.cpp | 17 +-
.../exprs/lambda_function/varray_map_function.cpp | 123 +++-
.../exprs/lambda_function/varray_sort_function.cpp | 5 +
.../aggregate_function_exception_test.cpp | 324 ++++++++++-
.../exprs/function/function_array_split_test.cpp | 224 ++++++++
be/test/exprs/function/function_array_zip_test.cpp | 396 +++++++++++++
.../lambda_function/array_map_function_test.cpp | 619 ++++++++++++++++++++-
.../array_functions/test_array_map_function.out | 11 +
.../array_functions/test_array_split.out | 7 +
.../array_functions/test_array_map_function.groovy | 44 ++
.../array_functions/test_array_split.groovy | 16 +-
17 files changed, 2388 insertions(+), 176 deletions(-)
diff --git a/be/src/exprs/aggregate/aggregate_function_foreach.h b/be/src/exprs/aggregate/aggregate_function_foreach.h
index d595e91a9f7..628c31009fd 100644
--- a/be/src/exprs/aggregate/aggregate_function_foreach.h
+++ b/be/src/exprs/aggregate/aggregate_function_foreach.h
@@ -20,6 +20,8 @@
#pragma once
+#include <utility>
+
#include "common/status.h"
#include "core/assert_cast.h"
#include "core/column/column_nullable.h"
@@ -126,6 +128,121 @@ protected:
return state;
}
+private:
+ struct PreparedBatchColumns {
+ Columns owned_columns;
+ std::vector<const IColumn*> nested_columns;
+ const ColumnArray::Offsets64* offsets = nullptr;
+ };
+
+ void add_aligned(AggregateDataPtr __restrict place, const IColumn** nested,
+ const ColumnArray::Offsets64& offsets, size_t row_num, Arena& arena) const {
+ const size_t begin = offsets[row_num - 1];
+ const size_t end = offsets[row_num];
+ AggregateFunctionForEachData& state = ensure_aggregate_data(place, end - begin, arena);
+
+ char* nested_state = state.array_of_aggregate_datas;
+ for (size_t i = begin; i < end; ++i) {
+ nested_function->add(nested_state, nested, i, arena);
+ nested_state += nested_size_of_data;
+ }
+ }
+
+ // Validate all selected rows first. If their physical begins differ, compact every argument
+ // once for the batch and give them shared offsets;
+ // E.g. data [9,8,7,6]/[1,2,3,4,5], offsets [1,4]/[2,5], skip row 0
+ // -> data [8,7,6]/[3,4,5], shared offsets [0,3].
+ template <typename IsSelected>
+ Columns normalize_batch_columns(size_t batch_size, const IColumn** columns,
+ IsSelected&& is_selected) const {
+ bool offsets_aligned = true;
+ // First pass: reject size mismatches before allocating, and detect whether the
+ // original nested columns can use one shared element index.
+ for (size_t row = 0; row < batch_size; ++row) {
+ if (!is_selected(row)) {
+ continue;
+ }
+
+ const auto& first_array =
+ assert_cast<const ColumnArray&, TypeCheckOnRelease::DISABLE>(*columns[0]);
+ const auto& first_offsets = first_array.get_offsets();
+ const size_t first_begin = first_offsets[row - 1];
+ const size_t row_size = first_offsets[row] - first_begin;
+ for (size_t i = 1; i < num_arguments; ++i) {
+ const auto& array =
+ assert_cast<const ColumnArray&, TypeCheckOnRelease::DISABLE>(*columns[i]);
+ const auto& offsets = array.get_offsets();
+ const size_t begin = offsets[row - 1];
+ if (offsets[row] - begin != row_size) {
+ throw Exception(ErrorCode::INTERNAL_ERROR,
+ "Arrays passed to {} aggregate function have different sizes",
+ get_name());
+ }
+ offsets_aligned &= row_size == 0 || begin == first_begin;
+ }
+ }
+
+ // Fast path: all visible rows already address the same physical nested positions.
+ if (offsets_aligned) {
+ return {};
+ }
+
+ // Slow path: build the shared offsets once, copy only selected row slices, and preserve
+ // the outer row count by emitting repeated offsets for unselected rows.
+ auto offsets_column = ColumnArray::ColumnOffsets::create();
+ auto& normalized_offsets = offsets_column->get_data();
+ normalized_offsets.reserve(batch_size);
+ size_t normalized_offset = 0;
+ const auto& first_offsets =
+ assert_cast<const ColumnArray&, TypeCheckOnRelease::DISABLE>(*columns[0])
+ .get_offsets();
+ for (size_t row = 0; row < batch_size; ++row) {
+ if (is_selected(row)) {
+ normalized_offset += first_offsets[row] - first_offsets[row - 1];
+ }
+ normalized_offsets.push_back(normalized_offset);
+ }
+ ColumnPtr shared_offsets = std::move(offsets_column);
+
+ Columns normalized_columns(num_arguments);
+ for (size_t i = 0; i < num_arguments; ++i) {
+ const auto& array =
+ assert_cast<const ColumnArray&, TypeCheckOnRelease::DISABLE>(*columns[i]);
+ auto nested = array.get_data().clone_empty();
+ for (size_t row = 0; row < batch_size; ++row) {
+ if (is_selected(row)) {
+ const auto& offsets = array.get_offsets();
+ const size_t begin = offsets[row - 1];
+ const size_t row_size = offsets[row] - begin;
+ nested->insert_range_from(array.get_data(), begin, row_size);
+ }
+ }
+ ColumnPtr nested_column = std::move(nested);
+ normalized_columns[i] = ColumnArray::create(nested_column, shared_offsets);
+ }
+ return normalized_columns;
+ }
+
+ template <typename IsSelected>
+ PreparedBatchColumns prepare_batch_columns(size_t batch_size, const IColumn** columns,
+ IsSelected&& is_selected) const {
+ PreparedBatchColumns prepared;
+ prepared.owned_columns =
+ normalize_batch_columns(batch_size, columns, std::forward<IsSelected>(is_selected));
+ prepared.nested_columns.resize(num_arguments);
+ for (size_t i = 0; i < num_arguments; ++i) {
+ const IColumn* column =
+ prepared.owned_columns.empty() ? columns[i] : prepared.owned_columns[i].get();
+ const auto& array =
+ assert_cast<const ColumnArray&, TypeCheckOnRelease::DISABLE>(*column);
+ prepared.nested_columns[i] = &array.get_data();
+ if (i == 0) {
+ prepared.offsets = &array.get_offsets();
+ }
+ }
+ return prepared;
+ }
+
public:
constexpr static auto AGG_FOREACH_SUFFIX = "_foreach";
AggregateFunctionForEach(AggregateFunctionPtr nested_function_, const DataTypes& arguments)
@@ -149,6 +266,16 @@ public:
return std::make_shared<DataTypeArray>(nested_function->get_return_type());
}
+ // Use foreach-specific streaming handling to align array inputs once while keeping aggregate
+ // states row-local; prepare returns empty when the original columns are already aligned.
+ bool requires_batch_add_for_streaming() const { return true; }
+
+ template <typename IsSelected>
+ Columns prepare_batch_columns_for_streaming(size_t batch_size, const IColumn** columns,
+ IsSelected&& is_selected) const {
+ return normalize_batch_columns(batch_size, columns, std::forward<IsSelected>(is_selected));
+ }
+
void destroy(AggregateDataPtr __restrict place) const noexcept override {
AggregateFunctionForEachData& state = data(place);
@@ -245,22 +372,41 @@ public:
size_t begin = offsets[row_num - 1];
size_t end = offsets[row_num];
+ const size_t row_size = end - begin;
+ bool offsets_aligned = true;
/// Sanity check. NOTE We can implement specialization for a case with single argument, if the check will hurt performance.
for (size_t i = 1; i < num_arguments; ++i) {
const auto& ith_column =
assert_cast<const ColumnArray&, TypeCheckOnRelease::DISABLE>(*columns[i]);
const auto& ith_offsets = ith_column.get_offsets();
+ const size_t ith_begin = ith_offsets[row_num - 1];
+ const size_t ith_end = ith_offsets[row_num];
- if (ith_offsets[row_num] != end ||
- (row_num != 0 && ith_offsets[row_num - 1] != begin)) {
+ if (ith_end - ith_begin != row_size) {
throw Exception(ErrorCode::INTERNAL_ERROR,
"Arrays passed to {} aggregate function have different sizes",
get_name());
}
+ offsets_aligned &= ith_begin == begin;
}
- AggregateFunctionForEachData& state = ensure_aggregate_data(place, end - begin, arena);
+ std::vector<ColumnPtr> compacted_nested;
+ if (!offsets_aligned && row_size != 0) {
+ // The nested aggregate accepts one shared index, so align only mismatched row slices.
+ compacted_nested.reserve(num_arguments);
+ for (size_t i = 0; i < num_arguments; ++i) {
+ const auto& ith_column =
+ assert_cast<const ColumnArray&, TypeCheckOnRelease::DISABLE>(*columns[i]);
+ const size_t ith_begin = ith_column.get_offsets()[row_num - 1];
+ compacted_nested.emplace_back(nested[i]->cut(ith_begin, row_size));
+ nested[i] = compacted_nested.back().get();
+ }
+ begin = 0;
+ end = row_size;
+ }
+
+ AggregateFunctionForEachData& state = ensure_aggregate_data(place, row_size, arena);
char* nested_state = state.array_of_aggregate_datas;
for (size_t i = begin; i < end; ++i) {
@@ -268,6 +414,35 @@ public:
nested_state += nested_size_of_data;
}
}
+
+ void add_batch(size_t batch_size, AggregateDataPtr* places, size_t place_offset,
+ const IColumn** columns, Arena& arena, bool /*agg_many*/) const override {
+ auto prepared = prepare_batch_columns(batch_size, columns, [](size_t) { return true; });
+ for (size_t row = 0; row < batch_size; ++row) {
+ add_aligned(places[row] + place_offset, prepared.nested_columns.data(),
+ *prepared.offsets, row, arena);
+ }
+ }
+
+ void add_batch_selected(size_t batch_size, AggregateDataPtr* places, size_t place_offset,
+ const IColumn** columns, Arena& arena) const override {
+ auto prepared = prepare_batch_columns(batch_size, columns,
+ [&](size_t row) { return places[row] != nullptr; });
+ for (size_t row = 0; row < batch_size; ++row) {
+ if (places[row] != nullptr) {
+ add_aligned(places[row] + place_offset, prepared.nested_columns.data(),
+ *prepared.offsets, row, arena);
+ }
+ }
+ }
+
+ void add_batch_single_place(size_t batch_size, AggregateDataPtr place, const IColumn** columns,
+ Arena& arena) const override {
+ auto prepared = prepare_batch_columns(batch_size, columns, [](size_t) { return true; });
+ for (size_t row = 0; row < batch_size; ++row) {
+ add_aligned(place, prepared.nested_columns.data(), *prepared.offsets, row, arena);
+ }
+ }
};
} // namespace doris
diff --git a/be/src/exprs/aggregate/aggregate_function_null.h b/be/src/exprs/aggregate/aggregate_function_null.h
index 293451cf460..8b7f309431d 100644
--- a/be/src/exprs/aggregate/aggregate_function_null.h
+++ b/be/src/exprs/aggregate/aggregate_function_null.h
@@ -597,7 +597,121 @@ public:
arena);
}
+ void add_batch(size_t batch_size, AggregateDataPtr* places, size_t place_offset,
+ const IColumn** columns, Arena& arena, bool /*agg_many*/) const override {
+ add_batch_selected_impl(batch_size, columns, arena,
+ [&](size_t row) { return places[row] + place_offset; });
+ }
+
+ void add_batch_selected(size_t batch_size, AggregateDataPtr* places, size_t place_offset,
+ const IColumn** columns, Arena& arena) const override {
+ add_batch_selected_impl(batch_size, columns, arena, [&](size_t row) {
+ return places[row] == nullptr ? nullptr : places[row] + place_offset;
+ });
+ }
+
+ void add_batch_single_place(size_t batch_size, AggregateDataPtr place, const IColumn** columns,
+ Arena& arena) const override {
+ add_batch_selected_impl(batch_size, columns, arena, [&](size_t) { return place; });
+ }
+
+ void streaming_agg_serialize_to_column(const IColumn** columns, MutableColumnPtr& dst,
+ size_t num_rows, Arena& arena) const override {
+ if constexpr (!requires(const NestFuction& function) {
+ function.requires_batch_add_for_streaming();
+ }) {
+ AggregateFunctionNullBaseInline<
+ NestFuction, result_is_nullable,
+ AggregateFunctionNullVariadicInline<NestFuction, result_is_nullable>>::
+ streaming_agg_serialize_to_column(columns, dst, num_rows, arena);
+ } else {
+ std::vector<const IColumn*> nested_columns(number_of_arguments);
+ std::array<const ColumnNullable*, MAX_ARGS> nullable_columns {};
+ for (size_t i = 0; i < number_of_arguments; ++i) {
+ if (is_nullable[i]) {
+ nullable_columns[i] =
+ &assert_cast<const ColumnNullable&, TypeCheckOnRelease::DISABLE>(
+ *columns[i]);
+ nested_columns[i] = &nullable_columns[i]->get_nested_column();
+ } else {
+ nested_columns[i] = columns[i];
+ }
+ }
+
+ const auto is_selected = [&](size_t row) {
+ for (size_t i = 0; i < number_of_arguments; ++i) {
+ if (nullable_columns[i] != nullptr && nullable_columns[i]->is_null_at(row)) {
+ return false;
+ }
+ }
+ return true;
+ };
+ auto prepared_columns = this->nested_function->prepare_batch_columns_for_streaming(
+ num_rows, nested_columns.data(), is_selected);
+ if (!prepared_columns.empty()) {
+ for (size_t i = 0; i < number_of_arguments; ++i) {
+ nested_columns[i] = prepared_columns[i].get();
+ }
+ }
+
+ const size_t state_size = this->size_of_data();
+ std::vector<char> state(state_size);
+ VectorBufferWriter buf(assert_cast<ColumnString&>(*dst));
+ for (size_t row = 0; row < num_rows; ++row) {
+ this->create(state.data());
+ try {
+ if (is_selected(row)) {
+ this->set_flag(state.data());
+ this->nested_function->add(this->nested_place(state.data()),
+ nested_columns.data(), row, arena);
+ }
+ this->serialize(state.data(), buf);
+ buf.commit();
+ } catch (...) {
+ this->destroy(state.data());
+ throw;
+ }
+ this->destroy(state.data());
+ }
+ }
+ }
+
private:
+ template <typename GetPlace>
+ void add_batch_selected_impl(size_t batch_size, const IColumn** columns, Arena& arena,
+ GetPlace&& get_place) const {
+ std::vector<const IColumn*> nested_columns(number_of_arguments);
+ std::array<const ColumnNullable*, MAX_ARGS> nullable_columns {};
+ for (size_t i = 0; i < number_of_arguments; ++i) {
+ if (is_nullable[i]) {
+ nullable_columns[i] =
+ &assert_cast<const ColumnNullable&, TypeCheckOnRelease::DISABLE>(
+ *columns[i]);
+ nested_columns[i] = &nullable_columns[i]->get_nested_column();
+ } else {
+ nested_columns[i] = columns[i];
+ }
+ }
+
+ std::vector<AggregateDataPtr> nested_places(batch_size, nullptr);
+ for (size_t row = 0; row < batch_size; ++row) {
+ AggregateDataPtr place = get_place(row);
+ if (place == nullptr) {
+ continue;
+ }
+ bool has_null = false;
+ for (size_t i = 0; i < number_of_arguments; ++i) {
+ has_null |= nullable_columns[i] != nullptr && nullable_columns[i]->is_null_at(row);
+ }
+ if (!has_null) {
+ this->set_flag(place);
+ nested_places[row] = this->nested_place(place);
+ }
+ }
+ this->nested_function->add_batch_selected(batch_size, nested_places.data(), 0,
+ nested_columns.data(), arena);
+ }
+
// The array length is fixed in the implementation of some aggregate functions.
// Therefore we choose 256 as the appropriate maximum length limit.
static const size_t MAX_ARGS = 256;
diff --git a/be/src/exprs/function/array/function_array_enumerate_uniq.cpp b/be/src/exprs/function/array/function_array_enumerate_uniq.cpp
index 71beccd28f5..9ac19721c3e 100644
--- a/be/src/exprs/function/array/function_array_enumerate_uniq.cpp
+++ b/be/src/exprs/function/array/function_array_enumerate_uniq.cpp
@@ -26,6 +26,7 @@
#include <ostream>
#include <string>
#include <utility>
+#include <vector>
#include "common/status.h"
#include "core/arena.h"
@@ -41,6 +42,7 @@
#include "core/column/column_vector.h"
#include "core/data_type/data_type.h"
#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_nothing.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/define_primitive_type.h"
@@ -51,6 +53,7 @@
#include "exec/common/columns_hashing.h"
#include "exec/common/hash_table/hash.h"
#include "exec/common/hash_table/hash_map_context.h"
+#include "exec/common/util.hpp"
#include "exprs/aggregate/aggregate_function.h"
#include "exprs/function/function.h"
#include "exprs/function/function_helpers.h"
@@ -77,6 +80,8 @@ public:
bool is_variadic() const override { return true; }
size_t get_number_of_arguments() const override { return 1; }
+ bool use_default_implementation_for_nulls() const override { return false; }
+
DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
if (arguments.empty()) {
throw doris::Exception(
@@ -85,6 +90,9 @@ public:
}
bool is_nested_nullable = false;
for (size_t i = 0; i < arguments.size(); ++i) {
+ if (arguments[i]->is_null_literal()) {
+ return make_nullable(std::make_shared<DataTypeNothing>());
+ }
const DataTypeArray* array_type =
check_and_get_data_type<DataTypeArray>(remove_nullable(arguments[i]).get());
if (!array_type) {
@@ -99,10 +107,7 @@ public:
auto return_nested_type = std::make_shared<DataTypeInt64>();
DataTypePtr return_type = std::make_shared<DataTypeArray>(
is_nested_nullable ? make_nullable(return_nested_type) : return_nested_type);
- if (arguments[0]->is_nullable()) {
- return_type = make_nullable(return_type);
- }
- return return_type;
+ return have_nullable(arguments) ? make_nullable(return_type) : return_type;
}
// When compiling `FunctionArrayEnumerateUniq::_execute_by_hash`, `AllocatorWithStackMemory::free(buf)`
@@ -117,36 +122,107 @@ public:
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const override {
- ColumnRawPtrs data_columns(arguments.size());
- const ColumnArray::Offsets64* offsets = nullptr;
- ColumnPtr src_offsets;
- Columns src_columns; // to keep ownership
+ for (const auto argument : arguments) {
+ if (block.get_by_position(argument).column->only_null()) {
+ auto& result_column = block.get_by_position(result);
+ result_column.column =
+ result_column.type->create_column_const(input_rows_count, Field());
+ return Status::OK();
+ }
+ }
- const ColumnArray* first_column_array = nullptr;
+ ColumnUInt8::MutablePtr result_null_map;
+ ColumnUInt8::Container* result_null_map_data = nullptr;
+ if (block.get_by_position(result).type->is_nullable()) {
+ result_null_map = ColumnUInt8::create(input_rows_count, 0);
+ result_null_map_data = &result_null_map->get_data();
+ }
- for (size_t i = 0; i < arguments.size(); i++) {
- src_columns.emplace_back(
- block.get_by_position(arguments[i]).column->convert_to_full_column_if_const());
- ColumnPtr& cur_column = src_columns[i];
- const ColumnArray* array =
- check_and_get_column<ColumnArray>(remove_nullable(cur_column->get_ptr()).get());
+ std::vector<const ColumnArray*> array_columns(arguments.size());
+ Columns src_columns;
+ src_columns.reserve(arguments.size());
+ for (size_t i = 0; i < arguments.size(); ++i) {
+ auto cur_column =
+ block.get_by_position(arguments[i]).column->convert_to_full_column_if_const();
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(cur_column.get())) {
+ VectorizedUtils::update_null_map(*result_null_map_data,
+ nullable->get_null_map_data());
+ cur_column = nullable->get_nested_column_ptr();
+ }
+ src_columns.emplace_back(std::move(cur_column));
+ const auto* array = check_and_get_column<ColumnArray>(src_columns.back().get());
if (!array) {
return Status::RuntimeError(
fmt::format("Illegal column {}, of first argument of function {}",
- cur_column->get_name(), get_name()));
+ src_columns.back()->get_name(), get_name()));
}
+ array_columns[i] = array;
+ }
- const ColumnArray::Offsets64& cur_offsets = array->get_offsets();
- if (i == 0) {
- first_column_array = array;
- offsets = &cur_offsets;
- src_offsets = array->get_offsets_ptr();
- } else if (*offsets != cur_offsets) {
- return Status::RuntimeError(fmt::format(
- "lengths of all arrays of function {} must be equal.", get_name()));
+ bool offsets_aligned = true;
+ for (size_t row = 0; row < input_rows_count; ++row) {
+ const auto& first_offsets = array_columns[0]->get_offsets();
+ const size_t first_row_size = first_offsets[row] - first_offsets[row - 1];
+ for (size_t i = 1; i < arguments.size(); ++i) {
+ const auto& current_offsets = array_columns[i]->get_offsets();
+ if (result_null_map_data == nullptr || !(*result_null_map_data)[row]) {
+ if (current_offsets[row] - current_offsets[row - 1] != first_row_size) {
+ return Status::RuntimeError(fmt::format(
+ "lengths of all arrays of function {} must be equal.", get_name()));
+ }
+ }
+ offsets_aligned &= current_offsets[row] == first_offsets[row];
+ }
+ }
+
+ ColumnRawPtrs data_columns(arguments.size());
+ const ColumnArray::Offsets64* offsets = nullptr;
+ ColumnPtr result_offsets;
+ MutableColumns compacted_data_columns;
+ if (offsets_aligned) {
+ offsets = &array_columns[0]->get_offsets();
+ result_offsets = array_columns[0]->get_offsets_ptr();
+ for (size_t i = 0; i < arguments.size(); ++i) {
+ data_columns[i] = &array_columns[i]->get_data();
+ }
+ } else {
+ // Outer-NULL rows may retain payload and shift each input's physical offsets.
+ // Compact visible rows so the hash key getter can keep using one element index.
+ compacted_data_columns.resize(arguments.size());
+ for (size_t i = 0; i < arguments.size(); ++i) {
+ compacted_data_columns[i] = array_columns[i]->get_data().clone_empty();
+ }
+
+ auto compacted_offsets = ColumnArray::ColumnOffsets::create();
+ auto& compacted_offsets_data = compacted_offsets->get_data();
+ compacted_offsets_data.reserve(input_rows_count);
+ size_t compacted_offset = 0;
+ for (size_t row = 0; row < input_rows_count; ++row) {
+ if ((*result_null_map_data)[row]) {
+ compacted_offsets_data.push_back(compacted_offset);
+ continue;
+ }
+
+ size_t row_size = 0;
+ for (size_t i = 0; i < arguments.size(); ++i) {
+ const auto& current_offsets = array_columns[i]->get_offsets();
+ const size_t row_begin = current_offsets[row - 1];
+ const size_t current_row_size = current_offsets[row] - row_begin;
+ if (i == 0) {
+ row_size = current_row_size;
+ }
+ compacted_data_columns[i]->insert_range_from(array_columns[i]->get_data(),
+ row_begin, row_size);
+ }
+ compacted_offset += row_size;
+ compacted_offsets_data.push_back(compacted_offset);
+ }
+
+ for (size_t i = 0; i < arguments.size(); ++i) {
+ data_columns[i] = compacted_data_columns[i].get();
}
- const auto* array_data = &array->get_data();
- data_columns[i] = array_data;
+ offsets = &compacted_offsets_data;
+ result_offsets = std::move(compacted_offsets);
}
const NullMapType* null_map = nullptr;
@@ -158,7 +234,7 @@ public:
auto dst_nested_column = ColumnInt64::create();
ColumnInt64::Container& dst_values = dst_nested_column->get_data();
- dst_values.resize(offsets->back());
+ dst_values.resize(offsets->empty() ? 0 : offsets->back());
if (arguments.size() == 1) {
DataTypePtr src_column_type = block.get_by_position(arguments[0]).type;
@@ -189,18 +265,16 @@ public:
}
ColumnPtr nested_column = dst_nested_column->get_ptr();
- if (first_column_array->get_data().is_nullable()) {
+ const auto& result_array_type = assert_cast<const DataTypeArray&>(
+ *remove_nullable(block.get_by_position(result).type));
+ if (result_array_type.get_nested_type()->is_nullable()) {
nested_column = ColumnNullable::create(nested_column,
ColumnUInt8::create(nested_column->size(), 0));
}
- ColumnPtr res_column = ColumnArray::create(std::move(nested_column), src_offsets);
- if (arguments.size() == 1 && block.get_by_position(arguments[0]).column->is_nullable()) {
- auto left_column =
- block.get_by_position(arguments[0]).column->convert_to_full_column_if_const();
- const ColumnNullable* nullable =
- check_and_get_column<ColumnNullable>(left_column.get());
- res_column = ColumnNullable::create(
- res_column, nullable->get_null_map_column().clone_resized(nullable->size()));
+ ColumnPtr res_column =
+ ColumnArray::create(std::move(nested_column), std::move(result_offsets));
+ if (block.get_by_position(result).type->is_nullable()) {
+ res_column = ColumnNullable::create(std::move(res_column), std::move(result_null_map));
}
block.replace_by_position(result, std::move(res_column));
diff --git a/be/src/exprs/function/array/function_array_sortby.cpp b/be/src/exprs/function/array/function_array_sortby.cpp
index b62fa1d9999..e2d3c939a0c 100644
--- a/be/src/exprs/function/array/function_array_sortby.cpp
+++ b/be/src/exprs/function/array/function_array_sortby.cpp
@@ -116,26 +116,23 @@ public:
for (int row = 0; row < input_rows_count; ++row) {
unsigned long cur_src_element_num = src_offsets[row] - src_offsets[row - 1];
unsigned long cur_key_element_num = key_offsets[row] - key_offsets[row - 1];
- if (cur_src_element_num != cur_key_element_num) {
- if (key_null_map_data != nullptr && (*key_null_map_data)[row]) {
- // deal with this case if one of row like: ([1,2,3], NULL) --->([1,2,3])
- for (unsigned long pos = src_offsets[row - 1]; pos < src_offsets[row]; ++pos) {
- src_selector.push_back(pos);
- }
- null_step = null_step + cur_src_element_num;
- continue;
- } else if (src_null_map_data != nullptr && (*src_null_map_data)[row]) {
- // deal with this case if one of row like: (NULL, [1,2,3]) --->(NULL)
- null_step = null_step - cur_key_element_num;
- continue;
- } else {
- return Status::InternalError(
- "in array sortby function, the input column nested column data rows "
- "are not equal, the first size is {}, but with second size is {} at "
- "row {}.",
- src_offsets[row] - src_offsets[row - 1],
- key_offsets[row] - key_offsets[row - 1], row);
+ const bool src_is_null = src_null_map_data != nullptr && (*src_null_map_data)[row];
+ const bool key_is_null = key_null_map_data != nullptr && (*key_null_map_data)[row];
+ if (src_is_null || key_is_null) {
+ // Preserve the source row and align later key indices across hidden NULL payloads.
+ for (unsigned long pos = src_offsets[row - 1]; pos < src_offsets[row]; ++pos) {
+ src_selector.push_back(pos);
}
+ null_step = null_step + cur_src_element_num - cur_key_element_num;
+ continue;
+ }
+ if (cur_src_element_num != cur_key_element_num) {
+ return Status::InternalError(
+ "in array sortby function, the input column nested column data rows "
+ "are not equal, the first size is {}, but with second size is {} at "
+ "row {}.",
+ src_offsets[row] - src_offsets[row - 1],
+ key_offsets[row] - key_offsets[row - 1], row);
}
auto start = key_offsets[row - 1];
diff --git a/be/src/exprs/function/array/function_array_split.cpp b/be/src/exprs/function/array/function_array_split.cpp
index 24bf4876c8a..c29e6b37378 100644
--- a/be/src/exprs/function/array/function_array_split.cpp
+++ b/be/src/exprs/function/array/function_array_split.cpp
@@ -22,6 +22,7 @@
#include <cstddef>
#include <memory>
#include <utility>
+#include <variant>
#include "common/status.h"
#include "core/assert_cast.h"
@@ -36,6 +37,8 @@
#include "core/data_type/data_type_array.h"
#include "core/data_type/data_type_nullable.h"
#include "core/types.h"
+#include "exec/common/template_helpers.hpp"
+#include "exec/common/util.hpp"
#include "exprs/aggregate/aggregate_function.h"
#include "exprs/function/function.h"
#include "exprs/function/simple_function_factory.h"
@@ -55,34 +58,62 @@ public:
size_t get_number_of_arguments() const override { return 2; }
+ bool use_default_implementation_for_nulls() const override { return false; }
+
DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
- return std::make_shared<DataTypeArray>(make_nullable(arguments[0]));
+ auto result_type =
+ std::make_shared<DataTypeArray>(make_nullable(remove_nullable(arguments[0])));
+ return have_nullable(arguments) ? make_nullable(result_type) : result_type;
};
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const override {
- // <Nullable>(Array(<Nullable>(Int)))
+ for (const auto argument : arguments) {
+ if (block.get_by_position(argument).column->only_null()) {
+ auto& result_column = block.get_by_position(result);
+ result_column.column =
+ result_column.type->create_column_const(input_rows_count, Field());
+ return Status::OK();
+ }
+ }
+
+ ColumnUInt8::MutablePtr result_null_map;
+ ColumnUInt8::Container* result_null_map_data = nullptr;
+ if (block.get_by_position(result).type->is_nullable()) {
+ result_null_map = ColumnUInt8::create(input_rows_count, 0);
+ result_null_map_data = &result_null_map->get_data();
+ }
auto src_column =
block.get_by_position(arguments[0]).column->convert_to_full_column_if_const();
- auto spliter_column =
- block.get_by_position(arguments[1]).column->convert_to_full_column_if_const();
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(src_column.get())) {
+ VectorizedUtils::update_null_map(*result_null_map_data, nullable->get_null_map_data());
+ src_column = nullable->get_nested_column_ptr();
+ }
+
+ const auto& [unpacked_splitter_column, splitter_is_const] =
+ unpack_if_const(block.get_by_position(arguments[1]).column);
+ const IColumn* splitter_column = unpacked_splitter_column.get();
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(splitter_column)) {
+ VectorizedUtils::update_null_map(*result_null_map_data, nullable->get_null_map_data(),
+ splitter_is_const);
+ splitter_column = &nullable->get_nested_column();
+ }
// only change its split(i.e. offsets)
const auto& src_data = assert_cast<const ColumnArray&>(*src_column).get_data_ptr();
const auto& src_offsets = assert_cast<const ColumnArray&>(*src_column).get_offsets();
- auto split_col = assert_cast<const ColumnArray*>(spliter_column.get())->get_data_ptr();
- const auto& split_offsets = assert_cast<const ColumnArray&>(*spliter_column)
+ auto split_col = assert_cast<const ColumnArray*>(splitter_column)->get_data_ptr();
+ const auto& split_offsets = assert_cast<const ColumnArray&>(*splitter_column)
.get_offsets(); // for check uneven array
- const NullMap* null_map = nullptr;
- if (split_col->is_nullable()) {
+ const NullMap* split_element_null_map = nullptr;
+ if (const auto* nullable_split_col =
+ check_and_get_column<ColumnNullable>(split_col.get())) {
if (split_col->has_null()) {
- null_map =
- &assert_cast<const ColumnNullable*>(split_col.get())->get_null_map_data();
+ split_element_null_map = &nullable_split_col->get_null_map_data();
}
- split_col =
- assert_cast<const ColumnNullable*>(split_col.get())->get_nested_column_ptr();
+ split_col = nullable_split_col->get_nested_column_ptr();
}
const IColumn::Filter& cut = assert_cast<const ColumnBool*>(split_col.get())->get_data();
@@ -94,57 +125,79 @@ public:
offsets_inner.reserve(src_offsets.size()); // assume the actual size to be equal or larger
offsets_outer.reserve(src_offsets.size());
- if (null_map != nullptr) {
- RETURN_IF_ERROR(do_loop<true>(src_offsets, split_offsets, cut, null_map, offsets_inner,
- offsets_outer));
- } else {
- RETURN_IF_ERROR(do_loop<false>(src_offsets, split_offsets, cut, null_map, offsets_inner,
- offsets_outer));
- }
+ RETURN_IF_ERROR(std::visit(
+ [&](auto consider_element_null, auto consider_outer_null) {
+ return do_loop<consider_element_null, consider_outer_null>(
+ src_offsets, split_offsets, cut, split_element_null_map,
+ result_null_map_data, splitter_is_const, offsets_inner, offsets_outer);
+ },
+ make_bool_variant(split_element_null_map != nullptr),
+ make_bool_variant(result_null_map_data != nullptr)));
auto inner_result = ColumnArray::create(src_data, std::move(col_offsets_inner));
- auto outer_result = ColumnArray::create(
+ ColumnPtr result_column = ColumnArray::create(
ColumnNullable::create(std::move(inner_result),
ColumnUInt8::create(inner_result->size(), 0)),
std::move(col_offsets_outer));
- block.replace_by_position(result, std::move(outer_result));
+ if (block.get_by_position(result).type->is_nullable()) {
+ result_column =
+ ColumnNullable::create(std::move(result_column), std::move(result_null_map));
+ }
+ block.replace_by_position(result, std::move(result_column));
return Status::OK();
}
- template <bool CONSIDER_NULL>
+ template <bool CONSIDER_ELEMENT_NULL, bool CONSIDER_OUTER_NULL>
static Status do_loop(const IColumn::Offsets64& src_offsets,
const IColumn::Offsets64& split_offsets, const IColumn::Filter& cut,
- const NullMap* null_map, PaddedPODArray<IColumn::Offset64>& offsets_inner,
+ const NullMap* split_element_null_map, const NullMap* result_null_map,
+ bool splitter_is_const, PaddedPODArray<IColumn::Offset64>& offsets_inner,
PaddedPODArray<IColumn::Offset64>& offsets_outer) {
- size_t pos = 0;
+ size_t src_begin = 0;
for (auto i = 0; i < src_offsets.size(); i++) { // per cells
- auto in_offset = src_offsets[i];
- auto sp_offset = split_offsets[i];
- if (in_offset != sp_offset) [[unlikely]] {
+ const size_t src_end = src_offsets[i];
+ const size_t splitter_row = index_check_const(i, splitter_is_const);
+ const size_t split_begin = split_offsets[splitter_row - 1];
+ const size_t split_end = split_offsets[splitter_row];
+
+ if constexpr (CONSIDER_OUTER_NULL) {
+ if ((*result_null_map)[i]) {
+ // Preserve the hidden source payload as one segment so later rows remain aligned.
+ if (src_begin < src_end) {
+ offsets_inner.push_back(src_end);
+ }
+ offsets_outer.push_back(offsets_inner.size());
+ src_begin = src_end;
+ continue;
+ }
+ }
+
+ if (src_end - src_begin != split_end - split_begin) [[unlikely]] {
return Status::InvalidArgument("function {} has uneven arguments on row {}", name,
i);
}
// [1,2,3,4,5]
- if (pos < in_offset) { // values in a cell
- pos += !reverse;
- for (; pos < in_offset - reverse; ++pos) {
- if constexpr (CONSIDER_NULL) {
- if (cut[pos] && !(*null_map)[pos]) {
- offsets_inner.push_back(pos + reverse); // cut a array [1,2,3]
+ if (src_begin < src_end) { // values in a cell
+ size_t src_pos = src_begin + !reverse;
+ size_t split_pos = split_begin + !reverse;
+ for (; src_pos < src_end - reverse; ++src_pos, ++split_pos) {
+ if constexpr (CONSIDER_ELEMENT_NULL) {
+ if (cut[split_pos] && !(*split_element_null_map)[split_pos]) {
+ offsets_inner.push_back(src_pos + reverse); // cut a array [1,2,3]
}
} else {
- if (cut[pos]) {
- offsets_inner.push_back(pos + reverse); // cut a array [1,2,3]
+ if (cut[split_pos]) {
+ offsets_inner.push_back(src_pos + reverse); // cut a array [1,2,3]
}
}
}
- pos += reverse;
// put the tail offset, always last.
- offsets_inner.push_back(pos); // put [4,5]
+ offsets_inner.push_back(src_end); // put [4,5]
}
offsets_outer.push_back(offsets_inner.size());
+ src_begin = src_end;
}
return Status::OK();
}
diff --git a/be/src/exprs/function/array/function_array_zip.cpp b/be/src/exprs/function/array/function_array_zip.cpp
index 562801326ee..82c825ae73d 100644
--- a/be/src/exprs/function/array/function_array_zip.cpp
+++ b/be/src/exprs/function/array/function_array_zip.cpp
@@ -27,6 +27,7 @@
#include <ostream>
#include <string>
#include <utility>
+#include <vector>
#include "common/status.h"
#include "core/block/block.h"
@@ -34,14 +35,17 @@
#include "core/block/column_with_type_and_name.h"
#include "core/column/column.h"
#include "core/column/column_array.h"
+#include "core/column/column_const.h"
#include "core/column/column_nullable.h"
#include "core/column/column_struct.h"
#include "core/column/column_vector.h"
#include "core/data_type/data_type.h"
#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_nothing.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_struct.h"
#include "core/types.h"
+#include "exec/common/util.hpp"
#include "exprs/aggregate/aggregate_function.h"
#include "exprs/function/function.h"
#include "exprs/function/function_helpers.h"
@@ -67,67 +71,174 @@ public:
size_t get_number_of_arguments() const override { return 0; }
+ bool use_default_implementation_for_nulls() const override { return false; }
+
DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
DCHECK(arguments.size() > 0)
<< "function: " << get_name() << ", arguments should not be empty";
+ if (std::ranges::any_of(arguments,
+ [](const auto& type) { return type->is_null_literal(); })) {
+ return make_nullable(std::make_shared<DataTypeNothing>());
+ }
+
DataTypes res_data_types;
size_t num_elements = arguments.size();
+ bool result_is_nullable = false;
for (size_t i = 0; i < num_elements; ++i) {
- DCHECK(arguments[i]->get_primitive_type() == TYPE_ARRAY)
- << i << "-th element is not array type";
-
- const auto* array_type = check_and_get_data_type<DataTypeArray>(arguments[i].get());
- DCHECK(array_type) << "function: " << get_name() << " " << i + 1
- << "-th argument is not array";
+ const auto argument_type = remove_nullable(arguments[i]);
+ result_is_nullable |= arguments[i]->is_nullable();
+ const auto& array_type = assert_cast<const DataTypeArray&>(*argument_type);
res_data_types.emplace_back(
- make_nullable(remove_nullable((array_type->get_nested_type()))));
+ make_nullable(remove_nullable(array_type.get_nested_type())));
}
auto res = std::make_shared<DataTypeArray>(
make_nullable(std::make_shared<DataTypeStruct>(res_data_types)));
- return res;
+ return result_is_nullable ? make_nullable(res) : res;
}
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const override {
size_t num_element = arguments.size();
- // all the columns must have the same size as the first column
- ColumnPtr first_array_column;
- Columns tuple_columns(num_element);
+ ColumnUInt8::MutablePtr result_null_map;
+ ColumnUInt8::Container* result_null_map_data = nullptr;
+ if (block.get_by_position(result).type->is_nullable()) {
+ result_null_map = ColumnUInt8::create(input_rows_count, 0);
+ result_null_map_data = &result_null_map->get_data();
+ }
+ std::vector<const ColumnArray*> column_arrays(num_element);
+ std::vector<bool> column_is_const(num_element, false);
for (size_t i = 0; i < num_element; ++i) {
- auto col = block.get_by_position(arguments[i]).column;
- col = col->convert_to_full_column_if_const();
-
- const auto* column_array = check_and_get_column<ColumnArray>(col.get());
- if (!column_array) {
- return Status::RuntimeError(fmt::format(
- "execute failed, function {}'s {}-th argument should be array bet get {}",
- get_name(), i + 1, block.get_by_position(arguments[i]).type->get_name()));
+ const auto& input_column = block.get_by_position(arguments[i]).column;
+ if (input_column->only_null()) {
+ auto& result_column = block.get_by_position(result);
+ result_column.column =
+ result_column.type->create_column_const(input_rows_count, Field());
+ return Status::OK();
}
+ const auto& [unpacked_column, is_const] = unpack_if_const(input_column);
+ column_is_const[i] = is_const;
+ const IColumn* array_column = unpacked_column.get();
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(array_column)) {
+ VectorizedUtils::update_null_map(*result_null_map_data,
+ nullable->get_null_map_data(), is_const);
+ array_column = nullable->get_nested_column_ptr().get();
+ }
+
+ column_arrays[i] = &assert_cast<const ColumnArray&>(*array_column);
+ }
- if (i == 0) {
- first_array_column = col;
- } else if (!column_array->has_equal_offsets(
- static_cast<const ColumnArray&>(*first_array_column))) {
- return Status::RuntimeError(
- fmt::format("execute failed, function {}'s {}-th argument should have same "
- "offsets with first argument",
+ bool offsets_aligned = true;
+ for (size_t row = 0; row < input_rows_count; ++row) {
+ size_t first_row_size = 0;
+ size_t first_logical_offset = 0;
+ for (size_t i = 0; i < num_element; ++i) {
+ const size_t actual_row = index_check_const(row, column_is_const[i]);
+ const auto& offsets = column_arrays[i]->get_offsets();
+ const size_t row_begin = offsets[actual_row - 1];
+ const size_t row_size = offsets[actual_row] - row_begin;
+ const size_t logical_offset =
+ column_is_const[i] ? (row + 1) * row_size : offsets[row];
+
+ if (i == 0) {
+ first_row_size = row_size;
+ first_logical_offset = logical_offset;
+ } else {
+ if (result_null_map_data == nullptr || !(*result_null_map_data)[row]) {
+ if (row_size != first_row_size) {
+ return Status::RuntimeError(fmt::format(
+ "execute failed, function {}'s {}-th argument should have "
+ "same offsets with first argument",
get_name(), i + 1));
+ }
+ }
+ offsets_aligned &= logical_offset == first_logical_offset;
+ }
+ }
+ }
+
+ Columns tuple_columns(num_element);
+ ColumnPtr result_offsets;
+ if (offsets_aligned) {
+ Columns materialized_columns(num_element);
+ // Const arrays must be expanded, but prefer offsets already owned by a non-const input.
+ size_t offsets_source = 0;
+ while (offsets_source < num_element && column_is_const[offsets_source]) {
+ ++offsets_source;
+ }
+ if (offsets_source == num_element) {
+ offsets_source = 0;
+ }
+
+ for (size_t i = 0; i < num_element; ++i) {
+ if (!column_is_const[i]) {
+ continue;
+ }
+
+ auto column = block.get_by_position(arguments[i])
+ .column->convert_to_full_column_if_const();
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(column.get())) {
+ column = nullable->get_nested_column_ptr();
+ }
+ materialized_columns[i] = std::move(column);
+ column_arrays[i] = &assert_cast<const ColumnArray&>(*materialized_columns[i]);
+ }
+
+ for (size_t i = 0; i < num_element; ++i) {
+ tuple_columns[i] = column_arrays[i]->get_data_ptr();
+ }
+ result_offsets = column_arrays[offsets_source]->get_offsets_ptr();
+ } else {
+ MutableColumns mutable_tuple_columns(num_element);
+ for (size_t i = 0; i < num_element; ++i) {
+ mutable_tuple_columns[i] = column_arrays[i]->get_data().clone_empty();
+ }
+
+ auto mutable_result_offsets = ColumnArray::ColumnOffsets::create();
+ auto& result_offsets_data = mutable_result_offsets->get_data();
+ result_offsets_data.reserve(input_rows_count);
+ size_t result_offset = 0;
+ for (size_t row = 0; row < input_rows_count; ++row) {
+ if (result_null_map_data != nullptr && (*result_null_map_data)[row]) {
+ result_offsets_data.push_back(result_offset);
+ continue;
+ }
+
+ size_t row_size = 0;
+ for (size_t i = 0; i < num_element; ++i) {
+ const size_t actual_row = index_check_const(row, column_is_const[i]);
+ const auto& offsets = column_arrays[i]->get_offsets();
+ const size_t row_begin = offsets[actual_row - 1];
+ const size_t current_size = offsets[actual_row] - row_begin;
+ if (i == 0) {
+ row_size = current_size;
+ }
+ mutable_tuple_columns[i]->insert_range_from(column_arrays[i]->get_data(),
+ row_begin, row_size);
+ }
+ result_offset += row_size;
+ result_offsets_data.push_back(result_offset);
}
- tuple_columns[i] = column_array->get_data_ptr();
+ for (size_t i = 0; i < num_element; ++i) {
+ tuple_columns[i] = std::move(mutable_tuple_columns[i]);
+ }
+ result_offsets = std::move(mutable_result_offsets);
}
auto tuples = ColumnStruct::create(tuple_columns);
+ const size_t tuple_size = tuples->size();
auto nullable_tuples =
- ColumnNullable::create(std::move(tuples), ColumnUInt8::create(tuples->size(), 0));
- auto res_column = ColumnArray::create(
- std::move(nullable_tuples),
- static_cast<const ColumnArray&>(*first_array_column).get_offsets_ptr());
+ ColumnNullable::create(std::move(tuples), ColumnUInt8::create(tuple_size, 0));
+ ColumnPtr res_column =
+ ColumnArray::create(std::move(nullable_tuples), std::move(result_offsets));
+ if (block.get_by_position(result).type->is_nullable()) {
+ res_column = ColumnNullable::create(std::move(res_column), std::move(result_null_map));
+ }
block.replace_by_position(result, std::move(res_column));
return Status::OK();
}
@@ -137,4 +248,4 @@ void register_function_array_zip(SimpleFunctionFactory& factory) {
factory.register_function<FunctionArrayZip>();
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/exprs/lambda_function/varray_filter_function.cpp b/be/src/exprs/lambda_function/varray_filter_function.cpp
index 60aeb26f201..9d4f781a72b 100644
--- a/be/src/exprs/lambda_function/varray_filter_function.cpp
+++ b/be/src/exprs/lambda_function/varray_filter_function.cpp
@@ -68,20 +68,25 @@ public:
RETURN_IF_ERROR(children[1]->execute_column(context, block, expr_selector, output_count,
column_ptr_1));
+ if (column_ptr_0->only_null() || column_ptr_1->only_null()) {
+ result_column = result_type->create_column_const(output_count, Field());
+ return Status::OK();
+ }
+
//2. get first and second array column
auto first_column = column_ptr_0->convert_to_full_column_if_const();
auto second_column = column_ptr_1->convert_to_full_column_if_const();
auto input_rows = first_column->size();
- auto first_outside_null_map = ColumnUInt8::create(input_rows, 0);
+ auto result_outside_null_map = ColumnUInt8::create(input_rows, 0);
auto first_arg_column = first_column;
if (first_arg_column->is_nullable()) {
first_arg_column =
assert_cast<const ColumnNullable*>(first_column.get())->get_nested_column_ptr();
const auto& column_array_nullmap =
assert_cast<const ColumnNullable*>(first_column.get())->get_null_map_column();
- VectorizedUtils::update_null_map(first_outside_null_map->get_data(),
+ VectorizedUtils::update_null_map(result_outside_null_map->get_data(),
column_array_nullmap.get_data());
}
const auto& first_col_array = assert_cast<const ColumnArray&>(*first_arg_column);
@@ -97,13 +102,12 @@ public:
result_offset_data.reserve(input_rows);
auto second_arg_column = second_column;
- auto second_outside_null_map = ColumnUInt8::create(input_rows, 0);
if (second_arg_column->is_nullable()) {
second_arg_column = assert_cast<const ColumnNullable*>(second_column.get())
->get_nested_column_ptr();
const auto& column_array_nullmap =
assert_cast<const ColumnNullable*>(second_column.get())->get_null_map_column();
- VectorizedUtils::update_null_map(second_outside_null_map->get_data(),
+ VectorizedUtils::update_null_map(result_outside_null_map->get_data(),
column_array_nullmap.get_data());
}
const auto& second_col_array = assert_cast<const ColumnArray&>(*second_arg_column);
@@ -121,8 +125,7 @@ public:
//3. get the idx of second column data is not null and not 0
for (int row = 0; row < input_rows; ++row) {
//first or second column is null, so current row is invalid data
- if (first_outside_null_map->get_data()[row] ||
- second_outside_null_map->get_data()[row]) {
+ if (result_outside_null_map->get_data()[row]) {
result_offset_data.push_back(result_offset_data.back());
} else {
unsigned long count = 0;
@@ -151,7 +154,7 @@ public:
result_column =
ColumnNullable::create(ColumnArray::create(std::move(result_data_column),
std::move(result_offset_column)),
- std::move(first_outside_null_map));
+ std::move(result_outside_null_map));
} else {
DCHECK(!first_column->is_nullable());
DCHECK(!second_column->is_nullable());
diff --git a/be/src/exprs/lambda_function/varray_map_function.cpp b/be/src/exprs/lambda_function/varray_map_function.cpp
index 7d972159687..07c6ee4a03f 100644
--- a/be/src/exprs/lambda_function/varray_map_function.cpp
+++ b/be/src/exprs/lambda_function/varray_map_function.cpp
@@ -109,9 +109,9 @@ public:
// offset column
MutableColumnPtr array_column_offset;
size_t nested_array_column_rows = 0;
- ColumnPtr first_array_offsets = nullptr;
//2. get the result column from executed expr, and the needed is nested column of array
std::vector<ColumnPtr> lambda_datas(arguments.size());
+ std::vector<ColumnPtr> lambda_offsets(arguments.size());
DataTypes lambda_argument_types(arguments.size());
for (int i = 0; i < arguments.size(); ++i) {
@@ -139,35 +139,90 @@ public:
// here is the array column
const auto& col_array = assert_cast<const ColumnArray&>(*column_array);
+ lambda_offsets[i] = col_array.get_offsets_ptr();
if (i == 0) {
nested_array_column_rows = col_array.get_data_ptr()->size();
- first_array_offsets = col_array.get_offsets_ptr();
const auto& off_data = col_array.get_offsets_column();
array_column_offset = off_data.clone_resized(col_array.get_offsets_column().size());
args_info.offsets_ptr = &col_array.get_offsets();
- } else {
- // select array_map((x,y)->x+y,c_array1,[0,1,2,3]) from array_test2;
- // c_array1: [0,1,2,3,4,5,6,7,8,9]
- const auto& array_offsets =
- assert_cast<const ColumnArray::ColumnOffsets&>(*first_array_offsets)
- .get_data();
- if (nested_array_column_rows != col_array.get_data_ptr()->size() ||
- (!array_offsets.empty() &&
- memcmp(array_offsets.data(), col_array.get_offsets().data(),
- sizeof(array_offsets[0]) * array_offsets.size()) != 0)) {
- return Status::InvalidArgument(
- "in array map function, the input column size "
- "are "
- "not equal completely, nested column data rows 1st size is {}, {}th "
- "size is {}.",
- nested_array_column_rows, i + 1, col_array.get_data_ptr()->size());
- }
}
lambda_datas[i] = col_array.get_data_ptr();
const auto& col_type = assert_cast<const DataTypeArray&>(*type_array);
lambda_argument_types[i] = col_type.get_nested_type();
}
+
+ const auto& first_array_offsets =
+ assert_cast<const ColumnArray::ColumnOffsets&>(*lambda_offsets[0]).get_data();
+ const auto& null_map_data = outside_null_map->get_data();
+ const bool has_null =
+ std::ranges::any_of(null_map_data, [](uint8_t is_null) { return is_null; });
+ bool has_hidden_nested_data = false;
+ if (!has_null) {
+ // select array_map((x,y)->x+y,c_array1,[0,1,2,3]) from array_test2;
+ // c_array1: [0,1,2,3,4,5,6,7,8,9]
+ for (int i = 1; i < arguments.size(); ++i) {
+ const auto& offsets =
+ assert_cast<const ColumnArray::ColumnOffsets&>(*lambda_offsets[i])
+ .get_data();
+ if (nested_array_column_rows != lambda_datas[i]->size() ||
+ (!first_array_offsets.empty() &&
+ memcmp(first_array_offsets.data(), offsets.data(),
+ sizeof(first_array_offsets[0]) * first_array_offsets.size()) != 0)) {
+ return Status::InvalidArgument(
+ "in array map function, the input column size are not equal "
+ "completely, nested column data rows 1st size is {}, {}th size is {}.",
+ nested_array_column_rows, i + 1, lambda_datas[i]->size());
+ }
+ }
+ } else {
+ std::vector<size_t> previous_offsets(arguments.size(), 0);
+ for (size_t row = 0; row < count; ++row) {
+ const size_t first_row_size = first_array_offsets[row] - previous_offsets[0];
+ has_hidden_nested_data |= null_map_data[row] != 0 && first_row_size > 0;
+ for (int i = 1; i < arguments.size(); ++i) {
+ const auto& offsets =
+ assert_cast<const ColumnArray::ColumnOffsets&>(*lambda_offsets[i])
+ .get_data();
+ const size_t row_size = offsets[row] - previous_offsets[i];
+ has_hidden_nested_data |= null_map_data[row] != 0 && row_size > 0;
+ if (null_map_data[row] == 0 && first_row_size != row_size) {
+ return Status::InvalidArgument(
+ "in array map function, the input column size are not equal "
+ "completely at row {}, 1st size is {}, {}th size is {}.",
+ row, first_row_size, i + 1, row_size);
+ }
+ previous_offsets[i] = offsets[row];
+ }
+ previous_offsets[0] = first_array_offsets[row];
+ }
+ }
+
+ // NULL rows are skipped. If they retain hidden payload, rebuild only result offsets;
+ // the bounded execution path reads each argument through its own original offsets.
+ if (has_hidden_nested_data) {
+ auto res_offsets = ColumnArray::ColumnOffsets::create();
+ auto& res_offsets_data = res_offsets->get_data();
+ res_offsets_data.reserve(count);
+ size_t previous_offset = 0;
+ size_t compacted_rows = 0;
+ for (size_t row = 0; row < count; ++row) {
+ const size_t current_offset = first_array_offsets[row];
+ if (null_map_data[row] == 0) {
+ const size_t row_size = current_offset - previous_offset;
+ compacted_rows += row_size;
+ }
+ res_offsets_data.push_back(compacted_rows);
+ previous_offset = current_offset;
+ }
+
+ nested_array_column_rows = compacted_rows;
+ array_column_offset = std::move(res_offsets);
+ args_info.offsets_ptr =
+ &assert_cast<const ColumnArray::ColumnOffsets&>(*array_column_offset)
+ .get_data();
+ }
+
std::set<int> required_input_column_ids;
children[0]->collect_slot_column_ids(required_input_column_ids);
context->lambda_execution_context().collect_visible_binding_column_positions(
@@ -229,7 +284,7 @@ public:
// if column_array is NULL, we know the array_data_column will not write any data,
// so the column is empty. eg : (x) -> concat('|',x + "1"). if still execute the lambda function, will cause the bolck rows are not equal
// the x column is empty, but "|" is const literal, size of column is 1, so the block rows is 1, but the x column is empty, will be coredump.
- if (std::ranges::any_of(lambda_datas, [](const auto& v) { return v->empty(); })) {
+ if (nested_array_column_rows == 0) {
DataTypePtr nested_type;
bool is_nullable = result_type->is_nullable();
if (is_nullable) {
@@ -273,7 +328,11 @@ public:
columns = columns_guard.release();
} else {
for (int i = 0; i < column_size; i++) {
- columns[i] = data_types[i]->create_column();
+ if (i < lambda_argument_base && !materialized_input_columns[i]) {
+ columns[i] = ColumnNothing::create(0);
+ } else {
+ columns[i] = data_types[i]->create_column();
+ }
}
}
// batch_size of array nested data every time inorder to avoid memory overflow
@@ -281,18 +340,24 @@ public:
long max_step = batch_size - columns[lambda_argument_base]->size();
long current_step = std::min(
max_step, (long)(args_info.cur_size - args_info.current_offset_in_array));
- size_t pos = args_info.array_start + args_info.current_offset_in_array;
for (int i = 0; i < arguments.size() && current_step > 0; ++i) {
- columns[lambda_argument_base + i]->insert_range_from(*lambda_datas[i], pos,
- current_step);
+ const auto& source_offsets =
+ assert_cast<const ColumnArray::ColumnOffsets&>(*lambda_offsets[i])
+ .get_data();
+ const size_t source_pos = source_offsets[args_info.current_row_idx - 1] +
+ args_info.current_offset_in_array;
+ columns[lambda_argument_base + i]->insert_range_from(*lambda_datas[i],
+ source_pos, current_step);
}
args_info.current_offset_in_array += current_step;
args_info.current_repeat_times += current_step;
if (args_info.current_offset_in_array >= args_info.cur_size) {
args_info.current_row_eos = true;
}
- _repeat_input_columns(columns, block, args_info.current_repeat_times,
- materialized_input_columns, args_info.current_row_idx);
+ _repeat_input_columns(
+ columns, block, args_info.current_repeat_times, materialized_input_columns,
+ expr_selector == nullptr ? args_info.current_row_idx
+ : (*expr_selector)[args_info.current_row_idx]);
args_info.current_repeat_times = 0;
if (args_info.current_row_eos) {
//current row is end of array, move to next row
@@ -308,6 +373,12 @@ public:
}
}
+ // Trailing NULL or empty array rows can finish the input without filling a batch.
+ // Do not evaluate the lambda on an empty block.
+ if (columns[lambda_argument_base]->empty()) {
+ break;
+ }
+
if (!mem_reuse) {
for (int i = 0; i < column_size; ++i) {
lambda_block.insert(
diff --git a/be/src/exprs/lambda_function/varray_sort_function.cpp b/be/src/exprs/lambda_function/varray_sort_function.cpp
index 70a3fa34dd9..c8a3d6ef1f4 100644
--- a/be/src/exprs/lambda_function/varray_sort_function.cpp
+++ b/be/src/exprs/lambda_function/varray_sort_function.cpp
@@ -206,6 +206,11 @@ public:
const int lambda_result_base = static_cast<int>(lambda_block.columns());
for (int row = 0; row < input_rows; ++row) {
+ // Hidden nested values of an outer-NULL array must not invoke the lambda.
+ if (outside_null_map.get() != nullptr &&
+ assert_cast<const ColumnUInt8&>(*outside_null_map).get_data()[row]) {
+ continue;
+ }
auto start = off_data[row - 1];
auto end = off_data[row];
std::sort(&permutation[start], &permutation[end], [&](size_t i, size_t j) {
diff --git a/be/test/exprs/aggregate/aggregate_function_exception_test.cpp b/be/test/exprs/aggregate/aggregate_function_exception_test.cpp
index 21ee64dba4a..d79b0ee2888 100644
--- a/be/test/exprs/aggregate/aggregate_function_exception_test.cpp
+++ b/be/test/exprs/aggregate/aggregate_function_exception_test.cpp
@@ -17,10 +17,21 @@
#include <gtest/gtest.h>
+#include <algorithm>
+#include <memory>
+#include <string>
#include <vector>
#include "core/arena.h"
+#include "core/column/column_array.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_vector.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
#include "exprs/aggregate/aggregate_function.h"
+#include "exprs/aggregate/aggregate_function_foreach.h"
+#include "exprs/aggregate/aggregate_function_null.h"
namespace doris {
@@ -40,6 +51,113 @@ struct TrackingAggregateState {
int TrackingAggregateState::construct_count = 0;
int TrackingAggregateState::destroy_count = 0;
+struct PairSumAggregateState {
+ Int64 value = 0;
+};
+
+struct HeapStringAggregateState {
+ ~HeapStringAggregateState() { live_bytes -= value.size(); }
+
+ void assign(StringRef source) {
+ live_bytes -= value.size();
+ value.assign(source.data, source.size);
+ live_bytes += value.size();
+ peak_live_bytes = std::max(peak_live_bytes, live_bytes);
+ }
+
+ static void reset_counters() {
+ live_bytes = 0;
+ peak_live_bytes = 0;
+ }
+
+ static size_t live_bytes;
+ static size_t peak_live_bytes;
+ String value;
+};
+
+size_t HeapStringAggregateState::live_bytes = 0;
+size_t HeapStringAggregateState::peak_live_bytes = 0;
+
+class PairSumAggregateFunction final
+ : public IAggregateFunctionDataHelper<PairSumAggregateState, PairSumAggregateFunction> {
+public:
+ explicit PairSumAggregateFunction(std::vector<size_t>* observed_column_sizes = nullptr)
+ : IAggregateFunctionDataHelper<PairSumAggregateState, PairSumAggregateFunction>(
+ DataTypes {std::make_shared<DataTypeInt32>(),
+ std::make_shared<DataTypeInt32>()}),
+ observed_column_sizes(observed_column_sizes) {}
+
+ String get_name() const override { return "pair_sum"; }
+
+ DataTypePtr get_return_type() const override { return std::make_shared<DataTypeInt64>(); }
+
+ void add(AggregateDataPtr place, const IColumn** columns, ssize_t row_num,
+ Arena&) const override {
+ if (observed_column_sizes != nullptr) {
+ observed_column_sizes->push_back(columns[0]->size());
+ }
+ data(place).value += assert_cast<const ColumnInt32&>(*columns[0]).get_data()[row_num] +
+ assert_cast<const ColumnInt32&>(*columns[1]).get_data()[row_num];
+ }
+
+ void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena&) const override {
+ data(place).value += data(rhs).value;
+ }
+
+ void serialize(ConstAggregateDataPtr place, BufferWritable& buf) const override {
+ buf.write_binary(data(place).value);
+ }
+
+ void deserialize(AggregateDataPtr place, BufferReadable& buf, Arena&) const override {
+ buf.read_binary(data(place).value);
+ }
+
+ void insert_result_into(ConstAggregateDataPtr place, IColumn& to) const override {
+ assert_cast<ColumnInt64&>(to).insert_value(data(place).value);
+ }
+
+private:
+ std::vector<size_t>* observed_column_sizes;
+};
+
+class HeapStringAggregateFunction final
+ : public IAggregateFunctionDataHelper<HeapStringAggregateState,
+ HeapStringAggregateFunction> {
+public:
+ HeapStringAggregateFunction()
+ : IAggregateFunctionDataHelper<HeapStringAggregateState, HeapStringAggregateFunction>(
+ DataTypes {std::make_shared<DataTypeString>(),
+ std::make_shared<DataTypeInt32>()}) {}
+
+ String get_name() const override { return "heap_string"; }
+
+ DataTypePtr get_return_type() const override { return std::make_shared<DataTypeString>(); }
+
+ void add(AggregateDataPtr place, const IColumn** columns, ssize_t row_num,
+ Arena&) const override {
+ data(place).assign(assert_cast<const ColumnString&>(*columns[0]).get_data_at(row_num));
+ }
+
+ void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena&) const override {
+ data(place).assign(StringRef(data(rhs).value));
+ }
+
+ void serialize(ConstAggregateDataPtr place, BufferWritable& buf) const override {
+ buf.write_binary(data(place).value);
+ }
+
+ void deserialize(AggregateDataPtr place, BufferReadable& buf, Arena&) const override {
+ String value;
+ buf.read_binary(value);
+ data(place).assign(StringRef(value));
+ }
+
+ void insert_result_into(ConstAggregateDataPtr place, IColumn& to) const override {
+ assert_cast<ColumnString&>(to).insert_data(data(place).value.data(),
+ data(place).value.size());
+ }
+};
+
class ThrowOnDeserializeAggregateFunction final
: public IAggregateFunctionDataHelper<TrackingAggregateState,
ThrowOnDeserializeAggregateFunction> {
@@ -159,4 +277,208 @@ TEST_F(AggregateFunctionExceptionTest,
EXPECT_EQ(TrackingAggregateState::construct_count, TrackingAggregateState::destroy_count);
}
-} // namespace doris
\ No newline at end of file
+TEST_F(AggregateFunctionExceptionTest, ForEachReadsEachArgumentFromItsOwnRowOffset) {
+ auto nested_function = std::make_shared<PairSumAggregateFunction>();
+ auto input_type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt32>());
+ AggregateFunctionForEach foreach_function(nested_function, DataTypes {input_type, input_type});
+
+ auto compacted_data = ColumnInt32::create();
+ compacted_data->get_data().assign({30, 40});
+ auto compacted_offsets = ColumnArray::ColumnOffsets::create();
+ compacted_offsets->get_data().assign({0, 2});
+ auto compacted = ColumnArray::create(std::move(compacted_data), std::move(compacted_offsets));
+
+ auto original_data = ColumnInt32::create();
+ original_data->get_data().assign({10, 20, 300, 400});
+ auto original_offsets = ColumnArray::ColumnOffsets::create();
+ original_offsets->get_data().assign({2, 4});
+ auto original = ColumnArray::create(std::move(original_data), std::move(original_offsets));
+ const IColumn* columns[] = {compacted.get(), original.get()};
+
+ AggregateFunctionGuard state(&foreach_function);
+ ASSERT_NO_THROW(foreach_function.add(state.data(), columns, 1, arena));
+
+ auto result = foreach_function.get_return_type()->create_column();
+ foreach_function.insert_result_into(state.data(), *result);
+ const auto& result_array = assert_cast<const ColumnArray&>(*result);
+ const auto& result_data = assert_cast<const ColumnInt64&>(
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column());
+ EXPECT_EQ(result_data.get_data(), ColumnInt64::Container({330, 440}));
+}
+
+TEST_F(AggregateFunctionExceptionTest, ForEachNormalizesShiftedOffsetsOncePerBatch) {
+ std::vector<size_t> observed_column_sizes;
+ auto nested_function = std::make_shared<PairSumAggregateFunction>(&observed_column_sizes);
+ auto input_type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt32>());
+ AggregateFunctionForEach foreach_function(nested_function, DataTypes {input_type, input_type});
+
+ auto compacted_data = ColumnInt32::create();
+ compacted_data->get_data().assign({30, 40});
+ auto compacted_offsets = ColumnArray::ColumnOffsets::create();
+ compacted_offsets->get_data().assign({0, 1, 2});
+ auto compacted = ColumnArray::create(std::move(compacted_data), std::move(compacted_offsets));
+
+ auto original_data = ColumnInt32::create();
+ original_data->get_data().assign({10, 20, 300, 400});
+ auto original_offsets = ColumnArray::ColumnOffsets::create();
+ original_offsets->get_data().assign({2, 3, 4});
+ auto original = ColumnArray::create(std::move(original_data), std::move(original_offsets));
+ const IColumn* columns[] = {compacted.get(), original.get()};
+
+ auto normalized_columns = foreach_function.prepare_batch_columns_for_streaming(
+ 3, columns, [](size_t row) { return row != 0; });
+ ASSERT_EQ(normalized_columns.size(), 2);
+ const auto& normalized_first = assert_cast<const ColumnArray&>(*normalized_columns[0]);
+ const auto& normalized_second = assert_cast<const ColumnArray&>(*normalized_columns[1]);
+ EXPECT_EQ(normalized_first.get_offsets_ptr().get(), normalized_second.get_offsets_ptr().get());
+
+ AggregateFunctionGuard state(&foreach_function);
+ std::array<AggregateDataPtr, 3> places {nullptr, state.data(), state.data()};
+ foreach_function.add_batch_selected(places.size(), places.data(), 0, columns, arena);
+
+ EXPECT_EQ(observed_column_sizes, std::vector<size_t>({2, 2}));
+ auto result = foreach_function.get_return_type()->create_column();
+ foreach_function.insert_result_into(state.data(), *result);
+ const auto& result_array = assert_cast<const ColumnArray&>(*result);
+ const auto& result_data = assert_cast<const ColumnInt64&>(
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column());
+ EXPECT_EQ(result_data.get_data(), ColumnInt64::Container({770}));
+}
+
+TEST_F(AggregateFunctionExceptionTest, NullableForEachNormalizesVisibleRowsOncePerBatch) {
+ std::vector<size_t> observed_column_sizes;
+ auto nested_function = std::make_shared<PairSumAggregateFunction>(&observed_column_sizes);
+ auto input_type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt32>());
+ auto* foreach_function =
+ new AggregateFunctionForEach(nested_function, DataTypes {input_type, input_type});
+ AggregateFunctionNullVariadicInline<AggregateFunctionForEach, true> nullable_foreach(
+ foreach_function, DataTypes {make_nullable(input_type), make_nullable(input_type)},
+ false);
+
+ auto compacted_data = ColumnInt32::create();
+ compacted_data->get_data().assign({30, 40});
+ auto compacted_offsets = ColumnArray::ColumnOffsets::create();
+ compacted_offsets->get_data().assign({0, 1, 2});
+ auto compacted = ColumnArray::create(std::move(compacted_data), std::move(compacted_offsets));
+ auto compacted_nullable =
+ ColumnNullable::create(std::move(compacted), ColumnUInt8::create(3, 0));
+ compacted_nullable->get_null_map_data()[0] = 1;
+
+ auto original_data = ColumnInt32::create();
+ original_data->get_data().assign({10, 20, 300, 400});
+ auto original_offsets = ColumnArray::ColumnOffsets::create();
+ original_offsets->get_data().assign({2, 3, 4});
+ auto original = ColumnArray::create(std::move(original_data), std::move(original_offsets));
+ auto original_nullable = ColumnNullable::create(std::move(original), ColumnUInt8::create(3, 0));
+ original_nullable->get_null_map_data()[0] = 1;
+ const IColumn* columns[] = {compacted_nullable.get(), original_nullable.get()};
+
+ AggregateFunctionGuard state(&nullable_foreach);
+ nullable_foreach.add_batch_single_place(3, state.data(), columns, arena);
+
+ EXPECT_EQ(observed_column_sizes, std::vector<size_t>({2, 2}));
+ auto result = nullable_foreach.get_return_type()->create_column();
+ nullable_foreach.insert_result_into(state.data(), *result);
+ const auto& nullable_result = assert_cast<const ColumnNullable&>(*result);
+ ASSERT_EQ(nullable_result.get_null_map_data(), ColumnUInt8::Container({0}));
+ const auto& result_array = assert_cast<const ColumnArray&>(nullable_result.get_nested_column());
+ const auto& result_data = assert_cast<const ColumnInt64&>(
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column());
+ EXPECT_EQ(result_data.get_data(), ColumnInt64::Container({770}));
+}
+
+TEST_F(AggregateFunctionExceptionTest, NullableForEachStreamingNormalizesVisibleRowsOncePerBatch) {
+ std::vector<size_t> observed_column_sizes;
+ auto nested_function = std::make_shared<PairSumAggregateFunction>(&observed_column_sizes);
+ auto input_type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt32>());
+ auto* foreach_function =
+ new AggregateFunctionForEach(nested_function, DataTypes {input_type, input_type});
+ AggregateFunctionNullVariadicInline<AggregateFunctionForEach, true> nullable_foreach(
+ foreach_function, DataTypes {make_nullable(input_type), make_nullable(input_type)},
+ false);
+
+ auto compacted_data = ColumnInt32::create();
+ compacted_data->get_data().assign({30, 40});
+ auto compacted_offsets = ColumnArray::ColumnOffsets::create();
+ compacted_offsets->get_data().assign({0, 1, 2});
+ auto compacted = ColumnArray::create(std::move(compacted_data), std::move(compacted_offsets));
+ auto compacted_nullable =
+ ColumnNullable::create(std::move(compacted), ColumnUInt8::create(3, 0));
+ compacted_nullable->get_null_map_data()[0] = 1;
+
+ auto original_data = ColumnInt32::create();
+ original_data->get_data().assign({10, 20, 300, 400});
+ auto original_offsets = ColumnArray::ColumnOffsets::create();
+ original_offsets->get_data().assign({2, 3, 4});
+ auto original = ColumnArray::create(std::move(original_data), std::move(original_offsets));
+ auto original_nullable = ColumnNullable::create(std::move(original), ColumnUInt8::create(3, 0));
+ original_nullable->get_null_map_data()[0] = 1;
+ const IColumn* columns[] = {compacted_nullable.get(), original_nullable.get()};
+ MutableColumnPtr serialized = ColumnString::create();
+
+ nullable_foreach.streaming_agg_serialize_to_column(columns, serialized, 3, arena);
+
+ EXPECT_EQ(observed_column_sizes, std::vector<size_t>({2, 2}));
+ EXPECT_EQ(serialized->size(), 3);
+
+ auto result = nullable_foreach.get_return_type()->create_column();
+ const auto& serialized_column = assert_cast<const ColumnString&>(*serialized);
+ for (size_t row = 0; row < serialized_column.size(); ++row) {
+ AggregateFunctionGuard state(&nullable_foreach);
+ VectorBufferReader reader(serialized_column.get_data_at(row));
+ nullable_foreach.deserialize(state.data(), reader, arena);
+ nullable_foreach.insert_result_into(state.data(), *result);
+ }
+ const auto& nullable_result = assert_cast<const ColumnNullable&>(*result);
+ EXPECT_EQ(nullable_result.get_null_map_data(), ColumnUInt8::Container({1, 0, 0}));
+ const auto& result_array = assert_cast<const ColumnArray&>(nullable_result.get_nested_column());
+ const auto& result_data = assert_cast<const ColumnInt64&>(
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column());
+ EXPECT_EQ(result_data.get_data(), ColumnInt64::Container({330, 440}));
+}
+
+TEST_F(AggregateFunctionExceptionTest, NullableForEachStreamingReleasesHeapStatePerRow) {
+ constexpr size_t row_count = 128;
+ constexpr size_t array_size = 8;
+ constexpr size_t payload_size = 4096;
+
+ HeapStringAggregateState::reset_counters();
+ auto nested_function = std::make_shared<HeapStringAggregateFunction>();
+ auto string_array_type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
+ auto int_array_type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt32>());
+ auto* foreach_function = new AggregateFunctionForEach(
+ nested_function, DataTypes {string_array_type, int_array_type});
+ AggregateFunctionNullVariadicInline<AggregateFunctionForEach, true> nullable_foreach(
+ foreach_function,
+ DataTypes {make_nullable(string_array_type), make_nullable(int_array_type)}, false);
+
+ const String payload(payload_size, 'x');
+ auto string_data = ColumnString::create();
+ auto key_data = ColumnInt32::create();
+ auto string_offsets = ColumnArray::ColumnOffsets::create();
+ auto key_offsets = ColumnArray::ColumnOffsets::create();
+ for (size_t row = 0; row < row_count; ++row) {
+ for (size_t element = 0; element < array_size; ++element) {
+ string_data->insert_data(payload.data(), payload.size());
+ key_data->insert_value(static_cast<Int32>(element));
+ }
+ string_offsets->get_data().push_back((row + 1) * array_size);
+ key_offsets->get_data().push_back((row + 1) * array_size);
+ }
+ auto strings = ColumnNullable::create(
+ ColumnArray::create(std::move(string_data), std::move(string_offsets)),
+ ColumnUInt8::create(row_count, 0));
+ auto keys =
+ ColumnNullable::create(ColumnArray::create(std::move(key_data), std::move(key_offsets)),
+ ColumnUInt8::create(row_count, 0));
+ const IColumn* columns[] = {strings.get(), keys.get()};
+ MutableColumnPtr serialized = ColumnString::create();
+
+ nullable_foreach.streaming_agg_serialize_to_column(columns, serialized, row_count, arena);
+
+ EXPECT_EQ(serialized->size(), row_count);
+ EXPECT_LE(HeapStringAggregateState::peak_live_bytes, array_size * payload_size);
+ EXPECT_EQ(HeapStringAggregateState::live_bytes, 0);
+}
+
+} // namespace doris
diff --git a/be/test/exprs/function/function_array_split_test.cpp b/be/test/exprs/function/function_array_split_test.cpp
new file mode 100644
index 00000000000..a6991c5e6c2
--- /dev/null
+++ b/be/test/exprs/function/function_array_split_test.cpp
@@ -0,0 +1,224 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "common/status.h"
+#include "core/assert_cast.h"
+#include "core/block/block.h"
+#include "core/column/column_array.h"
+#include "core/column/column_const.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_vector.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "exprs/function/simple_function_factory.h"
+#include "runtime/memory/mem_tracker_limiter.h"
+#include "runtime/thread_context.h"
+#include "testutil/function_utils.h"
+
+namespace doris {
+
+namespace {
+
+using IntRows = std::vector<std::vector<Int32>>;
+
+ColumnPtr create_int_array_column(const IntRows& rows) {
+ auto data = ColumnInt32::create();
+ auto nested_null_map = ColumnUInt8::create();
+ auto offsets = ColumnArray::ColumnOffsets::create();
+
+ size_t offset = 0;
+ for (const auto& row : rows) {
+ for (const auto value : row) {
+ data->get_data().push_back(value);
+ nested_null_map->get_data().push_back(0);
+ }
+ offset += row.size();
+ offsets->get_data().push_back(offset);
+ }
+
+ return ColumnArray::create(ColumnNullable::create(std::move(data), std::move(nested_null_map)),
+ std::move(offsets));
+}
+
+ColumnPtr create_compacted_nullable_predicate(const std::vector<UInt8>& valid_row) {
+ auto data = ColumnUInt8::create();
+ data->get_data().assign(valid_row.begin(), valid_row.end());
+ auto nested_null_map = ColumnUInt8::create(valid_row.size(), 0);
+ auto offsets = ColumnArray::ColumnOffsets::create();
+ offsets->get_data().assign({0, valid_row.size()});
+ auto array =
+ ColumnArray::create(ColumnNullable::create(std::move(data), std::move(nested_null_map)),
+ std::move(offsets));
+ auto array_null_map = ColumnUInt8::create();
+ array_null_map->get_data().assign({1, 0});
+ return ColumnNullable::create(std::move(array), std::move(array_null_map));
+}
+
+ColumnPtr create_const_predicate(const std::vector<UInt8>& row, size_t rows) {
+ auto data = ColumnUInt8::create();
+ data->get_data().assign(row.begin(), row.end());
+ auto nested_null_map = ColumnUInt8::create(row.size(), 0);
+ auto offsets = ColumnArray::ColumnOffsets::create();
+ offsets->get_data().push_back(row.size());
+ auto array =
+ ColumnArray::create(ColumnNullable::create(std::move(data), std::move(nested_null_map)),
+ std::move(offsets));
+ auto nullable = ColumnNullable::create(std::move(array), ColumnUInt8::create(1, 0));
+ return ColumnConst::create(std::move(nullable), rows);
+}
+
+ColumnPtr create_all_null_predicate(size_t rows) {
+ auto offsets = ColumnArray::ColumnOffsets::create();
+ offsets->get_data().resize_fill(rows, 0);
+ auto array = ColumnArray::create(
+ ColumnNullable::create(ColumnUInt8::create(), ColumnUInt8::create()),
+ std::move(offsets));
+ return ColumnNullable::create(std::move(array), ColumnUInt8::create(rows, 1));
+}
+
+DataTypePtr int_array_type() {
+ return std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeInt32>()));
+}
+
+DataTypePtr nullable_bool_array_type() {
+ auto array = std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeUInt8>()));
+ return make_nullable(array);
+}
+
+Status execute_array_split(const std::string& name, ColumnPtr source, ColumnPtr predicate,
+ Block* block) {
+ const size_t input_rows_count = source->size();
+ auto source_type = int_array_type();
+ auto predicate_type = nullable_bool_array_type();
+ auto result_type = make_nullable(std::make_shared<DataTypeArray>(make_nullable(source_type)));
+ block->insert({std::move(source), source_type, "source"});
+ block->insert({std::move(predicate), predicate_type, "predicate"});
+
+ auto function = SimpleFunctionFactory::instance().get_function(
+ name, block->get_columns_with_type_and_name(), result_type);
+ EXPECT_NE(function, nullptr);
+
+ FunctionUtils fn_utils(result_type, {source_type, predicate_type}, false);
+ auto* fn_ctx = fn_utils.get_fn_ctx();
+ RETURN_IF_ERROR(function->open(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
+ RETURN_IF_ERROR(function->open(fn_ctx, FunctionContext::THREAD_LOCAL));
+ block->insert({nullptr, result_type, "result"});
+ auto status = function->execute(fn_ctx, *block, {0, 1}, 2, input_rows_count);
+ RETURN_IF_ERROR(function->close(fn_ctx, FunctionContext::THREAD_LOCAL));
+ RETURN_IF_ERROR(function->close(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
+ return status;
+}
+
+void expect_all_null_predicate_skips_const_source_expansion(const std::string& name) {
+ constexpr size_t row_count = 512;
+ constexpr size_t array_size = 4096;
+ constexpr int64_t max_execution_bytes = 1024 * 1024;
+
+ auto source = ColumnConst::create(create_int_array_column({std::vector<Int32>(array_size, 1)}),
+ row_count);
+ auto predicate = create_all_null_predicate(row_count);
+
+ auto tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER,
+ "ArraySplitAllNullFastPath");
+ auto switch_tracker = SwitchThreadMemTrackerLimiter(tracker);
+ thread_context()->thread_mem_tracker_mgr->flush_untracked_mem();
+ const int64_t baseline = tracker->consumption();
+
+ Block block;
+ auto status = execute_array_split(name, std::move(source), std::move(predicate), &block);
+ thread_context()->thread_mem_tracker_mgr->flush_untracked_mem();
+ const int64_t execution_peak = tracker->peak_consumption() - baseline;
+
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_TRUE(block.get_by_position(2).column->only_null());
+ EXPECT_EQ(block.get_by_position(2).column->size(), row_count);
+ EXPECT_LT(execution_peak, max_execution_bytes);
+}
+
+void expect_nullable_split_result(const std::string& name, const std::vector<UInt8>& predicate) {
+ Block block;
+ auto status = execute_array_split(name, create_int_array_column({{100, 101}, {1, 2}}),
+ create_compacted_nullable_predicate(predicate), &block);
+ ASSERT_TRUE(status.ok()) << status;
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(2).column);
+ EXPECT_EQ(result.get_null_map_data(), ColumnUInt8::Container({1, 0}));
+
+ const auto& outer_array = assert_cast<const ColumnArray&>(result.get_nested_column());
+ const auto& inner_array = assert_cast<const ColumnArray&>(
+ assert_cast<const ColumnNullable&>(outer_array.get_data()).get_nested_column());
+ const size_t valid_row_begin = outer_array.get_offsets()[0];
+ EXPECT_EQ(outer_array.get_offsets()[1] - valid_row_begin, 2);
+ EXPECT_EQ(inner_array.size_at(valid_row_begin), 1);
+ EXPECT_EQ(inner_array.size_at(valid_row_begin + 1), 1);
+}
+
+} // namespace
+
+TEST(function_array_split_test, skips_null_array_row_with_compacted_predicate) {
+ expect_nullable_split_result("array_split", {0, 1});
+}
+
+TEST(function_array_split_test, reverse_skips_null_array_row_with_compacted_predicate) {
+ expect_nullable_split_result("array_reverse_split", {1, 0});
+}
+
+TEST(function_array_split_test, rejects_mismatched_non_null_row_after_null_array_row) {
+ Block block;
+ auto status = execute_array_split("array_split", create_int_array_column({{100, 101}, {1, 2}}),
+ create_compacted_nullable_predicate({1}), &block);
+ ASSERT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("uneven arguments on row 1"), std::string::npos) << status;
+}
+
+TEST(function_array_split_test, reads_const_predicate_for_each_source_row) {
+ Block block;
+ auto status = execute_array_split("array_split", create_int_array_column({{1, 2}, {3, 4}}),
+ create_const_predicate({0, 1}, 2), &block);
+ ASSERT_TRUE(status.ok()) << status;
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(2).column);
+ const auto& outer_array = assert_cast<const ColumnArray&>(result.get_nested_column());
+ const auto& inner_array = assert_cast<const ColumnArray&>(
+ assert_cast<const ColumnNullable&>(outer_array.get_data()).get_nested_column());
+ ASSERT_EQ(outer_array.get_offsets(), ColumnArray::Offsets64({2, 4}));
+ ASSERT_EQ(inner_array.get_offsets(), ColumnArray::Offsets64({1, 2, 3, 4}));
+ EXPECT_EQ(inner_array.size_at(0), 1);
+ EXPECT_EQ(inner_array.size_at(1), 1);
+ EXPECT_EQ(inner_array.size_at(2), 1);
+ EXPECT_EQ(inner_array.size_at(3), 1);
+ const auto& values = assert_cast<const ColumnInt32&>(
+ assert_cast<const ColumnNullable&>(inner_array.get_data()).get_nested_column());
+ EXPECT_EQ(values.get_data(), ColumnInt32::Container({1, 2, 3, 4}));
+}
+
+TEST(function_array_split_test, all_null_predicate_skips_const_source_expansion) {
+ expect_all_null_predicate_skips_const_source_expansion("array_split");
+}
+
+TEST(function_array_split_test, reverse_all_null_predicate_skips_const_source_expansion) {
+ expect_all_null_predicate_skips_const_source_expansion("array_reverse_split");
+}
+
+} // namespace doris
diff --git a/be/test/exprs/function/function_array_zip_test.cpp b/be/test/exprs/function/function_array_zip_test.cpp
new file mode 100644
index 00000000000..1a6b5d6c441
--- /dev/null
+++ b/be/test/exprs/function/function_array_zip_test.cpp
@@ -0,0 +1,396 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <vector>
+
+#include "common/status.h"
+#include "core/assert_cast.h"
+#include "core/block/block.h"
+#include "core/column/column_array.h"
+#include "core/column/column_const.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_struct.h"
+#include "core/column/column_vector.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_factory.hpp"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_struct.h"
+#include "exprs/function/simple_function_factory.h"
+#include "runtime/memory/mem_tracker_limiter.h"
+#include "runtime/thread_context.h"
+#include "testutil/function_utils.h"
+
+namespace doris {
+
+namespace {
+
+using IntRows = std::vector<std::vector<Int32>>;
+
+ColumnPtr create_array_column(const IntRows& rows) {
+ auto data = ColumnInt32::create();
+ auto nested_null_map = ColumnUInt8::create();
+ auto offsets = ColumnArray::ColumnOffsets::create();
+
+ size_t offset = 0;
+ for (const auto& row : rows) {
+ for (const auto value : row) {
+ data->get_data().push_back(value);
+ nested_null_map->get_data().push_back(0);
+ }
+ offset += row.size();
+ offsets->get_data().push_back(offset);
+ }
+
+ return ColumnArray::create(ColumnNullable::create(std::move(data), std::move(nested_null_map)),
+ std::move(offsets));
+}
+
+DataTypePtr array_int_type() {
+ return std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeInt32>()));
+}
+
+DataTypePtr array_zip_result_type(size_t field_count, bool nullable) {
+ DataTypes fields(field_count, make_nullable(std::make_shared<DataTypeInt32>()));
+ auto tuple_type = make_nullable(std::make_shared<DataTypeStruct>(fields));
+ auto array_type = std::make_shared<DataTypeArray>(std::move(tuple_type));
+ return nullable ? make_nullable(array_type) : array_type;
+}
+
+Status execute_array_zip(ColumnPtr lhs, ColumnPtr rhs, const DataTypePtr& lhs_type,
+ const DataTypePtr& rhs_type, size_t rows, Block* block) {
+ auto result_type = array_zip_result_type(2, lhs_type->is_nullable() || rhs_type->is_nullable());
+ block->insert({std::move(lhs), lhs_type, "lhs"});
+ block->insert({std::move(rhs), rhs_type, "rhs"});
+
+ auto function = SimpleFunctionFactory::instance().get_function(
+ "array_zip", block->get_columns_with_type_and_name(), result_type);
+ EXPECT_NE(function, nullptr);
+
+ FunctionUtils fn_utils(result_type, {lhs_type, rhs_type}, false);
+ auto* fn_ctx = fn_utils.get_fn_ctx();
+ RETURN_IF_ERROR(function->open(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
+ RETURN_IF_ERROR(function->open(fn_ctx, FunctionContext::THREAD_LOCAL));
+ block->insert({nullptr, result_type, "result"});
+ auto status = function->execute(fn_ctx, *block, {0, 1}, 2, rows);
+ RETURN_IF_ERROR(function->close(fn_ctx, FunctionContext::THREAD_LOCAL));
+ RETURN_IF_ERROR(function->close(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
+ return status;
+}
+
+} // namespace
+
+TEST(function_array_zip_test, supports_untyped_null_literal) {
+ auto null_literal_type =
+ DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_NULL, true);
+ auto array_type = array_int_type();
+ Block block;
+
+ auto status = execute_array_zip(null_literal_type->create_column_const(2, Field()),
+ create_array_column({{10}, {20}}), null_literal_type,
+ array_type, 2, &block);
+
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_TRUE(block.get_by_position(2).column->only_null());
+ EXPECT_EQ(block.get_by_position(2).column->size(), 2);
+}
+
+TEST(function_array_zip_test, ignores_hidden_payload_of_outer_null_rows) {
+ auto lhs_array = create_array_column({{100, 101}, {1}, {200, 201}});
+ auto lhs_null_map = ColumnUInt8::create();
+ lhs_null_map->get_data().assign({1, 0, 1});
+ auto lhs = ColumnNullable::create(std::move(lhs_array), std::move(lhs_null_map));
+ auto rhs = create_array_column({{10}, {20}, {30}});
+
+ auto array_type = array_int_type();
+ Block block;
+ auto status = execute_array_zip(std::move(lhs), std::move(rhs), make_nullable(array_type),
+ array_type, 3, &block);
+ ASSERT_TRUE(status.ok()) << status;
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(2).column);
+ EXPECT_EQ(result.get_null_map_data(), ColumnUInt8::Container({1, 0, 1}));
+
+ const auto& result_array = assert_cast<const ColumnArray&>(result.get_nested_column());
+ EXPECT_EQ(result_array.get_offsets(), ColumnArray::Offsets64({0, 1, 1}));
+
+ const auto& nullable_tuple =
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column();
+ const auto& tuple = assert_cast<const ColumnStruct&>(nullable_tuple);
+ const auto& lhs_values = assert_cast<const ColumnInt32&>(
+ assert_cast<const ColumnNullable&>(*tuple.get_columns()[0]).get_nested_column());
+ const auto& rhs_values = assert_cast<const ColumnInt32&>(
+ assert_cast<const ColumnNullable&>(*tuple.get_columns()[1]).get_nested_column());
+ ASSERT_EQ(lhs_values.size(), 1);
+ ASSERT_EQ(rhs_values.size(), 1);
+ EXPECT_EQ(lhs_values.get_data()[0], 1);
+ EXPECT_EQ(rhs_values.get_data()[0], 20);
+}
+
+TEST(function_array_zip_test, equal_hidden_offsets_reuse_input_columns) {
+ auto lhs_array = create_array_column({{100}, {1}});
+ const auto& lhs_array_ref = assert_cast<const ColumnArray&>(*lhs_array);
+ const auto* lhs_data = lhs_array_ref.get_data_ptr().get();
+ const auto* lhs_offsets = lhs_array_ref.get_offsets_ptr().get();
+ auto lhs_null_map = ColumnUInt8::create();
+ lhs_null_map->get_data().assign({1, 0});
+ auto lhs = ColumnNullable::create(std::move(lhs_array), std::move(lhs_null_map));
+ auto rhs = create_array_column({{10}, {20}});
+ const auto& rhs_array = assert_cast<const ColumnArray&>(*rhs);
+ const auto* rhs_data = rhs_array.get_data_ptr().get();
+
+ auto array_type = array_int_type();
+ Block block;
+ auto status = execute_array_zip(std::move(lhs), std::move(rhs), make_nullable(array_type),
+ array_type, 2, &block);
+ ASSERT_TRUE(status.ok()) << status;
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(2).column);
+ EXPECT_EQ(result.get_null_map_data(), ColumnUInt8::Container({1, 0}));
+
+ const auto& result_array = assert_cast<const ColumnArray&>(result.get_nested_column());
+ EXPECT_EQ(result_array.get_offsets_ptr().get(), lhs_offsets);
+ EXPECT_EQ(result_array.get_offsets(), ColumnArray::Offsets64({1, 2}));
+
+ const auto& nullable_tuple =
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column();
+ const auto& tuple = assert_cast<const ColumnStruct&>(nullable_tuple);
+ EXPECT_EQ(tuple.get_columns()[0].get(), lhs_data);
+ EXPECT_EQ(tuple.get_columns()[1].get(), rhs_data);
+}
+
+TEST(function_array_zip_test, rejects_different_lengths_on_non_null_rows) {
+ auto array_type = array_int_type();
+ Block block;
+ auto status = execute_array_zip(create_array_column({{1, 2}}), create_array_column({{10}}),
+ array_type, array_type, 1, &block);
+ ASSERT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("same offsets"), std::string::npos) << status;
+}
+
+TEST(function_array_zip_test, validates_late_mismatch_before_compacting) {
+ constexpr size_t row_count = 64;
+ constexpr size_t row_size = 2048;
+ constexpr int64_t max_execution_bytes = 256 * 1024;
+
+ IntRows lhs_rows(row_count, std::vector<Int32>(row_size, 1));
+ IntRows rhs_rows(row_count, std::vector<Int32>(row_size, 2));
+ lhs_rows[0] = {999};
+ rhs_rows[0].clear();
+ rhs_rows.back().pop_back();
+
+ auto lhs_array = create_array_column(lhs_rows);
+ auto lhs_null_map = ColumnUInt8::create(row_count, 0);
+ lhs_null_map->get_data()[0] = 1;
+ auto lhs = ColumnNullable::create(std::move(lhs_array), std::move(lhs_null_map));
+ auto rhs = create_array_column(rhs_rows);
+
+ auto tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER,
+ "ArrayZipValidateBeforeCompact");
+ auto switch_tracker = SwitchThreadMemTrackerLimiter(tracker);
+ thread_context()->thread_mem_tracker_mgr->flush_untracked_mem();
+ const int64_t baseline = tracker->consumption();
+
+ auto array_type = array_int_type();
+ Block block;
+ auto status = execute_array_zip(std::move(lhs), std::move(rhs), make_nullable(array_type),
+ array_type, row_count, &block);
+ thread_context()->thread_mem_tracker_mgr->flush_untracked_mem();
+ const int64_t execution_peak = tracker->peak_consumption() - baseline;
+
+ ASSERT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("same offsets"), std::string::npos) << status;
+ EXPECT_LT(execution_peak, max_execution_bytes);
+}
+
+TEST(function_array_zip_test, aligned_offsets_reuse_input_columns) {
+ auto lhs = create_array_column({{1, 2}, {3}});
+ auto rhs = create_array_column({{10, 20}, {30}});
+ const auto& lhs_array = assert_cast<const ColumnArray&>(*lhs);
+ const auto& rhs_array = assert_cast<const ColumnArray&>(*rhs);
+ const auto* lhs_data = lhs_array.get_data_ptr().get();
+ const auto* rhs_data = rhs_array.get_data_ptr().get();
+ const auto* lhs_offsets = lhs_array.get_offsets_ptr().get();
+
+ auto array_type = array_int_type();
+ Block block;
+ auto status =
+ execute_array_zip(std::move(lhs), std::move(rhs), array_type, array_type, 2, &block);
+ ASSERT_TRUE(status.ok()) << status;
+
+ const auto& result_array = assert_cast<const ColumnArray&>(*block.get_by_position(2).column);
+ EXPECT_EQ(result_array.get_offsets(), ColumnArray::Offsets64({2, 3}));
+ EXPECT_EQ(result_array.get_offsets_ptr().get(), lhs_offsets);
+ const auto& nullable_tuple =
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column();
+ const auto& tuple = assert_cast<const ColumnStruct&>(nullable_tuple);
+ EXPECT_EQ(tuple.get_columns()[0].get(), lhs_data);
+ EXPECT_EQ(tuple.get_columns()[1].get(), rhs_data);
+ const auto& lhs_values = assert_cast<const ColumnInt32&>(
+ assert_cast<const ColumnNullable&>(*tuple.get_columns()[0]).get_nested_column());
+ const auto& rhs_values = assert_cast<const ColumnInt32&>(
+ assert_cast<const ColumnNullable&>(*tuple.get_columns()[1]).get_nested_column());
+ EXPECT_EQ(lhs_values.get_data(), ColumnInt32::Container({1, 2, 3}));
+ EXPECT_EQ(rhs_values.get_data(), ColumnInt32::Container({10, 20, 30}));
+}
+
+TEST(function_array_zip_test, empty_outer_null_rows_reuse_input_columns) {
+ auto lhs_array = create_array_column({{}, {1, 2}});
+ const auto& lhs_array_ref = assert_cast<const ColumnArray&>(*lhs_array);
+ const auto* lhs_data = lhs_array_ref.get_data_ptr().get();
+ const auto* lhs_offsets = lhs_array_ref.get_offsets_ptr().get();
+ auto lhs_null_map = ColumnUInt8::create();
+ lhs_null_map->get_data().assign({1, 0});
+ auto lhs = ColumnNullable::create(std::move(lhs_array), std::move(lhs_null_map));
+
+ auto rhs = create_array_column({{}, {10, 20}});
+ const auto& rhs_array = assert_cast<const ColumnArray&>(*rhs);
+ const auto* rhs_data = rhs_array.get_data_ptr().get();
+
+ auto array_type = array_int_type();
+ Block block;
+ auto status = execute_array_zip(std::move(lhs), std::move(rhs), make_nullable(array_type),
+ array_type, 2, &block);
+ ASSERT_TRUE(status.ok()) << status;
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(2).column);
+ EXPECT_EQ(result.get_null_map_data(), ColumnUInt8::Container({1, 0}));
+ const auto& result_array = assert_cast<const ColumnArray&>(result.get_nested_column());
+ EXPECT_EQ(result_array.get_offsets_ptr().get(), lhs_offsets);
+
+ const auto& nullable_tuple =
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column();
+ const auto& tuple = assert_cast<const ColumnStruct&>(nullable_tuple);
+ EXPECT_EQ(tuple.get_columns()[0].get(), lhs_data);
+ EXPECT_EQ(tuple.get_columns()[1].get(), rhs_data);
+}
+
+TEST(function_array_zip_test, const_array_does_not_copy_aligned_normal_input) {
+ auto lhs = ColumnConst::create(create_array_column({{1, 2}}), 2);
+ auto rhs = create_array_column({{10, 20}, {30, 40}});
+ const auto& rhs_array = assert_cast<const ColumnArray&>(*rhs);
+ const auto* rhs_data = rhs_array.get_data_ptr().get();
+ const auto* rhs_offsets = rhs_array.get_offsets_ptr().get();
+
+ auto array_type = array_int_type();
+ Block block;
+ auto status =
+ execute_array_zip(std::move(lhs), std::move(rhs), array_type, array_type, 2, &block);
+ ASSERT_TRUE(status.ok()) << status;
+
+ const auto& result_array = assert_cast<const ColumnArray&>(*block.get_by_position(2).column);
+ EXPECT_EQ(result_array.get_offsets(), ColumnArray::Offsets64({2, 4}));
+ EXPECT_EQ(result_array.get_offsets_ptr().get(), rhs_offsets);
+
+ const auto& nullable_tuple =
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column();
+ const auto& tuple = assert_cast<const ColumnStruct&>(nullable_tuple);
+ EXPECT_EQ(tuple.get_columns()[1].get(), rhs_data);
+ const auto& lhs_values = assert_cast<const ColumnInt32&>(
+ assert_cast<const ColumnNullable&>(*tuple.get_columns()[0]).get_nested_column());
+ EXPECT_EQ(lhs_values.get_data(), ColumnInt32::Container({1, 2, 1, 2}));
+}
+
+TEST(function_array_zip_test, supports_all_const_arrays) {
+ auto lhs = ColumnConst::create(create_array_column({{1, 2}}), 2);
+ auto rhs = ColumnConst::create(create_array_column({{10, 20}}), 2);
+
+ auto array_type = array_int_type();
+ Block block;
+ auto status =
+ execute_array_zip(std::move(lhs), std::move(rhs), array_type, array_type, 2, &block);
+ ASSERT_TRUE(status.ok()) << status;
+
+ const auto result_column = block.get_by_position(2).column->convert_to_full_column_if_const();
+ const auto& result_array = assert_cast<const ColumnArray&>(*result_column);
+ EXPECT_EQ(result_array.get_offsets(), ColumnArray::Offsets64({2, 4}));
+
+ const auto& nullable_tuple =
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column();
+ const auto& tuple = assert_cast<const ColumnStruct&>(nullable_tuple);
+ const auto& lhs_values = assert_cast<const ColumnInt32&>(
+ assert_cast<const ColumnNullable&>(*tuple.get_columns()[0]).get_nested_column());
+ const auto& rhs_values = assert_cast<const ColumnInt32&>(
+ assert_cast<const ColumnNullable&>(*tuple.get_columns()[1]).get_nested_column());
+ EXPECT_EQ(lhs_values.get_data(), ColumnInt32::Container({1, 2, 1, 2}));
+ EXPECT_EQ(rhs_values.get_data(), ColumnInt32::Container({10, 20, 10, 20}));
+}
+
+TEST(function_array_zip_test, const_array_ignores_mismatched_hidden_payload) {
+ auto lhs = ColumnConst::create(create_array_column({{1, 2}}), 3);
+ auto rhs_array = create_array_column({{10}, {20, 30}, {40, 50}});
+ auto rhs_null_map = ColumnUInt8::create();
+ rhs_null_map->get_data().assign({1, 0, 0});
+ auto rhs = ColumnNullable::create(std::move(rhs_array), std::move(rhs_null_map));
+
+ auto array_type = array_int_type();
+ Block block;
+ auto status = execute_array_zip(std::move(lhs), std::move(rhs), array_type,
+ make_nullable(array_type), 3, &block);
+ ASSERT_TRUE(status.ok()) << status;
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(2).column);
+ EXPECT_EQ(result.get_null_map_data(), ColumnUInt8::Container({1, 0, 0}));
+
+ const auto& result_array = assert_cast<const ColumnArray&>(result.get_nested_column());
+ EXPECT_EQ(result_array.get_offsets(), ColumnArray::Offsets64({0, 2, 4}));
+
+ const auto& nullable_tuple =
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column();
+ const auto& tuple = assert_cast<const ColumnStruct&>(nullable_tuple);
+ const auto& lhs_values = assert_cast<const ColumnInt32&>(
+ assert_cast<const ColumnNullable&>(*tuple.get_columns()[0]).get_nested_column());
+ const auto& rhs_values = assert_cast<const ColumnInt32&>(
+ assert_cast<const ColumnNullable&>(*tuple.get_columns()[1]).get_nested_column());
+ EXPECT_EQ(lhs_values.get_data(), ColumnInt32::Container({1, 2, 1, 2}));
+ EXPECT_EQ(rhs_values.get_data(), ColumnInt32::Container({20, 30, 40, 50}));
+}
+
+TEST(function_array_zip_test, nullable_const_array_reuses_aligned_normal_input) {
+ auto lhs_null_map = ColumnUInt8::create();
+ lhs_null_map->get_data().push_back(0);
+ auto lhs_value = ColumnNullable::create(create_array_column({{1, 2}}), std::move(lhs_null_map));
+ auto lhs = ColumnConst::create(std::move(lhs_value), 2);
+ auto rhs = create_array_column({{10, 20}, {30, 40}});
+ const auto& rhs_array = assert_cast<const ColumnArray&>(*rhs);
+ const auto* rhs_data = rhs_array.get_data_ptr().get();
+ const auto* rhs_offsets = rhs_array.get_offsets_ptr().get();
+
+ auto array_type = array_int_type();
+ Block block;
+ auto status = execute_array_zip(std::move(lhs), std::move(rhs), make_nullable(array_type),
+ array_type, 2, &block);
+ ASSERT_TRUE(status.ok()) << status;
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(2).column);
+ EXPECT_EQ(result.get_null_map_data(), ColumnUInt8::Container({0, 0}));
+
+ const auto& result_array = assert_cast<const ColumnArray&>(result.get_nested_column());
+ EXPECT_EQ(result_array.get_offsets_ptr().get(), rhs_offsets);
+ const auto& nullable_tuple =
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column();
+ const auto& tuple = assert_cast<const ColumnStruct&>(nullable_tuple);
+ EXPECT_EQ(tuple.get_columns()[1].get(), rhs_data);
+ const auto& lhs_values = assert_cast<const ColumnInt32&>(
+ assert_cast<const ColumnNullable&>(*tuple.get_columns()[0]).get_nested_column());
+ EXPECT_EQ(lhs_values.get_data(), ColumnInt32::Container({1, 2, 1, 2}));
+}
+
+} // namespace doris
diff --git a/be/test/exprs/lambda_function/array_map_function_test.cpp b/be/test/exprs/lambda_function/array_map_function_test.cpp
index 42469b52c3b..a93d706d404 100644
--- a/be/test/exprs/lambda_function/array_map_function_test.cpp
+++ b/be/test/exprs/lambda_function/array_map_function_test.cpp
@@ -32,20 +32,33 @@
#include "core/data_type/data_type_array.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
+#include "exprs/function/simple_function_factory.h"
#include "exprs/vcolumn_ref.h"
#include "exprs/vexpr_context.h"
#include "exprs/vlambda_function_call_expr.h"
#include "exprs/vlambda_function_expr.h"
#include "exprs/vslot_ref.h"
#include "runtime/descriptors.h"
+#include "runtime/memory/mem_tracker_limiter.h"
#include "runtime/runtime_state.h"
+#include "runtime/thread_context.h"
+#include "testutil/function_utils.h"
namespace doris {
-class MockColumnExpr final : public VExpr {
+// The type-only VExpr constructor leaves node_type unset for non-SlotRef expressions.
+// Give mock expressions a stable type so lambda binding never mistakes them for ColumnRef.
+class MockExprBase : public VExpr {
+public:
+ explicit MockExprBase(DataTypePtr type) : VExpr(std::move(type), false) {
+ set_node_type(TExprNodeType::FUNCTION_CALL);
+ }
+};
+
+class MockColumnExpr final : public MockExprBase {
public:
MockColumnExpr(ColumnPtr column, DataTypePtr type, std::string name)
- : VExpr(type, false),
+ : MockExprBase(type),
_column(std::move(column)),
_type(std::move(type)),
_name(std::move(name)) {}
@@ -67,10 +80,10 @@ private:
std::string _name;
};
-class MockConstColumnExpr final : public VExpr {
+class MockConstColumnExpr final : public MockExprBase {
public:
MockConstColumnExpr(ColumnPtr column, DataTypePtr type, std::string name)
- : VExpr(type, false),
+ : MockExprBase(type),
_column(std::move(column)),
_type(std::move(type)),
_name(std::move(name)) {}
@@ -92,10 +105,10 @@ private:
std::string _name;
};
-class MockBodyExpr final : public VExpr {
+class MockBodyExpr final : public MockExprBase {
public:
MockBodyExpr(DataTypePtr type, std::string name)
- : VExpr(type, false), _type(std::move(type)), _name(std::move(name)) {}
+ : MockExprBase(type), _type(std::move(type)), _name(std::move(name)) {}
const std::string& expr_name() const override { return _name; }
@@ -112,9 +125,40 @@ private:
std::string _name;
};
-class MockSubtractExpr final : public VExpr {
+class MockPositiveExpr final : public MockExprBase {
+public:
+ MockPositiveExpr(DataTypePtr type) : MockExprBase(type), _type(std::move(type)) {}
+
+ const std::string& expr_name() const override { return _name; }
+
+ Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector,
+ size_t count, ColumnPtr& result_column) const override {
+ ColumnPtr input;
+ RETURN_IF_ERROR(get_child(0)->execute_column(context, block, selector, count, input));
+ const IColumn* data = input.get();
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(data)) {
+ data = &nullable->get_nested_column();
+ }
+ const auto& values = assert_cast<const ColumnInt32&>(*data);
+ auto result = ColumnUInt8::create();
+ result->reserve(count);
+ for (size_t i = 0; i < count; ++i) {
+ result->insert_value(values.get_element(i) > 0);
+ }
+ result_column = std::move(result);
+ return Status::OK();
+ }
+
+ DataTypePtr execute_type(const Block* /*block*/) const override { return _type; }
+
+private:
+ DataTypePtr _type;
+ std::string _name = "mock_positive";
+};
+
+class MockSubtractExpr final : public MockExprBase {
public:
- explicit MockSubtractExpr(DataTypePtr type) : VExpr(type, false), _type(std::move(type)) {}
+ explicit MockSubtractExpr(DataTypePtr type) : MockExprBase(type), _type(std::move(type)) {}
const std::string& expr_name() const override { return _name; }
@@ -151,14 +195,20 @@ private:
std::string _name = "mock_subtract";
};
-class MockAddExpr final : public VExpr {
+class MockAddExpr final : public MockExprBase {
public:
- explicit MockAddExpr(DataTypePtr type) : VExpr(type, false), _type(std::move(type)) {}
+ explicit MockAddExpr(DataTypePtr type, std::vector<size_t>* observed_batch_sizes = nullptr)
+ : MockExprBase(type),
+ _type(std::move(type)),
+ _observed_batch_sizes(observed_batch_sizes) {}
const std::string& expr_name() const override { return _name; }
Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector,
size_t count, ColumnPtr& result_column) const override {
+ if (_observed_batch_sizes != nullptr) {
+ _observed_batch_sizes->push_back(count);
+ }
ColumnPtr left;
ColumnPtr right;
RETURN_IF_ERROR(get_child(0)->execute_column(context, block, selector, count, left));
@@ -187,12 +237,13 @@ private:
}
DataTypePtr _type;
+ std::vector<size_t>* _observed_batch_sizes;
std::string _name = "mock_add";
};
-class MockMultiplyExpr final : public VExpr {
+class MockMultiplyExpr final : public MockExprBase {
public:
- explicit MockMultiplyExpr(DataTypePtr type) : VExpr(type, false), _type(std::move(type)) {}
+ explicit MockMultiplyExpr(DataTypePtr type) : MockExprBase(type), _type(std::move(type)) {}
const std::string& expr_name() const override { return _name; }
@@ -229,9 +280,9 @@ private:
std::string _name = "mock_multiply";
};
-class MockCompareExpr final : public VExpr {
+class MockCompareExpr final : public MockExprBase {
public:
- explicit MockCompareExpr(DataTypePtr type) : VExpr(type, false), _type(std::move(type)) {}
+ explicit MockCompareExpr(DataTypePtr type) : MockExprBase(type), _type(std::move(type)) {}
const std::string& expr_name() const override { return _name; }
@@ -360,6 +411,16 @@ static ColumnPtr make_int_array_column(const std::vector<std::vector<int32_t>>&
return ColumnArray::create(std::move(nullable_int_column), std::move(offsets));
}
+static ColumnPtr make_nullable_int_array_column(const std::vector<std::vector<int32_t>>& rows,
+ const std::vector<uint8_t>& array_null_map) {
+ auto array_column = IColumn::mutate(make_int_array_column(rows));
+ auto null_map = ColumnUInt8::create();
+ for (uint8_t is_null : array_null_map) {
+ null_map->insert_value(is_null);
+ }
+ return ColumnNullable::create(std::move(array_column), std::move(null_map));
+}
+
static ColumnPtr make_nested_int_array_column() {
// Two input rows:
// row 0: [[1, 2], [3]]
@@ -425,6 +486,503 @@ static void open_expr(const VExprSPtr& expr, VExprContext* context) {
ASSERT_TRUE(expr->open(&state, context, FunctionContext::THREAD_LOCAL).ok());
}
+static void open_expr_with_batch_size(const VExprSPtr& expr, VExprContext* context,
+ int batch_size) {
+ RuntimeState state;
+ TQueryOptions query_options;
+ query_options.__set_batch_size(batch_size);
+ state.set_query_options(query_options);
+ RowDescriptor row_desc;
+ ASSERT_TRUE(expr->prepare(&state, row_desc, context).ok());
+ ASSERT_TRUE(expr->open(&state, context, FunctionContext::THREAD_LOCAL).ok());
+}
+
+TEST(ArrayFilterFunctionTest, NullableSecondaryLambdaArrayPropagatesToResult) {
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto bool_type = std::make_shared<DataTypeUInt8>();
+ auto array_int_type = std::make_shared<DataTypeArray>(int_type);
+ auto nullable_array_int_type = std::make_shared<DataTypeNullable>(array_int_type);
+ auto array_bool_type = std::make_shared<DataTypeArray>(make_nullable(bool_type));
+ auto nullable_array_bool_type = std::make_shared<DataTypeNullable>(array_bool_type);
+
+ auto filter = VLambdaFunctionCallExpr::create_shared(
+ make_lambda_call_node(nullable_array_int_type, 2, "array_filter"));
+ filter->add_child(std::make_shared<MockColumnExpr>(make_int_array_column({{1, 2}, {3, 4}}),
+ array_int_type, "source"));
+
+ auto map = VLambdaFunctionCallExpr::create_shared(
+ make_lambda_call_node(nullable_array_bool_type, 3));
+ auto lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(bool_type, {"x", "y"}));
+ auto body = std::make_shared<MockPositiveExpr>(bool_type);
+ body->add_child(VColumnRef::create_shared(make_column_ref_node(1, "y", int_type)));
+ lambda->add_child(body);
+ map->add_child(lambda);
+ map->add_child(std::make_shared<MockColumnExpr>(make_int_array_column({{1, 2}, {3, 4}}),
+ array_int_type, "source"));
+ map->add_child(std::make_shared<MockColumnExpr>(
+ make_nullable_int_array_column({{100, 101}, {10, 0}}, {1, 0}), nullable_array_int_type,
+ "secondary"));
+ filter->add_child(map);
+
+ VExprContext context(filter);
+ open_expr(filter, &context);
+
+ Block block;
+ ColumnPtr result;
+ auto status = filter->execute_column(&context, &block, nullptr, 2, result);
+ ASSERT_TRUE(status.ok()) << status.to_string();
+
+ const auto& nullable_result = assert_cast<const ColumnNullable&>(*result);
+ EXPECT_EQ(nullable_result.get_null_map_data(), ColumnUInt8::Container({1, 0}));
+ const auto& result_array = assert_cast<const ColumnArray&>(nullable_result.get_nested_column());
+ EXPECT_EQ(result_array.get_offsets(), ColumnArray::Offsets64({0, 1}));
+ const auto& values = assert_cast<const ColumnInt32&>(
+ assert_cast<const ColumnNullable&>(*result_array.get_data_ptr()).get_nested_column());
+ EXPECT_EQ(values.get_data(), ColumnInt32::Container({3}));
+}
+
+TEST(ArrayFilterFunctionTest, AllNullSourceSkipsConstantPredicateExpansion) {
+ constexpr size_t row_count = 2;
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto bool_type = std::make_shared<DataTypeUInt8>();
+ auto array_int_type = std::make_shared<DataTypeArray>(make_nullable(int_type));
+ auto nullable_array_int_type = make_nullable(array_int_type);
+ auto array_bool_type = std::make_shared<DataTypeArray>(make_nullable(bool_type));
+
+ auto filter = VLambdaFunctionCallExpr::create_shared(
+ make_lambda_call_node(nullable_array_int_type, row_count, "array_filter"));
+ filter->add_child(std::make_shared<MockColumnExpr>(
+ make_nullable_int_array_column({{}, {}}, {1, 1}), nullable_array_int_type, "source"));
+
+ auto predicate_data = ColumnUInt8::create();
+ predicate_data->get_data().assign({1, 1});
+ auto predicate_null_map = ColumnUInt8::create(2, 0);
+ auto predicate_offsets = ColumnArray::ColumnOffsets::create();
+ predicate_offsets->get_data().push_back(2);
+ auto predicate = ColumnArray::create(
+ ColumnNullable::create(std::move(predicate_data), std::move(predicate_null_map)),
+ std::move(predicate_offsets));
+ filter->add_child(std::make_shared<MockConstColumnExpr>(std::move(predicate), array_bool_type,
+ "predicate"));
+
+ VExprContext context(filter);
+ open_expr(filter, &context);
+
+ Block block;
+ ColumnPtr result;
+ auto status = filter->execute_column(&context, &block, nullptr, row_count, result);
+ ASSERT_TRUE(status.ok()) << status.to_string();
+ EXPECT_TRUE(is_column_const(*result));
+ EXPECT_TRUE(result->only_null());
+}
+
+TEST(ArrayEnumerateUniqFunctionTest, MappedAndOriginalNullableArraysUseLogicalRowOffsets) {
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto nullable_int_type = make_nullable(int_type);
+ auto array_int_type = std::make_shared<DataTypeArray>(nullable_int_type);
+ auto nullable_array_int_type = make_nullable(array_int_type);
+ auto source = make_nullable_int_array_column({{1, 2}, {3, 4}}, {1, 0});
+
+ auto map = VLambdaFunctionCallExpr::create_shared(
+ make_lambda_call_node(nullable_array_int_type, 2));
+ auto lambda =
+ VLambdaFunctionExpr::create_shared(make_lambda_expr_node(nullable_int_type, {"x"}));
+ lambda->add_child(VColumnRef::create_shared(make_column_ref_node(0, "x", nullable_int_type)));
+ map->add_child(lambda);
+ map->add_child(std::make_shared<MockColumnExpr>(source, nullable_array_int_type, "source"));
+
+ VExprContext map_context(map);
+ open_expr(map, &map_context);
+
+ Block input_block;
+ ColumnPtr mapped;
+ auto status = map->execute_column(&map_context, &input_block, nullptr, 2, mapped);
+ ASSERT_TRUE(status.ok()) << status.to_string();
+
+ const auto& mapped_nullable = assert_cast<const ColumnNullable&>(*mapped);
+ EXPECT_EQ(mapped_nullable.get_null_map_data(), ColumnUInt8::Container({1, 0}));
+ EXPECT_EQ(assert_cast<const ColumnArray&>(mapped_nullable.get_nested_column()).get_offsets(),
+ ColumnArray::Offsets64({0, 2}));
+
+ auto result_type = make_nullable(
+ std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeInt64>())));
+ Block block;
+ block.insert({std::move(mapped), nullable_array_int_type, "mapped"});
+ block.insert({std::move(source), nullable_array_int_type, "source"});
+ auto function = SimpleFunctionFactory::instance().get_function(
+ "array_enumerate_uniq", block.get_columns_with_type_and_name(), result_type);
+ ASSERT_NE(function, nullptr);
+
+ FunctionUtils function_utils(result_type, {nullable_array_int_type, nullable_array_int_type},
+ false);
+ auto* function_context = function_utils.get_fn_ctx();
+ ASSERT_TRUE(function->open(function_context, FunctionContext::FRAGMENT_LOCAL).ok());
+ ASSERT_TRUE(function->open(function_context, FunctionContext::THREAD_LOCAL).ok());
+ block.insert({nullptr, result_type, "result"});
+ status = function->execute(function_context, block, {0, 1}, 2, 2);
+ ASSERT_TRUE(function->close(function_context, FunctionContext::THREAD_LOCAL).ok());
+ ASSERT_TRUE(function->close(function_context, FunctionContext::FRAGMENT_LOCAL).ok());
+ ASSERT_TRUE(status.ok()) << status.to_string();
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(2).column);
+ EXPECT_EQ(result.get_null_map_data(), ColumnUInt8::Container({1, 0}));
+ const auto& result_array = assert_cast<const ColumnArray&>(result.get_nested_column());
+ EXPECT_EQ(result_array.get_offsets(), ColumnArray::Offsets64({0, 2}));
+ const auto& values = assert_cast<const ColumnInt64&>(
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column());
+ EXPECT_EQ(values.get_data(), ColumnInt64::Container({1, 1}));
+}
+
+TEST(ArrayEnumerateUniqFunctionTest, EqualHiddenOffsetsReuseInputOffsets) {
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto array_int_type = std::make_shared<DataTypeArray>(make_nullable(int_type));
+ auto nullable_array_int_type = make_nullable(array_int_type);
+ auto source = make_nullable_int_array_column({{100}, {1}}, {1, 0});
+ const auto& source_array = assert_cast<const ColumnArray&>(
+ assert_cast<const ColumnNullable&>(*source).get_nested_column());
+ const auto* source_offsets = source_array.get_offsets_ptr().get();
+
+ auto result_type = make_nullable(
+ std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeInt64>())));
+ Block block;
+ block.insert({source, nullable_array_int_type, "lhs"});
+ block.insert({source, nullable_array_int_type, "rhs"});
+ auto function = SimpleFunctionFactory::instance().get_function(
+ "array_enumerate_uniq", block.get_columns_with_type_and_name(), result_type);
+ ASSERT_NE(function, nullptr);
+
+ FunctionUtils function_utils(result_type, {nullable_array_int_type, nullable_array_int_type},
+ false);
+ auto* function_context = function_utils.get_fn_ctx();
+ ASSERT_TRUE(function->open(function_context, FunctionContext::FRAGMENT_LOCAL).ok());
+ ASSERT_TRUE(function->open(function_context, FunctionContext::THREAD_LOCAL).ok());
+ block.insert({nullptr, result_type, "result"});
+ auto status = function->execute(function_context, block, {0, 1}, 2, 2);
+ ASSERT_TRUE(function->close(function_context, FunctionContext::THREAD_LOCAL).ok());
+ ASSERT_TRUE(function->close(function_context, FunctionContext::FRAGMENT_LOCAL).ok());
+ ASSERT_TRUE(status.ok()) << status.to_string();
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(2).column);
+ EXPECT_EQ(result.get_null_map_data(), ColumnUInt8::Container({1, 0}));
+ const auto& result_array = assert_cast<const ColumnArray&>(result.get_nested_column());
+ EXPECT_EQ(result_array.get_offsets_ptr().get(), source_offsets);
+ EXPECT_EQ(result_array.get_offsets(), ColumnArray::Offsets64({1, 2}));
+}
+
+TEST(ArrayEnumerateUniqFunctionTest, ValidatesLateMismatchBeforeCompacting) {
+ constexpr size_t row_count = 64;
+ constexpr size_t row_size = 2048;
+ constexpr int64_t max_execution_bytes = 256 * 1024;
+
+ std::vector<std::vector<int32_t>> lhs_rows(row_count, std::vector<int32_t>(row_size, 1));
+ std::vector<std::vector<int32_t>> rhs_rows(row_count, std::vector<int32_t>(row_size, 2));
+ lhs_rows[0] = {999};
+ rhs_rows[0].clear();
+ rhs_rows.back().pop_back();
+
+ std::vector<uint8_t> lhs_null_map(row_count, 0);
+ lhs_null_map[0] = 1;
+ auto lhs = make_nullable_int_array_column(lhs_rows, lhs_null_map);
+ auto rhs = make_int_array_column(rhs_rows);
+
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto array_int_type = std::make_shared<DataTypeArray>(make_nullable(int_type));
+ auto nullable_array_int_type = make_nullable(array_int_type);
+ auto result_type = make_nullable(
+ std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeInt64>())));
+ Block block;
+ block.insert({std::move(lhs), nullable_array_int_type, "lhs"});
+ block.insert({std::move(rhs), array_int_type, "rhs"});
+ auto function = SimpleFunctionFactory::instance().get_function(
+ "array_enumerate_uniq", block.get_columns_with_type_and_name(), result_type);
+ ASSERT_NE(function, nullptr);
+
+ FunctionUtils function_utils(result_type, {nullable_array_int_type, array_int_type}, false);
+ auto* function_context = function_utils.get_fn_ctx();
+ ASSERT_TRUE(function->open(function_context, FunctionContext::FRAGMENT_LOCAL).ok());
+ ASSERT_TRUE(function->open(function_context, FunctionContext::THREAD_LOCAL).ok());
+ block.insert({nullptr, result_type, "result"});
+
+ auto tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER,
+ "ArrayEnumerateValidateBeforeCompact");
+ auto switch_tracker = SwitchThreadMemTrackerLimiter(tracker);
+ thread_context()->thread_mem_tracker_mgr->flush_untracked_mem();
+ const int64_t baseline = tracker->consumption();
+ auto status = function->execute(function_context, block, {0, 1}, 2, row_count);
+ thread_context()->thread_mem_tracker_mgr->flush_untracked_mem();
+ const int64_t execution_peak = tracker->peak_consumption() - baseline;
+
+ ASSERT_TRUE(function->close(function_context, FunctionContext::THREAD_LOCAL).ok());
+ ASSERT_TRUE(function->close(function_context, FunctionContext::FRAGMENT_LOCAL).ok());
+ ASSERT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("must be equal"), std::string::npos) << status;
+ EXPECT_LT(execution_peak, max_execution_bytes);
+}
+
+TEST(ArraySortByFunctionTest, MappedAndOriginalNullableArraysUseLogicalRowOffsets) {
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto nullable_int_type = make_nullable(int_type);
+ auto array_int_type = std::make_shared<DataTypeArray>(nullable_int_type);
+ auto nullable_array_int_type = make_nullable(array_int_type);
+ auto source = make_nullable_int_array_column({{9, 8}, {2, 1}}, {1, 0});
+
+ auto map = VLambdaFunctionCallExpr::create_shared(
+ make_lambda_call_node(nullable_array_int_type, 2));
+ auto lambda =
+ VLambdaFunctionExpr::create_shared(make_lambda_expr_node(nullable_int_type, {"x"}));
+ lambda->add_child(VColumnRef::create_shared(make_column_ref_node(0, "x", nullable_int_type)));
+ map->add_child(lambda);
+ map->add_child(std::make_shared<MockColumnExpr>(source, nullable_array_int_type, "source"));
+
+ VExprContext map_context(map);
+ open_expr(map, &map_context);
+ Block input_block;
+ ColumnPtr mapped;
+ auto status = map->execute_column(&map_context, &input_block, nullptr, 2, mapped);
+ ASSERT_TRUE(status.ok()) << status.to_string();
+
+ Block block;
+ block.insert({std::move(mapped), nullable_array_int_type, "mapped"});
+ block.insert({std::move(source), nullable_array_int_type, "source"});
+ auto function = SimpleFunctionFactory::instance().get_function(
+ "array_sortby", block.get_columns_with_type_and_name(), nullable_array_int_type);
+ ASSERT_NE(function, nullptr);
+
+ FunctionUtils function_utils(nullable_array_int_type,
+ {nullable_array_int_type, nullable_array_int_type}, false);
+ auto* function_context = function_utils.get_fn_ctx();
+ ASSERT_TRUE(function->open(function_context, FunctionContext::FRAGMENT_LOCAL).ok());
+ ASSERT_TRUE(function->open(function_context, FunctionContext::THREAD_LOCAL).ok());
+ block.insert({nullptr, nullable_array_int_type, "result"});
+ status = function->execute(function_context, block, {0, 1}, 2, 2);
+ ASSERT_TRUE(function->close(function_context, FunctionContext::THREAD_LOCAL).ok());
+ ASSERT_TRUE(function->close(function_context, FunctionContext::FRAGMENT_LOCAL).ok());
+ ASSERT_TRUE(status.ok()) << status.to_string();
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(2).column);
+ EXPECT_EQ(result.get_null_map_data(), ColumnUInt8::Container({1, 0}));
+ const auto& result_array = assert_cast<const ColumnArray&>(result.get_nested_column());
+ EXPECT_EQ(result_array.get_offsets(), ColumnArray::Offsets64({0, 2}));
+ const auto& values = assert_cast<const ColumnInt32&>(
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column());
+ EXPECT_EQ(values.get_data(), ColumnInt32::Container({1, 2}));
+}
+
+TEST(ArrayEnumerateUniqFunctionTest, AllNullLaterArgumentSkipsEarlierConstantExpansion) {
+ constexpr size_t row_count = 512;
+ constexpr size_t array_size = 4096;
+ constexpr int64_t max_execution_bytes = 1024 * 1024;
+
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto nullable_int_type = make_nullable(int_type);
+ auto array_int_type = std::make_shared<DataTypeArray>(nullable_int_type);
+ auto nullable_array_int_type = make_nullable(array_int_type);
+ auto result_type = make_nullable(
+ std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeInt64>())));
+
+ auto tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER,
+ "ArrayEnumerateUniqAllNullFastPath");
+ auto switch_tracker = SwitchThreadMemTrackerLimiter(tracker);
+ std::vector<int32_t> constant_values(array_size, 1);
+ auto constant_array = ColumnConst::create(make_int_array_column({constant_values}), row_count);
+ auto all_null_array = make_nullable_int_array_column(
+ std::vector<std::vector<int32_t>>(row_count), std::vector<uint8_t>(row_count, 1));
+
+ Block block;
+ block.insert({std::move(constant_array), array_int_type, "constant_array"});
+ block.insert({std::move(all_null_array), nullable_array_int_type, "all_null_array"});
+ auto function = SimpleFunctionFactory::instance().get_function(
+ "array_enumerate_uniq", block.get_columns_with_type_and_name(), result_type);
+ ASSERT_NE(function, nullptr);
+
+ FunctionUtils function_utils(result_type, {array_int_type, nullable_array_int_type}, false);
+ auto* function_context = function_utils.get_fn_ctx();
+ ASSERT_TRUE(function->open(function_context, FunctionContext::FRAGMENT_LOCAL).ok());
+ ASSERT_TRUE(function->open(function_context, FunctionContext::THREAD_LOCAL).ok());
+ block.insert({nullptr, result_type, "result"});
+ thread_context()->thread_mem_tracker_mgr->flush_untracked_mem();
+ const int64_t baseline = tracker->consumption();
+ auto status = function->execute(function_context, block, {0, 1}, 2, row_count);
+ thread_context()->thread_mem_tracker_mgr->flush_untracked_mem();
+ const int64_t execution_peak = tracker->peak_consumption() - baseline;
+ ASSERT_TRUE(function->close(function_context, FunctionContext::THREAD_LOCAL).ok());
+ ASSERT_TRUE(function->close(function_context, FunctionContext::FRAGMENT_LOCAL).ok());
+
+ ASSERT_TRUE(status.ok()) << status.to_string();
+ EXPECT_TRUE(block.get_by_position(2).column->only_null());
+ EXPECT_LT(execution_peak, max_execution_bytes);
+}
+
+static const ColumnInt32& get_int_array_values(const ColumnPtr& result) {
+ const auto& result_array = assert_cast<const ColumnArray&>(*result);
+ const auto& nullable_values = assert_cast<const ColumnNullable&>(*result_array.get_data_ptr());
+ return assert_cast<const ColumnInt32&>(nullable_values.get_nested_column());
+}
+
+TEST(ArrayMapFunctionTest, CapturedColumnExpansionUsesOuterSelector) {
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto array_int_type = std::make_shared<DataTypeArray>(int_type);
+
+ auto root = VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2));
+ auto lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"x"}));
+ auto add = std::make_shared<MockAddExpr>(int_type);
+ add->add_child(make_slot_ref(0, "captured", int_type));
+ add->add_child(VColumnRef::create_shared(make_column_ref_node(0, "x", int_type)));
+ lambda->add_child(add);
+ root->add_child(lambda);
+ root->add_child(make_slot_ref(1, "input_array", array_int_type));
+
+ VExprContext context(root);
+ open_expr(root, &context);
+
+ Block block;
+ block.insert({make_int_column({10, 20, 30, 40}), int_type, "captured"});
+ block.insert(
+ {make_int_array_column({{1, 2}, {3}, {}, {7, 8, 9}}), array_int_type, "input_array"});
+ Selector selector;
+ selector.push_back(1);
+ selector.push_back(3);
+
+ ColumnPtr result;
+ auto status = root->execute_column(&context, &block, &selector, selector.size(), result);
+ ASSERT_TRUE(status.ok()) << status.to_string();
+
+ const auto& result_array = assert_cast<const ColumnArray&>(*result);
+ ASSERT_EQ(result_array.size(), 2);
+ ASSERT_EQ(result_array.get_offsets()[0], 1);
+ ASSERT_EQ(result_array.get_offsets()[1], 4);
+ const auto& values = get_int_array_values(result);
+ ASSERT_EQ(values.size(), 4);
+ EXPECT_EQ(values.get_element(0), 23);
+ EXPECT_EQ(values.get_element(1), 47);
+ EXPECT_EQ(values.get_element(2), 48);
+ EXPECT_EQ(values.get_element(3), 49);
+}
+
+TEST(ArrayMapFunctionTest, HiddenPayloadAfterValidRowUsesOwnOffsetsAcrossBatchesAndSelector) {
+ constexpr int lambda_batch_size = 3;
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto array_int_type = std::make_shared<DataTypeArray>(int_type);
+ auto nullable_array_int_type = std::make_shared<DataTypeNullable>(array_int_type);
+ std::vector<size_t> observed_batch_sizes;
+
+ auto root = VLambdaFunctionCallExpr::create_shared(
+ make_lambda_call_node(nullable_array_int_type, 3));
+ auto lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"x", "y"}));
+ auto body = std::make_shared<MockAddExpr>(int_type, &observed_batch_sizes);
+ body->add_child(VColumnRef::create_shared(make_column_ref_node(0, "x", int_type)));
+ body->add_child(VColumnRef::create_shared(make_column_ref_node(1, "y", int_type)));
+ lambda->add_child(body);
+ root->add_child(lambda);
+ root->add_child(make_slot_ref(0, "left", nullable_array_int_type));
+ root->add_child(make_slot_ref(1, "right", array_int_type));
+
+ VExprContext context(root);
+ open_expr_with_batch_size(root, &context, lambda_batch_size);
+
+ Block block;
+ block.insert({make_nullable_int_array_column(
+ {{-100}, {1, 2}, {-200}, {900, 901, 902}, {-300}, {3, 4, 5, 6, 7}},
+ {0, 0, 0, 1, 0, 0}),
+ nullable_array_int_type, "left"});
+ block.insert({make_int_array_column(
+ {{-10}, {10, 20}, {-20}, {800, 801}, {-30}, {30, 40, 50, 60, 70}}),
+ array_int_type, "right"});
+
+ Selector selector {1, 3, 5};
+ ColumnPtr result;
+ auto status = root->execute_column(&context, &block, &selector, selector.size(), result);
+ ASSERT_TRUE(status.ok()) << status.to_string();
+
+ ASSERT_EQ(observed_batch_sizes.size(), 3);
+ EXPECT_EQ(observed_batch_sizes[0], 3);
+ EXPECT_EQ(observed_batch_sizes[1], 3);
+ EXPECT_EQ(observed_batch_sizes[2], 1);
+
+ const auto& nullable_result = assert_cast<const ColumnNullable&>(*result);
+ ASSERT_EQ(nullable_result.size(), 3);
+ EXPECT_FALSE(nullable_result.is_null_at(0));
+ EXPECT_TRUE(nullable_result.is_null_at(1));
+ EXPECT_FALSE(nullable_result.is_null_at(2));
+
+ const auto& result_array = assert_cast<const ColumnArray&>(nullable_result.get_nested_column());
+ EXPECT_EQ(result_array.get_offsets()[0], 2);
+ EXPECT_EQ(result_array.get_offsets()[1], 2);
+ EXPECT_EQ(result_array.get_offsets()[2], 7);
+
+ const auto& nullable_values = assert_cast<const ColumnNullable&>(*result_array.get_data_ptr());
+ const auto& values = assert_cast<const ColumnInt32&>(nullable_values.get_nested_column());
+ ASSERT_EQ(values.size(), 7);
+ EXPECT_EQ(values.get_element(0), 11);
+ EXPECT_EQ(values.get_element(1), 22);
+ EXPECT_EQ(values.get_element(2), 33);
+ EXPECT_EQ(values.get_element(3), 44);
+ EXPECT_EQ(values.get_element(4), 55);
+ EXPECT_EQ(values.get_element(5), 66);
+ EXPECT_EQ(values.get_element(6), 77);
+}
+
+TEST(ArrayMapFunctionTest, HiddenPayloadAtBatchBoundarySkipsEmptyLambdaBatch) {
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto array_type = std::make_shared<DataTypeArray>(int_type);
+ auto nullable_array_type = make_nullable(array_type);
+ std::vector<size_t> observed_batch_sizes;
+
+ auto root =
+ VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(nullable_array_type, 3));
+ auto lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"x", "y"}));
+ auto body = std::make_shared<MockAddExpr>(int_type, &observed_batch_sizes);
+ body->add_child(VColumnRef::create_shared(make_column_ref_node(0, "x", int_type)));
+ body->add_child(VColumnRef::create_shared(make_column_ref_node(1, "y", int_type)));
+ lambda->add_child(body);
+ root->add_child(lambda);
+ root->add_child(std::make_shared<MockColumnExpr>(
+ make_nullable_int_array_column({{1, 2, 3}, {900}}, {0, 1}), nullable_array_type,
+ "left"));
+ root->add_child(std::make_shared<MockColumnExpr>(
+ make_int_array_column({{10, 20, 30}, {800, 801}}), array_type, "right"));
+
+ VExprContext context(root);
+ open_expr_with_batch_size(root, &context, 3);
+ Block block;
+ ColumnPtr result;
+ auto status = root->execute_column(&context, &block, nullptr, 2, result);
+ ASSERT_TRUE(status.ok()) << status.to_string();
+ ASSERT_EQ(observed_batch_sizes.size(), 1);
+ EXPECT_EQ(observed_batch_sizes[0], 3);
+ const auto& nullable_result = assert_cast<const ColumnNullable&>(*result);
+ EXPECT_FALSE(nullable_result.is_null_at(0));
+ EXPECT_TRUE(nullable_result.is_null_at(1));
+ const auto& array_result = assert_cast<const ColumnArray&>(nullable_result.get_nested_column());
+ EXPECT_EQ(array_result.get_offsets()[0], 3);
+ EXPECT_EQ(array_result.get_offsets()[1], 3);
+}
+
+TEST(ArrayMapFunctionTest, NonNullLengthMismatchStillReturnsErrorWithNullArrayRow) {
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto array_int_type = std::make_shared<DataTypeArray>(int_type);
+ auto nullable_array_int_type = std::make_shared<DataTypeNullable>(array_int_type);
+
+ auto root = VLambdaFunctionCallExpr::create_shared(
+ make_lambda_call_node(nullable_array_int_type, 3));
+ auto lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"x", "y"}));
+ lambda->add_child(std::make_shared<MockBodyExpr>(int_type, "unused_body"));
+ root->add_child(lambda);
+ root->add_child(std::make_shared<MockColumnExpr>(
+ make_nullable_int_array_column({{100, 101}, {1}}, {1, 0}), nullable_array_int_type,
+ "left"));
+ root->add_child(std::make_shared<MockColumnExpr>(make_int_array_column({{200}, {10, 20}}),
+ array_int_type, "right"));
+
+ VExprContext context(root);
+ open_expr(root, &context);
+
+ Block block;
+ ColumnPtr result;
+ auto status = root->execute_column(&context, &block, nullptr, 2, result);
+ EXPECT_TRUE(status.is<ErrorCode::INVALID_ARGUMENT>()) << status.to_string();
+}
+
TEST(ArrayMapFunctionTest, NestedLambdaWithSameArgumentNameUsesInnerScope) {
auto int_type = std::make_shared<DataTypeInt32>();
auto array_int_type = std::make_shared<DataTypeArray>(int_type);
@@ -540,6 +1098,39 @@ TEST(ArrayMapFunctionTest, NamedLambdaWithFewerArgumentsThanArraysUsesDeclaredBi
EXPECT_EQ(values.get_element(2), 30);
}
+TEST(ArrayMapFunctionTest, ArraySortSkipsComparatorForOuterNullRow) {
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto int8_type = std::make_shared<DataTypeInt8>();
+ auto array_int_type = std::make_shared<DataTypeArray>(int_type);
+ auto nullable_array_int_type = std::make_shared<DataTypeNullable>(array_int_type);
+
+ auto root = VLambdaFunctionCallExpr::create_shared(
+ make_lambda_call_node(nullable_array_int_type, 2, "array_sort"));
+ auto lambda =
+ VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int8_type, {"left", "right"}));
+ lambda->add_child(std::make_shared<MockBodyExpr>(int8_type, "unused_comparator"));
+ root->add_child(lambda);
+ root->add_child(std::make_shared<MockColumnExpr>(make_nullable_int_array_column({{2, 1}}, {1}),
+ nullable_array_int_type, "input"));
+
+ VExprContext context(root);
+ open_expr(root, &context);
+
+ Block block;
+ ColumnPtr result;
+ auto status = root->execute_column(&context, &block, nullptr, 1, result);
+ ASSERT_TRUE(status.ok()) << status.to_string();
+
+ const auto& nullable_result = assert_cast<const ColumnNullable&>(*result);
+ EXPECT_TRUE(nullable_result.is_null_at(0));
+ const auto& result_array = assert_cast<const ColumnArray&>(nullable_result.get_nested_column());
+ ASSERT_EQ(result_array.get_offsets()[0], 2);
+ const auto& values = assert_cast<const ColumnInt32&>(
+ assert_cast<const ColumnNullable&>(result_array.get_data()).get_nested_column());
+ EXPECT_EQ(values.get_element(0), 2);
+ EXPECT_EQ(values.get_element(1), 1);
+}
+
TEST(ArrayMapFunctionTest, NestedArraySortInsideArrayMapSkipsArrayMapArgumentInference) {
auto int_type = std::make_shared<DataTypeInt32>();
auto int8_type = std::make_shared<DataTypeInt8>();
diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function.out
index 0e3935d131f..6b96a63b788 100644
--- a/regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function.out
+++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function.out
@@ -132,3 +132,14 @@
-- !select_28 --
[]
+-- !array_map_null_container --
+1 \N
+2 [42, 33]
+
+-- !array_sort_null_container --
+1 \N
+2 ["10", "20"]
+
+-- !array_zip_null_container --
+1 \N
+2 [{"col1":42, "col2":20}, {"col1":33, "col2":21}]
diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_split.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_split.out
index f06e4309c86..a43e3c595b2 100644
--- a/regression-test/data/query_p0/sql_functions/array_functions/test_array_split.out
+++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_split.out
@@ -58,3 +58,10 @@
1 [["2020-12-12 00:00:00.000000", "2013-12-12 00:00:00.000000", "2015-12-12 00:00:00.000000", null]]
2 [["2020-12-12 00:00:00.000000", "2013-12-12 00:00:00.000000", "2015-12-12 00:00:00.000000", null], ["2200-12-12 12:12:12.123456"]]
+-- !nullable_lambda_split --
+1 \N
+2 [[1], [2]]
+
+-- !nullable_lambda_reverse_split --
+1 \N
+2 [[1], [2]]
diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy
index 2c1c86e9475..7392d3006e3 100644
--- a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy
@@ -162,4 +162,48 @@ suite("test_array_map_function") {
qt_select_27 """ select QC_result_list, array_map( x -> concat( '|', x + "1" ), QC_result_list ) FROM db; """
qt_select_28 """ select array_map((x,y)->x,[],[]); """
+
+ sql "DROP TABLE IF EXISTS array_map_null_container"
+ sql """
+ CREATE TABLE array_map_null_container (
+ id INT,
+ string_values ARRAY<STRING>,
+ int_values ARRAY<INT>
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO array_map_null_container VALUES
+ (1, ['bad-number', 'also-bad'], [100, 101]),
+ (2, ['20', '10'], [20, 21])
+ """
+ sql "SET enable_strict_cast = true"
+ sql "SET short_circuit_evaluation = false"
+ order_qt_array_map_null_container """
+ SELECT id,
+ array_map((x, y) -> cast(x AS INT) + y + id,
+ if(id = 1, cast(NULL AS ARRAY<STRING>), string_values),
+ int_values)
+ FROM array_map_null_container
+ ORDER BY id
+ """
+ order_qt_array_sort_null_container """
+ SELECT id,
+ array_sort((x, y) -> if(cast(x AS INT) < cast(y AS INT), -1, 1),
+ if(id = 1, cast(NULL AS ARRAY<STRING>), string_values))
+ FROM array_map_null_container
+ ORDER BY id
+ """
+ order_qt_array_zip_null_container """
+ SELECT id,
+ array_zip(
+ array_map((x, y) -> cast(x AS INT) + y + id,
+ if(id = 1, cast(NULL AS ARRAY<STRING>), string_values),
+ int_values),
+ int_values)
+ FROM array_map_null_container
+ ORDER BY id
+ """
}
diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_split.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_split.groovy
index f74760f4cef..eaa380d9ee0 100644
--- a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_split.groovy
+++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_split.groovy
@@ -70,6 +70,20 @@ suite("test_array_split") {
from dt order by x; """
qt_dt_null """ select x, array_reverse_split(x->(null_or_empty(x)), k0) from dt order by x; """
+ sql "set short_circuit_evaluation = false;"
+ qt_nullable_lambda_split """
+ select id, array_split((x, y) -> y > 0, [1, 2],
+ if(id = 1, cast(null as array<int>), [10, 20]))
+ from (select 1 as id union all select 2 as id) t
+ order by id;
+ """
+ qt_nullable_lambda_reverse_split """
+ select id, array_reverse_split((x, y) -> y > 0, [1, 2],
+ if(id = 1, cast(null as array<int>), [10, 20]))
+ from (select 1 as id union all select 2 as id) t
+ order by id;
+ """
+
test {
sql " select array_split([1,2,3,4,5], [1,1,1]); "
exception "function array_split has uneven arguments on row 0"
@@ -78,4 +92,4 @@ suite("test_array_split") {
sql " select array_reverse_split((x,y)->(y), [1,2,3,4,5], [1,1,1]); "
exception "in array map function, the input column size are not equal completely"
}
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 09/35: [fix](be) Backport unlimited topn expansion rates to branch-4.1 (#68545)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 37352da41c13a161d7a87f178dcd062ff7624d4f
Author: HappenLee <ha...@selectdb.com>
AuthorDate: Tue Sep 29 17:40:40 2026 +0800
[fix](be) Backport unlimited topn expansion rates to branch-4.1 (#68545)
### What problem does this PR solve?
Issue Number: N/A
Related PR: #68246
Backport #68246 to `branch-4.1`. A zero `space_expand_rate` previously
produced a zero candidate capacity, so partial-state serialization
discarded every TOPN candidate and two-stage aggregation could return an
empty result. Treat non-positive rates as unlimited intermediate
candidate retention for `topn`, `topn_array`, and `topn_weighted`;
`top_num` still limits the final result. Positive and default rates
retain their existing behavior.
Branch-specific conflict resolution:
- Keep the production fix, standalone TOPN unit tests, regression suite,
and the expected output generated in the original PR.
- Omit modifications to `agg_state_parameters_test.cpp` and
`test_agg_state_parameters.groovy`: neither file nor the associated
aggregate-state parameter validation exists in branch-4.1. This backport
does not introduce that separate master feature.
- Remove `SET enable_bucketed_hash_agg = false` from the regression
suite because branch-4.1 has no such session variable. Keep coverage for
aggregation phases 1 and 2.
### Release note
`topn`, `topn_array`, and `topn_weighted` interpret a non-positive
`space_expand_rate` as unlimited intermediate candidate retention. The
final result remains limited by `top_num`. Retaining all distinct
candidates can increase intermediate-state memory and network traffic.
### Check List (For Author)
- Test: Unit Test / Regression test included; local execution not
completed
- Repository clang-format 16 formatting and check scripts passed.
- Diff whitespace check passed with the original generated `.out`
trailing blank line preserved.
- Attempted `./run-be-ut.sh -j 48 --run
--filter='AggregateFunctionTopN*.*:*/AggregateFunctionTopN*.*:AggTest.topn*'`.
Stopped during dependency preparation; the isolated checkout has no
installed third-party dependency bundle. No local 4.1 unit-test pass is
claimed.
- Regression tests were not run locally against a 4.1 cluster. Expected
output is copied unchanged from #68246.
- Branch-4.1 does not contain `build-support/check-build-hygiene.sh`; no
successful hygiene or clang-tidy run is claimed.
- Behavior changed: Yes. Non-positive expansion rates retain all
intermediate candidates.
- Does this need documentation: Yes. The function documentation should
describe non-positive expansion rates; the documentation follow-up noted
by #68246 remains applicable.
---
be/src/exprs/aggregate/aggregate_function_topn.h | 3 +-
be/test/exprs/aggregate/agg_topn_test.cpp | 125 +++++++++++++++++++++
.../agg_function/topn/topn_unlimited.out | 91 +++++++++++++++
.../agg_function/topn/topn_unlimited.groovy | 75 +++++++++++++
4 files changed, 293 insertions(+), 1 deletion(-)
diff --git a/be/src/exprs/aggregate/aggregate_function_topn.h b/be/src/exprs/aggregate/aggregate_function_topn.h
index 77c9e260d4a..aae1bc182c7 100644
--- a/be/src/exprs/aggregate/aggregate_function_topn.h
+++ b/be/src/exprs/aggregate/aggregate_function_topn.h
@@ -60,7 +60,8 @@ struct AggregateFunctionTopNData {
using DataType = typename PrimitiveTypeTraits<T>::CppType;
void set_paramenters(int input_top_num, int space_expand_rate = 50) {
top_num = input_top_num;
- capacity = (uint64_t)top_num * space_expand_rate;
+ // Non-positive expansion rates retain all candidates during serialization and merging.
+ capacity = space_expand_rate <= 0 ? UINT64_MAX : (uint64_t)top_num * space_expand_rate;
}
void add(const StringRef& value, const UInt64& increment = 1) {
diff --git a/be/test/exprs/aggregate/agg_topn_test.cpp b/be/test/exprs/aggregate/agg_topn_test.cpp
new file mode 100644
index 00000000000..324bb7272d9
--- /dev/null
+++ b/be/test/exprs/aggregate/agg_topn_test.cpp
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <string>
+
+#include "core/column/column_string.h"
+#include "core/column/column_vector.h"
+#include "core/string_buffer.hpp"
+#include "exprs/aggregate/aggregate_function_topn.h"
+
+namespace doris {
+namespace {
+
+template <PrimitiveType T>
+AggregateFunctionTopNData<T> round_trip(const AggregateFunctionTopNData<T>& state) {
+ auto column = ColumnString::create();
+ BufferWritable writer(*column);
+ state.write(writer);
+ writer.commit();
+ BufferReadable reader(column->get_data_at(0));
+ AggregateFunctionTopNData<T> result;
+ result.read(reader);
+ return result;
+}
+
+class AggregateFunctionTopNUnlimitedTest : public testing::TestWithParam<int> {};
+
+TEST_P(AggregateFunctionTopNUnlimitedTest, SerializeAndMergeStrings) {
+ AggregateFunctionTopNData<TYPE_STRING> lhs;
+ AggregateFunctionTopNData<TYPE_STRING> rhs;
+ lhs.set_paramenters(1, GetParam());
+ rhs.set_paramenters(1, GetParam());
+ // The global winner is not the most frequent value in either partial state.
+ lhs.add(std::string("a"), 3);
+ lhs.add(std::string("winner"), 2);
+ rhs.add(std::string("b"), 3);
+ rhs.add(std::string("winner"), 2);
+
+ auto partial = round_trip(lhs);
+ EXPECT_EQ(partial.counter_map, lhs.counter_map);
+ AggregateFunctionTopNData<TYPE_STRING> merged;
+ merged.merge(partial);
+ merged = round_trip(merged);
+ merged.merge(round_trip(rhs));
+ merged = round_trip(merged);
+
+ ASSERT_EQ(merged.counter_map.size(), 3);
+ EXPECT_EQ(merged.counter_map.at("a"), 3);
+ EXPECT_EQ(merged.counter_map.at("b"), 3);
+ EXPECT_EQ(merged.counter_map.at("winner"), 4);
+ EXPECT_EQ(merged.get(), R"({"winner":4})");
+
+ AggregateFunctionTopNData<TYPE_STRING> empty;
+ merged.merge(round_trip(empty));
+ EXPECT_EQ(merged.get(), R"({"winner":4})");
+
+ merged.reset();
+ EXPECT_EQ(round_trip(merged).get(), "{}");
+ merged.set_paramenters(1, GetParam());
+ merged.add(std::string("new"), 7);
+ EXPECT_EQ(round_trip(merged).get(), R"({"new":7})");
+}
+
+TEST_P(AggregateFunctionTopNUnlimitedTest, SerializeAndMergeWeightedIntegers) {
+ AggregateFunctionTopNData<TYPE_INT> lhs;
+ AggregateFunctionTopNData<TYPE_INT> rhs;
+ lhs.set_paramenters(2, GetParam());
+ rhs.set_paramenters(2, GetParam());
+ lhs.add(1, 10);
+ lhs.add(2, 7);
+ lhs.add(3, 6);
+ rhs.add(4, 11);
+ rhs.add(5, 8);
+ rhs.add(3, 6);
+
+ AggregateFunctionTopNData<TYPE_INT> merged;
+ merged.merge(round_trip(lhs));
+ merged.merge(round_trip(rhs));
+ merged = round_trip(merged);
+ ASSERT_EQ(merged.counter_map.size(), 5);
+ EXPECT_EQ(merged.counter_map.at(3), 12);
+ auto result = ColumnInt32::create();
+ merged.insert_result_into(*result);
+ ASSERT_EQ(result->size(), 2);
+ EXPECT_EQ(result->get_element(0), 3);
+ EXPECT_EQ(result->get_element(1), 4);
+}
+
+INSTANTIATE_TEST_SUITE_P(NonPositiveRates, AggregateFunctionTopNUnlimitedTest,
+ testing::Values(0, -1, INT32_MIN));
+
+TEST(AggregateFunctionTopNTest, PositiveRateStillLimitsSerializedCandidates) {
+ AggregateFunctionTopNData<TYPE_INT> state;
+ state.set_paramenters(1, 2);
+ state.add(1, 3);
+ state.add(2, 2);
+ state.add(3, 1);
+ auto partial = round_trip(state);
+ ASSERT_EQ(partial.counter_map.size(), 2);
+ EXPECT_EQ(partial.counter_map.at(1), 3);
+ EXPECT_EQ(partial.counter_map.at(2), 2);
+
+ state.set_paramenters(1);
+ EXPECT_EQ(round_trip(state).counter_map, state.counter_map);
+}
+
+} // namespace
+} // namespace doris
diff --git a/regression-test/data/nereids_function_p0/agg_function/topn/topn_unlimited.out b/regression-test/data/nereids_function_p0/agg_function/topn/topn_unlimited.out
new file mode 100644
index 00000000000..888ec8e5112
--- /dev/null
+++ b/regression-test/data/nereids_function_p0/agg_function/topn/topn_unlimited.out
@@ -0,0 +1,91 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !unlimited --
+{"a":3,"x":2} ["a", "x"] [1, 4] ["b", "y"] [2, 5]
+
+-- !grouped --
+1 {"a":3,"b":2} ["a", "b"] [1, 2] ["b", "a"] [2, 1]
+2 {"x":2,"y":1} ["x", "y"] [4, 5] ["y", "x"] [5, 4]
+3 \N \N \N \N \N
+
+-- !empty --
+\N \N \N
+
+-- !constant_input --
+{"a":2} ["a"]
+
+-- !unlimited --
+{"a":3,"x":2} ["a", "x"] [1, 4] ["b", "y"] [2, 5]
+
+-- !grouped --
+1 {"a":3,"b":2} ["a", "b"] [1, 2] ["b", "a"] [2, 1]
+2 {"x":2,"y":1} ["x", "y"] [4, 5] ["y", "x"] [5, 4]
+3 \N \N \N \N \N
+
+-- !empty --
+\N \N \N
+
+-- !constant_input --
+{"a":2} ["a"]
+
+-- !unlimited --
+{"a":3,"x":2} ["a", "x"] [1, 4] ["b", "y"] [2, 5]
+
+-- !grouped --
+1 {"a":3,"b":2} ["a", "b"] [1, 2] ["b", "a"] [2, 1]
+2 {"x":2,"y":1} ["x", "y"] [4, 5] ["y", "x"] [5, 4]
+3 \N \N \N \N \N
+
+-- !empty --
+\N \N \N
+
+-- !constant_input --
+{"a":2} ["a"]
+
+-- !positive_and_default --
+{"a":3,"x":2} {"a":3,"x":2} [1, 4] [1, 4] [2, 5] [2, 5]
+
+-- !unlimited --
+{"a":3,"x":2} ["a", "x"] [1, 4] ["b", "y"] [2, 5]
+
+-- !grouped --
+1 {"a":3,"b":2} ["a", "b"] [1, 2] ["b", "a"] [2, 1]
+2 {"x":2,"y":1} ["x", "y"] [4, 5] ["y", "x"] [5, 4]
+3 \N \N \N \N \N
+
+-- !empty --
+\N \N \N
+
+-- !constant_input --
+{"a":2} ["a"]
+
+-- !unlimited --
+{"a":3,"x":2} ["a", "x"] [1, 4] ["b", "y"] [2, 5]
+
+-- !grouped --
+1 {"a":3,"b":2} ["a", "b"] [1, 2] ["b", "a"] [2, 1]
+2 {"x":2,"y":1} ["x", "y"] [4, 5] ["y", "x"] [5, 4]
+3 \N \N \N \N \N
+
+-- !empty --
+\N \N \N
+
+-- !constant_input --
+{"a":2} ["a"]
+
+-- !unlimited --
+{"a":3,"x":2} ["a", "x"] [1, 4] ["b", "y"] [2, 5]
+
+-- !grouped --
+1 {"a":3,"b":2} ["a", "b"] [1, 2] ["b", "a"] [2, 1]
+2 {"x":2,"y":1} ["x", "y"] [4, 5] ["y", "x"] [5, 4]
+3 \N \N \N \N \N
+
+-- !empty --
+\N \N \N
+
+-- !constant_input --
+{"a":2} ["a"]
+
+-- !positive_and_default --
+{"a":3,"x":2} {"a":3,"x":2} [1, 4] [1, 4] [2, 5] [2, 5]
+
diff --git a/regression-test/suites/nereids_function_p0/agg_function/topn/topn_unlimited.groovy b/regression-test/suites/nereids_function_p0/agg_function/topn/topn_unlimited.groovy
new file mode 100644
index 00000000000..89546bfb5fa
--- /dev/null
+++ b/regression-test/suites/nereids_function_p0/agg_function/topn/topn_unlimited.groovy
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("topn_unlimited") {
+ sql "DROP TABLE IF EXISTS test_topn_unlimited"
+ sql """
+ CREATE TABLE test_topn_unlimited (
+ g INT,
+ s STRING,
+ v INT,
+ w BIGINT
+ ) DISTRIBUTED BY HASH(v) BUCKETS 3
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO test_topn_unlimited VALUES
+ (1, 'a', 1, 1), (1, 'b', 2, 10), (1, 'a', 1, 2),
+ (1, 'c', 3, 3), (1, 'a', 1, 1), (1, 'b', 2, 2),
+ (2, 'x', 4, 1), (2, 'y', 5, 5), (2, 'x', 4, 2),
+ (3, NULL, NULL, 1)
+ """
+
+ sql "SET parallel_pipeline_task_num = 1"
+ for (def phase : [1, 2]) {
+ sql "SET agg_phase = ${phase}"
+ // Non-positive rates retain every candidate, including after partial serialization.
+ for (def rate : [0, -1, -2147483648]) {
+ order_qt_unlimited """
+ SELECT topn(s, 2, ${rate}),
+ topn_array(s, 2, ${rate}),
+ topn_array(v, 2, ${rate}),
+ topn_weighted(s, w, 2, ${rate}),
+ topn_weighted(v, w, 2, ${rate})
+ FROM test_topn_unlimited
+ """
+ order_qt_grouped """
+ SELECT g, topn(s, 2, ${rate}),
+ topn_array(s, 2, ${rate}),
+ topn_array(v, 2, ${rate}),
+ topn_weighted(s, w, 2, ${rate}),
+ topn_weighted(v, w, 2, ${rate})
+ FROM test_topn_unlimited GROUP BY g
+ """
+ order_qt_empty """
+ SELECT topn(s, 1, ${rate}), topn_array(v, 1, ${rate}),
+ topn_weighted(v, w, 1, ${rate})
+ FROM test_topn_unlimited WHERE g = 4
+ """
+ order_qt_constant_input """
+ SELECT topn(s, 1, ${rate}), topn_array(s, 1, ${rate})
+ FROM (SELECT 'a' AS s UNION ALL SELECT 'b' UNION ALL SELECT 'a') t
+ """
+ }
+ order_qt_positive_and_default """
+ SELECT topn(s, 2), topn(s, 2, 50),
+ topn_array(v, 2), topn_array(v, 2, 50),
+ topn_weighted(v, w, 2), topn_weighted(v, w, 2, 50)
+ FROM test_topn_unlimited
+ """
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/35: [fix](iceberg) Preserve file specs after evolving to unpartitioned tables (#68481)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit aa2fa487ade96f85e21cc1f6c4f25cb4f3c0dc91
Author: Gabriel <li...@selectdb.com>
AuthorDate: Tue Sep 29 15:37:59 2026 +0800
[fix](iceberg) Preserve file specs after evolving to unpartitioned tables (#68481)
### What problem does this PR solve?
On `branch-4.1`, Iceberg scan splits only carry partition metadata when
the table's current spec is partitioned. After an external REPLACE
changes a partitioned table to an unpartitioned table, the old spec
remains in the table metadata but new files use a different spec ID.
Missing split metadata makes the BE default to spec 0, so DELETE can
fail with `No partition data for partitioned table` when committing the
position-delete files. Dropping the last partition field also affects
tables containing files from both specs.
Preserve each data file's spec ID regardless of the current table spec.
Generate partition JSON only for scans projecting the hidden row ID,
with the projection requirement cached once per scan. Partition-pruning
metadata remains independent of this write payload.
Historical BINARY/FIXED partition values round-trip through the target
branch's lossless `0x`-prefixed hexadecimal representation, with
matching decoding and fixed-length validation. UUID/TIME values also
round-trip through delete metadata. Unsupported partition values are
never serialized as NULL. Ordinary reads omit unavailable row-ID
partition metadata, while scans that materialize the hidden row ID
reject unsupported partition types before generating delete metadata.
This keeps historical files readable after their partition source column
is dropped. Non-batch split planning preserves the actionable
UserException even when Hadoop authentication wraps it, including the
spec ID and rewrite guidance. Negative fractional timestamp values use
floor-based seconds/nanoseconds normalization, and TIMESTAMPTZ transport
retains an explicit UTC offset across DST overlaps.
### Release note
Fix Iceberg row-level deletes after replacing a partitioned table with
an unpartitioned table or dropping its last partition field. Keep
historical files readable after dropping a partition source column, and
reject row-ID-dependent operations when the old partition type is
unavailable.
### Tests
- Added two unit tests using real Iceberg metadata transitions. Both
fail on the original code because the spec ID is missing from the
serialized scan range, and pass with the fix. They also validate
conversion to position-delete metadata for the file's own spec.
- Added dropped-source-column tests using real Iceberg metadata,
covering NULL/non-NULL historical values and both currently partitioned
and unpartitioned tables. They verify readable scan ranges without
fabricated partition JSON and explicit rejection when row IDs are
required.
- Added ordinary-read checks that partition JSON is absent while each
file retains its spec ID and applicable partition-pruning values.
Dropped-source-column error tests now call the public getSplits entry
point, including a real Hadoop doAs wrapper, and verify the
UserException type, message, spec ID, guidance, and original cause.
Existing delete metadata tests explicitly project the hidden row ID.
- Added six evolved-spec tests for BINARY, FIXED, UUID, TIME, TIMESTAMP,
and TIMESTAMPTZ. These reproduced the review findings before the
follow-up fix. Coverage includes empty/NULL binary values, fixed-length
validation, read-only/direct buffer bounds, and negative fractional
timestamps.
- Ran `IcebergScanNodeTest`, `IcebergUtilsTest`,
`IcebergWriterHelperTest`, and `IcebergTransactionTest`: **217 tests
passed**, no failures or skips.
- FE Checkstyle: passed during the test build with
`-Dcheckstyle.skip=false`.
- Added `test_iceberg_delete_unpartitioned_evolution` to the external
Iceberg regression suite, covering Spark REPLACE followed by repeated
Doris DELETE, plus mixed-spec DELETE/UPDATE after dropping the last
partition field. It checks both query results against Spark and
delete-file spec IDs, with Parquet and ORC coverage.
- Extended the external suite with historical binary-partition DELETE,
pre-epoch timestamp-partition DELETE/UPDATE, and Parquet/ORC
dropped-source-column reads with safe DELETE/UPDATE rejection and
unchanged query results.
- Regression Groovy syntax check passed. The external regression suite
has **not been run end-to-end locally**; it requires the Iceberg/Spark
test environment. Spark REPLACE exercises the metadata transition
reported with Trino RTAS.
### Check List (For Author)
- Test
- [x] Regression test
- [x] Unit Test
- Behavior changed:
- [x] Yes. Preserve per-file partition metadata for evolved
unpartitioned tables.
- Does this need documentation?
- [x] No.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../doris/datasource/iceberg/IcebergUtils.java | 11 +-
.../datasource/iceberg/source/IcebergScanNode.java | 46 ++-
.../doris/datasource/iceberg/IcebergUtilsTest.java | 23 ++
.../iceberg/source/IcebergScanNodeTest.java | 339 +++++++++++++++++++++
...t_iceberg_delete_unpartitioned_evolution.groovy | 198 ++++++++++++
5 files changed, 601 insertions(+), 16 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
index bd5d2c8cb65..92fc7744060 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
@@ -1104,13 +1104,8 @@ public class IcebergUtils {
for (int i = 0; i < fields.size(); i++) {
NestedField field = fields.get(i);
Object value = partitionData.get(i);
- try {
- partitionValues.add(serializePartitionValue(field.type(), value, timeZone));
- } catch (UnsupportedOperationException e) {
- LOG.warn("Failed to serialize Iceberg partition value for field {}: {}", field.name(),
- e.getMessage());
- partitionValues.add(null);
- }
+ // These values also identify delete-file partitions; an unsupported value must never become NULL.
+ partitionValues.add(serializePartitionValue(field.type(), value, timeZone));
}
return partitionValues;
}
@@ -1407,6 +1402,8 @@ public class IcebergUtils {
return (int) LocalDate.parse(valueStr, DateTimeFormatter.ISO_LOCAL_DATE).toEpochDay();
case TIMESTAMP:
return parseTimestampToMicros(valueStr, (TimestampType) icebergType);
+ case TIME:
+ return LocalTime.parse(valueStr, DateTimeFormatter.ISO_LOCAL_TIME).toNanoOfDay() / 1000;
case DECIMAL:
return new BigDecimal(valueStr);
default:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
index 75489d13b37..de8ce891662 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -200,6 +200,7 @@ public class IcebergScanNode extends FileQueryScanNode {
private boolean enableMappingVarbinaryForPartitionMetadata;
private boolean enableMappingTimestampTzForPartitionMetadata;
private boolean isPartitionedTable;
+ private Boolean requiresRowId;
private int formatVersion;
private ExecutionAuthenticator preExecutionAuthenticator;
private IcebergRuntimeContext runtimeContext;
@@ -1658,6 +1659,12 @@ public class IcebergScanNode extends FileQueryScanNode {
try {
return preExecutionAuthenticator.execute(() -> doGetSplits(numBackends));
} catch (Exception e) {
+ // Authentication can wrap user errors; keep their guidance instead of only the root cause.
+ for (Throwable cause : ExceptionUtils.getThrowableList(e)) {
+ if (cause instanceof UserException) {
+ throw (UserException) cause;
+ }
+ }
Optional<NotSupportedException> opt = checkNotSupportedException(e);
if (opt.isPresent()) {
throw opt.get();
@@ -2556,6 +2563,15 @@ public class IcebergScanNode extends FileQueryScanNode {
return LocationPath.of(path, storagePropertiesMap);
}
+ private boolean requiresRowId() {
+ if (requiresRowId == null) {
+ // Slots are finalized by split planning; inspect them once for all files in this scan.
+ requiresRowId = desc.getSlots().stream().anyMatch(slot -> slot.getColumn() != null
+ && Column.ICEBERG_ROWID_COL.equalsIgnoreCase(slot.getColumn().getName()));
+ }
+ return requiresRowId;
+ }
+
private Split createIcebergSplit(FileScanTask fileScanTask) throws UserException {
DataFile dataFile = fileScanTask.file();
String originalPath = dataFile.path().toString();
@@ -2589,16 +2605,28 @@ public class IcebergScanNode extends FileQueryScanNode {
}
split.setTableFormatType(TableFormatType.ICEBERG);
split.setTargetSplitSize(selectFeSplitSize(fileScanTask, targetSplitSize));
- if (isPartitionedTable) {
- int specId = fileScanTask.file().specId();
+ // REPLACE or partition evolution can leave an unpartitioned table with historical specs.
+ // Row-level deletes must retain each file's spec instead of defaulting to historical spec 0.
+ int specId = dataFile.specId();
+ split.setPartitionSpecId(specId);
+ PartitionData partitionData = (PartitionData) dataFile.partition();
+ boolean includePartitionData = requiresRowId();
+ if (partitionData != null && (includePartitionData || isPartitionedTable)) {
PartitionSpec partitionSpec = icebergTable.specs().get(specId);
Preconditions.checkNotNull(partitionSpec, "Partition spec with specId %s not found for table %s",
specId, icebergTable.name());
- PartitionData partitionData = (PartitionData) fileScanTask.file().partition();
- if (partitionData != null) {
- split.setPartitionSpecId(specId);
- split.setPartitionDataJson(IcebergUtils.getPartitionDataJson(
- partitionData, partitionSpec, sessionVariable.getTimeZone()));
+ // Only row-ID consumers need this JSON; ordinary reads still retain spec and pruning metadata.
+ if (includePartitionData) {
+ try {
+ split.setPartitionDataJson(IcebergUtils.getPartitionDataJson(
+ partitionData, partitionSpec, sessionVariable.getTimeZone()));
+ } catch (UnsupportedOperationException e) {
+ // A dropped source column can leave UNKNOWN; DML must not substitute a NULL partition.
+ throw new UserException("Cannot produce Iceberg row IDs with unsupported partition types in spec "
+ + specId + ". Rewrite historical data files before DELETE or UPDATE.", e);
+ }
+ }
+ if (isPartitionedTable) {
Map<String, String> partitionInfoMap = partitionMapInfos.computeIfAbsent(
Pair.of(specId, partitionData), k -> IcebergUtils.getIdentityPartitionInfoMap(
partitionData, partitionSpec, icebergTable, sessionVariable.getTimeZone(),
@@ -2610,9 +2638,9 @@ public class IcebergScanNode extends FileQueryScanNode {
if (!partitionInfoMap.isEmpty()) {
split.setIcebergPartitionValues(partitionInfoMap);
}
- } else {
- partitionMapInfos.put(Pair.of(specId, null), Collections.emptyMap());
}
+ } else if (partitionData == null && isPartitionedTable) {
+ partitionMapInfos.put(Pair.of(specId, null), Collections.emptyMap());
}
return split;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
index 66bb9fb2c66..3717bbc2af3 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
@@ -1016,6 +1016,29 @@ public class IcebergUtilsTest {
Assert.assertNull(partitionInfoMap);
}
+ @Test
+ public void testBinaryPartitionJsonPreservesBufferBounds() {
+ for (org.apache.iceberg.types.Type type : Arrays.asList(Types.BinaryType.get(), Types.FixedType.ofLength(4))) {
+ Schema schema = new Schema(Types.NestedField.optional(1, "partition_key", type));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).identity("partition_key").build();
+ ByteBuffer buffer = ByteBuffer.allocateDirect(6);
+ buffer.put(new byte[] {42, 0, (byte) 0xff, (byte) 0x80, 0x2f, 42});
+ buffer.position(1);
+ buffer.limit(5);
+ ByteBuffer value = buffer.asReadOnlyBuffer();
+ PartitionData partition = new PartitionData(spec.partitionType());
+ partition.set(0, value);
+ List<String> encoded = IcebergUtils.parsePartitionValuesFromJson(
+ IcebergUtils.getPartitionDataJson(partition, spec, "UTC"));
+ Assert.assertEquals(Collections.singletonList("0x00ff802f"), encoded);
+ Assert.assertEquals(1, value.position());
+ Assert.assertEquals(5, value.limit());
+ Assert.assertEquals(value, IcebergUtils.parsePartitionValueFromString(encoded.get(0), type));
+ }
+ Assert.assertThrows(IllegalArgumentException.class,
+ () -> IcebergUtils.parsePartitionValueFromString("0x00ff802f", Types.FixedType.ofLength(3)));
+ }
+
@Test
public void testGetIdentityPartitionColumnsIgnoresTransformPartitions() {
Schema schema = new Schema(
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
index a104f5982ce..3e17f77ac74 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
@@ -37,6 +37,7 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.common.UserException;
import org.apache.doris.common.security.authentication.ExecutionAuthenticator;
+import org.apache.doris.common.security.authentication.HadoopExecutionAuthenticator;
import org.apache.doris.common.util.LocationPath;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.ExternalScanNode;
@@ -49,12 +50,14 @@ import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot;
import org.apache.doris.datasource.iceberg.IcebergPartitionInfo;
+import org.apache.doris.datasource.iceberg.IcebergRowId;
import org.apache.doris.datasource.iceberg.IcebergRuntimeContext;
import org.apache.doris.datasource.iceberg.IcebergSnapshot;
import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue;
import org.apache.doris.datasource.iceberg.IcebergSysExternalTable;
import org.apache.doris.datasource.iceberg.IcebergTableCacheValue;
import org.apache.doris.datasource.iceberg.IcebergUtils;
+import org.apache.doris.datasource.iceberg.helper.IcebergWriterHelper;
import org.apache.doris.datasource.mvcc.MvccTableInfo;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.planner.PlanNodeId;
@@ -65,10 +68,13 @@ import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TAccessPathType;
import org.apache.doris.thrift.TColumnAccessPath;
import org.apache.doris.thrift.TDataAccessPath;
+import org.apache.doris.thrift.TFileContent;
import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.thrift.TFileRangeDesc;
import org.apache.doris.thrift.TFileScanRangeParams;
+import org.apache.doris.thrift.TIcebergCommitData;
import org.apache.doris.thrift.TIcebergDeleteFileDesc;
+import org.apache.doris.thrift.TIcebergFileDesc;
import org.apache.doris.thrift.TMetaAccessPath;
import org.apache.doris.thrift.TPushAggOp;
import org.apache.doris.thrift.schema.external.TField;
@@ -78,6 +84,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.iceberg.AppendFiles;
import org.apache.iceberg.BaseMetadataTable;
import org.apache.iceberg.BaseTable;
@@ -1880,6 +1887,338 @@ public class IcebergScanNodeTest {
.isSetEqualityDeleteSchema());
}
+ @Test
+ public void testDeleteSplitKeepsSpecAfterReplacingPartitionedTable() throws Exception {
+ HadoopTables tables = new HadoopTables(new Configuration());
+ Schema schema = new Schema(Types.NestedField.required(1, "record_key", Types.IntegerType.get()));
+ String location = temporaryFolder.newFolder("replace_partitioned").toURI().toString();
+ Table table = tables.create(schema, PartitionSpec.builderFor(schema).identity("record_key").build(),
+ Collections.singletonMap(TableProperties.FORMAT_VERSION, "2"), location);
+ table.newAppend().appendFile(DataFiles.builder(table.spec()).withPath("file:///warehouse/old.parquet")
+ .withPartitionPath("record_key=7").withRecordCount(1).withFileSizeInBytes(128).build()).commit();
+ org.apache.iceberg.Transaction replace = tables.buildTable(location, schema)
+ .withPartitionSpec(PartitionSpec.unpartitioned()).replaceTransaction();
+ replace.newAppend().appendFile(DataFiles.builder(replace.table().spec()).withPath("file:///warehouse/new.parquet")
+ .withRecordCount(1).withFileSizeInBytes(128).build()).commit();
+ replace.commitTransaction();
+ table.refresh();
+
+ Assert.assertTrue(table.spec().isUnpartitioned());
+ Assert.assertTrue(table.specs().get(0).isPartitioned());
+ Assert.assertNotEquals(0, table.spec().specId());
+ assertDeleteSplitPartitionMetadata(table);
+ }
+
+ @Test
+ public void testDeleteSplitKeepsOldAndNewSpecsAfterDroppingPartitionField() throws Exception {
+ Schema schema = new Schema(Types.NestedField.required(1, "record_key", Types.IntegerType.get()));
+ Table table = new HadoopTables(new Configuration()).create(schema,
+ PartitionSpec.builderFor(schema).identity("record_key").build(),
+ Collections.singletonMap(TableProperties.FORMAT_VERSION, "2"),
+ temporaryFolder.newFolder("drop_partition_field").toURI().toString());
+ table.newAppend().appendFile(DataFiles.builder(table.spec()).withPath("file:///warehouse/old.parquet")
+ .withPartitionPath("record_key=7").withRecordCount(1).withFileSizeInBytes(128).build()).commit();
+ table.updateSpec().removeField("record_key").commit();
+ table.newAppend().appendFile(DataFiles.builder(table.spec()).withPath("file:///warehouse/new.parquet")
+ .withRecordCount(1).withFileSizeInBytes(128).build()).commit();
+
+ Assert.assertTrue(table.spec().isUnpartitioned());
+ assertDeleteSplitPartitionMetadata(table);
+ }
+
+ @Test
+ public void testReadSplitsSkipPartitionJson() throws Exception {
+ for (boolean keepPartitioned : Arrays.asList(false, true)) {
+ Schema schema = new Schema(Types.NestedField.required(1, "record_key", Types.IntegerType.get()));
+ Table table = new HadoopTables(new Configuration()).create(schema,
+ PartitionSpec.builderFor(schema).identity("record_key").build(),
+ Collections.singletonMap(TableProperties.FORMAT_VERSION, "2"),
+ temporaryFolder.newFolder().toURI().toString());
+ table.newAppend().appendFile(DataFiles.builder(table.spec()).withPath("file:///warehouse/old.parquet")
+ .withPartitionPath("record_key=7").withRecordCount(1).withFileSizeInBytes(128).build()).commit();
+ if (!keepPartitioned) {
+ table.updateSpec().removeField("record_key").commit();
+ }
+ DataFiles.Builder newFile = DataFiles.builder(table.spec()).withPath("file:///warehouse/new.parquet")
+ .withRecordCount(1).withFileSizeInBytes(128);
+ if (keepPartitioned) {
+ newFile.withPartitionPath("record_key=9");
+ }
+ table.newAppend().appendFile(newFile.build()).commit();
+
+ TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable());
+ node.addSlot(0, new Column("record_key", Type.INT));
+ setIcebergTable(node, table);
+ setPrivateField(node, "isPartitionedTable", keepPartitioned);
+ setPrivateField(node, "storagePropertiesMap", Collections.emptyMap());
+ setPrivateField(node, "formatVersion", 2);
+ setPrivateField(node, "partitionMapInfos", new HashMap<>());
+ setPrivateField(node, "orderedPathPartitionKeys", Collections.emptyList());
+ setPrivateField(node, "orderedPartitionMetadataKeys", Collections.emptyList());
+ try (CloseableIterable<FileScanTask> tasks = table.newScan().planFiles()) {
+ int fileCount = 0;
+ for (FileScanTask task : tasks) {
+ IcebergSplit split = createIcebergSplit(node, task);
+ TFileRangeDesc range = new TFileRangeDesc();
+ setIcebergParams(node, range, split);
+ TIcebergFileDesc file = range.getTableFormatParams().getIcebergParams();
+ Assert.assertTrue(file.isSetPartitionSpecId());
+ Assert.assertEquals(task.file().specId(), file.getPartitionSpecId());
+ Assert.assertFalse(file.isSetPartitionDataJson());
+ if (keepPartitioned) {
+ Assert.assertEquals(Collections.singletonMap("record_key",
+ task.file().partition().get(0, Integer.class).toString()),
+ split.getIcebergPartitionValues());
+ }
+ fileCount++;
+ }
+ Assert.assertEquals(2, fileCount);
+ }
+ }
+ }
+
+ @Test
+ public void testReadSplitAfterDroppingPartitionSourceColumn() throws Exception {
+ for (boolean keepPartitioned : Arrays.asList(false, true)) {
+ for (Integer value : Arrays.asList(7, null)) {
+ assertSplitAfterDroppingPartitionSourceColumn(keepPartitioned, value, false);
+ }
+ }
+ }
+
+ @Test
+ public void testRowIdScanRejectsDroppedPartitionSourceColumn() throws Exception {
+ for (boolean keepPartitioned : Arrays.asList(false, true)) {
+ for (Integer value : Arrays.asList(7, null)) {
+ assertSplitAfterDroppingPartitionSourceColumn(keepPartitioned, value, true);
+ }
+ }
+ }
+
+ private void assertSplitAfterDroppingPartitionSourceColumn(
+ boolean keepPartitioned, Integer value, boolean requireRowId) throws Exception {
+ assertSplitAfterDroppingPartitionSourceColumn(
+ keepPartitioned, value, requireRowId, new ExecutionAuthenticator() {});
+ }
+
+ @Test
+ public void testRowIdScanPreservesErrorThroughHadoopAuthentication() throws Exception {
+ assertSplitAfterDroppingPartitionSourceColumn(false, 7, true,
+ new HadoopExecutionAuthenticator(() -> UserGroupInformation.createRemoteUser("test_user")));
+ }
+
+ private void assertSplitAfterDroppingPartitionSourceColumn(boolean keepPartitioned, Integer value,
+ boolean requireRowId, ExecutionAuthenticator authenticator) throws Exception {
+ Schema schema = new Schema(Types.NestedField.optional(1, "record_key", Types.IntegerType.get()),
+ Types.NestedField.optional(2, "partition_key", Types.IntegerType.get()),
+ Types.NestedField.optional(3, "retained_key", Types.IntegerType.get()));
+ PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema).identity("partition_key");
+ if (keepPartitioned) {
+ specBuilder.identity("retained_key");
+ }
+ HadoopTables tables = new HadoopTables(new Configuration());
+ String location = temporaryFolder.newFolder().toURI().toString();
+ Table table = tables.create(schema, specBuilder.build(),
+ Collections.singletonMap(TableProperties.FORMAT_VERSION, "2"), location);
+ PartitionData partition = new PartitionData(table.spec().partitionType());
+ partition.set(0, value);
+ if (keepPartitioned) {
+ partition.set(1, 3);
+ }
+ table.newAppend().appendFile(DataFiles.builder(table.spec()).withPath("file:///warehouse/old.parquet")
+ .withPartition(partition).withRecordCount(1).withFileSizeInBytes(128).build()).commit();
+ table.updateSpec().removeField("partition_key").commit();
+ table.updateSchema().deleteColumn("partition_key").commit();
+ // Reload metadata so historical specs bind to the schema without the dropped source column.
+ table = tables.load(location);
+ Assert.assertEquals(keepPartitioned, table.spec().isPartitioned());
+ TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable());
+ setIcebergTable(node, table);
+ setPrivateField(node, "isPartitionedTable", keepPartitioned);
+ setPrivateField(node, "storagePropertiesMap", Collections.emptyMap());
+ setPrivateField(node, "formatVersion", 2);
+ setPrivateField(node, "partitionMapInfos", new HashMap<>());
+ setPrivateField(node, "orderedPathPartitionKeys", Collections.emptyList());
+ setPrivateField(node, "orderedPartitionMetadataKeys", Collections.emptyList());
+ node.addSlot(0, new Column("record_key", Type.INT));
+ if (requireRowId) {
+ node.addSlot(1, IcebergRowId.createHiddenColumn());
+ }
+ try (CloseableIterable<FileScanTask> tasks = table.newScan().planFiles()) {
+ int fileCount = 0;
+ for (FileScanTask task : tasks) {
+ PartitionData actualPartition = (PartitionData) task.file().partition();
+ Assert.assertEquals(Types.UnknownType.get(), actualPartition.getPartitionType().fields().get(0).type());
+ Assert.assertEquals(value, actualPartition.get(0));
+ if (requireRowId) {
+ UserException thrown = Assert.assertThrows(UserException.class,
+ () -> getSplitsWithTask(node, task, authenticator));
+ Assert.assertTrue(thrown.getMessage().contains(
+ "Cannot produce Iceberg row IDs with unsupported partition types"));
+ Assert.assertTrue(thrown.getMessage().contains("spec " + task.file().specId()));
+ Assert.assertTrue(thrown.getMessage().contains("Rewrite historical data files"));
+ Assert.assertTrue(thrown.getCause() instanceof UnsupportedOperationException);
+ } else {
+ IcebergSplit split = createIcebergSplit(node, task);
+ TFileRangeDesc range = new TFileRangeDesc();
+ setIcebergParams(node, range, split);
+ TIcebergFileDesc file = range.getTableFormatParams().getIcebergParams();
+ Assert.assertEquals(task.file().specId(), file.getPartitionSpecId());
+ Assert.assertFalse(file.isSetPartitionDataJson());
+ if (keepPartitioned) {
+ Assert.assertEquals(Collections.singletonMap("retained_key", "3"),
+ split.getIcebergPartitionValues());
+ }
+ }
+ fileCount++;
+ }
+ Assert.assertEquals(1, fileCount);
+ }
+ }
+
+ private static void getSplitsWithTask(IcebergScanNode node, FileScanTask task,
+ ExecutionAuthenticator authenticator) throws Exception {
+ ConnectContext previous = ConnectContext.get();
+ ConnectContext context = new ConnectContext();
+ context.setStatementContext(new StatementContext());
+ context.getStatementContext().setIcebergRewriteFileScanTasks(Collections.singletonList(task));
+ context.setThreadLocalInfo();
+ setPreExecutionAuthenticator(node, authenticator);
+ try {
+ // Exercise the public non-batch entry point used by row-level DML, including error wrapping.
+ node.getSplits(1);
+ } finally {
+ ConnectContext.remove();
+ if (previous != null) {
+ previous.setThreadLocalInfo();
+ }
+ }
+ }
+
+ @Test
+ public void testDeleteSplitKeepsBinaryPartitionAfterDroppingField() throws Exception {
+ assertDeleteAfterDroppingPartitionField(Types.BinaryType.get(),
+ ByteBuffer.wrap(new byte[] {0, (byte) 0xff, (byte) 0x80, 0x2f}));
+ assertDeleteAfterDroppingPartitionField(Types.BinaryType.get(), ByteBuffer.allocate(0));
+ assertDeleteAfterDroppingPartitionField(Types.BinaryType.get(), null);
+ }
+
+ @Test
+ public void testDeleteSplitKeepsFixedPartitionAfterDroppingField() throws Exception {
+ assertDeleteAfterDroppingPartitionField(Types.FixedType.ofLength(4),
+ ByteBuffer.wrap(new byte[] {0, (byte) 0xff, (byte) 0x80, 0x2f}));
+ assertDeleteAfterDroppingPartitionField(Types.FixedType.ofLength(4), null);
+ }
+
+ @Test
+ public void testDeleteSplitKeepsUuidPartitionAfterDroppingField() throws Exception {
+ assertDeleteAfterDroppingPartitionField(Types.UUIDType.get(),
+ UUID.fromString("123e4567-e89b-12d3-a456-426614174000"));
+ }
+
+ @Test
+ public void testDeleteSplitKeepsTimePartitionAfterDroppingField() throws Exception {
+ assertDeleteAfterDroppingPartitionField(Types.TimeType.get(), 12_345_678_901L);
+ }
+
+ @Test
+ public void testDeleteSplitKeepsPreEpochTimestampAfterDroppingField() throws Exception {
+ assertDeleteAfterDroppingPartitionField(Types.TimestampType.withoutZone(), -1L);
+ assertDeleteAfterDroppingPartitionField(Types.TimestampType.withoutZone(), -1_000_001L);
+ }
+
+ @Test
+ public void testDeleteSplitKeepsPreEpochTimestamptzAfterDroppingField() throws Exception {
+ assertDeleteAfterDroppingPartitionField(Types.TimestampType.withZone(), -1L);
+ assertDeleteAfterDroppingPartitionField(Types.TimestampType.withZone(), -1_000_001L);
+ }
+
+ private void assertDeleteAfterDroppingPartitionField(org.apache.iceberg.types.Type type, Object value)
+ throws Exception {
+ Schema schema = new Schema(Types.NestedField.optional(1, "partition_key", type));
+ Table table = new HadoopTables(new Configuration()).create(schema,
+ PartitionSpec.builderFor(schema).identity("partition_key").build(),
+ Collections.singletonMap(TableProperties.FORMAT_VERSION, "2"),
+ temporaryFolder.newFolder().toURI().toString());
+ PartitionData partition = new PartitionData(table.spec().partitionType());
+ partition.set(0, value);
+ table.newAppend().appendFile(DataFiles.builder(table.spec()).withPath("file:///warehouse/old.parquet")
+ .withPartition(partition).withRecordCount(1).withFileSizeInBytes(128).build()).commit();
+ table.updateSpec().removeField("partition_key").commit();
+ table.newAppend().appendFile(DataFiles.builder(table.spec()).withPath("file:///warehouse/new.parquet")
+ .withRecordCount(1).withFileSizeInBytes(128).build()).commit();
+
+ Assert.assertTrue(table.spec().isUnpartitioned());
+ assertDeleteSplitPartitionMetadata(table);
+ }
+
+ private void assertDeleteSplitPartitionMetadata(Table table) throws Exception {
+ ConnectContext previousContext = ConnectContext.get();
+ ConnectContext context = new ConnectContext();
+ context.getSessionVariable().setTimeZone("UTC");
+ context.setThreadLocalInfo();
+ try {
+ assertDeleteSplitPartitionMetadata(table, context.getSessionVariable());
+ } finally {
+ if (previousContext == null) {
+ ConnectContext.remove();
+ } else {
+ previousContext.setThreadLocalInfo();
+ }
+ }
+ }
+
+ private void assertDeleteSplitPartitionMetadata(Table table, SessionVariable sessionVariable) throws Exception {
+ TestIcebergScanNode node = new TestIcebergScanNode(sessionVariable);
+ node.addSlot(0, IcebergRowId.createHiddenColumn());
+ setIcebergTable(node, table);
+ setPrivateField(node, "isPartitionedTable", table.spec().isPartitioned());
+ setPrivateField(node, "storagePropertiesMap", Collections.emptyMap());
+ setPrivateField(node, "formatVersion", 2);
+ setPrivateField(node, "partitionMapInfos", new HashMap<>());
+ setPrivateField(node, "orderedPathPartitionKeys", Collections.emptyList());
+ setPrivateField(node, "orderedPartitionMetadataKeys", Collections.emptyList());
+ try (CloseableIterable<FileScanTask> tasks = table.newScan().planFiles()) {
+ int fileCount = 0;
+ for (FileScanTask task : tasks) {
+ IcebergSplit split = createIcebergSplit(node, task);
+ TFileRangeDesc range = new TFileRangeDesc();
+ setIcebergParams(node, range, split);
+ byte[] bytes = new TSerializer(new TCompactProtocol.Factory()).serialize(range);
+ TFileRangeDesc restored = new TFileRangeDesc();
+ new TDeserializer(new TCompactProtocol.Factory()).deserialize(restored, bytes);
+ TIcebergFileDesc file = restored.getTableFormatParams().getIcebergParams();
+ Assert.assertTrue("Every data file must carry its spec id through Thrift", file.isSetPartitionSpecId());
+ Assert.assertEquals(task.file().specId(), file.getPartitionSpecId());
+ PartitionSpec spec = table.specs().get(file.getPartitionSpecId());
+ Assert.assertNotNull(file.getPartitionDataJson());
+ if (spec.isUnpartitioned()) {
+ Assert.assertEquals("[]", file.getPartitionDataJson());
+ }
+
+ TIcebergCommitData commit = new TIcebergCommitData();
+ commit.setFilePath("delete-" + fileCount + ".parquet");
+ commit.setFileSize(128);
+ commit.setRowCount(1);
+ commit.setFileContent(TFileContent.POSITION_DELETES);
+ commit.setPartitionSpecId(file.getPartitionSpecId());
+ commit.setPartitionDataJson(file.getPartitionDataJson());
+ DeleteFile delete = IcebergWriterHelper.convertToDeleteFiles(
+ FileFormat.PARQUET, spec, Collections.singletonList(commit)).get(0);
+ Assert.assertEquals(task.file().specId(), delete.specId());
+ Assert.assertEquals(task.file().partition().size(), delete.partition().size());
+ // PartitionData.equals compares binary backing arrays by identity, not byte content.
+ for (int i = 0; i < task.file().partition().size(); i++) {
+ Assert.assertEquals(task.file().partition().get(i, Object.class),
+ delete.partition().get(i, Object.class));
+ }
+ fileCount++;
+ }
+ Assert.assertEquals(table.currentSnapshot().summary().get("total-data-files"),
+ Integer.toString(fileCount));
+ }
+ }
+
@Test
public void testDeleteFileSizePropagatedToThrift() throws Exception {
Types.NestedField id = Types.NestedField.required(1, "id", Types.LongType.get());
diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_delete_unpartitioned_evolution.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_delete_unpartitioned_evolution.groovy
new file mode 100644
index 00000000000..138b3a6c82c
--- /dev/null
+++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_delete_unpartitioned_evolution.groovy
@@ -0,0 +1,198 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_delete_unpartitioned_evolution",
+ "p0,external,iceberg,external_docker,external_docker_iceberg") {
+ if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableIcebergTest"))) {
+ logger.info("disable iceberg test")
+ return
+ }
+
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ String catalogName = "test_iceberg_delete_unpartitioned_evolution"
+ String dbName = "iceberg_delete_unpartitioned_evolution_db"
+ sql """drop catalog if exists ${catalogName}"""
+ sql """
+ create catalog ${catalogName} properties (
+ "type" = "iceberg",
+ "iceberg.catalog.type" = "rest",
+ "uri" = "http://${externalEnvIp}:${restPort}",
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.endpoint" = "http://${externalEnvIp}:${minioPort}",
+ "s3.region" = "us-east-1"
+ )
+ """
+ sql """switch ${catalogName}"""
+ sql """create database if not exists ${dbName}"""
+ sql """use ${dbName}"""
+
+ def checkRows = { String table, def expected ->
+ sql """refresh table ${dbName}.${table}"""
+ spark_iceberg """refresh table demo.${dbName}.${table}"""
+ def actual = sql """select record_key, metric from ${table} order by record_key"""
+ assertSparkDorisResultEquals(expected, actual)
+ assertSparkDorisResultEquals(spark_iceberg("""
+ select record_key, metric from demo.${dbName}.${table} order by record_key
+ """), actual)
+ }
+
+ // REPLACE retains historical spec 0 while writing new files under an unpartitioned spec.
+ // Spark's replace transaction exercises the same Iceberg metadata transition as Trino RTAS.
+ spark_iceberg """drop table if exists demo.${dbName}.replace_to_unpartitioned"""
+ spark_iceberg """
+ create table demo.${dbName}.replace_to_unpartitioned (record_key int, metric int)
+ using iceberg partitioned by (record_key)
+ tblproperties ('format-version' = '2', 'write.format.default' = 'parquet')
+ """
+ sql """refresh database ${dbName}"""
+ sql """insert into replace_to_unpartitioned values (1, 10), (2, 20), (3, 30)"""
+ spark_iceberg """refresh table demo.${dbName}.replace_to_unpartitioned"""
+ spark_iceberg """
+ create or replace table demo.${dbName}.replace_to_unpartitioned using iceberg
+ tblproperties ('format-version' = '2', 'write.delete.mode' = 'merge-on-read')
+ as select * from demo.${dbName}.replace_to_unpartitioned
+ """
+ sql """refresh table ${dbName}.replace_to_unpartitioned"""
+ def replacedSpecs = sql """select distinct spec_id from replace_to_unpartitioned\$data_files"""
+ assertEquals(1, replacedSpecs.size())
+ assertTrue((replacedSpecs[0][0] as int) > 0)
+ def expectedAfterReplaceDelete = spark_iceberg """
+ select record_key, metric from demo.${dbName}.replace_to_unpartitioned
+ where record_key <> 1 order by record_key
+ """
+ sql """delete from replace_to_unpartitioned where record_key = 1"""
+ checkRows("replace_to_unpartitioned", expectedAfterReplaceDelete)
+ assertEquals(replacedSpecs, sql("""
+ select distinct spec_id from replace_to_unpartitioned\$delete_files
+ """))
+ def expectedAfterRefreshDelete = spark_iceberg """
+ select record_key, metric from demo.${dbName}.replace_to_unpartitioned
+ where record_key <> 2 order by record_key
+ """
+ sql """delete from replace_to_unpartitioned where record_key = 2"""
+ checkRows("replace_to_unpartitioned", expectedAfterRefreshDelete)
+
+ // Dropping the last partition field leaves both old partitioned and new unpartitioned files live.
+ spark_iceberg """drop table if exists demo.${dbName}.mixed_partition_specs"""
+ spark_iceberg """
+ create table demo.${dbName}.mixed_partition_specs (record_key int, metric int)
+ using iceberg partitioned by (record_key)
+ tblproperties ('format-version' = '2', 'write.format.default' = 'orc',
+ 'write.delete.mode' = 'merge-on-read', 'write.update.mode' = 'merge-on-read')
+ """
+ spark_iceberg """insert into demo.${dbName}.mixed_partition_specs values (1, 10), (2, 20)"""
+ spark_iceberg """alter table demo.${dbName}.mixed_partition_specs drop partition field record_key"""
+ spark_iceberg """insert into demo.${dbName}.mixed_partition_specs values (3, 30), (4, 40)"""
+ sql """refresh database ${dbName}"""
+ def mixedSpecs = sql """select distinct spec_id from mixed_partition_specs\$data_files order by spec_id"""
+ assertEquals(2, mixedSpecs.size())
+ def expectedAfterMixedDelete = spark_iceberg """
+ select record_key, metric from demo.${dbName}.mixed_partition_specs
+ where record_key not in (1, 3) order by record_key
+ """
+ sql """delete from mixed_partition_specs where record_key in (1, 3)"""
+ checkRows("mixed_partition_specs", expectedAfterMixedDelete)
+ assertEquals(mixedSpecs, sql("""
+ select distinct spec_id from mixed_partition_specs\$delete_files order by spec_id
+ """))
+ def expectedAfterUpdate = spark_iceberg """
+ select record_key, metric + 100 from demo.${dbName}.mixed_partition_specs order by record_key
+ """
+ sql """update mixed_partition_specs set metric = metric + 100"""
+ checkRows("mixed_partition_specs", expectedAfterUpdate)
+
+ // Dropping a source column makes its historical partition type unknown, even for non-null values.
+ for (String fileFormat : ["parquet", "orc"]) {
+ String table = "dropped_partition_source_${fileFormat}"
+ spark_iceberg """drop table if exists demo.${dbName}.${table}"""
+ spark_iceberg """
+ create table demo.${dbName}.${table} (record_key int, partition_key int, metric int)
+ using iceberg partitioned by (partition_key)
+ tblproperties ('format-version' = '2', 'write.format.default' = '${fileFormat}',
+ 'write.delete.mode' = 'merge-on-read', 'write.update.mode' = 'merge-on-read')
+ """
+ spark_iceberg """insert into demo.${dbName}.${table} values (1, 7, 10), (2, 7, 20), (3, null, 30)"""
+ spark_iceberg """alter table demo.${dbName}.${table} drop partition field partition_key"""
+ spark_iceberg """alter table demo.${dbName}.${table} drop column partition_key"""
+ spark_iceberg """insert into demo.${dbName}.${table} values (4, 40)"""
+ sql """refresh database ${dbName}"""
+ def expected = spark_iceberg """
+ select record_key, metric from demo.${dbName}.${table} order by record_key
+ """
+ checkRows(table, expected)
+ test {
+ sql """delete from ${table} where record_key = 1"""
+ exception "Cannot produce Iceberg row IDs with unsupported partition types"
+ }
+ test {
+ sql """update ${table} set metric = metric + 100 where record_key = 2"""
+ exception "Cannot produce Iceberg row IDs with unsupported partition types"
+ }
+ checkRows(table, expected)
+ assertEquals([[0L]], sql("""select count(*) from ${table}\$delete_files"""))
+ }
+
+ // Historical binary partitions must retain their bytes, not become NULL delete partitions.
+ spark_iceberg """drop table if exists demo.${dbName}.historical_binary"""
+ spark_iceberg """
+ create table demo.${dbName}.historical_binary (record_key int, partition_key binary, metric int)
+ using iceberg partitioned by (partition_key)
+ tblproperties ('format-version' = '2', 'write.delete.mode' = 'merge-on-read')
+ """
+ spark_iceberg """insert into demo.${dbName}.historical_binary values
+ (1, unhex('00ff802f'), 10), (2, unhex('00ff802f'), 20), (3, null, 30), (4, unhex(''), 40)"""
+ spark_iceberg """alter table demo.${dbName}.historical_binary drop partition field partition_key"""
+ spark_iceberg """insert into demo.${dbName}.historical_binary values (5, unhex('ff'), 50)"""
+ sql """refresh database ${dbName}"""
+ def expectedAfterBinaryDelete = spark_iceberg """
+ select record_key, metric from demo.${dbName}.historical_binary
+ where record_key not in (1, 3, 4, 5) order by record_key
+ """
+ sql """delete from historical_binary where record_key in (1, 3, 4, 5)"""
+ checkRows("historical_binary", expectedAfterBinaryDelete)
+
+ // Negative fractional epochs require floor-based splitting into seconds and nanoseconds.
+ spark_iceberg """drop table if exists demo.${dbName}.historical_timestamp"""
+ spark_iceberg """
+ create table demo.${dbName}.historical_timestamp
+ (record_key int, partition_key timestamp_ntz, metric int)
+ using iceberg partitioned by (partition_key)
+ tblproperties ('format-version' = '2', 'write.delete.mode' = 'merge-on-read',
+ 'write.update.mode' = 'merge-on-read')
+ """
+ spark_iceberg """insert into demo.${dbName}.historical_timestamp values
+ (1, cast('1969-12-31 23:59:59.999999' as timestamp_ntz), 10),
+ (2, cast('1969-12-31 23:59:58.999999' as timestamp_ntz), 20), (3, null, 30)"""
+ spark_iceberg """alter table demo.${dbName}.historical_timestamp drop partition field partition_key"""
+ spark_iceberg """insert into demo.${dbName}.historical_timestamp values
+ (4, cast('1970-01-01 00:00:00.000001' as timestamp_ntz), 40)"""
+ sql """refresh database ${dbName}"""
+ def expectedAfterTimestampDelete = spark_iceberg """
+ select record_key, metric from demo.${dbName}.historical_timestamp
+ where record_key not in (1, 4) order by record_key
+ """
+ sql """delete from historical_timestamp where record_key in (1, 4)"""
+ checkRows("historical_timestamp", expectedAfterTimestampDelete)
+ def expectedAfterTimestampUpdate = spark_iceberg """
+ select record_key, metric + 100 from demo.${dbName}.historical_timestamp order by record_key
+ """
+ sql """update historical_timestamp set metric = metric + 100"""
+ checkRows("historical_timestamp", expectedAfterTimestampUpdate)
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 17/35: branch-4.1: [fix](auth) Check column privileges held by grantor in GRANT/REVOKE (#68629)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2f9f1e253fa924815caf2642a372c66dfdd2ab7e
Author: Calvin Kirs <gu...@selectdb.com>
AuthorDate: Tue Sep 29 23:35:34 2026 +0800
branch-4.1: [fix](auth) Check column privileges held by grantor in GRANT/REVOKE (#68629)
https://github.com/apache/doris/pull/68555
---
.../plans/commands/GrantTablePrivilegeCommand.java | 25 ++++-
.../commands/GrantTablePrivilegeCommandTest.java | 87 ++++++++++++++++
.../data/account_p0/test_grant_col_priv.out | 7 ++
.../suites/account_p0/test_grant_col_priv.groovy | 109 +++++++++++++++++++++
4 files changed, 226 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/GrantTablePrivilegeCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/GrantTablePrivilegeCommand.java
index 16169ef8326..0672cf57be5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/GrantTablePrivilegeCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/GrantTablePrivilegeCommand.java
@@ -27,6 +27,7 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.common.UserException;
import org.apache.doris.mysql.privilege.AccessControllerManager;
import org.apache.doris.mysql.privilege.Auth;
import org.apache.doris.mysql.privilege.ColPrivilegeKey;
@@ -133,6 +134,7 @@ public class GrantTablePrivilegeCommand extends Command implements ForwardWithSy
* 3. Only the user with NODE_PRIV can grant NODE_PRIV to other user
* 4. Check that the current user has both grant_priv and the permissions to be assigned to others
* 5. col priv must assign to specific table
+ * 6. Check that the current user has the col privs to be assigned to others, on the table or on each col
*/
public static void checkTablePrivileges(Collection<Privilege> privileges, TablePattern tblPattern,
Map<ColPrivilegeKey, Set<String>> colPrivileges) throws AnalysisException {
@@ -153,8 +155,8 @@ public class GrantTablePrivilegeCommand extends Command implements ForwardWithSy
// Rule 4
PrivPredicate predicate = getPrivPredicate(privileges);
AccessControllerManager accessManager = Env.getCurrentEnv().getAccessManager();
- if (!accessManager.checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)
- && !checkTablePriv(ConnectContext.get(), predicate, tblPattern)) {
+ boolean isAdmin = accessManager.checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN);
+ if (!isAdmin && !checkTablePriv(ConnectContext.get(), predicate, tblPattern)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ALL_ACCESS_DENIED_ERROR,
predicate.getPrivs().toPrivilegeList());
}
@@ -163,6 +165,25 @@ public class GrantTablePrivilegeCommand extends Command implements ForwardWithSy
if (!MapUtils.isEmpty(colPrivileges) && "*".equals(tblPattern.getTbl())) {
throw new AnalysisException("Col auth must specify specific table");
}
+
+ // Rule 6
+ if (!isAdmin) {
+ checkColPrivs(ConnectContext.get(), colPrivileges);
+ }
+ }
+
+ private static void checkColPrivs(ConnectContext ctx, Map<ColPrivilegeKey, Set<String>> colPrivileges)
+ throws AnalysisException {
+ AccessControllerManager accessManager = Env.getCurrentEnv().getAccessManager();
+ for (Map.Entry<ColPrivilegeKey, Set<String>> entry : colPrivileges.entrySet()) {
+ ColPrivilegeKey key = entry.getKey();
+ PrivPredicate wanted = PrivPredicate.of(PrivBitSet.of(key.getPrivilege()), CompoundPredicate.Operator.OR);
+ try {
+ accessManager.checkColumnsPriv(ctx, key.getCtl(), key.getDb(), key.getTbl(), entry.getValue(), wanted);
+ } catch (UserException e) {
+ throw new AnalysisException(e.getMessage(), e);
+ }
+ }
}
private static PrivPredicate getPrivPredicate(Collection<Privilege> privileges) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/GrantTablePrivilegeCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/GrantTablePrivilegeCommandTest.java
index bd022f4e2af..65f80e75732 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/GrantTablePrivilegeCommandTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/GrantTablePrivilegeCommandTest.java
@@ -18,8 +18,10 @@
package org.apache.doris.nereids.trees.plans.commands;
import org.apache.doris.analysis.TablePattern;
+import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.AccessPrivilege;
import org.apache.doris.catalog.AccessPrivilegeWithCols;
+import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
@@ -104,4 +106,89 @@ public class GrantTablePrivilegeCommandTest extends TestWithFeService {
Assertions.assertTrue(plan instanceof GrantTablePrivilegeCommand);
Assertions.assertThrows(DdlException.class, () -> ((GrantTablePrivilegeCommand) plan).run(connectContext, null));
}
+
+ @Test
+ public void testGrantColPrivWithOnlyGrantPriv() throws Exception {
+ addUser("col_grantor1", true);
+ addUser("col_target1", true);
+ grantPriv("GRANT GRANT_PRIV ON test.test_table TO 'col_grantor1'");
+ try {
+ useUser("col_grantor1");
+ Assertions.assertThrows(AnalysisException.class,
+ () -> runCommand("GRANT SELECT_PRIV(k1) ON test.test_table TO 'col_target1'"));
+ Assertions.assertThrows(AnalysisException.class,
+ () -> runCommand("GRANT SELECT_PRIV(k1, k2) ON test.test_table TO 'col_grantor1'"));
+ Assertions.assertThrows(AnalysisException.class,
+ () -> runCommand("REVOKE SELECT_PRIV(k1) ON test.test_table FROM 'col_target1'"));
+ } finally {
+ connectContext.setCurrentUserIdentity(UserIdentity.ROOT);
+ }
+ }
+
+ @Test
+ public void testGrantColPrivWithColSelectPriv() throws Exception {
+ addUser("col_grantor2", true);
+ addUser("col_target2", true);
+ grantPriv("GRANT GRANT_PRIV, SELECT_PRIV(k1) ON test.test_table TO 'col_grantor2'");
+ try {
+ useUser("col_grantor2");
+ Assertions.assertDoesNotThrow(
+ () -> runCommand("GRANT SELECT_PRIV(k1) ON test.test_table TO 'col_target2'"));
+ AnalysisException e = Assertions.assertThrows(AnalysisException.class,
+ () -> runCommand("GRANT SELECT_PRIV(k2) ON test.test_table TO 'col_target2'"));
+ Assertions.assertTrue(e.getMessage().contains("k2"), e.getMessage());
+ Assertions.assertThrows(AnalysisException.class,
+ () -> runCommand("GRANT SELECT_PRIV(k1, k2) ON test.test_table TO 'col_target2'"));
+ Assertions.assertThrows(AnalysisException.class,
+ () -> runCommand("REVOKE SELECT_PRIV(k2) ON test.test_table FROM 'col_target2'"));
+ Assertions.assertDoesNotThrow(
+ () -> runCommand("REVOKE SELECT_PRIV(k1) ON test.test_table FROM 'col_target2'"));
+ } finally {
+ connectContext.setCurrentUserIdentity(UserIdentity.ROOT);
+ }
+ }
+
+ @Test
+ public void testGrantColPrivWithTableSelectPriv() throws Exception {
+ addUser("col_grantor3", true);
+ addUser("col_target3", true);
+ grantPriv("GRANT GRANT_PRIV, SELECT_PRIV ON test.test_table TO 'col_grantor3'");
+ try {
+ useUser("col_grantor3");
+ Assertions.assertDoesNotThrow(
+ () -> runCommand("GRANT SELECT_PRIV(k1, k2) ON test.test_table TO 'col_target3'"));
+ Assertions.assertDoesNotThrow(
+ () -> runCommand("REVOKE SELECT_PRIV(k2) ON test.test_table FROM 'col_target3'"));
+ } finally {
+ connectContext.setCurrentUserIdentity(UserIdentity.ROOT);
+ }
+ }
+
+ @Test
+ public void testGrantColPrivWithDbGrantPriv() throws Exception {
+ addUser("col_grantor4", true);
+ addUser("col_target4", true);
+ grantPriv("GRANT GRANT_PRIV ON test.* TO 'col_grantor4'");
+ try {
+ useUser("col_grantor4");
+ Assertions.assertThrows(AnalysisException.class,
+ () -> runCommand("GRANT SELECT_PRIV(k1) ON test.test_table TO 'col_target4'"));
+ } finally {
+ connectContext.setCurrentUserIdentity(UserIdentity.ROOT);
+ }
+ }
+
+ @Test
+ public void testGrantColPrivByAdmin() throws Exception {
+ addUser("col_target5", true);
+ Assertions.assertDoesNotThrow(
+ () -> runCommand("GRANT SELECT_PRIV(k1) ON test.test_table TO 'col_target5'"));
+ Assertions.assertDoesNotThrow(
+ () -> runCommand("REVOKE SELECT_PRIV(k1) ON test.test_table FROM 'col_target5'"));
+ }
+
+ private void runCommand(String sql) throws Exception {
+ LogicalPlan plan = new NereidsParser().parseSingle(sql);
+ ((Command) plan).run(connectContext, null);
+ }
}
diff --git a/regression-test/data/account_p0/test_grant_col_priv.out b/regression-test/data/account_p0/test_grant_col_priv.out
new file mode 100644
index 00000000000..e26ff33084a
--- /dev/null
+++ b/regression-test/data/account_p0/test_grant_col_priv.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_c1 --
+1
+
+-- !select_c1_c2 --
+1 2
+
diff --git a/regression-test/suites/account_p0/test_grant_col_priv.groovy b/regression-test/suites/account_p0/test_grant_col_priv.groovy
new file mode 100644
index 00000000000..fb19a7a7ca8
--- /dev/null
+++ b/regression-test/suites/account_p0/test_grant_col_priv.groovy
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_grant_col_priv") {
+ String pwd = 'C123_567p'
+
+ try_sql("drop user test_grant_col_priv_grantor")
+ try_sql("drop user test_grant_col_priv_target")
+ sql """drop database if exists test_grant_col_priv_db"""
+
+ sql """create database test_grant_col_priv_db"""
+ sql """
+ create table test_grant_col_priv_db.test_grant_col_priv_tbl (c1 int, c2 int)
+ duplicate key(c1)
+ distributed by hash(c1) buckets 1
+ properties ("replication_num" = "1")
+ """
+ sql """insert into test_grant_col_priv_db.test_grant_col_priv_tbl values (1, 2)"""
+
+ sql """create user 'test_grant_col_priv_grantor' identified by '${pwd}'"""
+ sql """create user 'test_grant_col_priv_target' identified by '${pwd}'"""
+ //cloud-mode
+ if (isCloudMode()) {
+ def clusters = sql " SHOW CLUSTERS; "
+ assertTrue(!clusters.isEmpty())
+ def validCluster = clusters[0][0]
+ sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO test_grant_col_priv_grantor"""
+ sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO test_grant_col_priv_target"""
+ }
+ // for login
+ sql """grant select_priv on regression_test to test_grant_col_priv_grantor"""
+ sql """grant select_priv on regression_test to test_grant_col_priv_target"""
+
+ // have grant_priv only, can not grant col select_priv
+ sql """grant grant_priv on test_grant_col_priv_db.test_grant_col_priv_tbl to test_grant_col_priv_grantor"""
+ connect('test_grant_col_priv_grantor', "${pwd}", context.config.jdbcUrl) {
+ test {
+ sql """grant select_priv(c1) on test_grant_col_priv_db.test_grant_col_priv_tbl to test_grant_col_priv_target"""
+ exception "denied"
+ }
+ test {
+ sql """grant select_priv(c1, c2) on test_grant_col_priv_db.test_grant_col_priv_tbl to test_grant_col_priv_grantor"""
+ exception "denied"
+ }
+ }
+ connect('test_grant_col_priv_target', "${pwd}", context.config.jdbcUrl) {
+ test {
+ sql """select c1 from test_grant_col_priv_db.test_grant_col_priv_tbl"""
+ exception "denied"
+ }
+ }
+
+ // have grant_priv and select_priv on c1, can grant/revoke select_priv on c1 but not on c2
+ sql """grant select_priv(c1) on test_grant_col_priv_db.test_grant_col_priv_tbl to test_grant_col_priv_grantor"""
+ connect('test_grant_col_priv_grantor', "${pwd}", context.config.jdbcUrl) {
+ sql """grant select_priv(c1) on test_grant_col_priv_db.test_grant_col_priv_tbl to test_grant_col_priv_target"""
+ test {
+ sql """grant select_priv(c2) on test_grant_col_priv_db.test_grant_col_priv_tbl to test_grant_col_priv_target"""
+ exception "denied"
+ }
+ test {
+ sql """grant select_priv(c1, c2) on test_grant_col_priv_db.test_grant_col_priv_tbl to test_grant_col_priv_target"""
+ exception "denied"
+ }
+ test {
+ sql """revoke select_priv(c2) on test_grant_col_priv_db.test_grant_col_priv_tbl from test_grant_col_priv_target"""
+ exception "denied"
+ }
+ }
+ connect('test_grant_col_priv_target', "${pwd}", context.config.jdbcUrl) {
+ order_qt_select_c1 """select c1 from test_grant_col_priv_db.test_grant_col_priv_tbl"""
+ test {
+ sql """select c2 from test_grant_col_priv_db.test_grant_col_priv_tbl"""
+ exception "denied"
+ }
+ }
+ connect('test_grant_col_priv_grantor', "${pwd}", context.config.jdbcUrl) {
+ sql """revoke select_priv(c1) on test_grant_col_priv_db.test_grant_col_priv_tbl from test_grant_col_priv_target"""
+ }
+ connect('test_grant_col_priv_target', "${pwd}", context.config.jdbcUrl) {
+ test {
+ sql """select c1 from test_grant_col_priv_db.test_grant_col_priv_tbl"""
+ exception "denied"
+ }
+ }
+
+ // have grant_priv and select_priv on the table, can grant select_priv on any col
+ sql """grant select_priv on test_grant_col_priv_db.test_grant_col_priv_tbl to test_grant_col_priv_grantor"""
+ connect('test_grant_col_priv_grantor', "${pwd}", context.config.jdbcUrl) {
+ sql """grant select_priv(c1, c2) on test_grant_col_priv_db.test_grant_col_priv_tbl to test_grant_col_priv_target"""
+ }
+ connect('test_grant_col_priv_target', "${pwd}", context.config.jdbcUrl) {
+ order_qt_select_c1_c2 """select c1, c2 from test_grant_col_priv_db.test_grant_col_priv_tbl"""
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 33/35: branch-4.1: [fix](be) Stop background workers before startup exit #68652 (#68653)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4c5f158be1ce6d08b728915d9d34637ad3d4d5ad
Author: Gavin Chou <ga...@selectdb.com>
AuthorDate: Thu Oct 8 10:13:24 2026 +0800
branch-4.1: [fix](be) Stop background workers before startup exit #68652 (#68653)
### What problem does this PR solve?
Issue Number: None
Related PR: #68652
Problem Summary:
This PR backports the startup-failure lifecycle fix from #68652 to
`branch-4.1`.
When a BE service fails to start with `enable_graceful_exit_check=true`,
the startup failure path calls `exit(1)` while StorageEngine background
workers are still running. Process-wide static destruction can then race
with a path GC worker that is using the function-local static RE2
matcher in `TabletManager::get_tablet_id_and_schema_hash_from_path()`,
causing a null-address crash in `re2::Prog::SearchOnePass`.
The fix stops and joins created server starters, releases the backend
service, and explicitly destroys `ExecEnv` before calling `exit(1)`. The
immediate `_exit(1)` path remains unchanged when graceful exit checking
is disabled.
The backport has the same stable patch-id as #68652.
### Release note
None
### Check List (For Author)
- Test:
- [x] Manual test
- Targeted clang-format 16 check passed for
`be/src/service/doris_main.cpp` on `branch-4.1`.
- A local BE build could not start because the macOS environment does
not have Homebrew; `run buildall` is requested below for full build
validation.
- Behavior changed:
- [ ] Yes
- [x] No. Only abnormal BE startup cleanup ordering changes.
- Does this need documentation?
- [x] No
- [ ] Yes
---
be/src/service/doris_main.cpp | 69 +++++++++++++++++++++++++++++++++----------
1 file changed, 54 insertions(+), 15 deletions(-)
diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp
index 346ab386b15..2f31e3055a0 100644
--- a/be/src/service/doris_main.cpp
+++ b/be/src/service/doris_main.cpp
@@ -326,10 +326,18 @@ struct Checker {
// running the leak check on this abnormal-exit path reports them as false-positive
// leaks. enable_graceful_exit_check is honored so memleak-check mode still runs LSAN.
[[noreturn]] static void exit_on_startup_failure() {
- google::FlushLogFiles(google::GLOG_INFO);
if (!doris::config::enable_graceful_exit_check) {
+ google::FlushLogFiles(google::GLOG_INFO);
_exit(1);
}
+
+ // exit() starts destroying function-local statics while background threads are still
+ // running. Tear down ExecEnv first so StorageEngine workers are stopped and joined before
+ // they can race with those destructors.
+ if (doris::ExecEnv::ready()) {
+ doris::ExecEnv::GetInstance()->destroy();
+ }
+ google::FlushLogFiles(google::GLOG_INFO);
exit(1);
}
@@ -635,11 +643,38 @@ int main(int argc, char** argv) {
doris::ThriftRpcHelper::setup(exec_env);
// 1. thrift server with be_port
std::shared_ptr<doris::BaseBackendService> service;
+ std::unique_ptr<doris::server::IServerStarter> backend_thrift_starter;
+ std::unique_ptr<doris::server::IServerStarter> brpc_starter;
+ std::unique_ptr<doris::server::IServerStarter> http_starter;
+ std::unique_ptr<doris::server::IServerStarter> heartbeat_thrift_starter;
+ std::unique_ptr<doris::server::IServerStarter> flight_starter;
+ bool backend_thrift_started = false;
+ bool brpc_started = false;
+ bool http_started = false;
+ bool heartbeat_thrift_started = false;
+ bool flight_started = false;
+ auto stop_and_join_server = [](std::unique_ptr<doris::server::IServerStarter>& starter,
+ bool started) {
+ if (starter != nullptr) {
+ if (started) {
+ starter->stop();
+ }
+ starter->join();
+ }
+ };
std::function<void(Status&, std::string_view)> stop_work_if_error = [&](Status& status,
std::string_view msg) {
if (!status.ok()) {
std::cerr << msg << '\n';
service->stop_works();
+ if (doris::config::enable_graceful_exit_check) {
+ stop_and_join_server(flight_starter, flight_started);
+ stop_and_join_server(heartbeat_thrift_starter, heartbeat_thrift_started);
+ stop_and_join_server(http_starter, http_started);
+ stop_and_join_server(backend_thrift_starter, backend_thrift_started);
+ stop_and_join_server(brpc_starter, brpc_started);
+ service.reset();
+ }
exit_on_startup_failure();
}
};
@@ -652,44 +687,48 @@ int main(int argc, char** argv) {
exec_env);
}
- std::unique_ptr<doris::server::IServerStarter> backend_thrift_starter;
- EXIT_IF_ERROR(doris::server::create_backend_thrift_starter(exec_env, doris::config::be_port,
- service, &backend_thrift_starter));
+ status = doris::server::create_backend_thrift_starter(exec_env, doris::config::be_port, service,
+ &backend_thrift_starter);
+ stop_work_if_error(status, "Failed to create BE server, exiting");
status = backend_thrift_starter->start();
+ backend_thrift_started = status.ok();
stop_work_if_error(status, "Doris BE server did not start correctly, exiting");
// 2. brpc service
- std::unique_ptr<doris::server::IServerStarter> brpc_starter;
- EXIT_IF_ERROR(doris::server::create_brpc_starter(
- exec_env, doris::config::brpc_port, doris::config::brpc_num_threads, &brpc_starter));
+ status = doris::server::create_brpc_starter(exec_env, doris::config::brpc_port,
+ doris::config::brpc_num_threads, &brpc_starter);
+ stop_work_if_error(status, "Failed to create BRPC service, exiting");
status = brpc_starter->start();
+ brpc_started = status.ok();
stop_work_if_error(status, "BRPC service did not start correctly, exiting");
// 3. http service
- std::unique_ptr<doris::server::IServerStarter> http_starter;
- EXIT_IF_ERROR(doris::server::create_http_starter(exec_env, doris::config::webserver_port,
- doris::config::webserver_num_workers,
- &http_starter));
+ status =
+ doris::server::create_http_starter(exec_env, doris::config::webserver_port,
+ doris::config::webserver_num_workers, &http_starter);
+ stop_work_if_error(status, "Failed to create BE HTTP service, exiting");
status = http_starter->start();
+ http_started = status.ok();
stop_work_if_error(status, "Doris Be http service did not start correctly, exiting");
// 4. heart beat server
doris::ClusterInfo* cluster_info = exec_env->cluster_info();
- std::unique_ptr<doris::server::IServerStarter> heartbeat_thrift_starter;
status = doris::server::create_heartbeat_thrift_starter(
exec_env, doris::config::heartbeat_service_port,
doris::config::heartbeat_service_thread_count, cluster_info, &heartbeat_thrift_starter);
stop_work_if_error(status, "Heartbeat services did not start correctly, exiting");
status = heartbeat_thrift_starter->start();
+ heartbeat_thrift_started = status.ok();
stop_work_if_error(status, "Doris BE HeartBeat Service did not start correctly, exiting: " +
status.to_string());
// 5. arrow flight service
- std::unique_ptr<doris::server::IServerStarter> flight_starter;
- EXIT_IF_ERROR(doris::server::create_flight_starter(doris::config::arrow_flight_sql_port,
- &flight_starter));
+ status = doris::server::create_flight_starter(doris::config::arrow_flight_sql_port,
+ &flight_starter);
+ stop_work_if_error(status, "Failed to create Arrow Flight service, exiting");
status = flight_starter->start();
+ flight_started = status.ok();
stop_work_if_error(
status, "Arrow Flight Service did not start correctly, exiting, " + status.to_string());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org