You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/10/10 02:49:27 UTC
(doris) branch branch-4.2 updated (cabcf45c1af -> d3063235507)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
from cabcf45c1af [branch-4.2](thrift) Renumber enable_prune_nested_column to 230 on branch-4.2 (#68782)
new 5acff117d6e [improvement](lance) Upgrade prewarm dependency and remove install guard (branch-4.1) (#68698)
new 0910ff2ebcc [fix](ci) Backport MinIO image migration to branch-4.1 (#68777)
new 88549d00336 [fix](catalog) Validate external table partition models on branch-4.1 (#68769)
new 51f857541b3 [fix](flight-sql) Report actual server and Arrow versions on branch-4.1 (#68770)
new 97df025c483 branch-4.1: [fix](iceberg) Resolve dropped equality-delete fields in manifest-cache planning (#68433)
new b621170f282 branch-4.1:[fix](rowid-fetch) fix TopN lazy materialization crash with duplicateprojected columns (#64372) (#67563)
new f9d755fcb6a branch-4.1: [fix](iceberg) keep identity partition values of old specs after evolving to unpartitioned(#68124) (#68259)
new 68e4aa38915 branch-4.1: [fix](be) Defer Bloom allocation for IN runtime filters #68763 (#68789)
new 409acf9dc53 branch-4.1: [fix](be) Prevent shared quantile state mutation with TDigest COW (#68498) (#68759)
new e48911e417e [feature](flight) Support native Variant V2 results for ADBC on branch-4.1 (#68667)
new c552df550f4 [feature](lance) Search historical versions, tags and branches with vector_search and full_text_search (#68707)
new cd3af8974c7 [fix](flight-sql) Support prepared query parameter binding on branch-4.1 (#68768)
new e2a64e268be [improvement](metrics) Unify build version metric labels #68655 (#68656)
new 2a6f1521f03 [fix](regression) Bound large Parquet map output batches (#68808)
new 18b626064b9 [fix](parquet) Handle empty filter pages and offset index fallback (#68780)
new 658f5958b33 branch-4.1: [fix](cloud) Delete versioned tablet indexes and metadata atomically #68322 (#68817)
new 3030d467957 branch-4.1: [fix](window) Recompute sliding frames for floating-point sum/avg #68113 (#68736)
new 89fa7feecc2 branch-4.1: [fix](test)(dynamic-partition) fix some unstable test cases and dynamic-partition logic (#63551) (#68820)
new 66f60c1da50 branch-4.1: [fix](expr) Canonicalize logical OR results (#68798)
new f5dfeb66974 [fix](be) Fix null-safe string joins in 4.1 (backport #65975) (#68809)
new 3aa1a3c0285 branch-4.1: [fix](regression-test) Make test_analyze independent of the SQL cache #68829 (#68843)
new 4981796ac38 branch-4.1: [fix](function) Sum integers exactly in array_avg #68632 (#68831)
new ada545de782 branch-4.1: [fix](function) Reject a non-constant seed in array_shuffle #68438 (#68832)
new 3f8decb3429 [fix](fe) Fix HTTP system forwarding on follower FE (#68827)
new 3c4b3d93a59 branch-4.1: [fix](load) Avoid false quorum failures with distributed commit reports #68275 (#68818)
new 8b59ed1ec22 branch-4.1: [refactor](lance) Remove the durable Lance index job framework (#66497) (#68781)
new 698a018d503 branch-4.1: [fix](function) Fix overflow in quarter arithmetic (#68410) (#68764)
new d3063235507 [feature](lance) Support distance bounds in vector_search (#68815)
The 28 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/build-thirdparty.yml | 3 -
be/src/common/config.cpp | 4 +
be/src/common/config.h | 3 +
.../data_type_serde/data_type_variant_serde.cpp | 6 +
.../data_type_serde/data_type_variant_v2_serde.cpp | 78 +-
be/src/core/string_ref.h | 4 +-
be/src/core/value/quantile_state.cpp | 122 ++-
be/src/core/value/quantile_state.h | 16 +-
be/src/exec/common/hash_table/hash_map_context.h | 19 +-
be/src/exec/rowid_fetcher.cpp | 267 +++--
be/src/exec/rowid_fetcher.h | 15 +-
.../exec/runtime_filter/runtime_filter_wrapper.cpp | 2 +-
be/src/exec/sink/writer/vtablet_writer_v2.cpp | 36 +-
be/src/exprs/aggregate/aggregate_function_avg.cpp | 17 +-
be/src/exprs/aggregate/aggregate_function_avg.h | 20 +-
.../aggregate_function_quantile_state.cpp | 4 +-
.../aggregate/aggregate_function_quantile_state.h | 24 +-
be/src/exprs/aggregate/aggregate_function_sum.h | 8 +-
.../function/array/function_array_aggregation.cpp | 3 +-
.../function/array/function_array_shuffle.cpp | 32 +-
be/src/exprs/function/datetime_errors.h | 2 +-
.../function_date_or_datetime_computation.h | 5 +-
be/src/exprs/function/functions_logical.h | 4 +-
be/src/exprs/vcompound_pred.h | 7 +-
be/src/exprs/vectorized_fn_call.cpp | 3 +-
be/src/format/arrow/arrow_block_convertor.cpp | 22 +
be/src/format/arrow/arrow_block_convertor.h | 7 +
be/src/format/arrow/arrow_row_batch.cpp | 57 +-
be/src/format/arrow/arrow_row_batch.h | 14 +-
.../parquet/reader/native/column_chunk_reader.cpp | 102 +-
.../parquet/reader/native/column_chunk_reader.h | 3 +
.../parquet/reader/native/column_reader.cpp | 51 +-
.../parquet/reader/native/column_reader.h | 1 +
.../format_v2/parquet/reader/native/page_reader.h | 6 +
.../parquet/reader/native_column_reader.cpp | 18 +-
be/src/format_v2/table/lance_reader.cpp | 174 ++-
be/src/format_v2/table/lance_reader.h | 8 +
be/src/load/channel/load_stream.cpp | 62 ++
be/src/load/channel/load_stream.h | 2 +
.../arrow_flight/arrow_flight_batch_reader.cpp | 1 +
be/src/service/doris_main.cpp | 2 +
be/src/util/debug_util.cpp | 39 +-
be/src/util/debug_util.h | 4 +
be/src/util/tdigest.h | 2 +
be/test/core/value/quantile_state_test.cpp | 294 +++++
be/test/exec/hash_map/hash_table_method_test.cpp | 27 +
.../exec/operator/analytic_sink_operator_test.cpp | 79 ++
be/test/exec/rowid_fetcher_test.cpp | 167 +++
.../runtime_filter/runtime_filter_wrapper_test.cpp | 42 +
be/test/exec/sink/vtablet_writer_v2_test.cpp | 74 ++
be/test/exprs/aggregate/agg_avg_test.cpp | 11 +
be/test/exprs/aggregate/agg_percentile_test.cpp | 171 +++
.../{agg_avg_test.cpp => agg_sum_test.cpp} | 19 +-
.../function/function_array_aggregation_test.cpp | 17 +
.../exprs/function/function_array_shuffle_test.cpp | 227 ++++
be/test/exprs/function/function_time_test.cpp | 46 +
be/test/exprs/function/functions_logical_test.cpp | 143 +++
be/test/format/arrow/arrow_flight_variant_test.cpp | 511 +++++++++
be/test/format/arrow/arrow_row_batch_test.cpp | 11 +-
be/test/format_v2/parquet/native_decoder_test.cpp | 1156 +++++++++++++++++++-
be/test/format_v2/parquet/parquet_scan_test.cpp | 226 ++++
.../auxiliary.idx | Bin 0 -> 3254 bytes
.../9634c881-df64-46a5-95dc-2468928008a5/index.idx | Bin 0 -> 329 bytes
.../0-388c379e-6e57-4a48-a567-603ce868bdea.txn | Bin 0 -> 205 bytes
.../1-831523ce-b62e-46b0-bcd2-501920af09b8.txn | Bin 0 -> 376 bytes
.../2-4546c7a5-c68b-47d3-8f54-1e3036ad6429.txn | Bin 0 -> 122 bytes
.../_versions/18446744073709551612.manifest | Bin 0 -> 824 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 997 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 486 bytes
.../_versions/latest_version_hint.json | 0
...100000101111111a29c2749f892029e3946d88007.lance | Bin 0 -> 661 bytes
...000101110010011f2593f448ba30e92dc54b90567.lance | Bin 0 -> 3100 bytes
.../auxiliary.idx | Bin 0 -> 3255 bytes
.../76cb567d-f24f-4dd0-85b1-b0f486361b5e/index.idx | Bin 0 -> 330 bytes
.../0-ec982602-f60a-425e-8b31-f77179548301.txn | Bin 0 -> 205 bytes
.../1-bd4b52a8-552e-4270-b52f-3f678df92fed.txn | Bin 0 -> 378 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 1001 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 487 bytes
.../_versions/latest_version_hint.json | 1 +
...000110101111101bccc4b4c19af77197f1a92b69c.lance | Bin 0 -> 3100 bytes
.../auxiliary.idx | Bin 0 -> 2805 bytes
.../16361222-c72d-4428-9826-beeb3bfd3667/index.idx | Bin 0 -> 327 bytes
.../0-2dfc92d6-8481-4406-8893-9fc051e167d2.txn | Bin 0 -> 205 bytes
.../1-f4a6b175-3626-4561-817e-c45c857597fa.txn | Bin 0 -> 407 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 1059 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 487 bytes
.../_versions/latest_version_hint.json | 1 +
...1010101000001014167dc46e9aa87a5e01e036b2a.lance | Bin 0 -> 4892 bytes
.../table/lance/distance_range_fixture.md | 93 ++
be/test/format_v2/table/lance_reader_test.cpp | 535 ++++++++-
be/test/util/debug_util_test.cpp | 63 ++
be/test/util/tdigest_test.cpp | 28 +
build.sh | 25 +-
cloud/src/common/CMakeLists.txt | 1 +
cloud/src/common/version_metrics.cpp | 72 ++
.../src/common/version_metrics.h | 18 +-
cloud/src/main.cpp | 16 +-
cloud/src/recycler/recycler.cpp | 68 +-
cloud/test/metric_test.cpp | 37 +
cloud/test/recycle_versioned_keys_test.cpp | 74 ++
.../thirdparties/docker-compose/hudi/hudi.yaml.tpl | 4 +-
.../docker-compose/iceberg/iceberg.yaml.tpl | 21 +-
.../scripts/lance_build_legacy_vector_index.py | 100 ++
.../scripts/lance_build_preinstalled_catalog.py | 41 +-
.../iceberg/scripts/lance_build_search_snapshot.py | 329 ++++++
.../iceberg/scripts/lance_build_unicode_branch.py | 135 +++
.../iceberg/scripts/lance_fixture_requirements.txt | 6 +
.../bc324f21-a15a-45bf-8d9e-09538f895f55/index.idx | Bin 0 -> 9341 bytes
.../0-c5e3be90-3de0-44fb-95b1-bd46928ae0f2.txn | Bin 0 -> 184 bytes
.../1-9c8d1a85-5125-46ad-aeb8-8370bba30617.txn | Bin 0 -> 99 bytes
.../legacy_vector_index.lance/_versions/1.manifest | Bin 0 -> 253 bytes
.../legacy_vector_index.lance/_versions/2.manifest | Bin 0 -> 315 bytes
.../02cdc0ff-ceee-44ef-bccd-6cb6f4b50779.lance | Bin 0 -> 10641 bytes
.../_deletions/0-5-1267712482458982649.arrow | Bin 0 -> 698 bytes
.../_deletions/1-5-6394601565774866848.arrow | Bin 0 -> 698 bytes
.../auxiliary.idx | Bin 0 -> 1201 bytes
.../85880db4-716d-49b3-8eb2-fdadcc900e25/index.idx | Bin 0 -> 332 bytes
.../auxiliary.idx | Bin 0 -> 1265 bytes
.../984593f1-89f1-4849-83f9-d3e45462c244/index.idx | Bin 0 -> 332 bytes
.../metadata.lance | Bin 0 -> 654 bytes
.../part_0_docs.lance | Bin 0 -> 626 bytes
.../part_0_invert.lance | Bin 0 -> 3479 bytes
.../part_0_tokens.lance | Bin 0 -> 731 bytes
.../search_snapshot.lance/_refs/branches/dev.json | 15 +
.../search_snapshot.lance/_refs/tags/dev_rel.json | 8 +
.../search_snapshot.lance/_refs/tags/rel.json | 8 +
.../0-154ed129-09cd-48de-89ac-618fc1060501.txn | Bin 0 -> 237 bytes
.../1-d6645060-096c-4335-8bd9-c595a08fdd34.txn | Bin 0 -> 124 bytes
.../2-1518405c-5551-4e8f-b615-a0ca0d754255.txn | Bin 0 -> 372 bytes
.../3-c7c138f8-03a8-4564-9fc5-d59c25c2223c.txn | Bin 0 -> 278 bytes
.../4-1cbab51a-2c2d-41c4-8025-30daa3cc11a4.txn | Bin 0 -> 124 bytes
.../5-630bd020-d3af-4be8-b5e8-33388c068adc.txn | Bin 0 -> 259 bytes
.../6-9a8f0e65-3b5d-49ac-b232-5990ddd60891.txn | Bin 0 -> 702 bytes
.../_versions/18446744073709551608.manifest | Bin 0 -> 1789 bytes
.../_versions/18446744073709551609.manifest | Bin 0 -> 1343 bytes
.../_versions/18446744073709551610.manifest | Bin 0 -> 1173 bytes
.../_versions/18446744073709551611.manifest | Bin 0 -> 1244 bytes
.../_versions/18446744073709551612.manifest | Bin 0 -> 1104 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 521 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 551 bytes
.../_versions/latest_version_hint.json | 1 +
...11110000001111167198a47f9880a55a8085ebb87.lance | Bin 0 -> 1211 bytes
...010111000101110284ca44e90bfc0c56be666a32f.lance | Bin 0 -> 1211 bytes
...111011110010100fc3e5d485e9c40f5a217eaa59e.lance | Bin 0 -> 1211 bytes
.../metadata.lance | Bin 0 -> 654 bytes
.../part_0_docs.lance | Bin 0 -> 690 bytes
.../part_0_invert.lance | Bin 0 -> 2987 bytes
.../part_0_tokens.lance | Bin 0 -> 731 bytes
.../4-a99a47e7-aae1-46cf-87f7-4cc649db3aa1.txn | 1 +
.../4-d995fee3-982e-4e43-8e30-94537a9b0de2.txn | Bin 0 -> 124 bytes
.../5-6028c4ff-0a8e-4938-9d58-8a13524f6812.txn | Bin 0 -> 516 bytes
.../dev/_versions/18446744073709551609.manifest | Bin 0 -> 1632 bytes
.../dev/_versions/18446744073709551610.manifest | Bin 0 -> 1240 bytes
.../dev/_versions/18446744073709551611.manifest | Bin 0 -> 1128 bytes
...01010111100101014250f4aa0a70d3d926f124125.lance | Bin 0 -> 1211 bytes
.../auxiliary.idx | Bin 0 -> 1008 bytes
.../72e585d1-b843-4463-aba0-ffabc76017c1/index.idx | Bin 0 -> 332 bytes
.../0-fecb1851-f6b0-496c-bfb8-f4dbe7f649f1.txn | Bin 0 -> 237 bytes
.../1-5a98fef6-06a5-457c-b1b1-a9c830eafd2b.txn | Bin 0 -> 370 bytes
.../2-9bdddb95-2dc3-4e77-95b9-f67ed6e6387e.txn | Bin 0 -> 346 bytes
.../3-954a5d41-5fc9-4e39-96d9-864e1ac233bc.txn | 5 +
.../4-a41517b8-29a3-410f-b3ae-768c33b38cf5.txn | Bin 0 -> 126 bytes
.../_versions/18446744073709551610.manifest | Bin 0 -> 970 bytes
.../_versions/18446744073709551611.manifest | Bin 0 -> 956 bytes
.../_versions/18446744073709551612.manifest | Bin 0 -> 1099 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 1016 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 551 bytes
.../_versions/latest_version_hint.json | 0
...001001000111000634254483b82696db0b43cee8b.lance | Bin 0 -> 1576 bytes
...0111000011010107529f744f0b515d3cc0aaf4111.lance | Bin 0 -> 416 bytes
...0000010110100009494424b38a01956f50baa341d.lance | Bin 0 -> 1211 bytes
.../auxiliary.idx | Bin 0 -> 1008 bytes
.../961caf93-b3e9-45b9-b1a7-8944a37213ea/index.idx | Bin 0 -> 332 bytes
.../_refs/tags/kept.json | 8 +
.../1-5c4816b8-2e0a-46bb-aa6c-0879630399b4.txn | Bin 0 -> 370 bytes
.../2-b57c17ad-85a6-415b-a250-4b2efe8d3f8c.txn | Bin 0 -> 696 bytes
.../_versions/18446744073709551612.manifest | Bin 0 -> 1343 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 1017 bytes
.../_versions/latest_version_hint.json | 0
...110111001110011284b0f491a947f1b18c7abddff.lance | Bin 0 -> 1211 bytes
.../_refs/branches/dev%E1%B2%89.json | 15 +
.../0-420a1091-b61a-4cfb-8006-0b8f1e322ef8.txn | Bin 0 -> 237 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 552 bytes
.../_versions/latest_version_hint.json | 0
...101111000101000a13eb949eca41d69d17f9ea5a6.lance | Bin 0 -> 1146 bytes
.../1-1be00bd2-8443-4851-9ab8-4bdcee3ffa5e.txn" | Bin 0 -> 124 bytes
.../1-c1998c6f-6213-4b7e-ae7c-cc00cde3b7de.txn" | 1 +
.../_versions/18446744073709551613.manifest" | Bin 0 -> 585 bytes
.../_versions/18446744073709551614.manifest" | Bin 0 -> 476 bytes
...1000100001111040aa57401ab473a6a921298540.lance" | Bin 0 -> 1017 bytes
.../minio/minio-RELEASE.2024-11-07.yaml.tpl | 2 +-
.../docker-compose/polaris/docker-compose.yaml.tpl | 4 +-
fe/be-java-extensions/preload-extensions/pom.xml | 13 +-
.../main/java/org/apache/doris/common/Config.java | 31 +-
.../doris/common/LanceIndexConfigValidator.java | 31 +-
.../common/LanceIndexConfigValidatorTest.java | 65 +-
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 -
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 3 -
.../apache/doris/alter/SchemaChangeHandler.java | 33 +-
.../main/java/org/apache/doris/catalog/Env.java | 25 -
.../java/org/apache/doris/common/ErrorCode.java | 3 +-
.../org/apache/doris/datasource/CatalogMgr.java | 48 +-
.../doris/datasource/iceberg/IcebergUtils.java | 14 +
.../datasource/iceberg/source/IcebergScanNode.java | 67 +-
.../doris/datasource/lance/LanceCatalogClient.java | 84 +-
.../datasource/lance/LanceExternalCatalog.java | 38 +-
.../doris/datasource/lance/LanceExternalTable.java | 15 +-
.../datasource/lance/LanceIndexAdmission.java | 221 +---
.../doris/datasource/lance/LanceIndexFamilies.java | 1 -
.../lance/LanceIndexMutationValidator.java | 9 +-
.../lance/{job => }/LanceIndexNameNormalizer.java | 16 +-
.../lance/{job => }/LanceIndexSchemaContract.java | 2 +-
.../lance/LanceSchemaContractBuilder.java | 1 -
.../lance/index/LanceDatasetIndexDiscovery.java | 6 +-
.../lance/job/LanceIndexDatasetLocator.java | 143 ---
.../datasource/lance/job/LanceIndexFenceKey.java | 92 --
.../doris/datasource/lance/job/LanceIndexJob.java | 699 ------------
.../lance/job/LanceIndexJobCompletionReason.java | 28 -
.../datasource/lance/job/LanceIndexJobManager.java | 790 -------------
.../lance/job/LanceIndexJobMutationState.java | 47 -
.../datasource/lance/job/LanceIndexJobQuota.java | 189 ----
.../lance/job/LanceIndexJobRefreshState.java | 41 -
.../datasource/lance/job/LanceIndexJobResult.java | 112 --
.../lance/job/LanceIndexJobResultCode.java | 158 ---
.../lance/job/LanceIndexTerminationProof.java | 33 -
.../lance/metadata/LanceRefSelector.java | 11 +-
.../lance/metadata/LanceSnapshotResolver.java | 25 +
.../datasource/lance/source/LanceScanNode.java | 29 +-
.../datasource/lance/source/LanceScanPlan.java | 13 +-
.../datasource/lance/source/LanceScanPlanner.java | 26 +-
.../org/apache/doris/journal/JournalEntity.java | 14 +-
.../org/apache/doris/load/loadv2/LoadManager.java | 20 +
.../doris/nereids/parser/LogicalPlanBuilder.java | 37 +-
.../nereids/rules/analysis/ExpressionAnalyzer.java | 5 +-
.../functions/executable/DateTimeArithmetic.java | 6 +-
.../expressions/functions/scalar/ArrayShuffle.java | 28 +-
.../apache/doris/nereids/trees/plans/PlanType.java | 2 -
.../trees/plans/commands/AlterTableCommand.java | 35 +-
.../plans/commands/ShowLanceIndexJobCommand.java | 174 ---
.../plans/commands/ShowLanceIndexJobsCommand.java | 378 -------
.../trees/plans/commands/info/CreateTableInfo.java | 9 +-
.../plans/commands/info/PartitionTableInfo.java | 49 +
.../trees/plans/visitor/CommandVisitor.java | 10 -
.../java/org/apache/doris/persist/EditLog.java | 15 +-
.../org/apache/doris/persist/OperationType.java | 6 +-
.../doris/persist/meta/MetaPersistMethod.java | 6 -
.../doris/persist/meta/PersistMetaModules.java | 7 +-
.../java/org/apache/doris/planner/ResultSink.java | 2 +-
.../java/org/apache/doris/qe/ConnectContext.java | 176 ++-
.../java/org/apache/doris/qe/FEOpExecutor.java | 9 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 6 +
.../org/apache/doris/qe/runtime/LoadProcessor.java | 6 +
.../arrowflight/DorisFlightSqlProducer.java | 158 ++-
.../arrowflight/FlightSqlConnectProcessor.java | 38 +
.../service/arrowflight/FlightSqlParameters.java | 336 ++++++
.../service/arrowflight/FlightSqlQuerySchema.java | 96 +-
.../service/arrowflight/FlightSqlSchemaHelper.java | 39 +-
.../sessions/FlightSqlConnectPoolMgr.java | 3 +
.../FullTextSearchTableValuedFunction.java | 2 +-
.../LanceExternalSearchTableValuedFunction.java | 89 +-
.../VectorSearchTableValuedFunction.java | 44 +-
.../java/org/apache/iceberg/DeleteFileIndex.java | 3 +-
.../doris/datasource/CatalogMgrLanceGuardTest.java | 505 ++-------
.../doris/datasource/iceberg/IcebergUtilsTest.java | 19 +
.../iceberg/source/IcebergScanNodeTest.java | 157 +++
.../datasource/lance/LanceCatalogClientTest.java | 26 +
.../lance/LanceCatalogLifecycleTest.java | 20 -
.../datasource/lance/LanceIndexAdmissionTest.java | 400 ++-----
.../{job => }/LanceIndexNameNormalizerTest.java | 2 +-
.../lance/LanceManagedVersioningTest.java | 82 +-
.../LanceSchemaContractBuilderRealDatasetTest.java | 1 -
.../lance/LanceSchemaContractBuilderTest.java | 1 -
.../datasource/lance/LanceSearchSnapshotTest.java | 506 +++++++++
.../lance/job/LanceIndexDatasetLocatorTest.java | 183 ----
.../lance/job/LanceIndexJobManagerPersistTest.java | 457 --------
.../lance/job/LanceIndexJobManagerQueryTest.java | 131 ---
.../lance/job/LanceIndexJobManagerReplayTest.java | 628 -----------
.../lance/job/LanceIndexJobQuotaTest.java | 208 ----
.../lance/job/LanceIndexJobResultClassifyTest.java | 239 ----
.../lance/job/LanceIndexJobStateMachineTest.java | 640 -----------
.../datasource/lance/job/LanceIndexJobTest.java | 303 -----
.../lance/job/LanceIndexJobWiringTest.java | 78 --
.../datasource/lance/source/LanceScanNodeTest.java | 101 +-
.../datasource/paimon/PaimonMetadataOpsTest.java | 59 +
.../java/org/apache/doris/metric/MetricsTest.java | 24 +
.../doris/nereids/parser/NereidsParserTest.java | 13 +
.../parser/ShowLanceIndexJobsParserTest.java | 183 ----
.../functions/scalar/UniqueFunctionTest.java | 13 +
.../trees/plans/CreateTableCommandTest.java | 48 +
.../AlterTableCommandLanceAdmissionTest.java | 159 +--
.../commands/ShowLanceIndexJobCommandTest.java | 565 ----------
.../commands/ShowLanceIndexJobsCommandTest.java | 821 --------------
.../doris/qe/FEOpExecutorMysqlProtocolTest.java | 24 +
.../arrowflight/DorisFlightSqlSchemaTest.java | 98 +-
.../FlightSqlConnectProcessorSchemaTest.java | 8 +-
.../arrowflight/FlightSqlNativeVariantTest.java | 124 +++
.../arrowflight/FlightSqlParameterBindingTest.java | 41 +
.../arrowflight/FlightSqlPreparedQueryTest.java | 666 +++++++++++
.../arrowflight/FlightSqlServerInfoTest.java | 104 ++
.../tablefunction/LancePreparedSearchTest.java | 307 +++++-
.../tablefunction/LanceSearchSelectorTest.java | 189 ++++
.../VectorSearchTableValuedFunctionTest.java | 50 +-
fe/pom.xml | 7 +-
gensrc/thrift/PlanNodes.thrift | 3 +
.../data/doc/sql-manual/ArrayNullsafe.out | 2 +-
.../sql-function/test_array_function.out | 2 +-
.../hive/test_hive_topn_lazy_mat.out | 1047 ++++++++++++++++++
.../lance/test_lance_rest_catalog.out | 5 +
.../lance/test_lance_rest_search_snapshot.out | 59 +
.../lance/test_lance_rest_time_travel.out | 5 +
.../lance/test_lance_search_snapshot.out | 182 +++
.../test_lance_vector_search_legacy_index.out | 11 +
.../tvf/test_tvf_topn_lazy_mat.out | 72 ++
.../test_insert_quorum_split_reports.out | 6 +
.../nereids_function_p0/scalar_function/Array1.out | 24 +-
.../join/test_null_safe_eq_join_string_key.out | 66 ++
.../array_functions/test_array_avg_integer_sum.out | 21 +
.../array_functions/test_array_shuffle_seed.out | 26 +
.../test_json_extract_bool_null_payload.out | 41 +
.../test_quantile_state_function.out | 32 +
.../size_funciton/test_size_function.out | 12 +-
.../test_window_float_sliding_frame.out | 66 ++
regression-test/pipeline/p0/conf/fe.conf | 1 +
.../test_flight_cancel_cleanup.groovy | 4 +-
.../test_flight_native_variant.groovy | 215 ++++
.../test_prepared_query_parameters.groovy | 223 ++++
.../test_http_rest_v1_system_follower.groovy | 45 +
...st_file_cache_warmup_read_metrics_docker.groovy | 2 +-
.../test_external_partition_validation.groovy | 58 +
.../hive/ddl/test_hive_ddl.groovy | 20 +
.../hive/test_hive_topn_lazy_mat.groovy | 50 +-
.../test_iceberg_write_transform_partitions.groovy | 31 +
.../jdbc/test_doris_jdbc_catalog.groovy | 4 +
.../lance/test_lance_index_admission.groovy | 147 +--
.../lance/test_lance_index_ddl.groovy | 3 +-
.../lance/test_lance_rest_search_snapshot.groovy | 202 ++++
.../lance/test_lance_search_snapshot.groovy | 452 ++++++++
.../test_lance_vector_search_distance_range.groovy | 82 ++
.../test_lance_vector_search_legacy_index.groovy | 90 ++
.../test_paimon_create_partition_validation.groovy | 128 +++
.../tvf/test_hdfs_parquet_group0.groovy | 5 +-
.../tvf/test_tvf_topn_lazy_mat.groovy | 7 +
.../write/test_mc_write_partitions.groovy | 16 +
.../test_insert_quorum_split_reports.groovy | 101 ++
.../flink_connector_p0/flink_connector_type.groovy | 29 +-
.../suites/manager/test_manager_interface_1.groovy | 4 +-
.../suites/metrics_p0/test_version_metrics.groovy | 51 +-
.../nereids_p0/join/asof/test_asof_join.groovy | 24 +
.../join/test_null_safe_eq_join_string_key.groovy | 154 +++
.../test_array_avg_integer_sum.groovy | 60 +
.../array_functions/test_array_shuffle_seed.groovy | 101 ++
.../datetime_functions/test_quarters_add.groovy | 57 +-
.../test_json_extract_bool_null_payload.groovy | 153 +++
.../test_quantile_state_function.groovy | 90 ++
.../test_window_float_sliding_frame.groovy | 150 +++
.../suites/statistics/analyze_stats.groovy | 6 +
samples/arrow-flight-sql/python/README.md | 39 +-
samples/arrow-flight-sql/python/test_variant.py | 96 ++
.../deltalake_and_kudu/trinoconnector-compose.yml | 4 +-
samples/datalake/hudi/hudi-compose.yml | 4 +-
.../datalake/iceberg_and_paimon/docker-compose.yml | 4 +-
thirdparty/build-thirdparty.sh | 7 -
thirdparty/lance-install.sh | 38 -
thirdparty/patches/lance-c-foyer.patch | 22 +-
thirdparty/test/lance-install-test.sh | 176 ---
thirdparty/test/lance-prefilter-patch-test.sh | 7 +
thirdparty/vars.sh | 10 +-
367 files changed, 14545 insertions(+), 9706 deletions(-)
create mode 100644 be/test/exprs/aggregate/agg_percentile_test.cpp
copy be/test/exprs/aggregate/{agg_avg_test.cpp => agg_sum_test.cpp} (58%)
create mode 100644 be/test/exprs/function/function_array_shuffle_test.cpp
create mode 100644 be/test/exprs/function/functions_logical_test.cpp
create mode 100644 be/test/format/arrow/arrow_flight_variant_test.cpp
create mode 100644 be/test/format_v2/table/lance/data/distance_range.lance/_indices/9634c881-df64-46a5-95dc-2468928008a5/auxiliary.idx
create mode 100644 be/test/format_v2/table/lance/data/distance_range.lance/_indices/9634c881-df64-46a5-95dc-2468928008a5/index.idx
create mode 100644 be/test/format_v2/table/lance/data/distance_range.lance/_transactions/0-388c379e-6e57-4a48-a567-603ce868bdea.txn
create mode 100644 be/test/format_v2/table/lance/data/distance_range.lance/_transactions/1-831523ce-b62e-46b0-bcd2-501920af09b8.txn
create mode 100644 be/test/format_v2/table/lance/data/distance_range.lance/_transactions/2-4546c7a5-c68b-47d3-8f54-1e3036ad6429.txn
create mode 100644 be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551612.manifest
create mode 100644 be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551613.manifest
create mode 100644 be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551614.manifest
copy be/test/format_v2/table/lance/data/{all_types.lance => distance_range.lance}/_versions/latest_version_hint.json (100%)
create mode 100644 be/test/format_v2/table/lance/data/distance_range.lance/data/011111011100000101111111a29c2749f892029e3946d88007.lance
create mode 100644 be/test/format_v2/table/lance/data/distance_range.lance/data/100110001000101110010011f2593f448ba30e92dc54b90567.lance
create mode 100644 be/test/format_v2/table/lance/data/distance_range_dot.lance/_indices/76cb567d-f24f-4dd0-85b1-b0f486361b5e/auxiliary.idx
create mode 100644 be/test/format_v2/table/lance/data/distance_range_dot.lance/_indices/76cb567d-f24f-4dd0-85b1-b0f486361b5e/index.idx
create mode 100644 be/test/format_v2/table/lance/data/distance_range_dot.lance/_transactions/0-ec982602-f60a-425e-8b31-f77179548301.txn
create mode 100644 be/test/format_v2/table/lance/data/distance_range_dot.lance/_transactions/1-bd4b52a8-552e-4270-b52f-3f678df92fed.txn
create mode 100644 be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/18446744073709551613.manifest
create mode 100644 be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/18446744073709551614.manifest
create mode 100644 be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/latest_version_hint.json
create mode 100644 be/test/format_v2/table/lance/data/distance_range_dot.lance/data/110000101000110101111101bccc4b4c19af77197f1a92b69c.lance
create mode 100644 be/test/format_v2/table/lance/data/distance_range_sq.lance/_indices/16361222-c72d-4428-9826-beeb3bfd3667/auxiliary.idx
create mode 100644 be/test/format_v2/table/lance/data/distance_range_sq.lance/_indices/16361222-c72d-4428-9826-beeb3bfd3667/index.idx
create mode 100644 be/test/format_v2/table/lance/data/distance_range_sq.lance/_transactions/0-2dfc92d6-8481-4406-8893-9fc051e167d2.txn
create mode 100644 be/test/format_v2/table/lance/data/distance_range_sq.lance/_transactions/1-f4a6b175-3626-4561-817e-c45c857597fa.txn
create mode 100644 be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/18446744073709551613.manifest
create mode 100644 be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/18446744073709551614.manifest
create mode 100644 be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/latest_version_hint.json
create mode 100644 be/test/format_v2/table/lance/data/distance_range_sq.lance/data/0000100001010101000001014167dc46e9aa87a5e01e036b2a.lance
create mode 100644 be/test/format_v2/table/lance/distance_range_fixture.md
create mode 100644 be/test/util/debug_util_test.cpp
create mode 100644 cloud/src/common/version_metrics.cpp
rename fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationType.java => cloud/src/common/version_metrics.h (74%)
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/lance_build_legacy_vector_index.py
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/lance_build_search_snapshot.py
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/lance_build_unicode_branch.py
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_indices/bc324f21-a15a-45bf-8d9e-09538f895f55/index.idx
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_transactions/0-c5e3be90-3de0-44fb-95b1-bd46928ae0f2.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_transactions/1-9c8d1a85-5125-46ad-aeb8-8370bba30617.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_versions/1.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_versions/2.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/data/02cdc0ff-ceee-44ef-bccd-6cb6f4b50779.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_deletions/0-5-1267712482458982649.arrow
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_deletions/1-5-6394601565774866848.arrow
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/85880db4-716d-49b3-8eb2-fdadcc900e25/auxiliary.idx
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/85880db4-716d-49b3-8eb2-fdadcc900e25/index.idx
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/984593f1-89f1-4849-83f9-d3e45462c244/auxiliary.idx
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/984593f1-89f1-4849-83f9-d3e45462c244/index.idx
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/metadata.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/part_0_docs.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/part_0_invert.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/part_0_tokens.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_refs/branches/dev.json
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_refs/tags/dev_rel.json
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_refs/tags/rel.json
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/0-154ed129-09cd-48de-89ac-618fc1060501.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/1-d6645060-096c-4335-8bd9-c595a08fdd34.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/2-1518405c-5551-4e8f-b615-a0ca0d754255.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/3-c7c138f8-03a8-4564-9fc5-d59c25c2223c.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/4-1cbab51a-2c2d-41c4-8025-30daa3cc11a4.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/5-630bd020-d3af-4be8-b5e8-33388c068adc.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/6-9a8f0e65-3b5d-49ac-b232-5990ddd60891.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551608.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551609.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551610.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551611.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551612.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551613.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551614.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/latest_version_hint.json
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/data/01110111111110000001111167198a47f9880a55a8085ebb87.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/data/110001100010111000101110284ca44e90bfc0c56be666a32f.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/data/110100000111011110010100fc3e5d485e9c40f5a217eaa59e.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/metadata.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/part_0_docs.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/part_0_invert.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/part_0_tokens.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_transactions/4-a99a47e7-aae1-46cf-87f7-4cc649db3aa1.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_transactions/4-d995fee3-982e-4e43-8e30-94537a9b0de2.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_transactions/5-6028c4ff-0a8e-4938-9d58-8a13524f6812.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_versions/18446744073709551609.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_versions/18446744073709551610.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_versions/18446744073709551611.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/data/10010100101010111100101014250f4aa0a70d3d926f124125.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_indices/72e585d1-b843-4463-aba0-ffabc76017c1/auxiliary.idx
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_indices/72e585d1-b843-4463-aba0-ffabc76017c1/index.idx
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/0-fecb1851-f6b0-496c-bfb8-f4dbe7f649f1.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/1-5a98fef6-06a5-457c-b1b1-a9c830eafd2b.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/2-9bdddb95-2dc3-4e77-95b9-f67ed6e6387e.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/3-954a5d41-5fc9-4e39-96d9-864e1ac233bc.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/4-a41517b8-29a3-410f-b3ae-768c33b38cf5.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551610.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551611.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551612.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551613.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551614.manifest
copy {be/test/format_v2/table/lance/data/multivector_ivf_flat.lance => docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance}/_versions/latest_version_hint.json (100%)
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/data/011110000001001000111000634254483b82696db0b43cee8b.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/data/1011111000111000011010107529f744f0b515d3cc0aaf4111.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/data/1101110010000010110100009494424b38a01956f50baa341d.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_indices/961caf93-b3e9-45b9-b1a7-8944a37213ea/auxiliary.idx
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_indices/961caf93-b3e9-45b9-b1a7-8944a37213ea/index.idx
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_refs/tags/kept.json
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_transactions/1-5c4816b8-2e0a-46bb-aa6c-0879630399b4.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_transactions/2-b57c17ad-85a6-415b-a250-4b2efe8d3f8c.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_versions/18446744073709551612.manifest
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_versions/18446744073709551613.manifest
copy {be/test/format_v2/table/lance/data/all_types.lance => docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance}/_versions/latest_version_hint.json (100%)
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/data/101101000110111001110011284b0f491a947f1b18c7abddff.lance
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_refs/branches/dev%E1%B2%89.json
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_transactions/0-420a1091-b61a-4cfb-8006-0b8f1e322ef8.txn
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_versions/18446744073709551614.manifest
copy {be/test/format_v2/table/lance/data/multivector_dimensions.lance => docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance}/_versions/latest_version_hint.json (100%)
create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/data/001101010101111000101000a13eb949eca41d69d17f9ea5a6.lance
create mode 100644 "docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_transactions/1-1be00bd2-8443-4851-9ab8-4bdcee3ffa5e.txn"
create mode 100644 "docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_transactions/1-c1998c6f-6213-4b7e-ae7c-cc00cde3b7de.txn"
create mode 100644 "docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_versions/18446744073709551613.manifest"
create mode 100644 "docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_versions/18446744073709551614.manifest"
create mode 100644 "docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/data/01110100011000100001111040aa57401ab473a6a921298540.lance"
rename fe/fe-core/src/main/java/org/apache/doris/datasource/lance/{job => }/LanceIndexNameNormalizer.java (80%)
rename fe/fe-core/src/main/java/org/apache/doris/datasource/lance/{job => }/LanceIndexSchemaContract.java (99%)
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexFenceKey.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobCompletionReason.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationState.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobRefreshState.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResult.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultCode.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexTerminationProof.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobCommand.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobsCommand.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlParameters.java
rename fe/fe-core/src/test/java/org/apache/doris/datasource/lance/{job => }/LanceIndexNameNormalizerTest.java (99%)
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSearchSnapshotTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerQueryTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultClassifyTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ShowLanceIndexJobsParserTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobCommandTest.java
delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobsCommandTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlNativeVariantTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlParameterBindingTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlPreparedQueryTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlServerInfoTest.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/tablefunction/LanceSearchSelectorTest.java
create mode 100644 regression-test/data/external_table_p0/lance/test_lance_rest_search_snapshot.out
create mode 100644 regression-test/data/external_table_p0/lance/test_lance_search_snapshot.out
create mode 100644 regression-test/data/external_table_p0/lance/test_lance_vector_search_legacy_index.out
create mode 100644 regression-test/data/fault_injection_p0/test_insert_quorum_split_reports.out
create mode 100644 regression-test/data/query_p0/join/test_null_safe_eq_join_string_key.out
create mode 100644 regression-test/data/query_p0/sql_functions/array_functions/test_array_avg_integer_sum.out
create mode 100644 regression-test/data/query_p0/sql_functions/array_functions/test_array_shuffle_seed.out
create mode 100644 regression-test/data/query_p0/sql_functions/json_functions/test_json_extract_bool_null_payload.out
create mode 100644 regression-test/data/query_p0/sql_functions/window_functions/test_window_float_sliding_frame.out
create mode 100644 regression-test/suites/arrow_flight_sql_p0/test_flight_native_variant.groovy
create mode 100644 regression-test/suites/arrow_flight_sql_p0/test_prepared_query_parameters.groovy
create mode 100644 regression-test/suites/auth_p0/test_http_rest_v1_system_follower.groovy
create mode 100644 regression-test/suites/ddl_p0/test_external_partition_validation.groovy
create mode 100644 regression-test/suites/external_table_p0/lance/test_lance_rest_search_snapshot.groovy
create mode 100644 regression-test/suites/external_table_p0/lance/test_lance_search_snapshot.groovy
create mode 100644 regression-test/suites/external_table_p0/lance/test_lance_vector_search_distance_range.groovy
create mode 100644 regression-test/suites/external_table_p0/lance/test_lance_vector_search_legacy_index.groovy
create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_create_partition_validation.groovy
create mode 100644 regression-test/suites/fault_injection_p0/test_insert_quorum_split_reports.groovy
create mode 100644 regression-test/suites/query_p0/join/test_null_safe_eq_join_string_key.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/array_functions/test_array_avg_integer_sum.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/array_functions/test_array_shuffle_seed.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/json_functions/test_json_extract_bool_null_payload.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/window_functions/test_window_float_sliding_frame.groovy
create mode 100644 samples/arrow-flight-sql/python/test_variant.py
delete mode 100644 thirdparty/lance-install.sh
delete mode 100644 thirdparty/test/lance-install-test.sh
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 27/28: branch-4.1: [fix](function) Fix overflow in quarter arithmetic (#68410) (#68764)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 698a018d503214d09c683852dc189b05b9bf70cc
Author: linrrarity <li...@selectdb.com>
AuthorDate: Sat Oct 10 09:36:35 2026 +0800
branch-4.1: [fix](function) Fix overflow in quarter arithmetic (#68410) (#68764)
pick: https://github.com/apache/doris/pull/68410
---
be/src/exprs/function/datetime_errors.h | 2 +-
.../function_date_or_datetime_computation.h | 5 +-
be/test/exprs/function/function_time_test.cpp | 46 +++++++++++++++++
.../functions/executable/DateTimeArithmetic.java | 6 +--
.../datetime_functions/test_quarters_add.groovy | 57 +++++++++++++++++++++-
5 files changed, 109 insertions(+), 7 deletions(-)
diff --git a/be/src/exprs/function/datetime_errors.h b/be/src/exprs/function/datetime_errors.h
index 853106f4ffb..90d1753a05c 100644
--- a/be/src/exprs/function/datetime_errors.h
+++ b/be/src/exprs/function/datetime_errors.h
@@ -55,7 +55,7 @@ template <typename DateValueType>
// Throw for operations with a datelike and an integer (e.g. period)
template <typename DateValueType>
[[noreturn]] inline void throw_out_of_bound_date_int(const char* op, DateValueType arg0,
- Int32 delta) {
+ Int64 delta) {
throw Exception(ErrorCode::OUT_OF_BOUND, "Operation {} of {}, {} out of range", op,
datelike_to_string<DateValueType>(arg0), delta);
}
diff --git a/be/src/exprs/function/function_date_or_datetime_computation.h b/be/src/exprs/function/function_date_or_datetime_computation.h
index 279cc6b96e4..00cdae8de13 100644
--- a/be/src/exprs/function/function_date_or_datetime_computation.h
+++ b/be/src/exprs/function/function_date_or_datetime_computation.h
@@ -373,8 +373,9 @@ struct AddQuartersImpl {
static constexpr auto name = "quarters_add";
static constexpr auto is_nullable = false;
- static inline ReturnValueType execute(const InputValueType& t, Int32 delta) {
- return date_time_add<TimeUnit::MONTH, PType, Int32>(t, 3 * delta);
+ static inline ReturnValueType execute(const InputValueType& t, Int64 delta) {
+ // Preserve both the month offset and the negation of INT_MIN in quarters_sub.
+ return date_time_add<TimeUnit::MONTH, PType, Int64>(t, 3 * delta);
}
static DataTypes get_variadic_argument_types() {
diff --git a/be/test/exprs/function/function_time_test.cpp b/be/test/exprs/function/function_time_test.cpp
index afb622c459d..e01f0402012 100644
--- a/be/test/exprs/function/function_time_test.cpp
+++ b/be/test/exprs/function/function_time_test.cpp
@@ -37,6 +37,52 @@
namespace doris {
using namespace ut_type;
+template <typename Transform>
+void check_quarter_interval_overflow(const typename Transform::InputValueType& date) {
+ SCOPED_TRACE(Transform::name);
+ SCOPED_TRACE(Transform::ArgPType);
+ // Cover small wrapped month offsets and Int32 multiplication/negation boundaries.
+ for (Int32 quarters :
+ {1431655765, 1431655766, -1431655765, -1431655766, 715827882, 715827883, -715827882,
+ -715827883, std::numeric_limits<Int32>::min(), std::numeric_limits<Int32>::max()}) {
+ SCOPED_TRACE(quarters);
+ EXPECT_THROW(Transform::execute(date, quarters), Exception);
+ }
+}
+
+template <typename Transform>
+void check_quarter_sub_int_min(const typename Transform::InputValueType& date) {
+ SCOPED_TRACE(Transform::ArgPType);
+ try {
+ Transform::execute(date, std::numeric_limits<Int32>::min());
+ FAIL() << "Subtracting INT_MIN quarters must report a date-range error";
+ } catch (const Exception& e) {
+ EXPECT_EQ(e.code(), ErrorCode::OUT_OF_BOUND);
+ // Include the delimiter: a negative delta contains the same digits.
+ EXPECT_NE(e.to_string().find(", 6442450944 out of range"), std::string::npos)
+ << e.to_string();
+ }
+}
+
+TEST(VTimestampFunctionsTest, quarter_interval_overflow) {
+ DateV2Value<DateV2ValueType> date;
+ date.unchecked_set_time(2023, 1, 1, 0, 0, 0, 0);
+ DateV2Value<DateTimeV2ValueType> datetime;
+ datetime.unchecked_set_time(2023, 1, 1, 12, 34, 56, 123456);
+ TimestampTzValue timestamptz(datetime);
+ check_quarter_interval_overflow<AddQuartersImpl<TYPE_DATEV2>>(date);
+ check_quarter_interval_overflow<SubtractQuartersImpl<TYPE_DATEV2>>(date);
+ check_quarter_interval_overflow<AddQuartersImpl<TYPE_DATETIMEV2>>(datetime);
+ check_quarter_interval_overflow<SubtractQuartersImpl<TYPE_DATETIMEV2>>(datetime);
+ check_quarter_interval_overflow<AddQuartersImpl<TYPE_TIMESTAMPTZ>>(timestamptz);
+ check_quarter_interval_overflow<SubtractQuartersImpl<TYPE_TIMESTAMPTZ>>(timestamptz);
+
+ // Both signs exceed the date range, so EXPECT_THROW alone cannot detect narrowing.
+ check_quarter_sub_int_min<SubtractQuartersImpl<TYPE_DATEV2>>(date);
+ check_quarter_sub_int_min<SubtractQuartersImpl<TYPE_DATETIMEV2>>(datetime);
+ check_quarter_sub_int_min<SubtractQuartersImpl<TYPE_TIMESTAMPTZ>>(timestamptz);
+}
+
TEST(VTimestampFunctionsTest, day_of_week_test) {
std::string func_name = "dayofweek";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeArithmetic.java
index fa08569e000..458349d96c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeArithmetic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeArithmetic.java
@@ -746,17 +746,17 @@ public class DateTimeArithmetic {
*/
@ExecFunction(name = "quarters_sub")
public static Expression quartersSub(DateV2Literal date, IntegerLiteral quarter) {
- return quartersAdd(date, new IntegerLiteral(-quarter.getValue()));
+ return date.plusMonths(-3L * quarter.getValue());
}
@ExecFunction(name = "quarters_sub")
public static Expression quartersSub(DateTimeV2Literal date, IntegerLiteral quarter) {
- return quartersAdd(date, new IntegerLiteral(-quarter.getValue()));
+ return date.plusMonths(-3L * quarter.getValue());
}
@ExecFunction(name = "quarters_sub")
public static Expression quartersSub(TimestampTzLiteral date, IntegerLiteral quarter) {
- return quartersAdd(date, new IntegerLiteral(-quarter.getValue()));
+ return date.plusMonths(-3L * quarter.getValue());
}
/**
diff --git a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_quarters_add.groovy b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_quarters_add.groovy
index 915d2eabcb4..cb1bdfce776 100644
--- a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_quarters_add.groovy
+++ b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_quarters_add.groovy
@@ -216,4 +216,59 @@ suite("test_quarters_add") {
assertTrue(exception != null)
}
}
-}
\ No newline at end of file
+
+ // Int32 multiplication wraps these to -1 and +2 months. The BE unit test covers
+ // the full numeric boundary set; P0 covers folding and constant/vector execution.
+ def intervals = [1431655765, 1431655766]
+ def types = ["date", "datetime(6)", "timestamptz(6)"]
+ for (def skipFold : [false, true]) {
+ sql "set debug_skip_fold_constant = ${skipFold}"
+ for (def type : types) {
+ for (def function : ["quarters_add", "quarters_sub"]) {
+ for (def quarters : intervals) {
+ test {
+ sql "select ${function}(cast('2023-01-01' as ${type}), cast(${quarters} as int))"
+ exception "out of range"
+ }
+ }
+ }
+ }
+ test {
+ sql "select quarters_add(date '9999-12-01', 1)"
+ exception "out of range"
+ }
+ test {
+ sql "select quarters_sub(date '0000-01-01', 1)"
+ exception "out of range"
+ }
+ }
+
+ sql "set debug_skip_fold_constant = false"
+ sql "drop table if exists test_quarters_overflow"
+ sql """
+ create table test_quarters_overflow (
+ q int not null,
+ d date,
+ dt datetime(6),
+ tz timestamptz(6)
+ )
+ duplicate key(q)
+ distributed by hash(q) buckets 1
+ properties("replication_num" = "1")
+ """
+ def rows = intervals.collect { quarters ->
+ "(${quarters}, '2023-01-01', '2023-01-01 12:34:56.123456', " +
+ "'2023-01-01 12:34:56.123456')"
+ }
+ sql "insert into test_quarters_overflow values ${rows.join(',')}"
+ for (def column : ["d", "dt", "tz"]) {
+ for (def function : ["quarters_add", "quarters_sub"]) {
+ for (def quarters : intervals) {
+ test {
+ sql "select ${function}(${column}, q) from test_quarters_overflow where q = ${quarters}"
+ exception "out of range"
+ }
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/28: [fix](catalog) Validate external table partition models on branch-4.1 (#68769)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 88549d0033627aeb4ac25f2dfe61b28e6ab1c569
Author: Gabriel <li...@selectdb.com>
AuthorDate: Thu Oct 8 17:25:24 2026 +0800
[fix](catalog) Validate external table partition models on branch-4.1 (#68769)
### What problem does this PR solve?
On branch-4.1, external CREATE TABLE can silently discard unsupported
partition expressions or explicit partition definitions. For example, a
Paimon table declared with `date_trunc(ts, 'day')` may be created
without partition keys.
Validate partition models through a shared external-table entry point
before descriptor conversion and remote creation. Hive, Paimon, and
MaxCompute accept column partitions; Iceberg retains its supported
transforms and schema-specific validation. Reject explicit Doris
partition definitions for these formats, preserve Elasticsearch's
existing partition checks, and reject partition clauses for engines that
do not support them.
Also remove identical duplicate Nimbus and Hadoop COS dependency
declarations inherited from the base branch. Maven 3.10 rejects these
declarations before Checkstyle can run; dependency versions and
exclusions are preserved.
### Release note
External CREATE TABLE reports errors for unsupported partition models
instead of silently ignoring them.
### Check List (For Author)
- Test
- [x] Regression test: a standalone P0 suite covers invalid models for
all external engines without external services. Catalog suites cover
Paimon, Hive, Iceberg, and MaxCompute, including inferred engines and
supported partition behavior.
- [x] Unit Test: shared analysis checks, Paimon rejection before remote
creation, and supported partition syntax.
- Behavior changed:
- [x] Yes. Unsupported external partition models fail before table
creation.
- Does this need documentation?
- [x] No.
### Validation
- New rejection tests failed before the corresponding validation
changes.
- All 38 selected FE tests pass: 24 Paimon metadata tests, nine Iceberg
table-creation tests, and five external-table partition analysis tests.
- The full `mvn clean checkstyle:check` passes with Maven 3.10.0 (the CI
version), with zero violations across all 28 modules.
- All five new or updated Groovy suites compile with Groovy 4.0.19. SQL
regression suites were not executed locally; catalog integration tests
require their configured external services.
---
fe/be-java-extensions/preload-extensions/pom.xml | 13 +--
.../trees/plans/commands/info/CreateTableInfo.java | 9 +-
.../plans/commands/info/PartitionTableInfo.java | 49 ++++++++
.../datasource/paimon/PaimonMetadataOpsTest.java | 59 ++++++++++
.../trees/plans/CreateTableCommandTest.java | 48 ++++++++
fe/pom.xml | 6 +-
.../test_external_partition_validation.groovy | 58 ++++++++++
.../hive/ddl/test_hive_ddl.groovy | 20 ++++
.../test_iceberg_write_transform_partitions.groovy | 31 +++++
.../test_paimon_create_partition_validation.groovy | 128 +++++++++++++++++++++
.../write/test_mc_write_partitions.groovy | 16 +++
11 files changed, 414 insertions(+), 23 deletions(-)
diff --git a/fe/be-java-extensions/preload-extensions/pom.xml b/fe/be-java-extensions/preload-extensions/pom.xml
index 8113589e99c..772a98dd653 100644
--- a/fe/be-java-extensions/preload-extensions/pom.xml
+++ b/fe/be-java-extensions/preload-extensions/pom.xml
@@ -124,6 +124,7 @@ under the License.
<artifactId>HikariCP</artifactId>
</dependency>
<!-- For BE CosN Access -->
+ <!-- Keep a single declaration: Maven 3.10+ rejects duplicate dependency coordinates. -->
<dependency>
<groupId>com.qcloud.cos</groupId>
<artifactId>hadoop-cos</artifactId>
@@ -139,18 +140,6 @@ under the License.
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
- <!-- For BE CosN Access -->
- <dependency>
- <groupId>com.qcloud.cos</groupId>
- <artifactId>hadoop-cos</artifactId>
- <version>${tencentcos.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.json</groupId>
- <artifactId>json</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<!-- these dependencies are for iceberg-aws -->
<dependency>
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index c34f2c6d2d3..f702023d153 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -1131,6 +1131,9 @@ public class CreateTableInfo {
* analyzeEngine
*/
public void analyzeEngine() {
+ if (isExternal) {
+ partitionTableInfo.validateExternalPartition(engineName);
+ }
this.partitionDesc = partitionTableInfo.convertToPartitionDesc(isExternal);
this.distributionDesc =
distribution != null ? distribution.translateToCatalogStyle() : null;
@@ -1147,12 +1150,6 @@ public class CreateTableInfo {
throw new AnalysisException("Create " + engineName
+ " table should not contain distribution desc");
}
- if (!engineName.equals(ENGINE_HIVE) && !engineName.equals(ENGINE_ICEBERG)
- && !engineName.equals(ENGINE_PAIMON) && !engineName.equals(ENGINE_MAXCOMPUTE)
- && partitionDesc != null) {
- throw new AnalysisException("Create " + engineName
- + " table should not contain partition desc");
- }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionTableInfo.java
index 8c54b56018e..ec829dcebf5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionTableInfo.java
@@ -103,6 +103,55 @@ public class PartitionTableInfo {
return partitionType;
}
+ /**
+ * Validate external partition models before conversion can discard unsupported information.
+ * Schema-specific checks, including Iceberg transform binding, remain with the metadata builders.
+ */
+ public void validateExternalPartition(String engineName) {
+ if (partitionType.equalsIgnoreCase(PartitionType.UNPARTITIONED.name())) {
+ return;
+ }
+
+ String formatName;
+ boolean supportsTransforms = false;
+ switch (engineName) {
+ case CreateTableInfo.ENGINE_HIVE:
+ if (!partitionType.equalsIgnoreCase(PartitionType.LIST.name())) {
+ throw new AnalysisException("Only support 'LIST' partition type in hive catalog.");
+ }
+ formatName = "Hive";
+ break;
+ case CreateTableInfo.ENGINE_PAIMON:
+ formatName = "Paimon";
+ break;
+ case CreateTableInfo.ENGINE_MAXCOMPUTE:
+ formatName = "MaxCompute";
+ break;
+ case CreateTableInfo.ENGINE_ICEBERG:
+ formatName = "Iceberg";
+ supportsTransforms = true;
+ break;
+ case CreateTableInfo.ENGINE_ELASTICSEARCH:
+ // Elasticsearch mapping partitions are validated by EsUtil after descriptor conversion.
+ return;
+ default:
+ throw new AnalysisException("Create " + engineName + " table should not contain partition desc");
+ }
+
+ // Column-only formats must not silently lose function expressions from their partition keys.
+ if (!supportsTransforms && partitionList != null
+ && partitionList.stream().anyMatch(expr -> !(expr instanceof UnboundSlot))) {
+ throw new AnalysisException(formatName + " only supports partitioning by columns; "
+ + "partition expressions are not supported");
+ }
+ if (partitionDefs != null && !partitionDefs.isEmpty()) {
+ if (engineName.equals(CreateTableInfo.ENGINE_HIVE)) {
+ throw new AnalysisException("Partition values expressions is not supported in hive catalog.");
+ }
+ throw new AnalysisException(formatName + " does not support explicit partition definitions");
+ }
+ }
+
/**
* check partitions types.
*/
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java
index ac8b33281c7..5ce12fe698e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java
@@ -26,6 +26,7 @@ import org.apache.doris.datasource.CatalogFactory;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalDatabase;
import org.apache.doris.datasource.ExternalTable;
+import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
@@ -338,6 +339,64 @@ public class PaimonMetadataOpsTest {
Assert.assertEquals(1, table.primaryKeys().size());
}
+ @Test
+ public void testRejectPartitionExpressionsBeforeCreatingTable() throws Exception {
+ for (String partitionClause : Arrays.asList(
+ "auto partition by range(date_trunc(ts, 'day')) ()",
+ "auto partition by list(date_trunc(ts, 'day')) ()",
+ "partition by (date_trunc(ts, 'day')) ()",
+ "partition by (dt, date_trunc(ts, 'day')) ()")) {
+ assertPartitionRejected(partitionClause, "Paimon only supports partitioning by columns");
+ }
+ }
+
+ @Test
+ public void testRejectPartitionDefinitionsBeforeCreatingTable() throws Exception {
+ for (String partitionClause : Arrays.asList(
+ "partition by range(dt) (partition p1 values less than ('2026-01-02'))",
+ "partition by range(dt) (partition p1 values [('2026-01-01'), ('2026-01-02')))",
+ "partition by list(dt) (partition p1 values in ('2026-01-01'))",
+ "partition by range(dt) (from ('2026-01-01') to ('2026-01-03') interval 1 day)",
+ "auto partition by range(dt) (partition p1 values less than ('2026-01-02'))")) {
+ assertPartitionRejected(partitionClause, "Paimon does not support explicit partition definitions");
+ }
+ }
+
+ private void assertPartitionRejected(String partitionClause, String message) throws Exception {
+ String tableName = getTableName();
+ String sql = "create table " + dbName + "." + tableName
+ + " (id int, ts datetime not null, dt date not null) engine = paimon " + partitionClause;
+ AnalysisException exception = Assert.assertThrows(AnalysisException.class, () -> createTable(sql));
+ Assert.assertTrue(exception.getMessage(), exception.getMessage().contains(message));
+ Assert.assertThrows(Catalog.TableNotExistException.class,
+ () -> ops.getCatalog().getTable(new Identifier(dbName, tableName)));
+ }
+
+ @Test
+ public void testIdentityPartitionSyntaxes() throws Exception {
+ for (String partitionClause : Arrays.asList(
+ "partition by (dt) ()",
+ "partition by range(dt) ()",
+ "partition by list(dt) ()",
+ "auto partition by range(dt) ()",
+ "auto partition by list(dt) ()")) {
+ String tableName = getTableName();
+ createTable("create table " + dbName + "." + tableName
+ + " (id int, dt date not null) engine = paimon " + partitionClause);
+ Assert.assertEquals(Arrays.asList("dt"),
+ ops.getCatalog().getTable(new Identifier(dbName, tableName)).partitionKeys());
+ }
+ }
+
+ @Test
+ public void testMultipleIdentityPartitionColumns() throws Exception {
+ String tableName = getTableName();
+ createTable("create table " + dbName + "." + tableName
+ + " (id int, region string, dt date not null) engine = paimon partition by (region, dt) ()");
+ Assert.assertEquals(Arrays.asList("region", "dt"),
+ ops.getCatalog().getTable(new Identifier(dbName, tableName)).partitionKeys());
+ }
+
@Test
public void testPartitionPreservesNonLowercaseColumnNames() throws Exception {
String tableName = getTableName();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java
index e423c30b507..b630f15f1b4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java
@@ -838,6 +838,54 @@ public class CreateTableCommandTest extends TestWithFeService {
return createTableInfo.getPartitionDesc();
}
+ @Test
+ public void testExternalIdentityPartitionValidation() {
+ for (String engine : new String[] {"hive", "paimon", "maxcompute"}) {
+ String prefix = "create table partition_validation (id int, ts datetime, dt date) engine=" + engine;
+ PartitionDesc partition = getCreateTableStmt(prefix + " partition by (dt) ()");
+ Assertions.assertEquals(java.util.Collections.singletonList("dt"), partition.getPartitionColNames());
+ AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
+ () -> getCreateTableStmt(prefix + " partition by (date_trunc(ts, 'day')) ()"));
+ Assertions.assertTrue(exception.getMessage().contains("only supports partitioning by columns"));
+ }
+ }
+
+ @Test
+ public void testExternalPartitionDefinitionsRejected() {
+ for (String engine : new String[] {"hive", "paimon", "maxcompute", "iceberg"}) {
+ AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
+ () -> getCreateTableStmt("create table partition_validation (id int, dt date) engine=" + engine
+ + " partition by list(dt) (partition p1 values in ('2026-01-01'))"));
+ String expected = engine.equals("hive") ? "Partition values expressions is not supported in hive catalog"
+ : "does not support explicit partition definitions";
+ Assertions.assertTrue(exception.getMessage().contains(expected), exception.getMessage());
+ }
+ }
+
+ @Test
+ public void testExternalPartitionModelRestrictions() {
+ AnalysisException hiveException = Assertions.assertThrows(AnalysisException.class,
+ () -> getCreateTableStmt("create table partition_validation (id int, dt date) engine=hive"
+ + " partition by range(dt) ()"));
+ Assertions.assertTrue(hiveException.getMessage().contains("Only support 'LIST' partition type in hive catalog"));
+
+ PartitionDesc iceberg = getCreateTableStmt("create table partition_validation (id int, ts datetime)"
+ + " engine=iceberg partition by (bucket(4, id), day(ts)) ()");
+ Assertions.assertEquals(2, iceberg.getPartitionExprs().size());
+
+ PartitionDesc elasticsearch = getCreateTableStmt("create table partition_validation (id int, dt date)"
+ + " engine=elasticsearch partition by range(dt) ()");
+ Assertions.assertEquals(java.util.Collections.singletonList("dt"), elasticsearch.getPartitionColNames());
+ Assertions.assertThrows(AnalysisException.class,
+ () -> getCreateTableStmt("create table partition_validation (id int, dt date) engine=elasticsearch"
+ + " partition by list(dt) ()"));
+ for (String engine : new String[] {"jdbc", "odbc", "mysql", "broker"}) {
+ Assertions.assertThrows(AnalysisException.class,
+ () -> getCreateTableStmt("create table partition_validation (id int, dt date) engine=" + engine
+ + " partition by list(dt) ()"));
+ }
+ }
+
@Test
public void testPartitionCheckForHive() {
try {
diff --git a/fe/pom.xml b/fe/pom.xml
index 002e3e60552..5f1223aa793 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -627,6 +627,7 @@ under the License.
<artifactId>quartz</artifactId>
<version>${quartz.version}</version>
</dependency>
+ <!-- Maven 3.10+ rejects duplicate coordinates in dependencyManagement. -->
<dependency>
<groupId>com.nimbusds</groupId>
<artifactId>nimbus-jose-jwt</artifactId>
@@ -1928,11 +1929,6 @@ under the License.
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>com.nimbusds</groupId>
- <artifactId>nimbus-jose-jwt</artifactId>
- <version>${nimbusds.version}</version>
- </dependency>
<dependency>
<groupId>jakarta.annotation</groupId>
<artifactId>jakarta.annotation-api</artifactId>
diff --git a/regression-test/suites/ddl_p0/test_external_partition_validation.groovy b/regression-test/suites/ddl_p0/test_external_partition_validation.groovy
new file mode 100644
index 00000000000..d89e2248498
--- /dev/null
+++ b/regression-test/suites/ddl_p0/test_external_partition_validation.groovy
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_external_partition_validation", "p0") {
+ // Invalid external partition models must fail in analysis without contacting a remote catalog.
+ def rejectPartition = { String engine, String partition, String message ->
+ String tableName = "test_external_partition_validation_${engine}"
+ sql """DROP TABLE IF EXISTS `${tableName}`"""
+ try {
+ test {
+ sql """
+ CREATE TABLE `${tableName}` (id INT, ts DATETIME, dt DATE)
+ ENGINE=${engine} ${partition}
+ """
+ exception message
+ }
+ assertEquals([], sql("""SHOW TABLES LIKE '${tableName}'"""))
+ } finally {
+ sql """DROP TABLE IF EXISTS `${tableName}`"""
+ }
+ }
+
+ ["hive", "paimon", "maxcompute"].each { engine ->
+ rejectPartition(engine, "PARTITION BY (date_trunc(ts, 'day')) ()",
+ "only supports partitioning by columns")
+ }
+ ["hive", "paimon", "maxcompute", "iceberg"].each { engine ->
+ String message = engine == "hive" ? "Partition values expressions is not supported in hive catalog"
+ : "does not support explicit partition definitions"
+ rejectPartition(engine, "PARTITION BY LIST(dt) (PARTITION p1 VALUES IN ('2026-01-01'))", message)
+ }
+ ["paimon", "maxcompute", "iceberg"].each { engine ->
+ rejectPartition(engine, "PARTITION BY RANGE(dt) (PARTITION p1 VALUES LESS THAN ('2026-01-02'))",
+ "does not support explicit partition definitions")
+ }
+ rejectPartition("hive", "PARTITION BY RANGE(dt) ()", "Only support 'LIST' partition type in hive catalog")
+ rejectPartition("elasticsearch", "PARTITION BY LIST(dt) ()", "Elasticsearch table only permit range partition")
+ rejectPartition("elasticsearch", "PARTITION BY RANGE(id, dt) ()",
+ "Elasticsearch table's partition column could only be a single column")
+ rejectPartition("jdbc", "PARTITION BY LIST(dt) ()", "Create jdbc table should not contain partition desc")
+ ["mysql", "odbc", "broker"].each { engine ->
+ rejectPartition(engine, "PARTITION BY LIST(dt) ()", "odbc, mysql and broker table is no longer supported")
+ }
+}
diff --git a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy
index a3bf1410fbf..34efc5a86fe 100644
--- a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy
+++ b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy
@@ -488,6 +488,26 @@ suite("test_hive_ddl", "p0,external,hive,external_docker,external_docker_hive")
sql """ create database if not exists `test_hive_db_tbl` """;
sql """use `${catalog_name}`.`test_hive_db_tbl`"""
+ // Inferred Hive engines must reject unsupported models before creating HMS metadata.
+ String rejectedTable = "partition_model_rejected_${file_format}"
+ sql """DROP TABLE IF EXISTS `${rejectedTable}`"""
+ try {
+ [
+ ["PARTITION BY (date_trunc(ts, 'day')) ()", "Hive only supports partitioning by columns"],
+ ["PARTITION BY RANGE(dt) ()", "Only support 'LIST' partition type in hive catalog"],
+ ["PARTITION BY LIST(dt) (PARTITION p1 VALUES IN ('2026-01-01'))",
+ "Partition values expressions is not supported in hive catalog"]
+ ].each { entry ->
+ test {
+ sql """CREATE TABLE `${rejectedTable}` (id INT, ts DATETIME, dt DATE) ${entry[0]}"""
+ exception entry[1]
+ }
+ assertEquals([], sql("""SHOW TABLES LIKE '${rejectedTable}'"""))
+ }
+ } finally {
+ sql """DROP TABLE IF EXISTS `${rejectedTable}`"""
+ }
+
sql """ drop table if exists unpart_tbl_${file_format}"""
sql """
CREATE TABLE unpart_tbl_${file_format}(
diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_transform_partitions.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_transform_partitions.groovy
index 2d8e265222e..dc8eb69a81d 100644
--- a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_transform_partitions.groovy
+++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_transform_partitions.groovy
@@ -50,6 +50,37 @@ suite("test_iceberg_write_transform_partitions", "p0,external,iceberg,external_d
}
try {
+ String validationTable = "create_partition_validation"
+ sql """DROP TABLE IF EXISTS `${validationTable}`"""
+ try {
+ // Iceberg supports transforms, but explicit Doris partition bounds have no Iceberg equivalent.
+ [
+ ["PARTITION BY LIST(dt) (PARTITION p1 VALUES IN ('2026-01-01'))",
+ "Iceberg does not support explicit partition definitions"],
+ ["PARTITION BY RANGE(dt) (PARTITION p1 VALUES LESS THAN ('2026-01-02'))",
+ "Iceberg does not support explicit partition definitions"],
+ ["PARTITION BY (date_trunc(ts, 'day')) ()", "unsupported partition for date_trunc"]
+ ].each { entry ->
+ test {
+ sql """CREATE TABLE `${validationTable}` (id INT, ts DATETIME, dt DATE) ${entry[0]}"""
+ exception entry[1]
+ }
+ assertEquals([], sql("""SHOW TABLES LIKE '${validationTable}'"""))
+ }
+
+ sql """
+ CREATE TABLE `${validationTable}` (id INT, ts DATETIME)
+ PARTITION BY (bucket(4, id), day(ts)) ()
+ """
+ String ddl = sql("""SHOW CREATE TABLE `${validationTable}`""")[0][1].toString().toLowerCase()
+ assertTrue(ddl.contains("bucket"))
+ assertTrue(ddl.contains("day"))
+ sql """INSERT INTO `${validationTable}` VALUES (1, '2026-01-01 12:00:00')"""
+ assertEquals(1, sql("""SELECT COUNT(*) FROM `${validationTable}`""")[0][0].intValue())
+ } finally {
+ sql """DROP TABLE IF EXISTS `${validationTable}`"""
+ }
+
sql """ set time_zone = 'Asia/Shanghai'; """
test_write_transform_partitions("bucket_int_4");
test_write_transform_partitions("bucket_bigint_4");
diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_create_partition_validation.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_create_partition_validation.groovy
new file mode 100644
index 00000000000..15682a782db
--- /dev/null
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_create_partition_validation.groovy
@@ -0,0 +1,128 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_paimon_create_partition_validation",
+ "p0,external,paimon,external_docker,external_docker_paimon") {
+ String enabled = context.config.otherConfigs.get("enablePaimonTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable paimon test")
+ return
+ }
+
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ String catalogName = "test_paimon_create_partition_validation"
+ String dbName = "paimon_create_partition_validation_db"
+ String expressionError = "Paimon only supports partitioning by columns"
+ String definitionError = "Paimon does not support explicit partition definitions"
+
+ sql """DROP CATALOG IF EXISTS `${catalogName}`"""
+ sql """
+ CREATE CATALOG `${catalogName}` PROPERTIES (
+ 'type' = 'paimon',
+ 'paimon.catalog.type' = 'filesystem',
+ 'warehouse' = 's3://warehouse/wh',
+ 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}',
+ 's3.access_key' = 'admin',
+ 's3.secret_key' = 'password',
+ 's3.path.style.access' = 'true'
+ )
+ """
+
+ try {
+ sql """SWITCH `${catalogName}`"""
+ sql """DROP DATABASE IF EXISTS `${dbName}` FORCE"""
+ sql """CREATE DATABASE `${dbName}`"""
+ sql """USE `${dbName}`"""
+
+ // Both expressions and explicit bounds used to disappear from the remote schema.
+ def unsupportedPartitions = [
+ ["expr_range", "AUTO PARTITION BY RANGE(date_trunc(ts, 'day')) ()", expressionError],
+ ["expr_list", "AUTO PARTITION BY LIST(date_trunc(ts, 'day')) ()", expressionError],
+ ["expr_plain", "PARTITION BY (date_trunc(ts, 'day')) ()", expressionError],
+ ["expr_mixed", "PARTITION BY (dt, date_trunc(ts, 'day')) ()", expressionError],
+ ["expr_bucket", "PARTITION BY (bucket(4, id)) ()", expressionError],
+ ["range_less", "PARTITION BY RANGE(dt) (PARTITION p1 VALUES LESS THAN ('2026-01-02'))",
+ definitionError],
+ ["range_fixed", "PARTITION BY RANGE(dt) (PARTITION p1 VALUES [('2026-01-01'), ('2026-01-02')))",
+ definitionError],
+ ["list_values", "PARTITION BY LIST(dt) (PARTITION p1 VALUES IN ('2026-01-01'))", definitionError],
+ ["range_step", "PARTITION BY RANGE(dt) (FROM ('2026-01-01') TO ('2026-01-03') INTERVAL 1 DAY)",
+ definitionError],
+ ["auto_bounds", "AUTO PARTITION BY RANGE(dt) (PARTITION p1 VALUES LESS THAN ('2026-01-02'))",
+ definitionError]
+ ]
+ unsupportedPartitions.each { entry ->
+ test {
+ sql """
+ CREATE TABLE `${entry[0]}` (id INT, ts DATETIME NOT NULL, dt DATE NOT NULL)
+ ENGINE=paimon ${entry[1]}
+ """
+ exception entry[2]
+ }
+ assertEquals([], sql("""SHOW TABLES LIKE '${entry[0]}'"""))
+ }
+
+ // Catalog-inferred engines and CTAS must use the same validation before publishing metadata.
+ test {
+ sql """
+ CREATE TABLE inferred_engine (id INT, ts DATETIME NOT NULL)
+ AUTO PARTITION BY RANGE(date_trunc(ts, 'day')) ()
+ """
+ exception expressionError
+ }
+ test {
+ sql """
+ CREATE TABLE ctas_expression ENGINE=paimon
+ AUTO PARTITION BY RANGE(date_trunc(ts, 'day')) ()
+ AS SELECT 1 AS id, CAST('2026-01-01 12:00:00' AS DATETIME) AS ts
+ """
+ exception expressionError
+ }
+ assertEquals([], sql("SHOW TABLES"))
+ assertEquals([], spark_paimon("""SHOW TABLES IN paimon.${dbName}"""))
+
+ def supportedPartitions = [
+ ["unpartitioned", "", []],
+ ["identity_plain", "PARTITION BY (dt) ()", ["dt"]],
+ ["identity_range", "PARTITION BY RANGE(dt) ()", ["dt"]],
+ ["identity_list", "PARTITION BY LIST(dt) ()", ["dt"]],
+ ["identity_auto_range", "AUTO PARTITION BY RANGE(dt) ()", ["dt"]],
+ ["identity_auto_list", "AUTO PARTITION BY LIST(dt) ()", ["dt"]],
+ ["identity_multiple", "PARTITION BY (region, dt) ()", ["region", "dt"]]
+ ]
+ supportedPartitions.each { entry ->
+ sql """
+ CREATE TABLE `${entry[0]}` (id INT, region STRING, dt DATE NOT NULL)
+ ENGINE=paimon ${entry[1]}
+ """
+ def schemas = sql """
+ SELECT partition_keys FROM `${entry[0]}\$schemas`
+ ORDER BY schema_id DESC LIMIT 1
+ """
+ assertEquals(1, schemas.size())
+ assertEquals(entry[2], parseJson(schemas[0][0].toString()))
+ }
+ } finally {
+ try {
+ sql """DROP DATABASE IF EXISTS `${dbName}` FORCE"""
+ } finally {
+ sql "SWITCH internal"
+ sql """DROP CATALOG IF EXISTS `${catalogName}`"""
+ }
+ }
+}
diff --git a/regression-test/suites/external_table_p2/maxcompute/write/test_mc_write_partitions.groovy b/regression-test/suites/external_table_p2/maxcompute/write/test_mc_write_partitions.groovy
index 9bcc48ba199..9613d13e162 100644
--- a/regression-test/suites/external_table_p2/maxcompute/write/test_mc_write_partitions.groovy
+++ b/regression-test/suites/external_table_p2/maxcompute/write/test_mc_write_partitions.groovy
@@ -50,6 +50,22 @@ suite("test_mc_write_partitions", "p2,external,maxcompute,external_remote,extern
sql """use ${db}"""
try {
+ // Inferred MaxCompute engines must not silently ignore transforms or partition bounds.
+ String rejectedTable = "partition_model_rejected_${uuid}"
+ [
+ ["PARTITION BY (date_trunc(ts, 'day')) ()", "MaxCompute only supports partitioning by columns"],
+ ["PARTITION BY LIST(ds) (PARTITION p1 VALUES IN ('20260101'))",
+ "MaxCompute does not support explicit partition definitions"],
+ ["PARTITION BY RANGE(ds) (PARTITION p1 VALUES LESS THAN ('20260102'))",
+ "MaxCompute does not support explicit partition definitions"]
+ ].each { entry ->
+ test {
+ sql """CREATE TABLE `${rejectedTable}` (id INT, ts DATETIME, ds STRING) ${entry[0]}"""
+ exception entry[1]
+ }
+ assertEquals([], sql("""SHOW TABLES LIKE '${rejectedTable}'"""))
+ }
+
// Test 1: Single partition column INSERT
String tb1 = "single_part_${uuid}"
sql """DROP TABLE IF EXISTS ${tb1}"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/28: [fix](flight-sql) Report actual server and Arrow versions on branch-4.1 (#68770)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 51f857541b30304359ae50b01fc452c1de63c37f
Author: Gabriel <li...@selectdb.com>
AuthorDate: Thu Oct 8 17:28:00 2026 +0800
[fix](flight-sql) Report actual server and Arrow versions on branch-4.1 (#68770)
### What problem does this PR solve?
Flight SQL GetSqlInfo always reports server version `1.0` and Arrow
version `18.2.0`, so ADBC clients receive stale metadata after upgrades.
Report the running FE build version and short commit hash, using the
same format as FE heartbeat metadata, and read the Arrow version from
the loaded Flight SQL JAR.
If the Arrow manifest has no implementation version, report `unknown`.
Keep the info code registered because Arrow's SqlInfoBuilder rejects
explicit requests for unregistered codes.
### Release note
Fix stale server and Arrow library versions returned by Flight SQL
GetSqlInfo.
### Validation
- Added three Flight RPC tests covering all SQL info, the server version
alone, and the Arrow version alone.
- All three tests reproduce the stale version bug before the fix and
pass after it.
- All three also pass with the Flight SQL dependency loaded from an
exploded directory without its manifest; Arrow allocator debug checks
are enabled.
- Compiled the changed producer and tests against the existing FE
dependency classpath. No full FE/BE build or deployed-cluster test was
run.
- FE reactor Checkstyle validation passed with zero violations.
### Check List (For Author)
- [x] Unit Test
- Behavior changed:
- [x] Yes. Version metadata follows the running server and Arrow
library.
- Does this need documentation?
- [x] No.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../arrowflight/DorisFlightSqlProducer.java | 11 ++-
.../arrowflight/FlightSqlServerInfoTest.java | 104 +++++++++++++++++++++
2 files changed, 113 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java
index 153701dbd2d..2aae87d527b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java
@@ -22,6 +22,7 @@ package org.apache.doris.service.arrowflight;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.Status;
+import org.apache.doris.common.Version;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.Util;
import org.apache.doris.mysql.MysqlCommand;
@@ -120,11 +121,17 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
this.location = location;
this.flightSessionsManager = flightSessionsManager;
sqlInfoBuilder = new SqlInfoBuilder();
- sqlInfoBuilder.withFlightSqlServerName("DorisFE").withFlightSqlServerVersion("1.0")
- .withFlightSqlServerArrowVersion("18.2.0").withFlightSqlServerReadOnly(false)
+ // Keep discovery metadata aligned with the running FE build and its loaded Arrow dependency.
+ sqlInfoBuilder.withFlightSqlServerName("DorisFE")
+ .withFlightSqlServerVersion(Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH)
+ .withFlightSqlServerReadOnly(false)
.withSqlIdentifierQuoteChar("`").withSqlDdlCatalog(true).withSqlDdlSchema(false).withSqlDdlTable(false)
.withSqlIdentifierCase(SqlSupportedCaseSensitivity.SQL_CASE_SENSITIVITY_CASE_INSENSITIVE)
.withSqlQuotedIdentifierCase(SqlSupportedCaseSensitivity.SQL_CASE_SENSITIVITY_CASE_INSENSITIVE);
+ String arrowVersion = SqlInfoBuilder.class.getPackage().getImplementationVersion();
+ // Keep the info code registered even without a manifest: SqlInfoBuilder rejects unknown codes.
+ sqlInfoBuilder.withFlightSqlServerArrowVersion(
+ arrowVersion == null || arrowVersion.isEmpty() ? "unknown" : arrowVersion);
}
private static ByteBuffer serializeMetadata(final Schema schema) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlServerInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlServerInfoTest.java
new file mode 100644
index 00000000000..16f07211132
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlServerInfoTest.java
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service.arrowflight;
+
+import org.apache.doris.common.Version;
+import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager;
+
+import org.apache.arrow.flight.FlightClient;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightServer;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.Location;
+import org.apache.arrow.flight.sql.FlightSqlClient;
+import org.apache.arrow.flight.sql.FlightSqlProducer.Schemas;
+import org.apache.arrow.flight.sql.SqlInfoBuilder;
+import org.apache.arrow.flight.sql.impl.FlightSql.SqlInfo;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class FlightSqlServerInfoTest {
+ @Test
+ public void reportsBuildVersionsInAllSqlInfo() throws Exception {
+ Map<Integer, Object> info = readSqlInfo();
+ Assert.assertEquals("DorisFE", info.get(SqlInfo.FLIGHT_SQL_SERVER_NAME_VALUE));
+ Assert.assertEquals(Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH,
+ info.get(SqlInfo.FLIGHT_SQL_SERVER_VERSION_VALUE));
+ assertArrowVersion(info);
+ Assert.assertEquals(false, info.get(SqlInfo.FLIGHT_SQL_SERVER_READ_ONLY_VALUE));
+ }
+
+ @Test
+ public void reportsOnlyRequestedServerVersion() throws Exception {
+ Map<Integer, Object> info = readSqlInfo(SqlInfo.FLIGHT_SQL_SERVER_VERSION);
+ Assert.assertEquals(1, info.size());
+ Assert.assertEquals(Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH,
+ info.get(SqlInfo.FLIGHT_SQL_SERVER_VERSION_VALUE));
+ }
+
+ @Test
+ public void reportsOnlyRequestedArrowVersion() throws Exception {
+ Map<Integer, Object> info = readSqlInfo(SqlInfo.FLIGHT_SQL_SERVER_ARROW_VERSION);
+ assertArrowVersion(info);
+ Assert.assertEquals(1, info.size());
+ }
+
+ private static void assertArrowVersion(Map<Integer, Object> info) {
+ String version = SqlInfoBuilder.class.getPackage().getImplementationVersion();
+ if (version == null || version.isEmpty()) {
+ // Exploded dependencies may omit the manifest; unknown metadata must not be fabricated.
+ Assert.assertEquals("unknown", info.get(SqlInfo.FLIGHT_SQL_SERVER_ARROW_VERSION_VALUE));
+ } else {
+ Assert.assertEquals(version, info.get(SqlInfo.FLIGHT_SQL_SERVER_ARROW_VERSION_VALUE));
+ }
+ }
+
+ private static Map<Integer, Object> readSqlInfo(SqlInfo... requestedInfo) throws Exception {
+ Location location = Location.forGrpcInsecure("127.0.0.1", 0);
+ FlightSessionsManager sessions = Mockito.mock(FlightSessionsManager.class);
+ try (DorisFlightSqlProducer producer = new DorisFlightSqlProducer(location, sessions);
+ RootAllocator allocator = new RootAllocator();
+ FlightServer server = FlightServer.builder(allocator, location, producer).build().start();
+ FlightClient client = FlightClient.builder(allocator,
+ Location.forGrpcInsecure("127.0.0.1", server.getPort())).build()) {
+ FlightSqlClient sql = new FlightSqlClient(client);
+ FlightInfo info = sql.getSqlInfo(requestedInfo);
+ Assert.assertEquals(Schemas.GET_SQL_INFO_SCHEMA, info.getSchema());
+ Assert.assertEquals(1, info.getEndpoints().size());
+ Map<Integer, Object> values = new HashMap<>();
+ try (FlightStream stream = client.getStream(info.getEndpoints().get(0).getTicket())) {
+ while (stream.next()) {
+ VectorSchemaRoot root = stream.getRoot();
+ for (int row = 0; row < root.getRowCount(); ++row) {
+ int code = ((Number) root.getVector("info_name").getObject(row)).intValue();
+ Object value = root.getVector("value").getObject(row);
+ values.put(code, value instanceof Boolean ? value : value.toString());
+ }
+ }
+ }
+ Mockito.verifyNoInteractions(sessions);
+ return values;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 09/28: branch-4.1: [fix](be) Prevent shared quantile state mutation with TDigest COW (#68498) (#68759)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 409acf9dc5398c9fdb9e46e22dc0ce1e6ceee77c
Author: linrrarity <li...@selectdb.com>
AuthorDate: Fri Oct 9 09:31:06 2026 +0800
branch-4.1: [fix](be) Prevent shared quantile state mutation with TDigest COW (#68498) (#68759)
pick: https://github.com/apache/doris/pull/68498
---
be/src/core/value/quantile_state.cpp | 122 +++++++--
be/src/core/value/quantile_state.h | 16 +-
.../aggregate_function_quantile_state.cpp | 4 +-
.../aggregate/aggregate_function_quantile_state.h | 24 +-
be/src/util/tdigest.h | 2 +
be/test/core/value/quantile_state_test.cpp | 294 +++++++++++++++++++++
be/test/exprs/aggregate/agg_percentile_test.cpp | 171 ++++++++++++
be/test/util/tdigest_test.cpp | 28 ++
.../test_quantile_state_function.out | 32 +++
.../test_quantile_state_function.groovy | 90 +++++++
10 files changed, 754 insertions(+), 29 deletions(-)
diff --git a/be/src/core/value/quantile_state.cpp b/be/src/core/value/quantile_state.cpp
index 95598c2b38a..9ca0f0a0ca8 100644
--- a/be/src/core/value/quantile_state.cpp
+++ b/be/src/core/value/quantile_state.cpp
@@ -19,7 +19,9 @@
#include <string.h>
#include <cmath>
+#include <mutex>
#include <ostream>
+#include <shared_mutex>
#include <utility>
#include "common/logging.h"
@@ -28,8 +30,69 @@
#include "util/tdigest.h"
#include "util/unaligned.h"
+#ifdef BE_TEST
+#include "cpp/sync_point.h"
+#endif
+
namespace doris {
#include "common/compile_check_begin.h"
+
+// Shares a digest across QuantileState copies and detaches it before sample
+// writes. Readers use shared locks; compression uses an exclusive lock.
+struct QuantileState::TDigestHolder {
+ explicit TDigestHolder(float compression) : digest(compression) {}
+ TDigestHolder(const TDigestHolder& other) : digest(other.digest) {}
+
+ std::shared_lock<std::shared_mutex> lock_processed_digest() {
+ std::shared_lock read_lock(mutex);
+ if (digest.haveUnprocessed()) {
+ read_lock.unlock();
+ {
+ std::unique_lock write_lock(mutex);
+ if (digest.haveUnprocessed()) {
+ digest.compress();
+ }
+ }
+ read_lock.lock();
+ }
+ return read_lock;
+ }
+
+ TDigest digest;
+ std::shared_mutex mutex;
+};
+
+QuantileState QuantileState::copy_for_result() const {
+ if (_type != TDIGEST) {
+ return *this;
+ }
+ QuantileState result(_compression);
+ result._type = TDIGEST;
+ {
+ // Reuse processed centroids on the next row instead of sorting the prefix again.
+ auto lock = _tdigest_ptr->lock_processed_digest();
+ // Vector copies retain the elements, not the accumulator's spare write capacity.
+ result._tdigest_ptr = std::make_shared<TDigestHolder>(*_tdigest_ptr);
+ }
+ return result;
+}
+
+TDigest& QuantileState::_mutable_tdigest() {
+ if (_tdigest_ptr.use_count() == 1) {
+ return _tdigest_ptr->digest;
+ }
+ std::shared_ptr<TDigestHolder> detached;
+ {
+ std::shared_lock lock(_tdigest_ptr->mutex);
+#ifdef BE_TEST
+ TEST_SYNC_POINT("QuantileState::detach:source_locked");
+#endif
+ detached = std::make_shared<TDigestHolder>(*_tdigest_ptr);
+ }
+ _tdigest_ptr = std::move(detached);
+ return _tdigest_ptr->digest;
+}
+
QuantileState::QuantileState() : _type(EMPTY), _compression(QUANTILE_STATE_COMPRESSION_MIN) {}
QuantileState::QuantileState(float compression) : _type(EMPTY), _compression(compression) {}
@@ -51,10 +114,14 @@ size_t QuantileState::get_serialized_size() const {
case EXPLICIT:
size += sizeof(uint16_t) + sizeof(double) * _explicit_data.size();
break;
- case TDIGEST:
- size += _tdigest_ptr->serialized_size();
+ case TDIGEST: {
+ // Compress before sizing so concurrent queries cannot change the size
+ // before serialize(); writes through shared copies detach via COW.
+ auto lock = _tdigest_ptr->lock_processed_digest();
+ size += _tdigest_ptr->digest.serialized_size();
break;
}
+ }
return size;
}
@@ -138,7 +205,8 @@ double QuantileState::get_value_by_percentile(float percentile) const {
return get_explicit_value_by_percentile(percentile);
}
case TDIGEST: {
- return _tdigest_ptr->quantile(percentile);
+ auto lock = _tdigest_ptr->lock_processed_digest();
+ return _tdigest_ptr->digest.quantileProcessed(percentile);
}
default:
break;
@@ -187,8 +255,8 @@ bool QuantileState::deserialize(const Slice& slice) {
}
case TDIGEST: {
// 4: Tdigest object value
- _tdigest_ptr = std::make_shared<TDigest>(0);
- _tdigest_ptr->unserialize(ptr);
+ _tdigest_ptr = std::make_shared<TDigestHolder>(0);
+ _tdigest_ptr->digest.unserialize(ptr);
break;
}
default:
@@ -225,7 +293,8 @@ size_t QuantileState::serialize(uint8_t* dst) const {
}
case TDIGEST: {
*ptr++ = TDIGEST;
- size_t tdigest_size = _tdigest_ptr->serialize(ptr);
+ auto lock = _tdigest_ptr->lock_processed_digest();
+ size_t tdigest_size = _tdigest_ptr->digest.serialize(ptr);
ptr += tdigest_size;
break;
}
@@ -236,6 +305,11 @@ size_t QuantileState::serialize(uint8_t* dst) const {
}
void QuantileState::merge(const QuantileState& other) {
+ if (this == &other) {
+ const QuantileState source(other);
+ merge(source);
+ return;
+ }
switch (other._type) {
case EMPTY:
break;
@@ -257,23 +331,25 @@ void QuantileState::merge(const QuantileState& other) {
case EXPLICIT:
if (_explicit_data.size() + other._explicit_data.size() > QUANTILE_STATE_EXPLICIT_NUM) {
_type = TDIGEST;
- _tdigest_ptr = std::make_shared<TDigest>(_compression);
+ _tdigest_ptr = std::make_shared<TDigestHolder>(_compression);
for (int i = 0; i < _explicit_data.size(); i++) {
- _tdigest_ptr->add((float)_explicit_data[i]);
+ _tdigest_ptr->digest.add((float)_explicit_data[i]);
}
for (int i = 0; i < other._explicit_data.size(); i++) {
- _tdigest_ptr->add((float)other._explicit_data[i]);
+ _tdigest_ptr->digest.add((float)other._explicit_data[i]);
}
} else {
_explicit_data.insert(_explicit_data.end(), other._explicit_data.begin(),
other._explicit_data.end());
}
break;
- case TDIGEST:
+ case TDIGEST: {
+ auto& digest = _mutable_tdigest();
for (int i = 0; i < other._explicit_data.size(); i++) {
- _tdigest_ptr->add((float)other._explicit_data[i]);
+ digest.add((float)other._explicit_data[i]);
}
break;
+ }
default:
break;
}
@@ -288,18 +364,26 @@ void QuantileState::merge(const QuantileState& other) {
case SINGLE:
_type = TDIGEST;
_tdigest_ptr = other._tdigest_ptr;
- _tdigest_ptr->add((float)_single_data);
+ _mutable_tdigest().add((float)_single_data);
break;
- case EXPLICIT:
+ case EXPLICIT: {
_type = TDIGEST;
_tdigest_ptr = other._tdigest_ptr;
+ auto& digest = _mutable_tdigest();
for (int i = 0; i < _explicit_data.size(); i++) {
- _tdigest_ptr->add((float)_explicit_data[i]);
+ digest.add((float)_explicit_data[i]);
}
break;
- case TDIGEST:
- _tdigest_ptr->merge(other._tdigest_ptr.get());
+ }
+ case TDIGEST: {
+ auto& digest = _mutable_tdigest();
+ std::shared_lock lock(other._tdigest_ptr->mutex);
+#ifdef BE_TEST
+ TEST_SYNC_POINT("QuantileState::merge:source_locked");
+#endif
+ digest.merge(&other._tdigest_ptr->digest);
break;
+ }
default:
break;
}
@@ -323,9 +407,9 @@ void QuantileState::add_value(const double& value) {
break;
case EXPLICIT:
if (_explicit_data.size() == QUANTILE_STATE_EXPLICIT_NUM) {
- _tdigest_ptr = std::make_shared<TDigest>(_compression);
+ _tdigest_ptr = std::make_shared<TDigestHolder>(_compression);
for (int i = 0; i < _explicit_data.size(); i++) {
- _tdigest_ptr->add((float)_explicit_data[i]);
+ _tdigest_ptr->digest.add((float)_explicit_data[i]);
}
_explicit_data.clear();
_explicit_data.shrink_to_fit();
@@ -336,7 +420,7 @@ void QuantileState::add_value(const double& value) {
}
break;
case TDIGEST:
- _tdigest_ptr->add((float)value);
+ _mutable_tdigest().add((float)value);
break;
}
}
diff --git a/be/src/core/value/quantile_state.h b/be/src/core/value/quantile_state.h
index 50868290976..fc540848c26 100644
--- a/be/src/core/value/quantile_state.h
+++ b/be/src/core/value/quantile_state.h
@@ -47,11 +47,14 @@ public:
QuantileState();
explicit QuantileState(float compression);
explicit QuantileState(const Slice& slice);
- QuantileState& operator=(const QuantileState& other) noexcept = default;
- QuantileState(const QuantileState& other) noexcept = default;
+ QuantileState& operator=(const QuantileState& other) = default;
+ QuantileState(const QuantileState& other) = default;
QuantileState& operator=(QuantileState&& other) noexcept = default;
QuantileState(QuantileState&& other) noexcept = default;
+ // A compact, independent value for retained window results.
+ QuantileState copy_for_result() const;
+
void set_compression(float compression);
bool deserialize(const Slice& slice);
size_t serialize(uint8_t* dst) const;
@@ -70,9 +73,14 @@ public:
~QuantileState() = default;
private:
+ // Copies share a digest until the first sample write. Concurrent const
+ // operations are supported; mutating the same state requires exclusive access.
+ struct TDigestHolder;
+ TDigest& _mutable_tdigest();
+
QuantileStateType _type = EMPTY;
- std::shared_ptr<TDigest> _tdigest_ptr;
- double _single_data;
+ std::shared_ptr<TDigestHolder> _tdigest_ptr;
+ double _single_data = 0;
std::vector<double> _explicit_data;
float _compression;
};
diff --git a/be/src/exprs/aggregate/aggregate_function_quantile_state.cpp b/be/src/exprs/aggregate/aggregate_function_quantile_state.cpp
index 8e59828db3d..f31cb0d0cad 100644
--- a/be/src/exprs/aggregate/aggregate_function_quantile_state.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_quantile_state.cpp
@@ -32,11 +32,11 @@ AggregateFunctionPtr create_aggregate_function_quantile_state_union(
if (arg_is_nullable) {
return std::make_shared<
AggregateFunctionQuantileStateOp<true, AggregateFunctionQuantileStateUnionOp>>(
- argument_types);
+ argument_types, attr.is_window_function);
} else {
return std::make_shared<
AggregateFunctionQuantileStateOp<false, AggregateFunctionQuantileStateUnionOp>>(
- argument_types);
+ argument_types, attr.is_window_function);
}
}
diff --git a/be/src/exprs/aggregate/aggregate_function_quantile_state.h b/be/src/exprs/aggregate/aggregate_function_quantile_state.h
index fb03d3eef8f..e85bd4ec40b 100644
--- a/be/src/exprs/aggregate/aggregate_function_quantile_state.h
+++ b/be/src/exprs/aggregate/aggregate_function_quantile_state.h
@@ -103,10 +103,11 @@ public:
String get_name() const override { return Op::name; }
- AggregateFunctionQuantileStateOp(const DataTypes& argument_types_)
+ AggregateFunctionQuantileStateOp(const DataTypes& argument_types_, bool is_window_function)
: IAggregateFunctionDataHelper<AggregateFunctionQuantileStateData<Op>,
AggregateFunctionQuantileStateOp<arg_is_nullable, Op>>(
- argument_types_) {}
+ argument_types_),
+ _is_window_function(is_window_function) {}
DataTypePtr get_return_type() const override {
return std::make_shared<DataTypeQuantileState>();
@@ -144,11 +145,26 @@ public:
}
void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
- auto& column = assert_cast<ColVecResult&>(to);
- column.get_data().push_back(this->data(place).get());
+ auto& column = assert_cast<ColVecResult&, TypeCheckOnRelease::DISABLE>(to);
+ const auto& value = this->data(place).get();
+ column.get_data().push_back(_is_window_function ? value.copy_for_result() : value);
+ }
+
+ void insert_result_into_range(ConstAggregateDataPtr __restrict place, IColumn& to,
+ const size_t start, const size_t end) const override {
+ if (start == end) {
+ return;
+ }
+ insert_result_into(place, to);
+ auto& data = assert_cast<ColVecResult&, TypeCheckOnRelease::DISABLE>(to).get_data();
+ // Rows with the same window result share one compact digest.
+ data.insert(data.end(), end - start - 1, data.back());
}
void reset(AggregateDataPtr __restrict place) const override { this->data(place).reset(); }
+
+private:
+ const bool _is_window_function;
};
AggregateFunctionPtr create_aggregate_function_quantile_state_union(
diff --git a/be/src/util/tdigest.h b/be/src/util/tdigest.h
index 5dad6d5bac9..158dc748ce9 100644
--- a/be/src/util/tdigest.h
+++ b/be/src/util/tdigest.h
@@ -171,6 +171,8 @@ public:
return w;
}
+ TDigest(const TDigest&) = default;
+
TDigest& operator=(TDigest&& o) {
_compression = o._compression;
_max_processed = o._max_processed;
diff --git a/be/test/core/value/quantile_state_test.cpp b/be/test/core/value/quantile_state_test.cpp
index ae3fc43b1ed..c025294caa3 100644
--- a/be/test/core/value/quantile_state_test.cpp
+++ b/be/test/core/value/quantile_state_test.cpp
@@ -20,10 +20,304 @@
#include <gtest/gtest-message.h>
#include <gtest/gtest-test-part.h>
+#include <atomic>
+#include <barrier>
+#include <chrono>
+#include <cstring>
+#include <future>
+#include <thread>
+
+#include "cpp/sync_point.h"
#include "gtest/gtest_pred_impl.h"
+#include "util/tdigest.h"
namespace doris {
+TEST(QuantileStateTest, SharedDigestWritesDoNotChangeSource) {
+ QuantileState source;
+ for (int i = 0; i < 4096; ++i) {
+ source.add_value(10);
+ }
+ QuantileState copied = source;
+ EXPECT_EQ(copied._tdigest_ptr, source._tdigest_ptr);
+ copied.add_value(110);
+ EXPECT_NE(copied._tdigest_ptr, source._tdigest_ptr);
+ EXPECT_EQ(110, copied.get_value_by_percentile(1));
+ EXPECT_EQ(10, source.get_value_by_percentile(1));
+
+ QuantileState merged;
+ merged.merge(source);
+ merged.add_value(-10);
+ EXPECT_EQ(-10, merged.get_value_by_percentile(0));
+ EXPECT_EQ(10, source.get_value_by_percentile(0));
+}
+
+static QuantileState constant_state(double value, int count = 4096) {
+ QuantileState state;
+ for (int i = 0; i < count; ++i) {
+ state.add_value(value);
+ }
+ return state;
+}
+
+TEST(QuantileStateTest, MergeDoesNotModifyDigestSourceForAnyTargetType) {
+ for (int count : {0, 1, 2, 4096}) {
+ auto source = constant_state(10);
+ auto target = constant_state(-10, count);
+ target.merge(source);
+ EXPECT_EQ(count == 0 ? 10 : -10, target.get_value_by_percentile(0));
+ EXPECT_EQ(10, target.get_value_by_percentile(1));
+ target.add_value(110);
+ EXPECT_EQ(110, target.get_value_by_percentile(1));
+ EXPECT_EQ(10, source.get_value_by_percentile(0));
+ EXPECT_EQ(10, source.get_value_by_percentile(1));
+ }
+}
+
+TEST(QuantileStateTest, AssignmentAndExplicitMergeDetachOnlyOnce) {
+ auto source = constant_state(10);
+ QuantileState target;
+ target = source;
+ auto extra = constant_state(-10, 2);
+ target.merge(extra);
+ auto* detached = target._tdigest_ptr.get();
+ EXPECT_NE(detached, source._tdigest_ptr.get());
+ for (int i = 0; i < 100; ++i) {
+ target.add_value(110);
+ EXPECT_EQ(detached, target._tdigest_ptr.get());
+ }
+ EXPECT_EQ(-10, target.get_value_by_percentile(0));
+ EXPECT_EQ(110, target.get_value_by_percentile(1));
+ EXPECT_EQ(10, source.get_value_by_percentile(0));
+ EXPECT_EQ(10, source.get_value_by_percentile(1));
+}
+
+TEST(QuantileStateTest, WindowReadsReuseDigestUntilResultIsCopied) {
+ auto state = constant_state(10);
+ auto* original = state._tdigest_ptr.get();
+ for (int i = 11; i < 100; ++i) {
+ state.add_value(i);
+ EXPECT_EQ(i, state.get_value_by_percentile(1));
+ EXPECT_EQ(original, state._tdigest_ptr.get());
+ }
+ auto saved_result = state;
+ state.add_value(110);
+ EXPECT_EQ(99, saved_result.get_value_by_percentile(1));
+ EXPECT_EQ(110, state.get_value_by_percentile(1));
+}
+
+TEST(QuantileStateTest, WritesReuseDigestAfterLastCopyIsDestroyed) {
+ auto state = constant_state(10);
+ auto* original = state._tdigest_ptr.get();
+ {
+ auto copy = state;
+ EXPECT_EQ(original, copy._tdigest_ptr.get());
+ }
+ state.add_value(110);
+ EXPECT_EQ(original, state._tdigest_ptr.get());
+ EXPECT_EQ(110, state.get_value_by_percentile(1));
+}
+
+TEST(QuantileStateTest, WritesReuseDigestAfterOtherCopyDetaches) {
+ auto state = constant_state(10);
+ auto* original = state._tdigest_ptr.get();
+ auto copy = state;
+ copy.add_value(110);
+ state.add_value(-10);
+ EXPECT_EQ(original, state._tdigest_ptr.get());
+ EXPECT_EQ(10, state.get_value_by_percentile(1));
+ EXPECT_EQ(110, copy.get_value_by_percentile(1));
+ EXPECT_EQ(10, copy.get_value_by_percentile(0));
+}
+
+TEST(QuantileStateTest, SerializedSizeRemainsStableAcrossSharedQueries) {
+ auto source = constant_state(10);
+ auto copy = source;
+ const auto size = source.get_serialized_size();
+ EXPECT_EQ(10, copy.get_value_by_percentile(0.5));
+ std::vector<uint8_t> bytes(size);
+ ASSERT_EQ(size, source.serialize(bytes.data()));
+ QuantileState restored(Slice(reinterpret_cast<char*>(bytes.data()), bytes.size()));
+ EXPECT_EQ(10, restored.get_value_by_percentile(0.5));
+}
+
+static long serialized_digest_weight(const QuantileState& state) {
+ std::vector<uint8_t> bytes(state.get_serialized_size());
+ state.serialize(bytes.data());
+ TDigest digest(0);
+ digest.unserialize(bytes.data() + sizeof(float) + sizeof(uint8_t));
+ return digest.totalWeight();
+}
+
+TEST(QuantileStateTest, SelfMergeAndSharedSourceMerge) {
+ for (int count : {2, 4096}) {
+ auto state = constant_state(10, count);
+ auto unchanged = state;
+ state.merge(state);
+ state.merge(unchanged);
+ if (count == 2) {
+ EXPECT_EQ(6, state._explicit_data.size());
+ } else {
+ EXPECT_EQ(3 * serialized_digest_weight(unchanged), serialized_digest_weight(state));
+ }
+ state.add_value(110);
+ EXPECT_EQ(10, state.get_value_by_percentile(0));
+ EXPECT_EQ(110, state.get_value_by_percentile(1));
+ EXPECT_EQ(10, unchanged.get_value_by_percentile(1));
+ }
+}
+
+static void expect_serialized_maximum(const QuantileState& state, double expected) {
+ std::vector<uint8_t> bytes(state.get_serialized_size());
+ ASSERT_EQ(bytes.size(), state.serialize(bytes.data()));
+ QuantileState restored(Slice(reinterpret_cast<char*>(bytes.data()), bytes.size()));
+ EXPECT_EQ(expected, restored.get_value_by_percentile(1));
+}
+
+TEST(QuantileStateTest, ConcurrentReadsSerializationAndIndependentWrites) {
+ auto source = constant_state(10);
+ std::vector<QuantileState> writers(4, source);
+ std::barrier start(8);
+ std::vector<std::thread> threads;
+ for (int i = 0; i < 2; ++i) {
+ threads.emplace_back([&] {
+ start.arrive_and_wait();
+ for (int j = 0; j < 100; ++j) {
+ EXPECT_EQ(10, source.get_value_by_percentile(0.5));
+ }
+ });
+ }
+ threads.emplace_back([&] {
+ start.arrive_and_wait();
+ for (int j = 0; j < 100; ++j) {
+ expect_serialized_maximum(source, 10);
+ }
+ });
+ for (int i = 0; i < 4; ++i) {
+ threads.emplace_back([&, i] {
+ start.arrive_and_wait();
+ for (int j = 0; j < 100; ++j) {
+ writers[i].add_value(110 + i);
+ writers[i].merge(source);
+ EXPECT_EQ(110 + i, writers[i].get_value_by_percentile(1));
+ }
+ });
+ }
+ start.arrive_and_wait();
+ for (auto& thread : threads) {
+ thread.join();
+ }
+ EXPECT_EQ(10, source.get_value_by_percentile(1));
+}
+
+TEST(QuantileStateTest, SharedSourceDetachesAndReadsCanOverlap) {
+ auto source = constant_state(10);
+ // A processed query only needs a shared lock; compression still needs an exclusive lock.
+ ASSERT_EQ(10, source.get_value_by_percentile(0.5));
+ auto first = source;
+ auto second = source;
+ std::promise<void> first_entered;
+ std::promise<void> release_first;
+ auto released = release_first.get_future();
+ std::atomic<int> arrivals {0};
+ auto* sync = SyncPoint::get_instance();
+ SyncPoint::CallbackGuard guard;
+ sync->set_call_back(
+ "QuantileState::detach:source_locked",
+ [&](auto&&) {
+ if (arrivals.fetch_add(1) == 0) {
+ first_entered.set_value();
+ released.wait();
+ }
+ },
+ &guard);
+ sync->enable_processing();
+ std::thread first_thread([&] { first.add_value(-10); });
+ auto first_ready = first_entered.get_future().wait_for(std::chrono::seconds(10));
+ std::promise<void> second_finished;
+ std::thread second_thread([&] {
+ second.add_value(110);
+ second_finished.set_value();
+ });
+ std::promise<double> read_finished;
+ auto read_result = read_finished.get_future();
+ std::thread reader([&] { read_finished.set_value(source.get_value_by_percentile(0.5)); });
+ auto second_ready = second_finished.get_future().wait_for(std::chrono::seconds(10));
+ auto read_ready = read_result.wait_for(std::chrono::seconds(10));
+ release_first.set_value();
+ first_thread.join();
+ second_thread.join();
+ reader.join();
+ sync->disable_processing();
+ EXPECT_EQ(std::future_status::ready, first_ready);
+ EXPECT_EQ(std::future_status::ready, second_ready);
+ EXPECT_EQ(std::future_status::ready, read_ready);
+ EXPECT_EQ(10, read_result.get());
+ EXPECT_EQ(-10, first.get_value_by_percentile(0));
+ EXPECT_EQ(10, first.get_value_by_percentile(1));
+ EXPECT_EQ(10, second.get_value_by_percentile(0));
+ EXPECT_EQ(110, second.get_value_by_percentile(1));
+ EXPECT_EQ(10, source.get_value_by_percentile(0.5));
+}
+
+TEST(QuantileStateTest, SharedSourceMergesCanOverlap) {
+ auto source = constant_state(10);
+ auto first = constant_state(-10);
+ auto second = constant_state(20);
+ std::promise<void> first_entered;
+ std::promise<void> second_entered;
+ std::promise<void> release_first;
+ auto released = release_first.get_future();
+ std::atomic<int> arrivals {0};
+ auto* sync = SyncPoint::get_instance();
+ SyncPoint::CallbackGuard guard;
+ sync->set_call_back(
+ "QuantileState::merge:source_locked",
+ [&](auto&&) {
+ if (arrivals.fetch_add(1) == 0) {
+ first_entered.set_value();
+ released.wait();
+ } else {
+ second_entered.set_value();
+ }
+ },
+ &guard);
+ sync->enable_processing();
+ std::thread first_thread([&] { first.merge(source); });
+ auto first_ready = first_entered.get_future().wait_for(std::chrono::seconds(10));
+ std::thread second_thread([&] { second.merge(source); });
+ auto second_ready = second_entered.get_future().wait_for(std::chrono::seconds(10));
+ release_first.set_value();
+ first_thread.join();
+ second_thread.join();
+ sync->disable_processing();
+ EXPECT_EQ(std::future_status::ready, first_ready);
+ EXPECT_EQ(std::future_status::ready, second_ready);
+ EXPECT_EQ(-10, first.get_value_by_percentile(0));
+ EXPECT_EQ(20, second.get_value_by_percentile(1));
+ EXPECT_EQ(10, source.get_value_by_percentile(0.5));
+}
+
+TEST(QuantileStateTest, ReadsLegacyUnprocessedDigestAndKeepsWireLayout) {
+ TDigest legacy(2048);
+ for (int i = 0; i < 4096; ++i) {
+ legacy.add(10);
+ }
+ constexpr size_t header_size = sizeof(float) + sizeof(uint8_t);
+ std::vector<uint8_t> bytes(header_size + legacy.serialized_size());
+ const float compression = 2048;
+ memcpy(bytes.data(), &compression, sizeof(compression));
+ bytes[sizeof(float)] = TDIGEST;
+ legacy.serialize(bytes.data() + header_size);
+ QuantileState state(Slice(reinterpret_cast<char*>(bytes.data()), bytes.size()));
+ EXPECT_EQ(10, state.get_value_by_percentile(0.5));
+ bytes.resize(state.get_serialized_size());
+ ASSERT_EQ(bytes.size(), state.serialize(bytes.data()));
+ legacy.unserialize(bytes.data() + header_size);
+ EXPECT_EQ(4096, legacy.totalWeight());
+ EXPECT_EQ(10, legacy.quantile(0.5));
+}
+
TEST(QuantileStateTest, merge) {
QuantileState empty;
EXPECT_EQ(EMPTY, empty._type);
diff --git a/be/test/exprs/aggregate/agg_percentile_test.cpp b/be/test/exprs/aggregate/agg_percentile_test.cpp
new file mode 100644
index 00000000000..97f74de21fc
--- /dev/null
+++ b/be/test/exprs/aggregate/agg_percentile_test.cpp
@@ -0,0 +1,171 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <vector>
+
+#include "core/column/column_complex.h"
+#include "core/data_type/data_type_quantilestate.h"
+#include "exprs/aggregate/aggregate_function_quantile_state.h"
+#include "util/defer_op.h"
+#include "util/tdigest.h"
+
+namespace doris {
+namespace {
+
+std::vector<double> expected_quantiles(const std::vector<double>& values,
+ const std::vector<double>& quantiles, float compression) {
+ TDigest digest(compression);
+ for (double value : values) {
+ digest.add(value);
+ }
+ std::vector<double> result;
+ result.reserve(quantiles.size());
+ for (double quantile : quantiles) {
+ result.push_back(digest.quantile(quantile));
+ }
+ return result;
+}
+
+} // namespace
+
+TEST(AggregateFunctionQuantileStateTest, GrowingWindowKeepsResultsCompact) {
+ auto type = std::make_shared<DataTypeQuantileState>();
+ auto function = create_aggregate_function_quantile_state_union(
+ "quantile_union", {type}, type, false,
+ {.is_window_function = true, .column_names = {}});
+ std::unique_ptr<char[]> memory(new char[function->size_of_data()]);
+ auto* place = memory.get();
+ function->create(place);
+ Defer destroy([&] { function->destroy(place); });
+ Arena arena;
+ auto input = ColumnQuantileState::create();
+ QuantileState seed(10000);
+ constexpr size_t seed_count = 4096;
+ for (size_t i = 0; i < seed_count; ++i) {
+ seed.add_value(10);
+ }
+ input->insert_value(std::move(seed));
+ const IColumn* columns[] = {input.get()};
+ function->add(place, columns, 0, arena);
+ input->clear();
+ using Data = AggregateFunctionQuantileStateData<AggregateFunctionQuantileStateUnionOp>;
+ auto& accumulator = reinterpret_cast<Data*>(place)->value;
+ auto* original = accumulator._tdigest_ptr.get();
+ const size_t initial_unprocessed = accumulator._mutable_tdigest().unprocessed().size();
+ auto results = ColumnQuantileState::create();
+ constexpr size_t result_count = 12000;
+ results->reserve(result_count);
+ bool reused_accumulator = true;
+ size_t unprocessed_centroids = 0;
+ for (size_t row = 0; row < result_count; ++row) {
+ QuantileState value;
+ value.add_value(20 + row);
+ input->clear();
+ input->insert_value(std::move(value));
+ function->add(place, columns, 0, arena);
+ unprocessed_centroids += accumulator._mutable_tdigest().unprocessed().size();
+ function->insert_result_into(place, *results);
+ reused_accumulator &= accumulator._tdigest_ptr.get() == original;
+ }
+ EXPECT_TRUE(reused_accumulator);
+ // Each sample should enter result-insertion sorting only once across the window.
+ EXPECT_EQ(initial_unprocessed + result_count, unprocessed_centroids);
+ RecordProperty("unprocessed_centroids", std::to_string(unprocessed_centroids));
+ EXPECT_NE(accumulator._tdigest_ptr, results->get_element(result_count - 1)._tdigest_ptr);
+ // Inspect actual capacities independently of the column's approximate accounting.
+ size_t digest_bytes = 0;
+ for (auto& result : results->get_data()) {
+ auto& digest = result._mutable_tdigest();
+ ASSERT_EQ(0, digest._unprocessed.capacity());
+ ASSERT_EQ(digest._processed.size(), digest._processed.capacity());
+ ASSERT_EQ(digest._cumulative.size(), digest._cumulative.capacity());
+ digest_bytes +=
+ (digest._processed.capacity() + digest._unprocessed.capacity()) * sizeof(Centroid) +
+ digest._cumulative.capacity() * sizeof(Weight);
+ }
+ RecordProperty("retained_digest_bytes", std::to_string(digest_bytes));
+ EXPECT_LT(digest_bytes, result_count * 160 * 1024);
+ for (size_t row : {size_t(0), result_count / 2, result_count - 1}) {
+ auto& result = results->get_element(row);
+ EXPECT_EQ(0, result._mutable_tdigest().unprocessed().capacity());
+ EXPECT_EQ(10, result.get_value_by_percentile(0));
+ EXPECT_EQ(20 + row, result.get_value_by_percentile(1));
+ std::vector<double> values(seed_count, 10);
+ for (size_t i = 0; i <= row; ++i) {
+ values.push_back(20 + i);
+ }
+ const std::vector<double> quantiles {0.5, 0.9, 0.99};
+ const auto expected = expected_quantiles(values, quantiles, 10000);
+ for (size_t i = 0; i < quantiles.size(); ++i) {
+ EXPECT_NEAR(expected[i], result.get_value_by_percentile(quantiles[i]), 1.0);
+ }
+ }
+ EXPECT_GE(accumulator._mutable_tdigest().unprocessed().capacity(), 80001);
+ EXPECT_FALSE(accumulator._mutable_tdigest().haveUnprocessed());
+}
+
+class AggregateFunctionQuantileStateRangeTest : public testing::TestWithParam<bool> {};
+
+TEST_P(AggregateFunctionQuantileStateRangeTest, RangeResultsShareOneSavedDigest) {
+ const bool is_window = GetParam();
+ auto type = std::make_shared<DataTypeQuantileState>();
+ auto function = create_aggregate_function_quantile_state_union(
+ "quantile_union", {type}, type, false,
+ {.is_window_function = is_window, .column_names = {}});
+ std::unique_ptr<char[]> memory(new char[function->size_of_data()]);
+ auto* place = memory.get();
+ function->create(place);
+ Defer destroy([&] { function->destroy(place); });
+ Arena arena;
+ auto input = ColumnQuantileState::create();
+ QuantileState state(10000);
+ for (int i = 0; i < 4096; ++i) {
+ state.add_value(10);
+ }
+ input->insert_value(state);
+ const IColumn* columns[] = {input.get()};
+ function->add(place, columns, 0, arena);
+ auto results = ColumnQuantileState::create();
+ results->insert_many_defaults(3);
+ function->insert_result_into_range(place, *results, 3, 12003);
+ function->insert_result_into_range(place, *results, 12003, 12003);
+ ASSERT_EQ(12003, results->size());
+ const size_t column_bytes = results->allocated_bytes();
+ const auto& first = results->get_element(3);
+ for (size_t row = 3; row < results->size(); ++row) {
+ EXPECT_EQ(first._tdigest_ptr, results->get_element(row)._tdigest_ptr);
+ }
+ if (is_window) {
+ EXPECT_NE(state._tdigest_ptr, first._tdigest_ptr);
+ } else {
+ EXPECT_EQ(state._tdigest_ptr, first._tdigest_ptr);
+ }
+ auto modified = first;
+ modified.add_value(110);
+ EXPECT_EQ(110, modified.get_value_by_percentile(1));
+ EXPECT_EQ(10, first.get_value_by_percentile(1));
+ EXPECT_EQ(10, state.get_value_by_percentile(1));
+ EXPECT_EQ(column_bytes, results->allocated_bytes());
+}
+
+INSTANTIATE_TEST_SUITE_P(AggregateAndWindow, AggregateFunctionQuantileStateRangeTest,
+ testing::Bool());
+
+} // namespace doris
diff --git a/be/test/util/tdigest_test.cpp b/be/test/util/tdigest_test.cpp
index efb40774324..0804fa3ad7f 100644
--- a/be/test/util/tdigest_test.cpp
+++ b/be/test/util/tdigest_test.cpp
@@ -76,6 +76,34 @@ static double quantile(const double q, const std::vector<double>& values) {
return q1;
}
+TEST_F(TDigestTest, CopyPreservesValuesWithoutSpareCapacity) {
+ const auto allocated_bytes = [](const TDigest& digest) {
+ return (digest._processed.capacity() + digest._unprocessed.capacity()) * sizeof(Centroid) +
+ digest._cumulative.capacity() * sizeof(Weight);
+ };
+ TDigest source(10000);
+ for (int i = 0; i < 300; ++i) {
+ source.add(i);
+ }
+ source.compress();
+ TDigest processed_copy(source);
+ EXPECT_EQ(0, processed_copy.unprocessed().capacity());
+ EXPECT_LT(allocated_bytes(processed_copy), 16 * 1024);
+ source.add(1000);
+ TDigest copy(source);
+ EXPECT_EQ(301, copy.totalWeight());
+ EXPECT_LT(allocated_bytes(copy), 16 * 1024);
+ // Continued writes may grow the copy's buffers, without changing the source.
+ for (int i = 0; i < 1000; ++i) {
+ copy.add(2000);
+ }
+ EXPECT_EQ(1301, copy.totalWeight());
+ EXPECT_EQ(0, copy.quantile(0));
+ EXPECT_EQ(2000, copy.quantile(1));
+ EXPECT_EQ(1000, source.quantile(1));
+ EXPECT_EQ(299, processed_copy.quantile(1));
+}
+
TEST_F(TDigestTest, CrashAfterMerge) {
TDigest digest(1000);
std::uniform_real_distribution<> reals(0.0, 1.0);
diff --git a/regression-test/data/query_p0/sql_functions/quantile_state_functions/test_quantile_state_function.out b/regression-test/data/query_p0/sql_functions/quantile_state_functions/test_quantile_state_function.out
index 1e9b9630e05..42cd7965e97 100644
--- a/regression-test/data/query_p0/sql_functions/quantile_state_functions/test_quantile_state_function.out
+++ b/regression-test/data/query_p0/sql_functions/quantile_state_functions/test_quantile_state_function.out
@@ -35,3 +35,35 @@ AAAARQEAAAAAAAAkQA==
-- !sql_quantile_state_base64_12 --
true
+-- !cow_independent --
+0 10 10 10 10 110
+1 110 110 110 10 110
+
+-- !cow_shared --
+0 10 10 10 10 110
+1 110 110 110 10 110
+
+-- !cow_window --
+0 0 10 10
+0 1 10 10
+0 2 10 10
+0 3 10 10
+1 0 10 110
+1 1 10 110
+1 2 10 110
+1 3 10 110
+
+-- !cow_join_expand --
+0 10
+1 110
+
+-- !cow_explode_expand --
+0 10
+1 110
+
+-- !cow_long_high_compression_window --
+0 10 10
+1 10 20
+6000 10 6019
+12000 10 12019
+
diff --git a/regression-test/suites/query_p0/sql_functions/quantile_state_functions/test_quantile_state_function.groovy b/regression-test/suites/query_p0/sql_functions/quantile_state_functions/test_quantile_state_function.groovy
index b1480f0971b..d01983f5c69 100644
--- a/regression-test/suites/query_p0/sql_functions/quantile_state_functions/test_quantile_state_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/quantile_state_functions/test_quantile_state_function.groovy
@@ -79,4 +79,94 @@ suite("test_quantile_state_function") {
)
) = quantile_state_to_base64(to_quantile_state(10.0, 2048))
"""
+
+ sql "DROP TABLE IF EXISTS test_quantile_state_cow"
+ sql """
+ CREATE TABLE test_quantile_state_cow (
+ topic INT NOT NULL,
+ chunk INT NOT NULL,
+ q QUANTILE_STATE QUANTILE_UNION NOT NULL
+ ) AGGREGATE KEY(topic, chunk)
+ DISTRIBUTED BY HASH(topic, chunk) BUCKETS 4
+ PROPERTIES("replication_num" = "1")
+ """
+ // Each stored state has 4096 inputs, exceeding the EXPLICIT-state limit.
+ sql """
+ INSERT INTO test_quantile_state_cow
+ SELECT number % 2, (number DIV 2) % 4,
+ quantile_union(to_quantile_state(10 + 100 * (number % 2), 2048))
+ FROM numbers("number" = "32768")
+ GROUP BY 1, 2
+ """
+
+ def query = """
+ WITH shared AS (SELECT * FROM test_quantile_state_cow),
+ topics AS (SELECT topic, quantile_union(q) AS q FROM shared GROUP BY topic),
+ total AS (SELECT quantile_union(q) AS q FROM shared)
+ SELECT topic, quantile_percent(topics.q, 0), quantile_percent(topics.q, 0.5),
+ quantile_percent(topics.q, 1),
+ quantile_percent(total.q, 0), quantile_percent(total.q, 1)
+ FROM topics CROSS JOIN total ORDER BY topic
+ """
+ sql "SET enable_cte_materialize = false"
+ qt_cow_independent query
+ sql "SET enable_cte_materialize = true"
+ sql "SET inline_cte_referenced_threshold = 0"
+ explain {
+ sql(query)
+ contains "MultiCastDataSinks"
+ }
+ // Both plans are checked against fixed results: topic values are 10 or 110.
+ qt_cow_shared query
+
+ qt_cow_window """
+ SELECT topic, chunk, quantile_percent(q, 0), quantile_percent(q, 1)
+ FROM (
+ SELECT topic, chunk,
+ quantile_union(q) OVER (ORDER BY topic, chunk
+ ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS q
+ FROM test_quantile_state_cow
+ ) t ORDER BY topic, chunk
+ """
+
+ // Each state has 4096 samples. Process the shared 10 state first so both
+ // groups start from it before group 1 merges 110; group 0 must stay at 10.
+ qt_cow_join_expand """
+ SELECT g.k, quantile_percent(quantile_union(t.q), 1)
+ FROM (
+ SELECT number % 2 AS id,
+ quantile_union(to_quantile_state(10 + 100 * (number % 2), 2048)) AS q
+ FROM numbers("number" = "8192") GROUP BY 1 ORDER BY 1 LIMIT 2
+ ) t
+ JOIN (SELECT 0 AS k UNION ALL SELECT 1) g ON t.id = 0 OR g.k = 1
+ GROUP BY g.k ORDER BY g.k
+ """
+ qt_cow_explode_expand """
+ SELECT k, quantile_percent(quantile_union(q), 1)
+ FROM (
+ SELECT number % 2 AS id,
+ quantile_union(to_quantile_state(10 + 100 * (number % 2), 2048)) AS q
+ FROM numbers("number" = "8192") GROUP BY 1 ORDER BY 1 LIMIT 2
+ ) t
+ LATERAL VIEW explode(IF(id = 0, [0, 1], [1])) e AS k
+ GROUP BY k ORDER BY k
+ """
+
+ qt_cow_long_high_compression_window """
+ SELECT id, quantile_percent(q, 0), quantile_percent(q, 1)
+ FROM (
+ SELECT id, quantile_union(q) OVER (
+ ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS q
+ FROM (
+ SELECT 0 AS id, quantile_union(to_quantile_state(10, 10000)) AS q
+ FROM numbers("number" = "4096")
+ UNION ALL
+ SELECT number + 1 AS id, to_quantile_state(20 + number, 10000) AS q
+ FROM numbers("number" = "12000")
+ ) input
+ ) window_results
+ WHERE id IN (0, 1, 6000, 12000)
+ ORDER BY id
+ """
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 05/28: branch-4.1: [fix](iceberg) Resolve dropped equality-delete fields in manifest-cache planning (#68433)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 97df025c483310a9a6674f7c3b2587b730fa5557
Author: daidai <ch...@selectdb.com>
AuthorDate: Thu Oct 8 18:13:53 2026 +0800
branch-4.1: [fix](iceberg) Resolve dropped equality-delete fields in manifest-cache planning (#68433)
### What problem does this PR solve?
Issue Number: None
Related PR: #67479, #67687
Problem Summary:
#67479 lets `DeleteFileIndex` resolve equality-delete fields that were
later dropped, by looking them up in historical schemas. It covers the
default planner but misses the manifest-cache planner in
`IcebergScanNode`, which still passes only `specsById`. When the
manifest cache is enabled, planning fails with `Cannot find field for ID
N` and falls back to the SDK scan: results are correct, but each query
logs a WARN and skips the cache.
The master port (#67687) already covers this path. This PR applies the
same one-line change to branch-4.1: pass `icebergTable.schemas()` to the
index.
### Release note
None
### Check List (For Author)
- Test
- [ ] Regression test
- [x] Unit Test
- [x] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason
Manual test: with the manifest cache enabled, `EXPLAIN VERBOSE` shows
`failures=0` instead of `1`, and the fallback WARN is gone.
- Behavior changed:
- [x] No.
- [ ] Yes.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../datasource/iceberg/source/IcebergScanNode.java | 11 ++-
.../java/org/apache/iceberg/DeleteFileIndex.java | 3 +-
.../iceberg/source/IcebergScanNodeTest.java | 84 ++++++++++++++++++++++
3 files changed, 94 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
index de8ce891662..85f51f07d2c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -2440,10 +2440,15 @@ public class IcebergScanNode extends FileQueryScanNode {
}
// Build delete file index for efficient lookup of deletes applicable to each data file
- DeleteFileIndex deleteIndex = DeleteFileIndex.builderFor(deleteFiles)
+ DeleteFileIndex.Builder deleteIndexBuilder = DeleteFileIndex.builderFor(deleteFiles)
.specsById(specsById)
- .caseSensitive(caseSensitive)
- .build();
+ .caseSensitive(caseSensitive);
+ // Equality deletes may reference fields dropped from the current schema, so resolve their
+ // field IDs against every historical schema. Other scans skip indexing the schema history.
+ if (deleteFiles.stream().anyMatch(file -> file.content() == FileContent.EQUALITY_DELETES)) {
+ deleteIndexBuilder.schemasById(icebergTable.schemas());
+ }
+ DeleteFileIndex deleteIndex = deleteIndexBuilder.build();
// ========== Phase 2: Load data files and create scan tasks ==========
List<FileScanTask> tasks = new ArrayList<>();
diff --git a/fe/fe-core/src/main/java/org/apache/iceberg/DeleteFileIndex.java b/fe/fe-core/src/main/java/org/apache/iceberg/DeleteFileIndex.java
index 630f19c81a4..da7f7803835 100644
--- a/fe/fe-core/src/main/java/org/apache/iceberg/DeleteFileIndex.java
+++ b/fe/fe-core/src/main/java/org/apache/iceberg/DeleteFileIndex.java
@@ -400,7 +400,8 @@ public class DeleteFileIndex {
return this;
}
- Builder schemasById(Map<Integer, Schema> newSchemasById) {
+ // changed to public method: equality deletes may reference fields dropped from the current schema.
+ public Builder schemasById(Map<Integer, Schema> newSchemasById) {
this.schemasById = newSchemasById;
return this;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
index 3e17f77ac74..3d02dd83341 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
@@ -30,6 +30,7 @@ import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.StructField;
import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.TableIf;
@@ -40,7 +41,10 @@ import org.apache.doris.common.security.authentication.ExecutionAuthenticator;
import org.apache.doris.common.security.authentication.HadoopExecutionAuthenticator;
import org.apache.doris.common.util.LocationPath;
import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.ExternalMetaCacheMgr;
import org.apache.doris.datasource.ExternalScanNode;
+import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.FederationBackendPolicy;
import org.apache.doris.datasource.SplitAssignment;
import org.apache.doris.datasource.SplitGenerator;
@@ -57,6 +61,8 @@ import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue;
import org.apache.doris.datasource.iceberg.IcebergSysExternalTable;
import org.apache.doris.datasource.iceberg.IcebergTableCacheValue;
import org.apache.doris.datasource.iceberg.IcebergUtils;
+import org.apache.doris.datasource.iceberg.cache.IcebergManifestCacheLoader;
+import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue;
import org.apache.doris.datasource.iceberg.helper.IcebergWriterHelper;
import org.apache.doris.datasource.mvcc.MvccTableInfo;
import org.apache.doris.nereids.StatementContext;
@@ -97,6 +103,7 @@ import org.apache.iceberg.FileContent;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.FileMetadata;
import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.ManifestFiles;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.MetadataTableType;
import org.apache.iceberg.PartitionData;
@@ -125,6 +132,7 @@ import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.mockito.MockedStatic;
import org.mockito.Mockito;
import java.io.Closeable;
@@ -3594,6 +3602,82 @@ public class IcebergScanNodeTest {
}
}
+ @Test
+ public void testManifestCachePlanningResolvesDroppedEqualityDeleteField() throws Exception {
+ Schema schema = new Schema(
+ Types.NestedField.optional(1, "k", Types.IntegerType.get()),
+ Types.NestedField.optional(2, "row_tag", Types.StringType.get()));
+ String tableLocation = temporaryFolder.getRoot().toPath()
+ .resolve("manifest_cache_dropped_equality_key").toUri().toString();
+ Table table = new HadoopTables(new Configuration()).create(
+ schema, PartitionSpec.unpartitioned(), SortOrder.unsorted(),
+ ImmutableMap.of(TableProperties.FORMAT_VERSION, "2"), tableLocation);
+ table.newFastAppend().appendFile(DataFiles.builder(table.spec())
+ .withPath(tableLocation + "/data/old.parquet")
+ .withFormat(FileFormat.PARQUET)
+ .withFileSizeInBytes(10)
+ .withRecordCount(2)
+ .build()).commit();
+ table.newRowDelta().addDeletes(FileMetadata.deleteFileBuilder(table.spec())
+ .ofEqualityDeletes(1)
+ .withPath(tableLocation + "/data/eq-delete.parquet")
+ .withFormat(FileFormat.PARQUET)
+ .withFileSizeInBytes(10)
+ .withRecordCount(1)
+ .build()).commit();
+ // The live equality delete still references field 1 after it leaves the current schema.
+ table.updateSchema().deleteColumn("k").commit();
+ table.newFastAppend().appendFile(DataFiles.builder(table.spec())
+ .withPath(tableLocation + "/data/new.parquet")
+ .withFormat(FileFormat.PARQUET)
+ .withFileSizeInBytes(10)
+ .withRecordCount(1)
+ .build()).commit();
+
+ TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable());
+ IcebergSource source = Mockito.mock(IcebergSource.class);
+ Mockito.when(source.getCatalog()).thenReturn(Mockito.mock(ExternalCatalog.class));
+ Mockito.when(source.getTargetTable()).thenReturn(Mockito.mock(ExternalTable.class));
+ setPrivateField(node, "source", source);
+ setPrivateField(node, "icebergTable", table);
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(Mockito.mock(ExternalMetaCacheMgr.class));
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class);
+ MockedStatic<IcebergManifestCacheLoader> loader =
+ Mockito.mockStatic(IcebergManifestCacheLoader.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ loader.when(() -> IcebergManifestCacheLoader.loadDeleteFilesWithCache(Mockito.any(), Mockito.any(),
+ Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
+ .thenAnswer(invocation -> {
+ List<DeleteFile> files = new ArrayList<>();
+ try (CloseableIterable<DeleteFile> reader = ManifestFiles.readDeleteManifest(
+ invocation.getArgument(2), table.io(), table.specs())) {
+ reader.forEach(file -> files.add(file.copy()));
+ }
+ return ManifestCacheValue.forDeleteFiles(files);
+ });
+ loader.when(() -> IcebergManifestCacheLoader.loadDataFilesWithCache(Mockito.any(), Mockito.any(),
+ Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
+ .thenAnswer(invocation -> {
+ List<DataFile> files = new ArrayList<>();
+ try (CloseableIterable<DataFile> reader = ManifestFiles.read(
+ invocation.getArgument(2), table.io(), table.specs())) {
+ reader.forEach(file -> files.add(file.copy()));
+ }
+ return ManifestCacheValue.forDataFiles(files);
+ });
+
+ List<FileScanTask> tasks = node.loadFileScanTasksWithManifestCache(
+ table.newScan(), table.currentSnapshot());
+
+ Map<String, Integer> deleteCounts = new HashMap<>();
+ tasks.forEach(task -> deleteCounts.put(task.file().location(), task.deletes().size()));
+ Assert.assertEquals(ImmutableMap.of(
+ tableLocation + "/data/old.parquet", 1,
+ tableLocation + "/data/new.parquet", 0), deleteCounts);
+ }
+ }
+
@Test
public void testPinnedBranchUsesFrozenSnapshotWithCurrentSchema() throws Exception {
Schema snapshotSchema = new Schema(11, ImmutableList.of(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 07/28: branch-4.1: [fix](iceberg) keep identity partition values of old specs after evolving to unpartitioned(#68124) (#68259)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f9d755fcb6ae248e9ef28d0ff1fcea4870af159b
Author: daidai <ch...@selectdb.com>
AuthorDate: Thu Oct 8 19:10:48 2026 +0800
branch-4.1: [fix](iceberg) keep identity partition values of old specs after evolving to unpartitioned(#68124) (#68259)
Cherry-picked from #68124.
---
.../doris/datasource/iceberg/IcebergUtils.java | 14 +++++
.../datasource/iceberg/source/IcebergScanNode.java | 56 +++++++++++++----
.../doris/datasource/iceberg/IcebergUtilsTest.java | 19 ++++++
.../iceberg/source/IcebergScanNodeTest.java | 73 ++++++++++++++++++++++
4 files changed, 151 insertions(+), 11 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
index 92fc7744060..dc775bb948c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
@@ -982,6 +982,20 @@ public class IcebergUtils {
return partitionInfoMap;
}
+ /**
+ * Whether <b>any</b> partition spec of the table is partitioned. Scan planning must use this instead of
+ * the current default spec: after evolving to an unpartitioned spec, data files written under an older
+ * partitioned spec still carry their partition metadata (identity values, spec id, partition data).
+ */
+ public static boolean hasPartitionedSpec(Table table) {
+ for (PartitionSpec spec : table.specs().values()) {
+ if (spec.isPartitioned()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
public static List<String> getIdentityPartitionColumns(Table table) {
return getIdentityPartitionColumns(table, false, false);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
index 85f51f07d2c..f8a7f00a23d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -199,8 +199,15 @@ public class IcebergScanNode extends FileQueryScanNode {
private List<String> orderedPartitionMetadataKeys;
private boolean enableMappingVarbinaryForPartitionMetadata;
private boolean enableMappingTimestampTzForPartitionMetadata;
+ // Whether the table's CURRENT default spec is partitioned. Display/accounting only: it gates what
+ // counts as a scanned partition (selectedPartitionNum, numApproximateSplits), preserving the legacy
+ // numbers for a table that evolved to an unpartitioned spec.
private boolean isPartitionedTable;
private Boolean requiresRowId;
+ // Whether ANY spec of the table is partitioned. This is the READ gate: data files written under an
+ // older partitioned spec still carry their identity values, and must keep sending them even after
+ // the default spec evolved to unpartitioned.
+ private boolean hasPartitionedSpec;
private int formatVersion;
private ExecutionAuthenticator preExecutionAuthenticator;
private IcebergRuntimeContext runtimeContext;
@@ -318,6 +325,7 @@ public class IcebergScanNode extends FileQueryScanNode {
partitionMapInfos = new HashMap<>();
initializePartitionMetadata();
isPartitionedTable = icebergTable.spec().isPartitioned();
+ hasPartitionedSpec = IcebergUtils.hasPartitionedSpec(icebergTable);
// Metadata tables (system tables) are not BaseTable instances, so we need to handle this case
if (icebergTable instanceof BaseTable) {
formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion();
@@ -510,18 +518,26 @@ public class IcebergScanNode extends FileQueryScanNode {
}
enableMappingVarbinaryForPartitionMetadata = getEnableMappingVarbinary();
enableMappingTimestampTzForPartitionMetadata = getEnableMappingTimestampTz();
- orderedPathPartitionKeys = Collections.unmodifiableList(
- IcebergUtils.getCommonIdentityPartitionColumns(icebergTable,
- enableMappingVarbinaryForPartitionMetadata,
- enableMappingTimestampTzForPartitionMetadata));
if (sessionVariable.enableFileScannerV2) {
- orderedPartitionMetadataKeys = Collections.unmodifiableList(
+ // Classify every identity column of EVERY spec as a partition key, like master does. A file
+ // written under an older identity spec carries its value in the manifest only, and BE uses a
+ // split partition value just for a column FE marked as a partition key, so the intersection
+ // below would leave that value unused and the column read as NULL. Files that carry no value
+ // for the column (a spec without it) are unaffected: the v2 column mapper falls back to the
+ // physical field when the split has no partition value for a partition key.
+ orderedPathPartitionKeys = Collections.unmodifiableList(
IcebergUtils.getIdentityPartitionColumns(icebergTable,
enableMappingVarbinaryForPartitionMetadata,
enableMappingTimestampTzForPartitionMetadata));
} else {
- orderedPartitionMetadataKeys = orderedPathPartitionKeys;
+ // v1 has no such per-file fallback: a partition key it cannot fill from the split is not read
+ // from the file either. Keep the columns common to all specs so those files stay readable.
+ orderedPathPartitionKeys = Collections.unmodifiableList(
+ IcebergUtils.getCommonIdentityPartitionColumns(icebergTable,
+ enableMappingVarbinaryForPartitionMetadata,
+ enableMappingTimestampTzForPartitionMetadata));
}
+ orderedPartitionMetadataKeys = orderedPathPartitionKeys;
}
@VisibleForTesting
@@ -2616,7 +2632,10 @@ public class IcebergScanNode extends FileQueryScanNode {
split.setPartitionSpecId(specId);
PartitionData partitionData = (PartitionData) dataFile.partition();
boolean includePartitionData = requiresRowId();
- if (partitionData != null && (includePartitionData || isPartitionedTable)) {
+ // Gate identity values on the table's spec HISTORY, not on the current default spec: a file written
+ // under an older identity spec keeps its partition values in the manifest, and they are the only
+ // source for an identity partition column that the physical file does not store.
+ if (partitionData != null && (includePartitionData || hasPartitionedSpec)) {
PartitionSpec partitionSpec = icebergTable.specs().get(specId);
Preconditions.checkNotNull(partitionSpec, "Partition spec with specId %s not found for table %s",
specId, icebergTable.name());
@@ -2631,7 +2650,10 @@ public class IcebergScanNode extends FileQueryScanNode {
+ specId + ". Rewrite historical data files before DELETE or UPDATE.", e);
}
}
- if (isPartitionedTable) {
+ // The cache doubles as the scanned-partition count only for a partitioned current spec. Otherwise
+ // cache only specs with identity fields, or transformed-only old partitions would grow it uselessly.
+ if (isPartitionedTable || partitionSpec.fields().stream()
+ .anyMatch(field -> field.transform().isIdentity())) {
Map<String, String> partitionInfoMap = partitionMapInfos.computeIfAbsent(
Pair.of(specId, partitionData), k -> IcebergUtils.getIdentityPartitionInfoMap(
partitionData, partitionSpec, icebergTable, sessionVariable.getTimeZone(),
@@ -2813,7 +2835,7 @@ public class IcebergScanNode extends FileQueryScanNode {
for (FileScanTask task : customFileScanTasks) {
splits.add(createIcebergSplit(task));
}
- selectedPartitionNum = partitionMapInfos.size();
+ selectedPartitionNum = scannedPartitionNum();
recordManifestCacheProfile();
return splits;
}
@@ -2850,7 +2872,7 @@ public class IcebergScanNode extends FileQueryScanNode {
}
}
- selectedPartitionNum = partitionMapInfos.size();
+ selectedPartitionNum = scannedPartitionNum();
recordManifestCacheProfile();
return splits;
}
@@ -3233,9 +3255,21 @@ public class IcebergScanNode extends FileQueryScanNode {
((IcebergSplit) splits.get(size - 1)).setTableLevelRowCount(countPerSplit + totalCount % size);
}
+ /**
+ * The number of scanned partitions reported to EXPLAIN ({@code partition=N/M}), the
+ * {@code sql_block_rule} {@code partition_num} guard and the batch-mode split estimate. A table whose
+ * CURRENT spec is unpartitioned reports no partitions at all, so the partitions that files of an older
+ * spec still carry must not start being counted — {@code partitionMapInfos} now collects them because
+ * it doubles as the per-(spec, partition) identity-value cache for the read path.
+ */
+ private int scannedPartitionNum() {
+ return isPartitionedTable ? partitionMapInfos.size() : 0;
+ }
+
@Override
public int numApproximateSplits() {
- return NUM_SPLITS_PER_PARTITION * partitionMapInfos.size() > 0 ? partitionMapInfos.size() : 1;
+ int partitions = scannedPartitionNum();
+ return NUM_SPLITS_PER_PARTITION * partitions > 0 ? partitions : 1;
}
private Optional<NotSupportedException> checkNotSupportedException(Exception e) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
index 3717bbc2af3..a9f31393d93 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
@@ -427,6 +427,25 @@ public class IcebergUtilsTest {
Assert.assertTrue(exception.getMessage().contains("db.tbl"));
}
+ @Test
+ public void testHasPartitionedSpecLooksAtEverySpecNotOnlyTheCurrentOne() {
+ Schema schema = new Schema(
+ Types.NestedField.required(1, "id", Types.IntegerType.get()),
+ Types.NestedField.required(2, "p", Types.IntegerType.get()));
+ PartitionSpec identity = PartitionSpec.builderFor(schema).identity("p").withSpecId(0).build();
+ PartitionSpec unpartitioned = PartitionSpec.unpartitioned();
+
+ // A table that evolved identity(p) -> unpartitioned still has files carrying p in their
+ // partition metadata, so scan planning must keep sending those values.
+ Table evolved = Mockito.mock(Table.class);
+ Mockito.when(evolved.specs()).thenReturn(ImmutableMap.of(0, identity, 1, unpartitioned));
+ Assert.assertTrue(IcebergUtils.hasPartitionedSpec(evolved));
+
+ Table neverPartitioned = Mockito.mock(Table.class);
+ Mockito.when(neverPartitioned.specs()).thenReturn(ImmutableMap.of(0, unpartitioned));
+ Assert.assertFalse(IcebergUtils.hasPartitionedSpec(neverPartitioned));
+ }
+
@Test
public void testEmptyNameMappingStillParsesAsAuthoritativeMapping() throws Exception {
Table table = Mockito.mock(Table.class);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
index 3d02dd83341..84d5f625d53 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
@@ -1824,6 +1824,69 @@ public class IcebergScanNodeTest {
IcebergUtils.getSerializedInitialDefaults(fields, false).get(7));
}
+ @Test
+ public void testSplitKeepsOldSpecIdentityValuesAfterEvolvingToUnpartitioned() throws Exception {
+ // A file written under identity(p) keeps p=7 only in its manifest partition metadata. Gating on
+ // the CURRENT spec drops that value once the default spec evolves to unpartitioned, and keying
+ // the partition-key classification off the columns common to all specs leaves BE reading p from
+ // a file that does not store it, so the column comes back NULL in both cases.
+ Schema schema = new Schema(
+ Types.NestedField.required(1, "id", Types.IntegerType.get()),
+ Types.NestedField.required(2, "p", Types.IntegerType.get()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).identity("p").build();
+ String location = temporaryFolder.newFolder("identity_evolved_table").toURI().toString();
+ Table table = new HadoopTables(new Configuration()).create(schema, spec, location);
+ table.newFastAppend()
+ .appendFile(DataFiles.builder(table.spec())
+ .withPath(location + "data/p=7/old.parquet")
+ .withFileSizeInBytes(1024L)
+ .withRecordCount(2L)
+ .withFormat(FileFormat.PARQUET)
+ .withPartitionPath("p=7")
+ .build())
+ .commit();
+ table.updateSpec().removeField("p").commit();
+ Assert.assertTrue(table.spec().isUnpartitioned());
+
+ SessionVariable sessionVariable = new SessionVariable();
+ sessionVariable.enableFileScannerV2 = true;
+ TestIcebergScanNode node = new TestIcebergScanNode(sessionVariable);
+ setIcebergTable(node, table);
+ setPrivateField(node, "formatVersion", 2);
+ setPrivateField(node, "storagePropertiesMap", Collections.emptyMap());
+ setPrivateField(node, "partitionMapInfos", new HashMap<>());
+ // Exactly what doInitialize computes for this table.
+ setPrivateField(node, "isPartitionedTable", table.spec().isPartitioned());
+ setPrivateField(node, "hasPartitionedSpec", IcebergUtils.hasPartitionedSpec(table));
+
+ // p must stay a partition key even though the current spec no longer mentions it; otherwise BE
+ // ignores the split value and reads the (absent) physical column.
+ Assert.assertEquals(ImmutableList.of("p"), orderedPathPartitionKeys(node));
+
+ FileScanTask task;
+ try (CloseableIterable<FileScanTask> tasks = table.newScan().planFiles()) {
+ Iterator<FileScanTask> iterator = tasks.iterator();
+ Assert.assertTrue(iterator.hasNext());
+ task = iterator.next();
+ }
+
+ IcebergSplit split = createIcebergSplit(node, task);
+ Assert.assertEquals(ImmutableMap.of("p", "7"), split.getIcebergPartitionValues());
+ Assert.assertEquals(Integer.valueOf(0), split.getPartitionSpecId());
+ // A plain read does not project the row ID, so no row-ID partition JSON is generated.
+ Assert.assertNull(split.getPartitionDataJson());
+
+ // What BE actually receives: p=7 as a columns-from-path constant for this range.
+ TFileRangeDesc rangeDesc = new TFileRangeDesc();
+ setIcebergParams(node, rangeDesc, split);
+ Assert.assertEquals(ImmutableList.of("p"), rangeDesc.getColumnsFromPathKeys());
+ Assert.assertEquals(ImmutableList.of("7"), rangeDesc.getColumnsFromPath());
+
+ // Display parity: the CURRENT spec is unpartitioned, so the table still reports no scanned
+ // partitions and the batch-mode estimate stays at the unpartitioned default.
+ Assert.assertEquals(1, node.numApproximateSplits());
+ }
+
@Test
public void testBatchSplitCarriesDroppedEqualitySchemaThroughThrift() throws Exception {
Types.NestedField id = Types.NestedField.required(1, "id", Types.LongType.get());
@@ -1958,6 +2021,7 @@ public class IcebergScanNodeTest {
node.addSlot(0, new Column("record_key", Type.INT));
setIcebergTable(node, table);
setPrivateField(node, "isPartitionedTable", keepPartitioned);
+ setPrivateField(node, "hasPartitionedSpec", IcebergUtils.hasPartitionedSpec(table));
setPrivateField(node, "storagePropertiesMap", Collections.emptyMap());
setPrivateField(node, "formatVersion", 2);
setPrivateField(node, "partitionMapInfos", new HashMap<>());
@@ -2043,6 +2107,7 @@ public class IcebergScanNodeTest {
TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable());
setIcebergTable(node, table);
setPrivateField(node, "isPartitionedTable", keepPartitioned);
+ setPrivateField(node, "hasPartitionedSpec", IcebergUtils.hasPartitionedSpec(table));
setPrivateField(node, "storagePropertiesMap", Collections.emptyMap());
setPrivateField(node, "formatVersion", 2);
setPrivateField(node, "partitionMapInfos", new HashMap<>());
@@ -2181,6 +2246,7 @@ public class IcebergScanNodeTest {
node.addSlot(0, IcebergRowId.createHiddenColumn());
setIcebergTable(node, table);
setPrivateField(node, "isPartitionedTable", table.spec().isPartitioned());
+ setPrivateField(node, "hasPartitionedSpec", IcebergUtils.hasPartitionedSpec(table));
setPrivateField(node, "storagePropertiesMap", Collections.emptyMap());
setPrivateField(node, "formatVersion", 2);
setPrivateField(node, "partitionMapInfos", new HashMap<>());
@@ -4097,6 +4163,13 @@ public class IcebergScanNodeTest {
return task;
}
+ @SuppressWarnings("unchecked")
+ private static List<String> orderedPathPartitionKeys(IcebergScanNode node) throws Exception {
+ Method method = IcebergScanNode.class.getDeclaredMethod("getOrderedPathPartitionKeys");
+ method.setAccessible(true);
+ return (List<String>) method.invoke(node);
+ }
+
private static IcebergSplit createIcebergSplit(IcebergScanNode node, FileScanTask task)
throws Exception {
Method method = IcebergScanNode.class.getDeclaredMethod("createIcebergSplit", FileScanTask.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 23/28: branch-4.1: [fix](function) Reject a non-constant seed in array_shuffle #68438 (#68832)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ada545de782659d506ab4f795a1be842fc5fdac6
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Sat Oct 10 09:32:09 2026 +0800
branch-4.1: [fix](function) Reject a non-constant seed in array_shuffle #68438 (#68832)
Cherry-picked from #68438
Co-authored-by: Chenyang Sun <su...@selectdb.com>
Co-authored-by: Claude Opus 5.5 (1M context) <no...@anthropic.com>
---
.../function/array/function_array_shuffle.cpp | 32 ++-
be/src/exprs/vectorized_fn_call.cpp | 3 +-
.../exprs/function/function_array_shuffle_test.cpp | 227 +++++++++++++++++++++
.../expressions/functions/scalar/ArrayShuffle.java | 28 ++-
.../functions/scalar/UniqueFunctionTest.java | 13 ++
.../sql-function/test_array_function.out | 2 +-
.../array_functions/test_array_shuffle_seed.out | 26 +++
.../size_funciton/test_size_function.out | 12 +-
.../array_functions/test_array_shuffle_seed.groovy | 101 +++++++++
9 files changed, 425 insertions(+), 19 deletions(-)
diff --git a/be/src/exprs/function/array/function_array_shuffle.cpp b/be/src/exprs/function/array/function_array_shuffle.cpp
index 38096df32ca..41f25b0b87c 100644
--- a/be/src/exprs/function/array/function_array_shuffle.cpp
+++ b/be/src/exprs/function/array/function_array_shuffle.cpp
@@ -16,10 +16,9 @@
// under the License.
#include <fmt/format.h>
#include <glog/logging.h>
-#include <stdint.h>
-#include <time.h>
#include <algorithm>
+#include <cstdint>
#include <memory>
#include <ostream>
#include <random>
@@ -65,21 +64,40 @@ public:
return arguments[0];
}
+ // Shuffle a constant array on each row too, so every row gets its own order.
+ bool use_default_implementation_for_constants() const override { return false; }
+
+ Status open(FunctionContext* context, FunctionContext::FunctionStateScope scope) override {
+ // The rows of a block draw from one random sequence that starts from the seed, so a
+ // per-row seed would be ignored. Reject a non-constant seed instead.
+ if (scope == FunctionContext::THREAD_LOCAL && context->get_num_args() == 2 &&
+ !context->is_col_constant(1)) {
+ return Status::InvalidArgument("The seed of {} must be a constant", get_name());
+ }
+ return Status::OK();
+ }
+
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const override {
ColumnPtr src_column =
block.get_by_position(arguments[0]).column->convert_to_full_column_if_const();
const auto& src_column_array = assert_cast<const ColumnArray&>(*src_column);
- size_t seed = time(nullptr);
+ uint64_t seed = 0;
if (arguments.size() == 2) {
+ // open() makes sure the seed is a constant, so read it from the first row.
ColumnPtr seed_column =
block.get_by_position(arguments[1]).column->convert_to_full_column_if_const();
- seed = assert_cast<const ColumnInt64*>(seed_column.get())->get_element(0);
+ // Use all 64 bits, so any BIGINT works, a negative one too.
+ seed = static_cast<uint64_t>(
+ assert_cast<const ColumnInt64*>(seed_column.get())->get_element(0));
+ } else {
+ // Give each block its own random seed, so blocks do not repeat the same orders.
+ std::random_device random_device;
+ seed = (static_cast<uint64_t>(random_device()) << 32) | random_device();
}
- // time() and seed will not exceed the range of uint32.
- std::mt19937 g(cast_set<uint32_t>(seed));
+ std::mt19937_64 g(seed);
auto dest_column_ptr = _execute(src_column_array, g);
if (!dest_column_ptr) {
return Status::RuntimeError(
@@ -92,7 +110,7 @@ public:
}
private:
- ColumnPtr _execute(const ColumnArray& src_column_array, std::mt19937& g) const {
+ ColumnPtr _execute(const ColumnArray& src_column_array, std::mt19937_64& g) const {
const auto& src_offsets = src_column_array.get_offsets();
const auto src_nested_column = src_column_array.get_data_ptr();
diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp
index 929b61f43b0..160679d3563 100644
--- a/be/src/exprs/vectorized_fn_call.cpp
+++ b/be/src/exprs/vectorized_fn_call.cpp
@@ -662,8 +662,9 @@ bool VectorizedFnCall::can_push_down_to_index() const {
}
bool VectorizedFnCall::is_deterministic() const {
+ // array_shuffle gives the rows of a block different orders, even with a seed.
static const std::set<std::string> NON_DETERMINISTIC_FUNCTIONS = {
- "random", "rand", "random_bytes", "uuid", "uuid_numeric"};
+ "random", "rand", "random_bytes", "uuid", "uuid_numeric", "array_shuffle", "shuffle"};
return !NON_DETERMINISTIC_FUNCTIONS.contains(_function_name) && VExpr::is_deterministic();
}
diff --git a/be/test/exprs/function/function_array_shuffle_test.cpp b/be/test/exprs/function/function_array_shuffle_test.cpp
new file mode 100644
index 00000000000..d4f1313cde3
--- /dev/null
+++ b/be/test/exprs/function/function_array_shuffle_test.cpp
@@ -0,0 +1,227 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <limits>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "core/block/block.h"
+#include "core/column/column_const.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "exprs/function/function_test_util.h"
+#include "exprs/function/simple_function_factory.h"
+#include "exprs/vectorized_fn_call.h"
+#include "gen_cpp/Exprs_types.h"
+
+namespace doris {
+
+static const TestArray kArray = {Int32(1), Int32(2), Int32(3), Int32(4), Int32(5)};
+
+// Two random orders of 20 elements are the same only by a tiny chance, so tests that expect
+// different orders use this array.
+static const TestArray kLongArray = [] {
+ TestArray array;
+ for (int32_t i = 0; i < 20; ++i) {
+ array.emplace_back(Int32(i));
+ }
+ return array;
+}();
+
+// Runs array_shuffle on one block with one row per element of arrays, and puts each result row
+// as a string in results. With const_array, the arrays must all be the same and the array column
+// is a ColumnConst. An empty seeds runs array_shuffle(array), one seed is a constant seed, and
+// more seeds are a seed column with one seed per row.
+// Returns the first failed status of open() and execute().
+static Status run_array_shuffle(const std::vector<TestArray>& arrays, bool const_array,
+ const std::vector<int64_t>& seeds,
+ std::vector<std::string>* results) {
+ auto array_type =
+ std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeInt32>()));
+ auto seed_type = std::make_shared<DataTypeInt64>();
+ const size_t row_size = arrays.size();
+ // Empty rows make a failed call fail the checks instead of reading past the end.
+ results->assign(row_size, "");
+
+ MutableColumnPtr array_column = array_type->create_column();
+ for (size_t i = 0; i < (const_array ? 1 : row_size); ++i) {
+ EXPECT_TRUE(insert_cell(array_column, array_type, arrays[i]));
+ }
+ ColumnPtr array_ptr = std::move(array_column);
+ if (const_array) {
+ array_ptr = ColumnConst::create(array_ptr, row_size);
+ }
+
+ Block block;
+ block.insert({array_ptr, array_type, "array"});
+ DataTypes arg_types = {array_type};
+ ColumnNumbers arguments = {0};
+ std::vector<std::shared_ptr<ColumnPtrWrapper>> constant_cols = {nullptr};
+ if (const_array) {
+ constant_cols[0] = std::make_shared<ColumnPtrWrapper>(array_ptr);
+ }
+ if (!seeds.empty()) {
+ MutableColumnPtr seed_column = seed_type->create_column();
+ for (auto seed : seeds) {
+ EXPECT_TRUE(insert_cell(seed_column, seed_type, seed));
+ }
+ ColumnPtr seed_ptr = std::move(seed_column);
+ constant_cols.push_back(nullptr);
+ if (seeds.size() == 1) {
+ seed_ptr = ColumnConst::create(seed_ptr, row_size);
+ constant_cols[1] = std::make_shared<ColumnPtrWrapper>(seed_ptr);
+ } else {
+ EXPECT_EQ(seeds.size(), row_size);
+ }
+ block.insert({seed_ptr, seed_type, "seed"});
+ arg_types.push_back(seed_type);
+ arguments.push_back(1);
+ }
+
+ DataTypePtr return_type = array_type;
+ FunctionBasePtr func = SimpleFunctionFactory::instance().get_function(
+ "array_shuffle", block.get_columns_with_type_and_name(), return_type);
+ EXPECT_NE(func, nullptr);
+
+ FunctionUtils fn_utils(return_type, arg_types, false);
+ auto* fn_ctx = fn_utils.get_fn_ctx();
+ fn_ctx->set_constant_cols(constant_cols);
+ RETURN_IF_ERROR(func->open(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
+ RETURN_IF_ERROR(func->open(fn_ctx, FunctionContext::THREAD_LOCAL));
+
+ block.insert({nullptr, return_type, "result"});
+ auto result_idx = block.columns() - 1;
+ RETURN_IF_ERROR(func->execute(fn_ctx, block, arguments, result_idx, row_size));
+ static_cast<void>(func->close(fn_ctx, FunctionContext::THREAD_LOCAL));
+ static_cast<void>(func->close(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
+
+ const auto& result_column = *block.get_by_position(result_idx).column;
+ for (size_t i = 0; i < row_size; ++i) {
+ (*results)[i] = return_type->to_string(result_column, i);
+ }
+ return Status::OK();
+}
+
+// Builds an array_shuffle(array) call under the given function name.
+static VExprSPtr array_shuffle_call(const std::string& function_name) {
+ DataTypePtr array_type =
+ std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeInt32>()));
+ TFunctionName fn_name;
+ fn_name.__set_function_name(function_name);
+ TFunction fn;
+ fn.__set_name(fn_name);
+ fn.__set_binary_type(TFunctionBinaryType::BUILTIN);
+ fn.__set_arg_types({array_type->to_thrift()});
+ fn.__set_ret_type(array_type->to_thrift());
+ fn.__set_has_var_args(true);
+
+ TExprNode node;
+ node.__set_node_type(TExprNodeType::FUNCTION_CALL);
+ node.__set_type(array_type->to_thrift());
+ node.__set_fn(fn);
+ node.__set_num_children(1);
+ node.__set_is_nullable(true);
+ return VectorizedFnCall::create_shared(node);
+}
+
+// Runs array_shuffle(array, seed) with a constant seed, which must succeed.
+static std::vector<std::string> shuffle_with_const_seed(const std::vector<TestArray>& arrays,
+ int64_t seed, bool const_array = false) {
+ std::vector<std::string> results;
+ auto st = run_array_shuffle(arrays, const_array, {seed}, &results);
+ EXPECT_TRUE(st.ok()) << st;
+ return results;
+}
+
+// Runs array_shuffle(array) on a constant array, which must succeed.
+static std::vector<std::string> shuffle_const_array_without_seed(const TestArray& array,
+ size_t row_size) {
+ std::vector<std::string> results;
+ auto st = run_array_shuffle(std::vector<TestArray>(row_size, array), true, {}, &results);
+ EXPECT_TRUE(st.ok()) << st;
+ return results;
+}
+
+// The rows of a block draw from one random sequence that starts from the seed, so a per-row
+// seed would be ignored. A non-constant seed is rejected instead.
+TEST(function_array_shuffle_test, non_constant_seed) {
+ std::vector<std::string> results;
+ auto st = run_array_shuffle({kArray, kArray}, false, {1, 2}, &results);
+ EXPECT_TRUE(st.is<ErrorCode::INVALID_ARGUMENT>()) << st;
+ EXPECT_NE(st.to_string().find("must be a constant"), std::string::npos) << st;
+}
+
+// A constant seed gives the same result each time, and the first row gives the same result as
+// running that row alone.
+TEST(function_array_shuffle_test, const_seed) {
+ auto seed1 = shuffle_with_const_seed({kArray}, 1);
+ auto results = shuffle_with_const_seed({kArray, kArray, kArray}, 1);
+ EXPECT_EQ(results[0], seed1[0]);
+ EXPECT_EQ(shuffle_with_const_seed({kArray, kArray, kArray}, 1), results);
+ EXPECT_NE(shuffle_with_const_seed({kLongArray}, 2), shuffle_with_const_seed({kLongArray}, 1));
+}
+
+// Any BIGINT is a valid seed, a negative one too. All 64 bits are used, so seeds with the same
+// low 32 bits still give different results.
+TEST(function_array_shuffle_test, any_bigint_seed) {
+ EXPECT_NE(shuffle_with_const_seed({kLongArray}, -1),
+ shuffle_with_const_seed({kLongArray}, 4294967295));
+ EXPECT_NE(shuffle_with_const_seed({kLongArray}, 4294967301),
+ shuffle_with_const_seed({kLongArray}, 5));
+ EXPECT_NE(shuffle_with_const_seed({kLongArray}, std::numeric_limits<int64_t>::min()),
+ shuffle_with_const_seed({kLongArray}, 0));
+ EXPECT_NE(shuffle_with_const_seed({kLongArray}, std::numeric_limits<int64_t>::max()),
+ shuffle_with_const_seed({kLongArray}, -1));
+}
+
+// Arrays with 0 or 1 element stay the same.
+TEST(function_array_shuffle_test, short_arrays) {
+ const TestArray empty_array = {};
+ const TestArray one_element = {Int32(7)};
+ auto results = shuffle_with_const_seed({empty_array, one_element, kArray}, 1);
+ EXPECT_EQ(results[0], "[]");
+ EXPECT_EQ(results[1], "[7]");
+}
+
+// A constant array is still shuffled on each row, so it gives the same rows as the same arrays
+// in a column, and the rows do not all get the same order.
+TEST(function_array_shuffle_test, const_array) {
+ const std::vector<TestArray> arrays(3, kLongArray);
+ auto from_column = shuffle_with_const_seed(arrays, 1);
+ EXPECT_EQ(shuffle_with_const_seed(arrays, 1, true), from_column);
+ EXPECT_NE(from_column[0], from_column[1]);
+}
+
+// Without a seed, each row and each call gets a new random order.
+TEST(function_array_shuffle_test, no_seed) {
+ auto first = shuffle_const_array_without_seed(kLongArray, 2);
+ auto second = shuffle_const_array_without_seed(kLongArray, 2);
+ EXPECT_NE(first[0], first[1]);
+ EXPECT_NE(first[0], second[0]);
+}
+
+// Running array_shuffle again on other rows gives other orders, so it is not deterministic, and
+// a scan must not run it twice, for example as a file-local filter copy and again in the scanner.
+TEST(function_array_shuffle_test, not_deterministic) {
+ EXPECT_FALSE(array_shuffle_call("array_shuffle")->is_deterministic());
+ EXPECT_FALSE(array_shuffle_call("shuffle")->is_deterministic());
+}
+
+} // namespace doris
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayShuffle.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayShuffle.java
index 3eef9c608bb..f9039ace214 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayShuffle.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayShuffle.java
@@ -18,7 +18,9 @@
package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.VolatileIdentity;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
@@ -36,7 +38,7 @@ import java.util.List;
* ScalarFunction 'array_shuffle'
* with 1 or 2 arguments : array_shuffle(arr) or array_shuffle(arr, seed)
*/
-public class ArrayShuffle extends ScalarFunction
+public class ArrayShuffle extends UniqueFunction
implements BinaryExpression, ExplicitlyCastableSignature, PropagateNullable {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
@@ -49,18 +51,22 @@ public class ArrayShuffle extends ScalarFunction
* constructor with 1 arguments.
*/
public ArrayShuffle(Expression arg) {
- super("array_shuffle", arg);
+ this(VolatileIdentity.newVolatileIdentity(), ImmutableList.of(arg));
}
/**
* constructor with 2 arguments.
*/
public ArrayShuffle(Expression arg, Expression arg1) {
- super("array_shuffle", arg, arg1);
+ this(VolatileIdentity.newVolatileIdentity(), ImmutableList.of(arg, arg1));
+ }
+
+ private ArrayShuffle(VolatileIdentity volatileIdentity, List<Expression> children) {
+ super("array_shuffle", volatileIdentity, children);
}
/** constructor for withChildren and reuse signature */
- private ArrayShuffle(ScalarFunctionParams functionParams) {
+ private ArrayShuffle(UniqueFunctionParams functionParams) {
super(functionParams);
}
@@ -73,6 +79,20 @@ public class ArrayShuffle extends ScalarFunction
return new ArrayShuffle(getFunctionParams(children));
}
+ @Override
+ public ArrayShuffle withIgnoreUniqueId(boolean ignoreUniqueId) {
+ return new ArrayShuffle(volatileIdentity.withIgnoreUniqueId(ignoreUniqueId), children);
+ }
+
+ @Override
+ public void checkLegalityBeforeTypeCoercion() {
+ // The rows of a block draw from one random sequence that starts from the seed,
+ // so a per-row seed would be ignored.
+ if (arity() == 2 && !getArgument(1).isConstant()) {
+ throw new AnalysisException("The seed of array_shuffle must be a constant: " + toSql());
+ }
+ }
+
@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitArrayShuffle(this, context);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UniqueFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UniqueFunctionTest.java
index c0dd9dc0d14..97b3a3b0f96 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UniqueFunctionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UniqueFunctionTest.java
@@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.WindowExpression;
import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
+import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral;
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
@@ -97,6 +98,18 @@ class UniqueFunctionTest extends SqlTestBase {
Assertions.assertNotEquals(uuidNum, new UuidNumeric());
Assertions.assertEquals(uuidNum.withIgnoreUniqueId(true), new UuidNumeric().withIgnoreUniqueId(true));
Assertions.assertEquals(uuidNum, uuidNum.withChildren());
+
+ ArrayLiteral array = new ArrayLiteral(ImmutableList.of(new BigIntLiteral(1L), new BigIntLiteral(2L)));
+ ArrayShuffle shuffle1 = new ArrayShuffle(array);
+ ArrayShuffle shuffle2 = new ArrayShuffle(array, new BigIntLiteral(10L));
+ Assertions.assertNotEquals(shuffle1, new ArrayShuffle(array));
+ Assertions.assertEquals(shuffle1.withIgnoreUniqueId(true), new ArrayShuffle(array).withIgnoreUniqueId(true));
+ Assertions.assertEquals(shuffle1, shuffle1.withChildren(array));
+ Assertions.assertEquals(shuffle1, shuffle1.withChildren(array, new BigIntLiteral(10L))); // only compare unique id
+ Assertions.assertNotEquals(shuffle2, new ArrayShuffle(array, new BigIntLiteral(10L)));
+ Assertions.assertEquals(shuffle2.withIgnoreUniqueId(true),
+ new ArrayShuffle(array, new BigIntLiteral(10L)).withIgnoreUniqueId(true));
+ Assertions.assertEquals(shuffle2, shuffle2.withChildren(array, new BigIntLiteral(10L)));
}
@Test
diff --git a/regression-test/data/doc/sql-manual/sql-function/test_array_function.out b/regression-test/data/doc/sql-manual/sql-function/test_array_function.out
index a8b38ff8c67..3d380e1474b 100644
--- a/regression-test/data/doc/sql-manual/sql-function/test_array_function.out
+++ b/regression-test/data/doc/sql-manual/sql-function/test_array_function.out
@@ -427,7 +427,7 @@ false
[[1, 2], [3]]
-- !sql --
-[3, 5, 4, 1, 2]
+[3, 1, 2, 4, 5]
-- !sql --
3
diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_shuffle_seed.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_shuffle_seed.out
new file mode 100644
index 00000000000..0ca7a1e70a7
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_shuffle_seed.out
@@ -0,0 +1,26 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !const_seed --
+[4, 1, 5, 3, 2] [1, 2, 5, 4, 3] [4, 1, 5, 3, 2] [1, 2, 5, 4, 3]
+
+-- !bigint_seed --
+[4, 5, 2, 6, 9, 0, 1, 3, 8, 7] [5, 3, 8, 6, 7, 9, 0, 4, 2, 1] [0, 6, 3, 1, 2, 5, 9, 8, 7, 4] [4, 7, 2, 3, 8, 0, 1, 5, 9, 6] [7, 2, 5, 4, 6, 9, 8, 0, 1, 3]
+
+-- !null_seed --
+\N
+
+-- !table --
+1 [1, 2, 3, 4, 5] 5
+2 [1, 2, 3, 4, 5] 5
+3 \N \N
+4 [] 0
+5 [42] 1
+
+-- !const_array --
+100 100
+
+-- !const_array_fold --
+100
+
+-- !no_seed_blocks --
+100000
+
diff --git a/regression-test/data/query_p0/sql_functions/size_funciton/test_size_function.out b/regression-test/data/query_p0/sql_functions/size_funciton/test_size_function.out
index cc453fbea27..285028cec10 100644
--- a/regression-test/data/query_p0/sql_functions/size_funciton/test_size_function.out
+++ b/regression-test/data/query_p0/sql_functions/size_funciton/test_size_function.out
@@ -1,21 +1,21 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql --
-4 ["aaa", "bbb", null, "fff"] ["aaa", "bbb", null, "fff"]
+4 [null, "aaa", "bbb", "fff"] [null, "aaa", "bbb", "fff"]
-- !sql --
-4 ["2020-01-02", "2021-01-01", "2022-01-03", "1996-04-17"] ["2020-01-02", "2021-01-01", "2022-01-03", "1996-04-17"]
+4 ["2022-01-03", "2020-01-02", "2021-01-01", "1996-04-17"] ["2022-01-03", "2020-01-02", "2021-01-01", "1996-04-17"]
-- !sql --
-4 ["aaa", "bbb", null, "fff"] ["aaa", "bbb", null, "fff"]
+4 [null, "aaa", "bbb", "fff"] [null, "aaa", "bbb", "fff"]
-- !sql --
-4 ["2020-01-02", "2021-01-01", "2022-01-03", "1996-04-17"] ["2020-01-02", "2021-01-01", "2022-01-03", "1996-04-17"]
+4 ["2022-01-03", "2020-01-02", "2021-01-01", "1996-04-17"] ["2022-01-03", "2020-01-02", "2021-01-01", "1996-04-17"]
-- !sql --
-4 ["aaa", "bbb", null, "fff"] ["aaa", "bbb", null, "fff"]
+4 [null, "aaa", "bbb", "fff"] [null, "aaa", "bbb", "fff"]
-- !sql --
-4 ["2020-01-02", "2021-01-01", "2022-01-03", "1996-04-17"] ["2020-01-02", "2021-01-01", "2022-01-03", "1996-04-17"]
+4 ["2022-01-03", "2020-01-02", "2021-01-01", "1996-04-17"] ["2022-01-03", "2020-01-02", "2021-01-01", "1996-04-17"]
-- !sql --
2
diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_shuffle_seed.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_shuffle_seed.groovy
new file mode 100644
index 00000000000..e61776a1deb
--- /dev/null
+++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_shuffle_seed.groovy
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_array_shuffle_seed") {
+ sql "DROP TABLE IF EXISTS test_array_shuffle_seed"
+ sql """
+ CREATE TABLE test_array_shuffle_seed (
+ k INT,
+ a ARRAY<INT> NULL,
+ s BIGINT NULL
+ ) DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO test_array_shuffle_seed VALUES
+ (1, [1, 2, 3, 4, 5], 1),
+ (2, [1, 2, 3, 4, 5], 2),
+ (3, NULL, 1),
+ (4, [], 1),
+ (5, [42], 1)
+ """
+
+ // The rows of a block draw from one random sequence that starts from the seed, so a per-row
+ // seed would be ignored. A non-constant seed is rejected instead.
+ test {
+ sql "SELECT k, array_shuffle(a, s) FROM test_array_shuffle_seed"
+ exception "must be a constant"
+ }
+ test {
+ sql "SELECT k, shuffle(a, k) FROM test_array_shuffle_seed"
+ exception "must be a constant"
+ }
+ test {
+ sql """
+ WITH t AS (SELECT [1, 2, 3, 4, 5] a, 1 seed UNION ALL SELECT [1, 2, 3, 4, 5], 2)
+ SELECT seed, array_shuffle(a, seed) FROM t ORDER BY seed
+ """
+ exception "must be a constant"
+ }
+ test {
+ sql "SELECT array_shuffle([1, 2, 3, 4, 5], cast(random() * 10 as bigint))"
+ exception "must be a constant"
+ }
+
+ // A constant seed gives a fixed result. A constant expression works as a seed too.
+ order_qt_const_seed """
+ SELECT array_shuffle([1, 2, 3, 4, 5], 1), array_shuffle([1, 2, 3, 4, 5], 2),
+ shuffle([1, 2, 3, 4, 5], 1), array_shuffle([1, 2, 3, 4, 5], 1 + 1)
+ """
+ // Any BIGINT is a valid seed, a negative one too. All 64 bits are used, so -1 and 4294967295
+ // (same low 32 bits) give different results, and so do -9223372036854775808 and 0.
+ order_qt_bigint_seed """
+ SELECT array_shuffle(array_range(10), -1), array_shuffle(array_range(10), 4294967295),
+ array_shuffle(array_range(10), -9223372036854775808),
+ array_shuffle(array_range(10), 0),
+ array_shuffle(array_range(10), 9223372036854775807)
+ """
+ // A NULL seed gives NULL.
+ order_qt_null_seed "SELECT array_shuffle([1, 2, 3, 4, 5], NULL)"
+ // Shuffling keeps the elements, so sorting them back gives a stable result.
+ order_qt_table """
+ SELECT k, array_sort(array_shuffle(a, 1)), array_size(shuffle(a, -1))
+ FROM test_array_shuffle_seed
+ """
+
+ // A constant array is still shuffled on each row, so the rows do not all get the same order.
+ // Each block starts again from the seed, so keep all 100 rows in one block.
+ order_qt_const_array """
+ SELECT /*+ SET_VAR(batch_size = 4064) */
+ count(DISTINCT cast(array_shuffle(array_range(20), 1) AS string)),
+ count(DISTINCT cast(array_shuffle(array_range(20)) AS string))
+ FROM numbers("number" = "100")
+ """
+ // array_shuffle is not folded into one constant, even when BE folds the constants.
+ order_qt_const_array_fold """
+ SELECT /*+ SET_VAR(enable_fold_constant_by_be = true) */
+ count(DISTINCT cast(array_shuffle(array_range(20)) AS string))
+ FROM numbers("number" = "100")
+ """
+ // Without a seed, each block gets its own random seed, so the blocks do not repeat the
+ // same orders.
+ order_qt_no_seed_blocks """
+ SELECT count(DISTINCT cast(array_shuffle(array_range(20 + number * 0)) AS string))
+ FROM numbers("number" = "100000")
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 16/28: branch-4.1: [fix](cloud) Delete versioned tablet indexes and metadata atomically #68322 (#68817)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 658f5958b33ea293644a36178f66cbc7dd33d4a5
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Fri Oct 9 16:07:59 2026 +0800
branch-4.1: [fix](cloud) Delete versioned tablet indexes and metadata atomically #68322 (#68817)
Cherry-picked from #68322
Co-authored-by: Yixuan Wang <wa...@selectdb.com>
---
cloud/src/recycler/recycler.cpp | 68 +++++++++++++++------------
cloud/test/recycle_versioned_keys_test.cpp | 74 ++++++++++++++++++++++++++++++
2 files changed, 112 insertions(+), 30 deletions(-)
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index 76a1f048b59..7595d80317d 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -3361,9 +3361,6 @@ int InstanceRecycler::recycle_orphan_partitions() {
int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id,
RecyclerMetricsContext& metrics_context,
int64_t partition_id) {
- bool is_multi_version =
- instance_info_.has_multi_version_status() &&
- instance_info_.multi_version_status() != MultiVersionStatus::MULTI_VERSION_DISABLED;
int64_t num_scanned = 0;
std::atomic_long num_recycled = 0;
@@ -3500,7 +3497,43 @@ int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id,
}
}
}
- if (is_multi_version) {
+ if (should_recycle_versioned_keys()) {
+ // Remove tablet indexes in the same transaction as tablet metadata.
+ std::vector<std::string> versioned_idx_keys;
+ versioned_idx_keys.reserve(tablets_info.size());
+ for (const auto& tablet_info : tablets_info) {
+ versioned_idx_keys.push_back(
+ versioned::tablet_index_key({instance_id_, tablet_info.tablet_id}));
+ }
+ std::vector<std::optional<std::string>> tablet_index_vals;
+ TxnErrorCode err = txn->batch_get(&tablet_index_vals, versioned_idx_keys);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG_WARNING("failed to batch get tablet index kv")
+ .tag("instance_id", instance_id_)
+ .tag("num_tablets", tablets_info.size())
+ .tag("err", err);
+ return -1;
+ }
+ DCHECK_EQ(tablet_index_vals.size(), versioned_idx_keys.size());
+ for (size_t i = 0; i < tablets_info.size(); ++i) {
+ if (!tablet_index_vals[i].has_value()) {
+ continue;
+ }
+ const auto& tablet_info = tablets_info[i];
+ TabletIndexPB tablet_index_pb;
+ if (!tablet_index_pb.ParseFromString(tablet_index_vals[i].value())) {
+ LOG_WARNING("failed to parse tablet index pb")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", tablet_info.tablet_id);
+ return -1;
+ }
+ std::string versioned_inverted_idx_key = versioned::tablet_inverted_index_key(
+ {instance_id_, tablet_index_pb.db_id(), tablet_index_pb.table_id(),
+ tablet_index_pb.index_id(), tablet_index_pb.partition_id(),
+ tablet_info.tablet_id});
+ txn->remove(versioned_inverted_idx_key);
+ txn->remove(versioned_idx_keys[i]);
+ }
for (auto& tablet_info : tablets_info) {
// Remove all versions of tablet compact stats for recycled tablet
auto k = versioned::tablet_compact_stats_key({instance_id_, tablet_info.tablet_id});
@@ -3534,6 +3567,7 @@ int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id,
for (auto& k : init_rs_keys) {
txn->remove(k);
}
+ TEST_SYNC_POINT_CALLBACK("InstanceRecycler::recycle_tablets.before_commit", txn.get());
if (TxnErrorCode err = txn->commit(); err != TxnErrorCode::TXN_OK) {
LOG(WARNING) << "failed to delete kvs related to tablets, instance_id=" << instance_id_
<< ", err=" << err;
@@ -5470,32 +5504,6 @@ int InstanceRecycler::recycle_versioned_tablet(int64_t tablet_id,
LOG(INFO) << "remove delete bitmap kv, tablet=" << tablet_id << ", begin=" << hex(dbm_start_key)
<< " end=" << hex(dbm_end_key);
- std::string versioned_idx_key = versioned::tablet_index_key({instance_id_, tablet_id});
- std::string tablet_index_val;
- err = txn->get(versioned_idx_key, &tablet_index_val);
- if (err != TxnErrorCode::TXN_KEY_NOT_FOUND && err != TxnErrorCode::TXN_OK) {
- LOG_WARNING("failed to get tablet index kv")
- .tag("instance_id", instance_id_)
- .tag("tablet_id", tablet_id)
- .tag("err", err);
- ret = -1;
- } else if (err == TxnErrorCode::TXN_OK) {
- // If the tablet index kv exists, we need to delete it
- TabletIndexPB tablet_index_pb;
- if (!tablet_index_pb.ParseFromString(tablet_index_val)) {
- LOG_WARNING("failed to parse tablet index pb")
- .tag("instance_id", instance_id_)
- .tag("tablet_id", tablet_id);
- ret = -1;
- } else {
- std::string versioned_inverted_idx_key = versioned::tablet_inverted_index_key(
- {instance_id_, tablet_index_pb.db_id(), tablet_index_pb.table_id(),
- tablet_index_pb.index_id(), tablet_index_pb.partition_id(), tablet_id});
- txn->remove(versioned_inverted_idx_key);
- txn->remove(versioned_idx_key);
- }
- }
-
err = txn->commit();
if (err != TxnErrorCode::TXN_OK) {
LOG(WARNING) << "failed to delete rowset kv of tablet " << tablet_id << ", err=" << err;
diff --git a/cloud/test/recycle_versioned_keys_test.cpp b/cloud/test/recycle_versioned_keys_test.cpp
index 1551f895c30..763749dafb4 100644
--- a/cloud/test/recycle_versioned_keys_test.cpp
+++ b/cloud/test/recycle_versioned_keys_test.cpp
@@ -32,6 +32,7 @@
#include "common/defer.h"
#include "common/util.h"
+#include "cpp/sync_point.h"
#include "meta-service/meta_service.h"
#include "meta-store/codec.h"
#include "meta-store/document_message.h"
@@ -1537,6 +1538,79 @@ TEST(RecycleVersionedKeysTest, RecycleTabletWithRowsetRefCountConcurrent) {
}
}
+TEST(RecycleVersionedKeysTest, RecycleTabletMetadataAndIndexesAtomically) {
+ auto meta_service = get_meta_service();
+ auto txn_kv = meta_service->txn_kv();
+ std::string instance_id = "recycle_tablet_metadata_and_indexes";
+ std::string cloud_unique_id = fmt::format("1:{}:0", instance_id);
+ ASSERT_NO_FATAL_FAILURE(create_and_refresh_instance(meta_service.get(), instance_id));
+
+ int64_t db_id = 1, table_id = 2, index_id = 3, partition_id = 4, tablet_id = 5;
+ ASSERT_NO_FATAL_FAILURE(prepare_and_commit_index(meta_service.get(), cloud_unique_id, db_id,
+ table_id, index_id));
+ ASSERT_NO_FATAL_FAILURE(prepare_and_commit_partition(meta_service.get(), cloud_unique_id, db_id,
+ table_id, partition_id, index_id));
+ ASSERT_NO_FATAL_FAILURE(create_tablet(meta_service.get(), cloud_unique_id, db_id, table_id,
+ index_id, partition_id, tablet_id));
+
+ auto check_tablet_keys = [&](bool exists) {
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
+ for (const auto& key :
+ {meta_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id}),
+ meta_tablet_idx_key({instance_id, tablet_id}),
+ versioned::tablet_index_key({instance_id, tablet_id}),
+ versioned::tablet_inverted_index_key(
+ {instance_id, db_id, table_id, index_id, partition_id, tablet_id})}) {
+ std::string value;
+ EXPECT_EQ(txn->get(key, &value),
+ exists ? TxnErrorCode::TXN_OK : TxnErrorCode::TXN_KEY_NOT_FOUND)
+ << hex(key);
+ }
+ for (const auto& key : {versioned::meta_tablet_key({instance_id, tablet_id}),
+ versioned::tablet_load_stats_key({instance_id, tablet_id}),
+ versioned::tablet_compact_stats_key({instance_id, tablet_id})}) {
+ std::vector<std::pair<std::string, Versionstamp>> values;
+ ASSERT_NO_FATAL_FAILURE(versioned_get_all(txn_kv.get(), key, values));
+ EXPECT_EQ(values.size(), exists ? 1 : 0) << hex(key);
+ }
+ };
+ ASSERT_NO_FATAL_FAILURE(check_tablet_keys(true));
+
+ InstanceInfoPB instance_info;
+ ASSERT_NO_FATAL_FAILURE(get_instance(meta_service.get(), cloud_unique_id, instance_info));
+ auto recycler = get_instance_recycler(meta_service.get(), instance_info);
+ auto* sp = SyncPoint::get_instance();
+ DORIS_CLOUD_DEFER {
+ sp->clear_all_call_backs();
+ sp->disable_processing();
+ };
+ bool commit_attempted = false;
+ sp->set_call_back("InstanceRecycler::recycle_tablets.before_commit", [&](auto&& args) {
+ commit_attempted = true;
+ // Force a real commit conflict after data recycling without changing tablet keys.
+ auto* txn = try_any_cast<Transaction*>(args[0]);
+ std::string key = instance_key(instance_id);
+ std::string value;
+ ASSERT_EQ(txn->get(key, &value), TxnErrorCode::TXN_OK);
+ std::unique_ptr<Transaction> conflicting_txn;
+ ASSERT_EQ(txn_kv->create_txn(&conflicting_txn), TxnErrorCode::TXN_OK);
+ conflicting_txn->put(key, value);
+ ASSERT_EQ(conflicting_txn->commit(), TxnErrorCode::TXN_OK);
+ });
+ sp->enable_processing();
+
+ RecyclerMetricsContext ctx;
+ ASSERT_EQ(recycler->recycle_tablets(table_id, index_id, ctx), -1);
+ ASSERT_TRUE(commit_attempted);
+ ASSERT_NO_FATAL_FAILURE(check_tablet_keys(true));
+
+ sp->disable_processing();
+ recycler = get_instance_recycler(meta_service.get(), instance_info);
+ ASSERT_EQ(recycler->recycle_tablets(table_id, index_id, ctx), 0);
+ ASSERT_NO_FATAL_FAILURE(check_tablet_keys(false));
+}
+
// A test that simulates a drop index operation.
TEST(RecycleVersionedKeysTest, RecycleIndex) {
auto meta_service = get_meta_service();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 15/28: [fix](parquet) Handle empty filter pages and offset index fallback (#68780)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 18b626064b99485739129911c035d22e3733629c
Author: Gabriel <li...@selectdb.com>
AuthorDate: Fri Oct 9 15:45:27 2026 +0800
[fix](parquet) Handle empty filter pages and offset index fallback (#68780)
### What problem does this PR solve?
Parquet V2 scans can fail when an indexed page contributes no predicate
rows: its `NONE` execution kind conflicts with the preceding page.
Parsing a selected page can also discard a stale OffsetIndex after
callers have already calculated its read range, leading to incorrect
projected rows or EOF accounting.
The fix ignores zero-row fragments when merging predicate execution
kinds, reconciles selected-page ranges in
ordinary/fixed-width/dictionary reads, and preserves physical-value
counts and nested row positions across safe sequential fallback.
### Read cost and fallback behavior
- Keep indexed skips lazy after the initial data-page reconciliation.
Unselected pages do not require header reads, page-cache lookups,
payload reads, decompression, or repetition-level scans solely for
validation.
- Do not drain the remaining repetition levels of a partially consumed
indexed V1 data page when advancing. Continue to check selected page
headers, row boundaries, and naturally exhausted level streams.
- Track whether indexed navigation skipped an unread header or an
unfinished nested V1 row span. If a later accessed header invalidates
the index, reject that unsafe sequential fallback instead of reusing
uncertain row positions or an incomplete physical-value prefix. Safe
fallback still succeeds.
- Retain the metadata-only byte-coverage check before publishing
optional page indexes. It adds no file I/O; gaps and uncovered suffixes
still conservatively disable the index pair, including auxiliary gaps or
writer padding.
This preserves the normal index trust boundary: structurally accepted
indexes describe unread pages. The PR does not eagerly verify every
index entry against physical headers/levels and does not claim to detect
every forged row span or coordinated index/header inconsistency.
Comprehensive corrupt-index validation is outside this fix's scope.
Changes remain confined to `format_v2` and its tests. The legacy reader
under `be/src/format` is unchanged.
### Release note
Fix Parquet V2 scans with empty predicate page fragments and
selected-page OffsetIndex fallback while preserving lazy indexed page
skipping. Reject unsafe sequential fallback after an unverified indexed
prefix.
### Check List (For Author)
- [x] Unit tests
- [x] No new configuration or documentation requirement
- Reproduced both eager-read regressions before the implementation
change: a three-page sparse seek made 4 or 5 stream reads rather than 3,
and advancing a header-only/partially consumed V1 page loaded or drained
its levels.
- All **747** related BE ASAN tests passed with
`*Parquet*:FileScannerV2Test.*:TableReaderTest.*`.
- Sparse-read tests cover flat ordinary/fixed-width/dictionary paths,
nested value/levels-only paths, all 16 four-page V1/V2 layouts, and
cache hits/misses. Native 3-page and 64-page seeks parse exactly two
headers and perform exactly three stream reads, independent of the
skipped page count.
- Retained physical-row result comparisons across mixed page versions,
nulls, repeated values, dictionary encoding, batch sizes and all
nonempty subsets of a seven-row fixture. Fallback cases assert exact
success or the explicitly required corruption error.
- Complete Arrow-written file regressions retain coverage for empty
predicate fragments, selected-page fallback, incomplete index pairs and
pruning behavior.
- clang-format 16 on all eight affected C++ files and `git diff --check`
passed.
These are deterministic work-count and correctness checks, not
remote-storage throughput benchmarks.
---
.../parquet/reader/native/column_chunk_reader.cpp | 102 +-
.../parquet/reader/native/column_chunk_reader.h | 3 +
.../parquet/reader/native/column_reader.cpp | 51 +-
.../parquet/reader/native/column_reader.h | 1 +
.../format_v2/parquet/reader/native/page_reader.h | 6 +
.../parquet/reader/native_column_reader.cpp | 18 +-
be/test/format_v2/parquet/native_decoder_test.cpp | 1156 +++++++++++++++++++-
be/test/format_v2/parquet/parquet_scan_test.cpp | 226 ++++
8 files changed, 1491 insertions(+), 72 deletions(-)
diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp
index d1677c66676..f524713f107 100644
--- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp
+++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp
@@ -240,7 +240,10 @@ bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunk
// to the owning metadata so page-to-row mapping cannot silently move by one physical page.
const uint64_t chunk_begin = chunk_range.offset;
const uint64_t chunk_end = chunk_begin + chunk_range.length;
- uint64_t previous_end = chunk_begin;
+ // A gap can hide an omitted data page even when ColumnIndex has the same entry count.
+ // Decline such indexes before pruning can lose rows; auxiliary pages and writer padding
+ // conservatively use sequential traversal. A dictionary prefix ends at data_page_offset.
+ uint64_t previous_end = static_cast<uint64_t>(data_page_offset);
int64_t previous_row = -1;
for (const auto& location : index.page_locations) {
if (location.first_row_index <= previous_row || location.first_row_index >= row_count ||
@@ -249,14 +252,14 @@ bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunk
}
const uint64_t begin = static_cast<uint64_t>(location.offset);
const uint64_t size = static_cast<uint64_t>(location.compressed_page_size);
- if (begin < chunk_begin || begin < previous_end || begin > chunk_end ||
+ if (begin < chunk_begin || begin != previous_end || begin > chunk_end ||
size > chunk_end - begin) {
return false;
}
previous_row = location.first_row_index;
previous_end = begin + size;
}
- return true;
+ return previous_end == chunk_end;
}
namespace {
@@ -1067,11 +1070,11 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::load_dictionary_page(bool
template <bool IN_COLLECTION, bool OFFSET_INDEX>
Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::ensure_first_data_page_parsed() {
- if (_first_data_page_parsed) {
+ if (_first_data_page_parsed && _page_reader->has_active_offset_index()) {
return Status::OK();
}
- // OffsetIndex row bounds are untrusted until page zero has been reconciled and its declared
- // cardinality checked, so no indexed skip may observe them before this one-time parse.
+ // OffsetIndex bounds need an initial reconciliation. If the index is discarded, every later
+ // page needs its header parsed before callers use its sequential row bounds.
return parse_page_header();
}
@@ -1096,19 +1099,17 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::parse_page_header() {
}
int32_t page_num_values = _page_reader->is_header_v2() ? header->data_page_header_v2.num_values
: header->data_page_header.num_values;
- if constexpr (IN_COLLECTION && OFFSET_INDEX) {
- if (!_page_reader->is_header_v2() && _page_reader->has_active_offset_index()) {
- // V1 nested pages do not declare their logical row count. An OffsetIndex span cannot
- // be trusted until repetition levels are decoded, so keep the sequential cursor path.
- _page_reader->discard_offset_index();
- _offset_index = nullptr;
- }
- }
+ // A later V1 page can still use a valid OffsetIndex in a mixed-version chunk. Its first
+ // repetition level is checked when loaded, and its row count when the levels are exhausted.
const bool active_offset_index = _page_reader->has_active_offset_index();
+ if (_skipped_unverified_indexed_page && !active_offset_index) {
+ // Lazy skips leave part of the physical prefix unknown. Once the index is discarded,
+ // neither its absolute row coordinate nor the parsed-value EOF count can be reused.
+ return Status::Corruption("Parquet OffsetIndex fallback after skipping an unverified page");
+ }
if (page_num_values < 0 || page_num_values > _metadata.num_values ||
- (!active_offset_index &&
- static_cast<uint64_t>(page_num_values) >
- static_cast<uint64_t>(_metadata.num_values) - _chunk_parsed_values)) {
+ static_cast<uint64_t>(page_num_values) >
+ static_cast<uint64_t>(_metadata.num_values) - _chunk_parsed_values) {
// Page counts are untrusted and feed both level decoders and scratch sizing. Bound each
// page by the column metadata before converting to unsigned counters.
return Status::Corruption("Parquet data page value count {} exceeds column total {}",
@@ -1142,10 +1143,9 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::parse_page_header() {
_remaining_def_nums = page_num_values;
_remaining_num_values = page_num_values;
- // no offset will parse all header.
- if (!active_offset_index) {
- _chunk_parsed_values += _remaining_num_values;
- }
+ // Count each parsed data header exactly once, including the indexed prefix. A safe fallback
+ // uses this physical value count for EOF; nested logical rows are not interchangeable with it.
+ _chunk_parsed_values += _remaining_num_values;
_first_data_page_parsed = true;
_state = HEADER_PARSED;
return Status::OK();
@@ -1155,6 +1155,17 @@ template <bool IN_COLLECTION, bool OFFSET_INDEX>
Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::next_page() {
if constexpr (OFFSET_INDEX) {
RETURN_IF_ERROR(ensure_first_data_page_parsed());
+ if (_page_reader->has_active_offset_index()) {
+ // Preserve header/payload-free indexed skips. An unread header or an unfinished V1
+ // row span prevents a later sequential fallback, but does not require eager I/O.
+ if (_state == INITIALIZED) {
+ _skipped_unverified_indexed_page = true;
+ } else if constexpr (IN_COLLECTION) {
+ if (!_page_reader->is_header_v2() && _remaining_rep_nums != 0) {
+ _skipped_unverified_indexed_page = true;
+ }
+ }
+ }
} else {
// Load dictionary state before advancing can jump past the physical dictionary page.
RETURN_IF_ERROR(_ensure_dictionary_page_loaded());
@@ -1171,7 +1182,18 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::next_page() {
_decompress_release_pending = false;
_decompress_release_threshold = std::numeric_limits<size_t>::max();
}
+ const bool indexed = _page_reader->has_active_offset_index();
+ if (!indexed && (_page_reader->is_header_v2() || !IN_COLLECTION)) {
+ _current_row = _page_reader->end_row();
+ }
RETURN_IF_ERROR(_page_reader->next_page());
+ if (indexed) {
+ _current_row = _page_reader->start_row();
+ } else if constexpr (IN_COLLECTION) {
+ // V1 row counts come from repetition levels, so PageReader cannot advance this coordinate
+ // from its header. Synchronize before a following V2 header derives its sequential bounds.
+ _page_reader->set_sequential_row_start(_current_row);
+ }
_state = INITIALIZED;
return Status::OK();
}
@@ -1396,6 +1418,18 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::load_page_data() {
_remaining_rep_nums));
}
}
+ if constexpr (IN_COLLECTION && OFFSET_INDEX) {
+ if (_max_rep_level > 0 && _remaining_rep_nums > 0 &&
+ _page_reader->has_active_offset_index()) {
+ // Indexed pages must start at a row boundary, including V1 pages after indexed skips.
+ // Reject continuations before a partial selection can expose values from the wrong row.
+ if (_rep_level_decoder.get_next() != 0) {
+ return Status::Corruption(
+ "Indexed Parquet nested page must start at a row boundary");
+ }
+ _rep_level_decoder.rewind_one();
+ }
+ }
if (_max_def_level > 0) {
SCOPED_RAW_TIMER(&_chunk_statistics.decode_level_time);
if (header->__isset.data_page_header_v2) {
@@ -2262,29 +2296,29 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::seek_to_nested_row(size_t
if constexpr (IN_COLLECTION && OFFSET_INDEX) {
RETURN_IF_ERROR(ensure_first_data_page_parsed());
}
- if constexpr (OFFSET_INDEX) {
- if (_page_reader->has_active_offset_index()) {
- while (true) {
+ while (true) {
+ if constexpr (OFFSET_INDEX) {
+ if (_page_reader->has_active_offset_index()) {
if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) {
- break;
+ RETURN_IF_ERROR(parse_page_header());
+ if (_page_reader->has_active_offset_index()) {
+ RETURN_IF_ERROR(load_page_data());
+ RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row));
+ _current_row = left_row;
+ return Status::OK();
+ }
+ // Parsing can discard the index and change this page's bounds. Re-enter the
+ // sequential decision below instead of seeking within the stale indexed span.
} else if (has_next_page()) {
RETURN_IF_ERROR(next_page());
- _current_row = _page_reader->start_row();
+ continue;
} else [[unlikely]] {
return Status::InternalError("no match seek row {}, current row {}", left_row,
_current_row);
}
}
-
- RETURN_IF_ERROR(parse_page_header());
- RETURN_IF_ERROR(load_page_data());
- RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row));
- _current_row = left_row;
- return Status::OK();
}
- }
- while (true) {
RETURN_IF_ERROR(parse_page_header());
if (_page_reader->is_header_v2() || !IN_COLLECTION) {
if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) {
diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h
index a50b7e4bc08..266bcca9b6d 100644
--- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h
+++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h
@@ -434,6 +434,8 @@ private:
LevelDecoder _rep_level_decoder;
LevelDecoder _def_level_decoder;
+ // Parsed headers contribute exactly once. This is a complete physical prefix only while
+ // no indexed page has been skipped unverified; otherwise sequential fallback is rejected.
size_t _chunk_parsed_values = 0;
// this page remaining rep/def nums
// if max_rep_level = 0 / max_def_level = 0, this value retail hava value.
@@ -454,6 +456,7 @@ private:
Slice _v2_def_levels;
bool _dict_checked = false;
bool _first_data_page_parsed = false;
+ bool _skipped_unverified_indexed_page = false;
bool _has_dict = false;
bool _nested_row_started = false;
Decoder* _page_decoder = nullptr;
diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp
index 314a2869041..1744966a061 100644
--- a/be/src/format_v2/parquet/reader/native/column_reader.cpp
+++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp
@@ -729,6 +729,27 @@ void ColumnReader::_generate_read_ranges(RowRange page_row_range, RowRanges* res
RowRanges::ranges_intersection(*result_ranges, _row_ranges, result_ranges);
}
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::_get_page_read_ranges(
+ int64_t* right_row, RowRanges* read_ranges) {
+ RETURN_IF_ERROR(_chunk_reader->ensure_first_data_page_parsed());
+ *right_row = _chunk_reader->page_end_row();
+ _generate_read_ranges(RowRange {_current_row_index, *right_row}, read_ranges);
+ if (read_ranges->count() == 0) {
+ return Status::OK();
+ }
+ RETURN_IF_ERROR(_chunk_reader->parse_page_header());
+ const auto parsed_end = _chunk_reader->page_end_row();
+ if (*right_row != parsed_end) {
+ // Parsing a selected later page can discard its OffsetIndex. Both the selection and the
+ // page-advance decision must use the reconciled bound, or unread values shift into the next page.
+ *right_row = parsed_end;
+ read_ranges->clear();
+ _generate_read_ranges(RowRange {_current_row_index, *right_row}, read_ranges);
+ }
+ return Status::OK();
+}
+
template <bool IN_COLLECTION, bool OFFSET_INDEX>
Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::init(
io::FileReaderSPtr file, NativeFieldSchema* field, size_t max_buf_size, RuntimeState* state,
@@ -1331,18 +1352,11 @@ Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::read_fixed_width_filter(
}
int64_t right_row = 0;
- if constexpr (OFFSET_INDEX == false) {
- RETURN_IF_ERROR(_chunk_reader->parse_page_header());
- } else {
- RETURN_IF_ERROR(_chunk_reader->ensure_first_data_page_parsed());
- }
- right_row = _chunk_reader->page_end_row();
RowRanges read_ranges;
- _generate_read_ranges(RowRange {_current_row_index, right_row}, &read_ranges);
+ RETURN_IF_ERROR(_get_page_read_ranges(&right_row, &read_ranges));
if (read_ranges.count() == 0) {
_current_row_index = right_row;
} else {
- RETURN_IF_ERROR(_chunk_reader->parse_page_header());
RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent());
if (!_chunk_reader->can_filter_fixed_width_values(conjuncts, column_id, _serde.get(),
&_decode_context)) {
@@ -1468,18 +1482,11 @@ Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::read_dictionary_filter(
}
int64_t right_row = 0;
- if constexpr (OFFSET_INDEX == false) {
- RETURN_IF_ERROR(_chunk_reader->parse_page_header());
- right_row = _chunk_reader->page_end_row();
- } else {
- right_row = _chunk_reader->page_end_row();
- }
RowRanges read_ranges;
- _generate_read_ranges(RowRange {_current_row_index, right_row}, &read_ranges);
+ RETURN_IF_ERROR(_get_page_read_ranges(&right_row, &read_ranges));
if (read_ranges.count() == 0) {
_current_row_index = right_row;
} else {
- RETURN_IF_ERROR(_chunk_reader->parse_page_header());
RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent());
if (_chunk_reader->current_encoding() != tparquet::Encoding::RLE_DICTIONARY) {
return Status::OK();
@@ -1764,17 +1771,10 @@ Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::read_column_data(
}
int64_t right_row = 0;
- if constexpr (OFFSET_INDEX == false) {
- RETURN_IF_ERROR(_chunk_reader->parse_page_header());
- } else {
- RETURN_IF_ERROR(_chunk_reader->ensure_first_data_page_parsed());
- }
- right_row = _chunk_reader->page_end_row();
+ RowRanges read_ranges;
+ RETURN_IF_ERROR(_get_page_read_ranges(&right_row, &read_ranges));
do {
- // generate the row ranges that should be read
- RowRanges read_ranges;
- _generate_read_ranges(RowRange {_current_row_index, right_row}, &read_ranges);
if (read_ranges.count() == 0) {
// skip the whole page
_current_row_index = right_row;
@@ -1800,7 +1800,6 @@ Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::read_column_data(
}
}
// load page data to decode or skip values
- RETURN_IF_ERROR(_chunk_reader->parse_page_header());
RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent());
size_t has_read = 0;
for (size_t idx = 0; idx < read_ranges.range_size(); idx++) {
diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h
index b43822cd165..0b1d578d611 100644
--- a/be/src/format_v2/parquet/reader/native/column_reader.h
+++ b/be/src/format_v2/parquet/reader/native/column_reader.h
@@ -444,6 +444,7 @@ private:
size_t retained_batch_scratch_bytes() const;
size_t active_batch_scratch_bytes() const;
+ Status _get_page_read_ranges(int64_t* right_row, RowRanges* read_ranges);
Status _skip_values(size_t num_values);
Status _read_values(size_t num_values, ColumnPtr& doris_column, const DataTypePtr& type,
FilterMap& filter_map, bool is_dict_filter);
diff --git a/be/src/format_v2/parquet/reader/native/page_reader.h b/be/src/format_v2/parquet/reader/native/page_reader.h
index da2c066891b..bf009f472f1 100644
--- a/be/src/format_v2/parquet/reader/native/page_reader.h
+++ b/be/src/format_v2/parquet/reader/native/page_reader.h
@@ -229,6 +229,12 @@ public:
size_t end_row() const { return _end_row; }
+ void set_sequential_row_start(size_t row) {
+ DCHECK(!has_active_offset_index());
+ _start_row = row;
+ _end_row = row;
+ }
+
bool has_active_offset_index() const {
if constexpr (OFFSET_INDEX) {
return _offset_index != nullptr;
diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp
index 666fdbba327..179a09ad345 100644
--- a/be/src/format_v2/parquet/reader/native_column_reader.cpp
+++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp
@@ -542,6 +542,16 @@ Status NativeColumnReader::read_with_fixed_width_filter(
row_filter->clear();
return Status::OK();
}
+ // A skipped page can advance the native cursor without evaluating a predicate. Its NONE
+ // execution kind must not replace or conflict with the preceding data page's kind.
+ if (loop_rows == 0) {
+ if (!eof && ++consecutive_empty_calls > _row_group_rows + 1) {
+ return Status::Corruption(
+ "Native parquet fixed-width predicate made no progress for column {}",
+ _name);
+ }
+ continue;
+ }
if (*execution_kind == DirectPredicateExecutionKind::NONE) {
*execution_kind = loop_kind;
} else if (loop_kind != DirectPredicateExecutionKind::DEFINITION_LEVEL) {
@@ -550,14 +560,6 @@ Status NativeColumnReader::read_with_fixed_width_filter(
*execution_kind = loop_kind;
}
row_filter->insert(row_filter->end(), loop_filter.begin(), loop_filter.end());
- if (loop_rows == 0 && !eof) {
- if (++consecutive_empty_calls > _row_group_rows + 1) {
- return Status::Corruption(
- "Native parquet fixed-width predicate made no progress for column {}",
- _name);
- }
- continue;
- }
consecutive_empty_calls = 0;
*rows_read += static_cast<int64_t>(loop_rows);
}
diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp
index af4a17d1b8a..7d8ca6a7176 100644
--- a/be/test/format_v2/parquet/native_decoder_test.cpp
+++ b/be/test/format_v2/parquet/native_decoder_test.cpp
@@ -43,6 +43,7 @@
#include "exprs/vectorized_fn_call.h"
#include "exprs/vliteral.h"
#include "exprs/vslot_ref.h"
+#include "format_v2/parquet/parquet_column_schema.h"
#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h"
#include "format_v2/parquet/reader/native/column_reader.h"
#include "format_v2/parquet/reader/native/decoder.h"
@@ -51,6 +52,8 @@
#include "format_v2/parquet/reader/native/level_decoder.h"
#include "format_v2/parquet/reader/native/level_reader.h"
#include "format_v2/parquet/reader/native/page_reader.h"
+#include "format_v2/parquet/reader/native_column_reader.h"
+#include "format_v2/parquet/reader/variant_column_reader.h"
#include "io/fs/buffered_reader.h"
#include "io/fs/file_reader.h"
#include "util/block_compression.h"
@@ -1316,7 +1319,7 @@ TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDateTimeSelectionBatchesPhys
ColumnChunkReaderStatistics statistics;
const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64,
.logical_type = ParquetLogicalType::TIMESTAMP,
- .time_unit = ParquetTimeUnit::MICROS};
+ .time_unit = ::doris::ParquetTimeUnit::MICROS};
ASSERT_TRUE(materialize_selected_plain_int64(physical_micros, LOGICAL_VALUES, null_runs, filter,
type, &column, &null_map, &statistics, context)
.ok());
@@ -1342,10 +1345,10 @@ TEST(ParquetV2NativeDecoderTest, NegativeNanosFloorAcrossPlainAndDictionaryTimes
"1970-01-01 00:00:00.000000", "1970-01-01 00:00:00.000001"};
const ParquetDecodeContext local_context {.physical_type = ParquetPhysicalType::INT64,
.logical_type = ParquetLogicalType::TIMESTAMP,
- .time_unit = ParquetTimeUnit::NANOS};
+ .time_unit = ::doris::ParquetTimeUnit::NANOS};
const ParquetDecodeContext utc_context {.physical_type = ParquetPhysicalType::INT64,
.logical_type = ParquetLogicalType::TIMESTAMP,
- .time_unit = ParquetTimeUnit::NANOS,
+ .time_unit = ::doris::ParquetTimeUnit::NANOS,
.timestamp_is_adjusted_to_utc = true};
for (const bool dictionary : {false, true}) {
@@ -3454,6 +3457,81 @@ TEST(ParquetV2NativeDecoderTest, LazyFixedWidthFilterUsesReconciledFirstPageRang
EXPECT_EQ(row_filter, (IColumn::Filter {1, 1}));
}
+// Exercise the adapter and native reader together: an empty fragment must not change the
+// predicate domain, and an index fallback must still refresh every subsequent page's row bounds.
+void check_fixed_width_filter_across_pages(bool shifted_index) {
+ const std::vector<std::vector<uint8_t>> pages {serialize_plain_int32_page({10, 11}),
+ serialize_plain_int32_page({20, 21}),
+ serialize_plain_int32_page({30, 31})};
+ std::vector<uint8_t> bytes;
+ tparquet::OffsetIndex offset_index;
+ for (size_t page = 0; page < pages.size(); ++page) {
+ tparquet::PageLocation location;
+ location.__set_offset(bytes.size() + (shifted_index && page > 0 ? 1 : 0));
+ location.__set_compressed_page_size(pages[page].size() +
+ (shifted_index && page == 0 ? 1 : 0));
+ location.__set_first_row_index(page * 2);
+ offset_index.page_locations.push_back(location);
+ bytes.insert(bytes.end(), pages[page].begin(), pages[page].end());
+ }
+ bytes.push_back(0);
+ tparquet::ColumnChunk chunk;
+ chunk.meta_data.__set_type(tparquet::Type::INT32);
+ chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED);
+ chunk.meta_data.__set_num_values(6);
+ chunk.meta_data.__set_total_compressed_size(bytes.size());
+ chunk.meta_data.__set_data_page_offset(0);
+ chunk.meta_data.__set_encodings({tparquet::Encoding::PLAIN});
+ NativeFieldSchema field;
+ field.physical_type = tparquet::Type::INT32;
+ field.data_type = std::make_shared<DataTypeInt32>();
+ field.parquet_schema.__set_type(tparquet::Type::INT32);
+ field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED);
+ auto ranges = ::doris::RowRanges::create_single(0, shifted_index ? 6 : 2);
+ if (!shifted_index) {
+ ranges.add({4, 6});
+ }
+ auto file = std::make_shared<NativeDecoderMemoryFileReader>(bytes);
+ auto native = std::make_unique<ScalarColumnReader<false, true>>(ranges, 6, chunk, &offset_index,
+ nullptr, nullptr);
+ ASSERT_TRUE(native->init(file, &field, bytes.size(), nullptr, "", ParquetReaderCompat {}, true)
+ .ok());
+ ParquetColumnSchema schema;
+ schema.name = "value";
+ schema.type = field.data_type;
+ NativeColumnReader reader(schema, schema.type, schema.type, nullptr, {});
+ reader._native_reader = std::move(native);
+ reader._row_group_rows = 6;
+ auto projected = ColumnInt32::create();
+ IColumn::Filter row_filter;
+ int64_t rows = 0;
+ bool used_filter = false;
+ DirectPredicateExecutionKind kind = DirectPredicateExecutionKind::NONE;
+ const auto status = reader.read_with_fixed_width_filter(
+ ranges.count(), nullptr, false,
+ {create_int32_raw_comparison(0, "gt", TExprOpcode::GT, 10)}, 0, projected.get(),
+ &row_filter, &rows, &used_filter, &kind);
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_TRUE(used_filter);
+ EXPECT_EQ(rows, ranges.count());
+ EXPECT_EQ(kind, DirectPredicateExecutionKind::CONVERTED_FIXED);
+ if (shifted_index) {
+ EXPECT_EQ(row_filter, (IColumn::Filter {0, 1, 1, 1, 1, 1}));
+ EXPECT_EQ(projected->get_data(), (ColumnInt32::Container {11, 20, 21, 30, 31}));
+ } else {
+ EXPECT_EQ(row_filter, (IColumn::Filter {0, 1, 1, 1}));
+ EXPECT_EQ(projected->get_data(), (ColumnInt32::Container {11, 30, 31}));
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, FixedWidthFilterSkipsEmptyPageFragments) {
+ check_fixed_width_filter_across_pages(false);
+}
+
+TEST(ParquetV2NativeDecoderTest, FixedWidthFilterContinuesAfterOffsetIndexFallback) {
+ check_fixed_width_filter_across_pages(true);
+}
+
TEST(ParquetV2NativeDecoderTest, LazyFlatIndexedFallbackUsesReconciledFirstPageRange) {
const auto first_page = serialize_plain_int32_page({10, 11});
const auto second_page = serialize_plain_int32_page({20});
@@ -3484,7 +3562,7 @@ TEST(ParquetV2NativeDecoderTest, LazyFlatIndexedFallbackUsesReconciledFirstPageR
field.parquet_schema.__set_type(tparquet::Type::INT32);
field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED);
auto file = std::make_shared<NativeDecoderMemoryFileReader>(bytes);
- const auto row_ranges = ::doris::RowRanges::create_single(0, 2);
+ const auto row_ranges = ::doris::RowRanges::create_single(0, 3);
ScalarColumnReader<false, true> reader(row_ranges, 3, chunk, &offset_index, nullptr, nullptr);
ASSERT_TRUE(reader.init(file, &field, bytes.size(), nullptr, "", ParquetReaderCompat {}, true)
.ok());
@@ -3500,6 +3578,12 @@ TEST(ParquetV2NativeDecoderTest, LazyFlatIndexedFallbackUsesReconciledFirstPageR
ASSERT_EQ(rows, 2);
EXPECT_EQ(assert_cast<const ColumnInt32&>(*values).get_data(),
(ColumnInt32::Container {10, 11}));
+ values = ColumnInt32::create();
+ ASSERT_TRUE(
+ reader.read_column_data(values, field.data_type, nullptr, filter, 1, &rows, &eof, false)
+ .ok());
+ ASSERT_EQ(rows, 1);
+ EXPECT_EQ(assert_cast<const ColumnInt32&>(*values).get_data(), (ColumnInt32::Container {20}));
}
TEST(ParquetV2NativeDecoderTest, LazyNestedV2SeekValidatesFirstPageRowRange) {
@@ -3559,6 +3643,1068 @@ TEST(ParquetV2NativeDecoderTest, LazyNestedV2SeekValidatesFirstPageRowRange) {
EXPECT_TRUE(status.is<ErrorCode::CORRUPTION>()) << status;
}
+TEST(ParquetV2NativeDecoderTest, SelectedLaterPageReconcilesRangesAfterIndexFallback) {
+ for (const bool shrinking : {false, true}) {
+ for (const int mode : {0, 1, 2}) {
+ for (const size_t batch_size : {1, 2, 3}) {
+ for (const bool cache_hit : {false, true}) {
+ SCOPED_TRACE(testing::Message()
+ << "mode=" << mode << ", batch=" << batch_size
+ << ", cache=" << cache_hit << ", shrinking=" << shrinking);
+ const std::vector<std::vector<int32_t>> values {
+ {10, 11}, {20, 21, 22}, {30, 31}};
+ std::vector<uint8_t> bytes(1, 0);
+ auto dictionary = ColumnInt32::create();
+ dictionary->get_data() = {10, 11, 20, 21, 22, 30, 31};
+ if (mode == 2) {
+ std::vector<uint8_t> payload(dictionary->size() * sizeof(int32_t));
+ memcpy(payload.data(), dictionary->get_data().data(), payload.size());
+ tparquet::PageHeader header;
+ header.__set_type(tparquet::PageType::DICTIONARY_PAGE);
+ header.__set_compressed_page_size(payload.size());
+ header.__set_uncompressed_page_size(payload.size());
+ tparquet::DictionaryPageHeader data;
+ data.__set_num_values(dictionary->size());
+ data.__set_encoding(tparquet::Encoding::PLAIN);
+ header.__set_dictionary_page_header(data);
+ auto page = serialize_page(header, payload);
+ bytes.insert(bytes.end(), page.begin(), page.end());
+ }
+ const size_t data_offset = bytes.size();
+ tparquet::OffsetIndex index;
+ std::vector<std::vector<uint8_t>> pages;
+ uint32_t next_id = 0;
+ for (size_t i = 0; i < values.size(); ++i) {
+ if (mode == 2) {
+ faststring ids;
+ RleEncoder<uint32_t> encoder(&ids, 3);
+ for (size_t j = 0; j < values[i].size(); ++j) {
+ encoder.Put(next_id++);
+ }
+ for (size_t j = values[i].size(); j < 8; ++j) {
+ encoder.Put(0);
+ }
+ encoder.Flush();
+ std::vector<uint8_t> payload(ids.size() + 1, 3);
+ memcpy(payload.data() + 1, ids.data(), ids.size());
+ tparquet::PageHeader header;
+ header.__set_type(tparquet::PageType::DATA_PAGE);
+ header.__set_compressed_page_size(payload.size());
+ header.__set_uncompressed_page_size(payload.size());
+ tparquet::DataPageHeader data;
+ data.__set_num_values(values[i].size());
+ data.__set_encoding(tparquet::Encoding::RLE_DICTIONARY);
+ data.__set_definition_level_encoding(tparquet::Encoding::RLE);
+ data.__set_repetition_level_encoding(tparquet::Encoding::RLE);
+ header.__set_data_page_header(data);
+ pages.push_back(serialize_page(header, payload));
+ } else {
+ pages.push_back(serialize_plain_int32_page(values[i]));
+ }
+ tparquet::PageLocation location;
+ location.__set_offset(bytes.size());
+ location.__set_compressed_page_size(pages.back().size() - (i == 1 ? 1 : 0));
+ // Reconciliation can expand a selected range or remove it entirely.
+ // Neither case may advance using the stale indexed page boundary.
+ location.__set_first_row_index(shrinking && i == 2 ? 6 : i * 2);
+ index.page_locations.push_back(location);
+ bytes.insert(bytes.end(), pages.back().begin(), pages.back().end());
+ }
+ const std::string cache_key =
+ fmt::format("selected-late-{}-{}-{}", mode, batch_size, shrinking);
+ if (cache_hit) {
+ auto* page = new DataPage(pages[1].size(), true, segment_v2::DATA_PAGE);
+ memcpy(page->data(), pages[1].data(), pages[1].size());
+ page->reset_size(pages[1].size());
+ PageCacheHandle handle;
+ StoragePageCache::instance()->insert(
+ StoragePageCache::CacheKey(cache_key, bytes.size(),
+ index.page_locations[1].offset),
+ page, &handle, segment_v2::DATA_PAGE);
+ }
+ tparquet::ColumnChunk chunk;
+ chunk.meta_data.__set_type(tparquet::Type::INT32);
+ chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED);
+ chunk.meta_data.__set_num_values(7);
+ chunk.meta_data.__set_total_compressed_size(bytes.size() - 1);
+ chunk.meta_data.__set_data_page_offset(data_offset);
+ chunk.meta_data.__set_encodings({tparquet::Encoding::PLAIN});
+ if (mode == 2) {
+ chunk.meta_data.__set_dictionary_page_offset(1);
+ chunk.meta_data.__set_encodings(
+ {tparquet::Encoding::PLAIN, tparquet::Encoding::RLE_DICTIONARY});
+ }
+ NativeFieldSchema field;
+ field.physical_type = tparquet::Type::INT32;
+ field.data_type = std::make_shared<DataTypeInt32>();
+ field.parquet_schema.__set_type(tparquet::Type::INT32);
+ field.parquet_schema.__set_repetition_type(
+ tparquet::FieldRepetitionType::REQUIRED);
+ const auto ranges =
+ ::doris::RowRanges::create_single(shrinking ? 5 : 2, shrinking ? 6 : 5);
+ const size_t expected_rows = shrinking ? 1 : 3;
+ auto file = std::make_shared<NativeDecoderMemoryFileReader>(bytes);
+ ScalarColumnReader<false, true> reader(ranges, 7, chunk, &index, nullptr,
+ nullptr);
+ ASSERT_TRUE(reader.init(file, &field, bytes.size(), nullptr,
+ cache_hit ? cache_key : "", ParquetReaderCompat {},
+ true)
+ .ok());
+ FilterMap filter;
+ ASSERT_TRUE(filter.init(nullptr, expected_rows, false).ok());
+ ColumnPtr ordinary = ColumnInt32::create();
+ auto projected = ColumnInt32::create();
+ size_t consumed = 0;
+ bool eof = false;
+ for (size_t call = 0; consumed < expected_rows && !eof; ++call) {
+ ASSERT_LT(call, 8);
+ size_t rows = 0;
+ Status status;
+ IColumn::Filter row_filter;
+ bool used = false;
+ const size_t count = std::min(batch_size, expected_rows - consumed);
+ if (mode == 0) {
+ status = reader.read_column_data(ordinary, field.data_type, nullptr,
+ filter, count, &rows, &eof, false);
+ } else if (mode == 1) {
+ DirectPredicateExecutionKind kind = DirectPredicateExecutionKind::NONE;
+ status = reader.read_fixed_width_filter(
+ {create_int32_raw_comparison(0, "gt", TExprOpcode::GT, 0)}, 0,
+ filter, count, projected.get(), &row_filter, &rows, &eof, &used,
+ &kind);
+ } else {
+ size_t survivors = 0;
+ bool direct = false;
+ status = reader.read_dictionary_filter(
+ IColumn::Filter(7, 1), filter, count, dictionary.get(),
+ projected.get(), nullptr, &row_filter, &survivors, &rows, &eof,
+ &direct, &used);
+ EXPECT_EQ(survivors, rows);
+ }
+ ASSERT_TRUE(status.ok()) << status;
+ if (mode != 0) {
+ EXPECT_TRUE(used);
+ EXPECT_EQ(row_filter, IColumn::Filter(rows, 1));
+ }
+ consumed += rows;
+ }
+ EXPECT_EQ(consumed, expected_rows);
+ const auto& actual =
+ mode == 0 ? assert_cast<const ColumnInt32&>(*ordinary) : *projected;
+ EXPECT_EQ(actual.get_data(), (shrinking ? ColumnInt32::Container {30}
+ : ColumnInt32::Container {20, 21, 22}));
+ EXPECT_EQ(reader.column_statistics().page_cache_hit_counter > 0, cache_hit);
+ }
+ }
+ }
+ }
+}
+
+// Serialize independent page versions and row shapes so navigation tests do not inherit a writer's
+// fixed page layout. Expected values are derived from the physical row sequence, never the index.
+struct NestedNavigationFile {
+ std::vector<uint8_t> bytes {0};
+ tparquet::ColumnChunk chunk;
+ tparquet::OffsetIndex index;
+ NativeFieldSchema field;
+ size_t rows = 0;
+ std::vector<bool> page_versions;
+ std::vector<std::vector<int32_t>> row_values;
+ std::vector<std::vector<uint8_t>> pages;
+
+ NestedNavigationFile(const std::vector<bool>& v2, const std::vector<size_t>& page_rows,
+ bool repeated = false, bool dictionary = false, bool nullable = false,
+ bool nested = true)
+ : page_versions(v2) {
+ field.physical_type = tparquet::Type::INT32;
+ field.data_type = std::make_shared<DataTypeInt32>();
+ field.repetition_level = nested ? 1 : 0;
+ field.definition_level = nullable ? 1 : 0;
+ if (nullable) {
+ field.data_type = make_nullable(field.data_type);
+ }
+ field.parquet_schema.__set_type(tparquet::Type::INT32);
+ field.parquet_schema.__set_repetition_type(
+ nullable ? tparquet::FieldRepetitionType::OPTIONAL
+ : tparquet::FieldRepetitionType::REQUIRED);
+ const size_t row_count = std::accumulate(page_rows.begin(), page_rows.end(), size_t {0});
+ const size_t value_count = row_count * (repeated ? 2 : 1);
+ if (dictionary) {
+ std::vector<int32_t> dictionary_values(value_count);
+ std::iota(dictionary_values.begin(), dictionary_values.end(), 100);
+ std::vector<uint8_t> payload(value_count * sizeof(int32_t));
+ memcpy(payload.data(), dictionary_values.data(), payload.size());
+ tparquet::PageHeader header;
+ header.__set_type(tparquet::PageType::DICTIONARY_PAGE);
+ header.__set_compressed_page_size(payload.size());
+ header.__set_uncompressed_page_size(payload.size());
+ tparquet::DictionaryPageHeader data;
+ data.__set_num_values(value_count);
+ data.__set_encoding(tparquet::Encoding::PLAIN);
+ header.__set_dictionary_page_header(data);
+ const auto page = serialize_page(header, payload);
+ bytes.insert(bytes.end(), page.begin(), page.end());
+ chunk.meta_data.__set_dictionary_page_offset(1);
+ }
+ chunk.meta_data.__set_data_page_offset(bytes.size());
+ size_t value_index = 0;
+ for (size_t i = 0; i < v2.size(); ++i) {
+ std::vector<int32_t> values;
+ std::vector<uint32_t> levels;
+ std::vector<uint32_t> definitions;
+ for (size_t row = 0; row < page_rows[i]; ++row) {
+ row_values.emplace_back();
+ for (size_t element = 0; element < (repeated ? 2 : 1); ++element) {
+ levels.push_back(element == 0 ? 0 : 1);
+ const bool is_null = nullable && value_index % 3 == 1;
+ const int32_t value = 100 + value_index++;
+ definitions.push_back(is_null ? 0 : 1);
+ if (!is_null) {
+ values.push_back(value);
+ }
+ row_values.back().push_back(is_null ? std::numeric_limits<int32_t>::min()
+ : value);
+ }
+ }
+ faststring encoded_levels;
+ std::vector<uint8_t> payload;
+ if (nested) {
+ RleEncoder<uint32_t> level_encoder(&encoded_levels, 1);
+ for (const auto level : levels) {
+ level_encoder.Put(level);
+ }
+ for (size_t padding = levels.size(); padding % 8 != 0; ++padding) {
+ level_encoder.Put(0);
+ }
+ level_encoder.Flush();
+ if (!v2[i]) {
+ const uint32_t size = encoded_levels.size();
+ const auto* raw = reinterpret_cast<const uint8_t*>(&size);
+ payload.insert(payload.end(), raw, raw + sizeof(size));
+ }
+ payload.insert(payload.end(), encoded_levels.data(),
+ encoded_levels.data() + encoded_levels.size());
+ }
+ faststring encoded_definitions;
+ if (nullable) {
+ RleEncoder<uint32_t> def_encoder(&encoded_definitions, 1);
+ for (const auto level : definitions) {
+ def_encoder.Put(level);
+ }
+ for (size_t padding = definitions.size(); padding % 8 != 0; ++padding) {
+ def_encoder.Put(0);
+ }
+ def_encoder.Flush();
+ if (!v2[i]) {
+ const uint32_t size = encoded_definitions.size();
+ const auto* raw = reinterpret_cast<const uint8_t*>(&size);
+ payload.insert(payload.end(), raw, raw + sizeof(size));
+ }
+ payload.insert(payload.end(), encoded_definitions.data(),
+ encoded_definitions.data() + encoded_definitions.size());
+ }
+ if (dictionary) {
+ faststring ids;
+ RleEncoder<uint32_t> encoder(&ids, 8);
+ for (const auto value : values) {
+ encoder.Put(value - 100);
+ }
+ for (size_t padding = values.size(); padding % 8 != 0; ++padding) {
+ encoder.Put(0);
+ }
+ encoder.Flush();
+ payload.push_back(8);
+ payload.insert(payload.end(), ids.data(), ids.data() + ids.size());
+ } else {
+ const auto* raw = reinterpret_cast<const uint8_t*>(values.data());
+ payload.insert(payload.end(), raw, raw + values.size() * sizeof(int32_t));
+ }
+ const auto encoding =
+ dictionary ? tparquet::Encoding::RLE_DICTIONARY : tparquet::Encoding::PLAIN;
+ tparquet::PageHeader header;
+ header.__set_type(v2[i] ? tparquet::PageType::DATA_PAGE_V2
+ : tparquet::PageType::DATA_PAGE);
+ header.__set_compressed_page_size(payload.size());
+ header.__set_uncompressed_page_size(payload.size());
+ if (v2[i]) {
+ tparquet::DataPageHeaderV2 data;
+ data.__set_num_values(levels.size());
+ data.__set_num_rows(page_rows[i]);
+ data.__set_num_nulls(levels.size() - values.size());
+ data.__set_encoding(encoding);
+ data.__set_repetition_levels_byte_length(encoded_levels.size());
+ data.__set_definition_levels_byte_length(encoded_definitions.size());
+ data.__set_is_compressed(false);
+ header.__set_data_page_header_v2(data);
+ } else {
+ tparquet::DataPageHeader data;
+ data.__set_num_values(levels.size());
+ data.__set_encoding(encoding);
+ data.__set_repetition_level_encoding(tparquet::Encoding::RLE);
+ data.__set_definition_level_encoding(tparquet::Encoding::RLE);
+ header.__set_data_page_header(data);
+ }
+ pages.push_back(serialize_page(header, payload));
+ tparquet::PageLocation location;
+ location.__set_offset(bytes.size());
+ location.__set_compressed_page_size(pages.back().size());
+ location.__set_first_row_index(rows);
+ index.page_locations.push_back(location);
+ rows += page_rows[i];
+ bytes.insert(bytes.end(), pages.back().begin(), pages.back().end());
+ }
+ chunk.meta_data.__set_type(tparquet::Type::INT32);
+ chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED);
+ chunk.meta_data.__set_num_values(value_count);
+ chunk.meta_data.__set_total_compressed_size(bytes.size() - 1);
+ chunk.meta_data.__set_encodings(
+ dictionary
+ ? std::vector<tparquet::Encoding::type> {tparquet::Encoding::PLAIN,
+ tparquet::Encoding::RLE_DICTIONARY}
+ : std::vector<tparquet::Encoding::type> {tparquet::Encoding::PLAIN});
+ }
+
+ Status read(const RowRanges& ranges, size_t batch_size, bool indexed, bool cache,
+ std::vector<int32_t>* actual, bool levels_only = false, int flat_mode = 0,
+ int64_t* parsed_headers = nullptr) {
+ static size_t cache_id = 0;
+ const std::string cache_key = cache ? fmt::format("nested-navigation-{}", ++cache_id) : "";
+ if (cache) {
+ for (size_t i = 0; i < pages.size(); ++i) {
+ auto* page = new DataPage(pages[i].size(), true, segment_v2::DATA_PAGE);
+ memcpy(page->data(), pages[i].data(), pages[i].size());
+ page->reset_size(pages[i].size());
+ PageCacheHandle handle;
+ StoragePageCache::instance()->insert(
+ StoragePageCache::CacheKey(cache_key, bytes.size(),
+ index.page_locations[i].offset),
+ page, &handle, segment_v2::DATA_PAGE);
+ }
+ }
+ auto file = std::make_shared<NativeDecoderMemoryFileReader>(bytes);
+ auto read_with = [&]<bool NESTED, bool INDEXED>() -> Status {
+ ScalarColumnReader<NESTED, INDEXED> reader(
+ ranges, rows, chunk, indexed ? &index : nullptr, nullptr, nullptr);
+ RETURN_IF_ERROR(reader.init(file, &field, bytes.size(), nullptr, cache_key,
+ ParquetReaderCompat {}, true));
+ if constexpr (NESTED) {
+ reader.set_column_in_nested();
+ }
+ FilterMap filter;
+ RETURN_IF_ERROR(filter.init(nullptr, rows, false));
+ bool eof = false;
+ size_t calls = 0;
+ while (!eof) {
+ if (++calls > rows + pages.size() + 1) {
+ return Status::InternalError("Nested navigation did not reach EOF");
+ }
+ ColumnPtr output = field.data_type->create_column();
+ size_t read_rows = 0;
+ if (levels_only) {
+ RETURN_IF_ERROR(
+ reader.read_column_levels(filter, batch_size, &read_rows, &eof));
+ actual->insert(actual->end(), read_rows, 0);
+ } else if (flat_mode != 0) {
+ auto projected = ColumnInt32::create();
+ IColumn::Filter row_filter;
+ bool used = false;
+ if (flat_mode == 1) {
+ DirectPredicateExecutionKind kind = DirectPredicateExecutionKind::NONE;
+ RETURN_IF_ERROR(reader.read_fixed_width_filter(
+ {create_int32_raw_comparison(0, "gt", TExprOpcode::GT, 0)}, 0,
+ filter, batch_size, projected.get(), &row_filter, &read_rows, &eof,
+ &used, &kind));
+ } else {
+ auto dictionary = ColumnInt32::create();
+ for (int32_t i = 0; i < chunk.meta_data.num_values; ++i) {
+ dictionary->insert_value(100 + i);
+ }
+ size_t survivors = 0;
+ bool direct = false;
+ RETURN_IF_ERROR(reader.read_dictionary_filter(
+ IColumn::Filter(chunk.meta_data.num_values, 1), filter, batch_size,
+ dictionary.get(), projected.get(), nullptr, &row_filter, &survivors,
+ &read_rows, &eof, &direct, &used));
+ EXPECT_EQ(survivors, read_rows);
+ }
+ EXPECT_TRUE(used);
+ EXPECT_EQ(row_filter, IColumn::Filter(read_rows, 1));
+ actual->insert(actual->end(), projected->get_data().begin(),
+ projected->get_data().end());
+ } else {
+ RETURN_IF_ERROR(reader.read_column_data(output, field.data_type, nullptr,
+ filter, batch_size, &read_rows, &eof,
+ false));
+ if (field.data_type->is_nullable()) {
+ const auto& nullable_output = assert_cast<const ColumnNullable&>(*output);
+ const auto& values =
+ assert_cast<const ColumnInt32&>(nullable_output.get_nested_column())
+ .get_data();
+ for (size_t i = 0; i < values.size(); ++i) {
+ actual->push_back(nullable_output.is_null_at(i)
+ ? std::numeric_limits<int32_t>::min()
+ : values[i]);
+ }
+ } else {
+ const auto& values = assert_cast<const ColumnInt32&>(*output).get_data();
+ actual->insert(actual->end(), values.begin(), values.end());
+ }
+ }
+ }
+ if (parsed_headers != nullptr) {
+ *parsed_headers = reader.column_statistics().parse_page_header_num;
+ }
+ return Status::OK();
+ };
+ if (field.repetition_level != 0) {
+ return indexed ? read_with.template operator()<true, true>()
+ : read_with.template operator()<true, false>();
+ }
+ return indexed ? read_with.template operator()<false, true>()
+ : read_with.template operator()<false, false>();
+ }
+
+ bool rejects_fallback(const RowRanges& ranges, int stale_page) const {
+ if (stale_page <= 0 || !page_versions.front()) {
+ return false;
+ }
+ auto selected = [&](size_t row) {
+ for (size_t i = 0; i < ranges.range_size(); ++i) {
+ if (ranges.get_range_from(i) <= row && row < ranges.get_range_to(i)) {
+ return true;
+ }
+ }
+ return false;
+ };
+ auto page_selected = [&](int page) {
+ const size_t start = index.page_locations[page].first_row_index;
+ const size_t end = page + 1 == page_versions.size()
+ ? rows
+ : index.page_locations[page + 1].first_row_index;
+ for (size_t row = start; row < end; ++row) {
+ if (selected(row)) {
+ return true;
+ }
+ }
+ return false;
+ };
+ if (!page_selected(stale_page)) {
+ return false;
+ }
+ for (int page = 0; page < stale_page; ++page) {
+ if (page > 0 && !page_selected(page)) {
+ return true;
+ }
+ const size_t end = index.page_locations[page + 1].first_row_index;
+ if (!page_versions[page] && !selected(end - 1)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ std::vector<int32_t> expected(const RowRanges& ranges) const {
+ std::vector<int32_t> result;
+ for (size_t i = 0; i < ranges.range_size(); ++i) {
+ for (size_t row = ranges.get_range_from(i); row < ranges.get_range_to(i); ++row) {
+ result.insert(result.end(), row_values[row].begin(), row_values[row].end());
+ }
+ }
+ return result;
+ }
+};
+
+TEST(ParquetV2NativeDecoderTest, IncompleteOffsetIndexCannotRelabelMixedPageRows) {
+ for (const bool dictionary : {false, true}) {
+ NestedNavigationFile file({true, true, false}, {1, 1, 2}, true, dictionary);
+ file.index.page_locations.erase(file.index.page_locations.begin() + 1);
+ file.index.page_locations.back().first_row_index = 1;
+ const ColumnChunkRange range {.offset = 1, .length = file.bytes.size() - 1};
+ const bool usable = validate_offset_index(file.index, range,
+ file.chunk.meta_data.data_page_offset, file.rows);
+ EXPECT_FALSE(usable);
+ const auto selection = RowRanges::create_single(1, 2);
+ std::vector<int32_t> actual;
+ ASSERT_TRUE(file.read(selection, 1, usable, false, &actual).ok());
+ EXPECT_EQ(actual, file.expected(selection));
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, IndexGapsWithAuxiliaryPagesOrPaddingRemainReadable) {
+ for (const bool auxiliary : {false, true}) {
+ for (const bool dictionary : {false, true}) {
+ NestedNavigationFile file({true, false, true}, {1, 1, 1}, true, dictionary);
+ std::vector<uint8_t> extra(16, 0);
+ if (auxiliary) {
+ tparquet::PageHeader header;
+ header.__set_type(tparquet::PageType::INDEX_PAGE);
+ header.__set_compressed_page_size(extra.size());
+ header.__set_uncompressed_page_size(extra.size());
+ header.__set_index_page_header(tparquet::IndexPageHeader {});
+ extra = serialize_page(header, extra);
+ const size_t offset = file.index.page_locations[1].offset;
+ file.bytes.insert(file.bytes.begin() + offset, extra.begin(), extra.end());
+ for (size_t i = 1; i < file.index.page_locations.size(); ++i) {
+ file.index.page_locations[i].offset += extra.size();
+ }
+ } else {
+ file.bytes.insert(file.bytes.end(), extra.begin(), extra.end());
+ }
+ file.chunk.meta_data.total_compressed_size += extra.size();
+ const bool usable = validate_offset_index(
+ file.index, {.offset = 1, .length = file.bytes.size() - 1},
+ file.chunk.meta_data.data_page_offset, file.rows);
+ EXPECT_FALSE(usable);
+ const auto ranges = RowRanges::create_single(0, 3);
+ std::vector<int32_t> actual;
+ ASSERT_TRUE(file.read(ranges, 1, usable, false, &actual).ok());
+ EXPECT_EQ(actual, file.expected(ranges));
+ }
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, OffsetIndexRequiresCompleteDataByteCoverage) {
+ const ColumnChunkRange range {.offset = 100, .length = 100};
+ tparquet::OffsetIndex index;
+ for (int i = 0; i < 3; ++i) {
+ tparquet::PageLocation location;
+ location.__set_offset(110 + i * 30);
+ location.__set_compressed_page_size(30);
+ location.__set_first_row_index(i);
+ index.page_locations.push_back(location);
+ }
+ // The prefix belongs to a dictionary; the complete data-page suffix is still indexed.
+ ASSERT_TRUE(validate_offset_index(index, range, 110, 3));
+ for (size_t omitted = 0; omitted < 3; ++omitted) {
+ auto incomplete = index;
+ incomplete.page_locations.erase(incomplete.page_locations.begin() + omitted);
+ EXPECT_FALSE(validate_offset_index(incomplete, range, 110, 3)) << omitted;
+ }
+ auto gap = index;
+ --gap.page_locations[0].compressed_page_size;
+ EXPECT_FALSE(validate_offset_index(gap, range, 110, 3));
+ // Unclassified trailing bytes could be padding, an auxiliary page, or an omitted data page.
+ // Decline the optional index instead of interpreting those bytes as harmless padding.
+ EXPECT_FALSE(validate_offset_index(index, {.offset = 100, .length = 101}, 110, 3));
+}
+
+TEST(ParquetV2NativeDecoderTest, SparseIndexedReadsKeepUnselectedPagesLazy) {
+ for (int versions = 0; versions < 16; ++versions) {
+ for (const bool nested : {false, true}) {
+ for (const bool cache : {false, true}) {
+ for (int mode = 0; mode < 3; ++mode) {
+ SCOPED_TRACE(testing::Message()
+ << "versions=" << versions << ", nested=" << nested
+ << ", cache=" << cache << ", mode=" << mode);
+ NestedNavigationFile file({bool(versions & 1), bool(versions & 2),
+ bool(versions & 4), bool(versions & 8)},
+ {1, 3, 1, 2}, nested, mode == 2, nested, nested);
+ const auto ranges = RowRanges::create_single(5, 6);
+ std::vector<int32_t> actual;
+ const bool levels_only = nested && mode == 1;
+ int64_t headers = 0;
+ const auto status = file.read(ranges, 1, true, cache, &actual, levels_only,
+ nested ? 0 : mode, &headers);
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(actual,
+ levels_only ? std::vector<int32_t>(1, 0) : file.expected(ranges));
+ // A first nested V1 page retains the existing sequential policy. Otherwise
+ // only the first and selected data headers (plus a dictionary) are needed.
+ EXPECT_EQ(headers, (nested && !(versions & 1) ? 4 : 2) + (mode == 2));
+ }
+ }
+ }
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, PartialNestedPageRejectsUnsafeFallback) {
+ for (const bool cache : {false, true}) {
+ for (const bool levels_only : {false, true}) {
+ NestedNavigationFile file({true, false, false}, {1, 3, 2}, true);
+ --file.index.page_locations.back().compressed_page_size;
+ auto ranges = RowRanges::create_single(1, 2);
+ ranges.add({4, 5});
+ std::vector<int32_t> actual;
+ const auto status = file.read(ranges, 1, true, cache, &actual, levels_only);
+ EXPECT_TRUE(status.is<ErrorCode::CORRUPTION>()) << status;
+ EXPECT_EQ(actual, levels_only ? std::vector<int32_t>(1, 0) : file.row_values[1]);
+ }
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, FinalNestedFallbackPreservesColumnValueCount) {
+ for (const bool repeated : {false, true}) {
+ for (const bool cache : {false, true}) {
+ NestedNavigationFile file({true, false}, {1, 1}, repeated);
+ --file.index.page_locations.back().compressed_page_size;
+ const auto ranges = RowRanges::create_single(0, 2);
+ std::vector<int32_t> actual;
+ const auto status = file.read(ranges, 1, true, cache, &actual);
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(actual, file.expected(ranges));
+ }
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, NestedNavigationPreservesRowsAcrossPageVersionTransitions) {
+ for (int versions = 0; versions < 8; ++versions) {
+ for (const bool indexed : {false, true}) {
+ for (const bool dictionary : {false, true}) {
+ for (const bool nullable : {false, true}) {
+ for (const bool cache : {false, true}) {
+ for (const size_t batch : {1, 2, 7}) {
+ for (int selection = 0; selection < 3; ++selection) {
+ SCOPED_TRACE(testing::Message()
+ << "versions=" << versions << ", indexed=" << indexed
+ << ", dictionary=" << dictionary
+ << ", nullable=" << nullable << ", cache=" << cache
+ << ", batch=" << batch << ", selection=" << selection);
+ NestedNavigationFile file({bool(versions & 1), bool(versions & 2),
+ bool(versions & 4)},
+ {2, 3, 2}, true, dictionary, nullable);
+ auto ranges = RowRanges::create_single(0, 7);
+ if (selection == 1) {
+ ranges = RowRanges::create_single(1, 2);
+ ranges.add({3, 4});
+ ranges.add({6, 7});
+ } else if (selection == 2) {
+ ranges = RowRanges::create_single(6, 7);
+ }
+ for (const bool levels_only : {false, true}) {
+ std::vector<int32_t> actual;
+ const auto status = file.read(ranges, batch, indexed, cache,
+ &actual, levels_only);
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(actual,
+ levels_only ? std::vector<int32_t>(ranges.count(), 0)
+ : file.expected(ranges));
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, NestedNavigationFallbackMatrix) {
+ for (int versions = 0; versions < 8; ++versions) {
+ for (int stale_page = 0; stale_page < 3; ++stale_page) {
+ for (const bool cache : {false, true}) {
+ for (const bool dictionary : {false, true}) {
+ for (const size_t batch : {1, 2, 7}) {
+ for (int selection = 0; selection < 3; ++selection) {
+ SCOPED_TRACE(testing::Message()
+ << "versions=" << versions << ", stale=" << stale_page
+ << ", cache=" << cache << ", dictionary=" << dictionary
+ << ", batch=" << batch << ", selection=" << selection);
+ NestedNavigationFile file(
+ {bool(versions & 1), bool(versions & 2), bool(versions & 4)},
+ {2, 3, 2}, true, dictionary, true);
+ --file.index.page_locations[stale_page].compressed_page_size;
+ auto ranges = RowRanges::create_single(0, 7);
+ if (selection == 1) {
+ ranges = RowRanges::create_single(1, 2);
+ ranges.add({3, 4});
+ ranges.add({6, 7});
+ } else if (selection == 2) {
+ ranges = RowRanges::create_single(6, 7);
+ }
+ for (const bool levels_only : {false, true}) {
+ std::vector<int32_t> actual;
+ const auto status =
+ file.read(ranges, batch, true, cache, &actual, levels_only);
+ if (file.rejects_fallback(ranges, stale_page)) {
+ EXPECT_TRUE(status.is<ErrorCode::CORRUPTION>()) << status;
+ continue;
+ }
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(actual, levels_only
+ ? std::vector<int32_t>(ranges.count(), 0)
+ : file.expected(ranges));
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, NestedFallbackReevaluatesSelectedPageRange) {
+ NestedNavigationFile file({true, true, false}, {2, 1, 3}, true);
+ file.index.page_locations[2].first_row_index = 4;
+ --file.index.page_locations[1].compressed_page_size;
+ const auto ranges = RowRanges::create_single(3, 4);
+ std::vector<int32_t> actual;
+ const auto status = file.read(ranges, 1, true, false, &actual);
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(actual, file.expected(ranges));
+}
+
+TEST(ParquetV2NativeDecoderTest, NestedNavigationAllRowSelections) {
+ for (int versions = 0; versions < 8; ++versions) {
+ for (unsigned mask = 1; mask < 128; ++mask) {
+ RowRanges ranges;
+ for (int64_t row = 0; row < 7; ++row) {
+ if (mask & (1U << row)) {
+ ranges.add({row, row + 1});
+ }
+ }
+ // Every nonempty row subset is checked against physical values. Vary encoding, nulls
+ // and cache independently of the navigation policy, including partial and whole skips.
+ for (int index_mode = -2; index_mode < 3; ++index_mode) {
+ NestedNavigationFile file(
+ {bool(versions & 1), bool(versions & 2), bool(versions & 4)}, {2, 3, 2},
+ true, mask & 1, mask & 2);
+ if (index_mode >= 0) {
+ --file.index.page_locations[index_mode].compressed_page_size;
+ }
+ for (const size_t batch : {1, 2, 7}) {
+ for (const bool levels_only : {false, true}) {
+ SCOPED_TRACE(testing::Message()
+ << "versions=" << versions << ", mask=" << mask
+ << ", index_mode=" << index_mode << ", batch=" << batch
+ << ", levels_only=" << levels_only);
+ std::vector<int32_t> actual;
+ const auto status = file.read(ranges, batch, index_mode != -2, mask & 4,
+ &actual, levels_only);
+ if (file.rejects_fallback(ranges, index_mode)) {
+ EXPECT_TRUE(status.is<ErrorCode::CORRUPTION>()) << status;
+ continue;
+ }
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(actual, levels_only ? std::vector<int32_t>(ranges.count(), 0)
+ : file.expected(ranges));
+ }
+ }
+ }
+ }
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, IndexedNestedFallbackRequiresCompleteV1RowSpan) {
+ for (const bool middle_v2 : {false, true}) {
+ for (const bool partial : {false, true}) {
+ NestedNavigationFile file({true, middle_v2, false}, {1, 3, 2}, true);
+ --file.index.page_locations.back().compressed_page_size;
+ MemoryBufferedReader stream(file.bytes);
+ ColumnChunkReader<true, true> reader(&stream, &file.chunk, &file.field, &file.index,
+ file.rows, nullptr, ParquetPageReadContext());
+ ASSERT_TRUE(reader.init().ok());
+ ASSERT_TRUE(reader.parse_page_header().ok());
+ ASSERT_TRUE(reader.next_page().ok());
+ ASSERT_TRUE(reader.parse_page_header().ok());
+ ASSERT_TRUE(reader.parse_page_header().ok());
+ ASSERT_TRUE(reader.load_page_data().ok());
+ std::vector<level_t> levels;
+ size_t rows = 0;
+ bool cross_page = false;
+ ASSERT_TRUE(
+ reader.load_page_nested_rows(levels, partial ? 1 : 3, &rows, &cross_page).ok());
+ ASSERT_TRUE(reader.next_page().ok());
+ const auto status = reader.parse_page_header();
+ if (partial && !middle_v2) {
+ EXPECT_TRUE(status.is<ErrorCode::CORRUPTION>()) << status;
+ } else {
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(reader.page_start_row(), 4);
+ EXPECT_EQ(reader._chunk_parsed_values, file.chunk.meta_data.num_values);
+ }
+ }
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, IndexedNestedNavigationSkipsHeadersAndPayloads) {
+ for (const bool middle_v2 : {false, true}) {
+ for (const size_t page_count : {3, 64}) {
+ std::vector<bool> versions(page_count, middle_v2);
+ versions.front() = true;
+ versions.back() = false;
+ NestedNavigationFile file(versions, std::vector<size_t>(page_count, 3), true);
+ MemoryBufferedReader stream(file.bytes);
+ ColumnChunkReader<true, true> reader(&stream, &file.chunk, &file.field, &file.index,
+ file.rows, nullptr, ParquetPageReadContext());
+ ASSERT_TRUE(reader.init().ok());
+ ASSERT_TRUE(reader.seek_to_nested_row(file.rows - 1).ok());
+ const auto& statistics = reader.chunk_statistics();
+ EXPECT_EQ(statistics.parse_page_header_num, 2);
+ EXPECT_EQ(statistics.skip_page_header_num, page_count - 2);
+ // Only the first and selected headers and the selected payload need reading, regardless
+ // of the skipped page count. Skipping V1 pages must not load their repetition levels.
+ EXPECT_EQ(stream.read_count(), 3);
+ }
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, IndexedNestedPartialPageAdvanceDoesNotDrainLevels) {
+ for (const bool partial : {false, true}) {
+ NestedNavigationFile file({true, false, true}, {1, 3, 2}, true);
+ MemoryBufferedReader stream(file.bytes);
+ ColumnChunkReader<true, true> reader(&stream, &file.chunk, &file.field, &file.index,
+ file.rows, nullptr, ParquetPageReadContext());
+ ASSERT_TRUE(reader.init().ok());
+ ASSERT_TRUE(reader.parse_page_header().ok());
+ ASSERT_TRUE(reader.next_page().ok());
+ ASSERT_TRUE(reader.parse_page_header().ok());
+ if (partial) {
+ ASSERT_TRUE(reader.load_page_data().ok());
+ std::vector<level_t> levels;
+ size_t rows = 0;
+ bool cross_page = false;
+ ASSERT_TRUE(reader.load_page_nested_rows(levels, 1, &rows, &cross_page).ok());
+ ASSERT_EQ(rows, 1);
+ }
+ const auto reads = stream.read_count();
+ const auto remaining = reader._remaining_rep_nums;
+ ASSERT_GT(remaining, 0);
+ ASSERT_TRUE(reader.next_page().ok());
+ EXPECT_EQ(stream.read_count(), reads);
+ EXPECT_EQ(reader._remaining_rep_nums, remaining);
+ EXPECT_EQ(reader.page_start_row(), 4);
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, IndexedNestedSeekReadsMixedPageVersions) {
+ for (const bool bad_span : {false, true}) {
+ for (const bool with_index : {false, true}) {
+ for (const bool continuation : {false, true}) {
+ if ((!with_index && (continuation || bad_span)) || (continuation && bad_span)) {
+ continue;
+ }
+ SCOPED_TRACE(testing::Message() << "index=" << with_index << ", continuation="
+ << continuation << ", bad_span=" << bad_span);
+ std::vector<uint8_t> bytes;
+ tparquet::OffsetIndex index;
+ for (int32_t i = 0; i < 3; ++i) {
+ const bool v1 = i == 2;
+ std::vector<uint8_t> payload;
+ if (v1) {
+ payload = {2, 0, 0, 0};
+ }
+ payload.push_back(2);
+ payload.push_back(v1 && continuation ? 1 : 0);
+ const int32_t value = (i + 1) * 10;
+ const auto* raw = reinterpret_cast<const uint8_t*>(&value);
+ payload.insert(payload.end(), raw, raw + sizeof(value));
+ tparquet::PageHeader header;
+ header.__set_type(v1 ? tparquet::PageType::DATA_PAGE
+ : tparquet::PageType::DATA_PAGE_V2);
+ header.__set_compressed_page_size(payload.size());
+ header.__set_uncompressed_page_size(payload.size());
+ if (v1) {
+ tparquet::DataPageHeader data;
+ data.__set_num_values(1);
+ data.__set_encoding(tparquet::Encoding::PLAIN);
+ data.__set_repetition_level_encoding(tparquet::Encoding::RLE);
+ data.__set_definition_level_encoding(tparquet::Encoding::RLE);
+ header.__set_data_page_header(data);
+ } else {
+ tparquet::DataPageHeaderV2 data;
+ data.__set_num_values(1);
+ data.__set_num_rows(1);
+ data.__set_num_nulls(0);
+ data.__set_encoding(tparquet::Encoding::PLAIN);
+ data.__set_repetition_levels_byte_length(2);
+ data.__set_definition_levels_byte_length(0);
+ data.__set_is_compressed(false);
+ header.__set_data_page_header_v2(data);
+ }
+ const auto page = serialize_page(header, payload);
+ tparquet::PageLocation location;
+ location.__set_offset(bytes.size());
+ location.__set_compressed_page_size(page.size());
+ location.__set_first_row_index(i);
+ index.page_locations.push_back(location);
+ bytes.insert(bytes.end(), page.begin(), page.end());
+ }
+ tparquet::ColumnChunk chunk;
+ chunk.meta_data.__set_type(tparquet::Type::INT32);
+ chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED);
+ chunk.meta_data.__set_num_values(3);
+ chunk.meta_data.__set_total_compressed_size(bytes.size());
+ chunk.meta_data.__set_data_page_offset(0);
+ NativeFieldSchema field;
+ field.physical_type = tparquet::Type::INT32;
+ field.repetition_level = 1;
+ MemoryBufferedReader stream(bytes);
+ ColumnChunkReader<true, true> reader(
+ &stream, &chunk, &field, with_index ? &index : nullptr, bad_span ? 4 : 3,
+ nullptr, ParquetPageReadContext(false, ""));
+ ASSERT_TRUE(reader.init().ok());
+ auto status = reader.seek_to_nested_row(2);
+ if (continuation) {
+ // An indexed page must start a row; accepting a continuation would relabel it.
+ EXPECT_TRUE(status.is<ErrorCode::CORRUPTION>()) << status;
+ continue;
+ }
+ ASSERT_TRUE(status.ok()) << status;
+ std::vector<level_t> reps;
+ size_t rows = 0;
+ bool cross_page = false;
+ status = reader.load_page_nested_rows(reps, 1, &rows, &cross_page);
+ if (bad_span) {
+ EXPECT_TRUE(status.is<ErrorCode::CORRUPTION>()) << status;
+ continue;
+ }
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(rows, 1);
+ EXPECT_EQ(reps, (std::vector<level_t> {0}));
+ EXPECT_FALSE(cross_page);
+ EXPECT_FALSE(reader.has_next_page());
+ std::vector<level_t> defs;
+ ASSERT_TRUE(reader.fill_def(defs).ok());
+ DataTypeInt32 type;
+ auto output = type.create_column();
+ ParquetDecodeContext context;
+ context.physical_type = ParquetPhysicalType::INT32;
+ ParquetMaterializationState state;
+ FilterMap filter;
+ ASSERT_TRUE(filter.init(nullptr, 1, false).ok());
+ ColumnSelectVector selection;
+ const std::vector<uint16_t> null_runs {1};
+ ASSERT_TRUE(selection.init(null_runs, 1, nullptr, &filter, 0).ok());
+ ASSERT_TRUE(reader.materialize_values(output, *type.get_serde(), context, state,
+ selection)
+ .ok());
+ EXPECT_EQ(assert_cast<const ColumnInt32&>(*output).get_data(),
+ (ColumnInt32::Container {30}));
+ }
+ }
+ }
+}
+
+TEST(ParquetV2NativeDecoderTest, LateOffsetIndexFallbackRequiresVerifiedRowCoordinates) {
+ auto check = []<bool NESTED>(bool skip_middle, bool stale_size, bool cache_hit) {
+ SCOPED_TRACE(testing::Message() << "nested=" << NESTED << ", skip=" << skip_middle
+ << ", stale=" << stale_size << ", cache=" << cache_hit);
+ const bool reject = skip_middle && stale_size;
+ const std::vector<std::vector<int32_t>> values {{10}, {20}, {30, 31}};
+ std::vector<uint8_t> bytes;
+ std::vector<std::vector<uint8_t>> pages;
+ tparquet::OffsetIndex index;
+ for (size_t page = 0; page < values.size(); ++page) {
+ std::vector<uint8_t> payload;
+ if constexpr (NESTED) {
+ payload = {cast_set<uint8_t>(values[page].size() * 2), 0};
+ }
+ const size_t level_bytes = payload.size();
+ const auto* raw = reinterpret_cast<const uint8_t*>(values[page].data());
+ payload.insert(payload.end(), raw, raw + values[page].size() * sizeof(int32_t));
+ tparquet::PageHeader header;
+ header.__set_type(tparquet::PageType::DATA_PAGE_V2);
+ header.__set_compressed_page_size(payload.size());
+ header.__set_uncompressed_page_size(payload.size());
+ header.__isset.data_page_header_v2 = true;
+ auto& data = header.data_page_header_v2;
+ data.__set_num_values(values[page].size());
+ data.__set_num_rows(values[page].size());
+ data.__set_num_nulls(0);
+ data.__set_encoding(tparquet::Encoding::PLAIN);
+ data.__set_repetition_levels_byte_length(level_bytes);
+ data.__set_definition_levels_byte_length(0);
+ data.__set_is_compressed(false);
+ pages.push_back(serialize_page(header, payload));
+ tparquet::PageLocation location;
+ location.__set_offset(bytes.size());
+ location.__set_compressed_page_size(pages.back().size() -
+ (stale_size && page == 2 ? 1 : 0));
+ // A skipped middle page hides the extra indexed row; fallback must not label physical
+ // row 2 as row 3 merely because the last page's own header is internally consistent.
+ location.__set_first_row_index(page == 2 && reject ? 3 : page);
+ index.page_locations.push_back(location);
+ bytes.insert(bytes.end(), pages.back().begin(), pages.back().end());
+ }
+ // Production loaders discard the short-size index early. Inject it directly here to
+ // retain coverage of the reader's defensive header reconciliation as well.
+ EXPECT_EQ(validate_offset_index(index, {.offset = 0, .length = bytes.size()}, 0, 4),
+ !stale_size);
+ const std::string cache_key =
+ fmt::format("late-index-{}-{}-{}", NESTED, skip_middle, stale_size);
+ if (cache_hit) {
+ auto* cached = new DataPage(pages.back().size(), true, segment_v2::DATA_PAGE);
+ memcpy(cached->data(), pages.back().data(), pages.back().size());
+ cached->reset_size(pages.back().size());
+ PageCacheHandle handle;
+ StoragePageCache::instance()->insert(
+ StoragePageCache::CacheKey(cache_key, bytes.size(),
+ index.page_locations.back().offset),
+ cached, &handle, segment_v2::DATA_PAGE);
+ }
+ tparquet::ColumnChunk chunk;
+ chunk.meta_data.__set_type(tparquet::Type::INT32);
+ chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED);
+ chunk.meta_data.__set_num_values(4);
+ chunk.meta_data.__set_total_compressed_size(bytes.size());
+ chunk.meta_data.__set_data_page_offset(0);
+ chunk.meta_data.__set_encodings({tparquet::Encoding::PLAIN});
+ NativeFieldSchema field;
+ field.physical_type = tparquet::Type::INT32;
+ field.data_type = std::make_shared<DataTypeInt32>();
+ field.repetition_level = NESTED ? 1 : 0;
+ field.parquet_schema.__set_type(tparquet::Type::INT32);
+ field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED);
+ const ParquetPageReadContext context(cache_hit, cache_key);
+ Status status;
+ if constexpr (NESTED) {
+ MemoryBufferedReader stream(bytes);
+ ColumnChunkReader<true, true> reader(&stream, &chunk, &field, &index, 4, nullptr,
+ context);
+ ASSERT_TRUE(reader.init().ok());
+ if (!skip_middle) {
+ ASSERT_TRUE(reader.seek_to_nested_row(1).ok());
+ }
+ status = reader.seek_to_nested_row(3);
+ EXPECT_EQ(reader.chunk_statistics().page_cache_hit_counter > 0, cache_hit);
+ } else {
+ auto ranges = ::doris::RowRanges::create_single(skip_middle ? 3 : 0, 4);
+ auto file = std::make_shared<NativeDecoderMemoryFileReader>(bytes);
+ ScalarColumnReader<false, true> reader(ranges, 4, chunk, &index, nullptr, nullptr);
+ ASSERT_TRUE(reader.init(file, &field, bytes.size(), nullptr, cache_hit ? cache_key : "",
+ ParquetReaderCompat {}, true)
+ .ok());
+ FilterMap filter;
+ ASSERT_TRUE(filter.init(nullptr, ranges.count(), false).ok());
+ ColumnPtr output = ColumnInt32::create();
+ size_t consumed = 0;
+ size_t calls = 0;
+ bool eof = false;
+ while (consumed < ranges.count() && !eof) {
+ ASSERT_LE(++calls, pages.size());
+ size_t rows = 0;
+ status = reader.read_column_data(output, field.data_type, nullptr, filter,
+ ranges.count() - consumed, &rows, &eof, false);
+ if (!status.ok()) {
+ break;
+ }
+ consumed += rows;
+ }
+ EXPECT_EQ(reader.column_statistics().page_cache_hit_counter > 0, cache_hit);
+ if (!reject) {
+ EXPECT_EQ(assert_cast<const ColumnInt32&>(*output).get_data(),
+ (skip_middle ? ColumnInt32::Container {31}
+ : ColumnInt32::Container {10, 20, 30, 31}));
+ } else {
+ EXPECT_EQ(output->size(), 0);
+ }
+ }
+ if (reject) {
+ EXPECT_TRUE(status.is<ErrorCode::CORRUPTION>()) << status;
+ } else {
+ EXPECT_TRUE(status.ok()) << status;
+ }
+ };
+ for (const bool skip_middle : {false, true}) {
+ for (const bool stale_size : {false, true}) {
+ for (const bool cache_hit : {false, true}) {
+ check.template operator()<false>(skip_middle, stale_size, cache_hit);
+ check.template operator()<true>(skip_middle, stale_size, cache_hit);
+ }
+ }
+ }
+}
+
TEST(ParquetV2NativeDecoderTest, FlatPagesRejectLogicalAndPhysicalCardinalityMismatch) {
auto init_chunk = [](tparquet::PageHeader header, bool with_offset_index) {
std::vector<uint8_t> payload(static_cast<size_t>(header.compressed_page_size), 0);
@@ -4492,6 +5638,8 @@ TEST(ParquetV2NativeDecoderTest, OffsetIndexValidationRejectsBackwardAndOverlapp
EXPECT_FALSE(validate_offset_index(index, range, 110, 20));
second.__set_first_row_index(10);
+ second.__set_offset(130);
+ second.__set_compressed_page_size(70);
index.page_locations = {first, second};
EXPECT_TRUE(validate_offset_index(index, range, 110, 20));
diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp
index c045683e424..d8934a4f690 100644
--- a/be/test/format_v2/parquet/parquet_scan_test.cpp
+++ b/be/test/format_v2/parquet/parquet_scan_test.cpp
@@ -1859,6 +1859,114 @@ void write_misdeclared_two_page_parquet_file(const std::string& file_path) {
DORIS_CHECK(output.good());
}
+void shorten_first_offset_index_page(const std::string& file_path) {
+ std::ifstream input(file_path, std::ios::binary | std::ios::ate);
+ ASSERT_TRUE(input.good());
+ const auto input_size = static_cast<std::streamoff>(input.tellg());
+ ASSERT_GE(input_size, 8);
+ std::vector<uint8_t> bytes(cast_set<size_t>(input_size));
+ input.seekg(0);
+ input.read(reinterpret_cast<char*>(bytes.data()), bytes.size());
+ ASSERT_TRUE(input.good());
+ input.close();
+ const uint32_t footer_size = decode_fixed32_le(bytes.data() + bytes.size() - 8);
+ const size_t footer_offset = bytes.size() - 8 - footer_size;
+ uint32_t thrift_size = footer_size;
+ tparquet::FileMetaData metadata;
+ ASSERT_TRUE(deserialize_thrift_msg(bytes.data() + footer_offset, &thrift_size, true, &metadata)
+ .ok());
+ auto& column = metadata.row_groups[0].columns[0];
+ ASSERT_TRUE(column.__isset.offset_index_offset);
+ tparquet::OffsetIndex index;
+ thrift_size = column.offset_index_length;
+ ASSERT_TRUE(deserialize_thrift_msg(bytes.data() + column.offset_index_offset, &thrift_size,
+ true, &index)
+ .ok());
+ ASSERT_GE(index.page_locations.size(), 3);
+ // Keep the Arrow-written data intact; only the optional index's first page size is stale.
+ --index.page_locations[0].compressed_page_size;
+ std::vector<uint8_t> serialized_index;
+ ThriftSerializer serializer(true, 1024);
+ ASSERT_TRUE(serializer.serialize(&index, &serialized_index).ok());
+ bytes.resize(footer_offset);
+ column.__set_offset_index_offset(bytes.size());
+ column.__set_offset_index_length(serialized_index.size());
+ bytes.insert(bytes.end(), serialized_index.begin(), serialized_index.end());
+ std::vector<uint8_t> footer;
+ ASSERT_TRUE(serializer.serialize(&metadata, &footer).ok());
+ bytes.insert(bytes.end(), footer.begin(), footer.end());
+ std::array<uint8_t, 4> length {};
+ encode_fixed32_le(length.data(), cast_set<uint32_t>(footer.size()));
+ bytes.insert(bytes.end(), length.begin(), length.end());
+ bytes.insert(bytes.end(), {'P', 'A', 'R', '1'});
+ std::ofstream output(file_path, std::ios::binary | std::ios::trunc);
+ output.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+ ASSERT_TRUE(output.good());
+}
+
+void omit_page_from_indexes(const std::string& file_path, int omitted, int32_t* matching_value) {
+ std::ifstream input(file_path, std::ios::binary | std::ios::ate);
+ ASSERT_TRUE(input.good());
+ std::vector<uint8_t> bytes(cast_set<size_t>(static_cast<std::streamoff>(input.tellg())));
+ input.seekg(0);
+ input.read(reinterpret_cast<char*>(bytes.data()), bytes.size());
+ ASSERT_TRUE(input.good());
+ input.close();
+ ASSERT_GE(bytes.size(), 8);
+ const uint32_t footer_size = decode_fixed32_le(bytes.data() + bytes.size() - 8);
+ const size_t footer_offset = bytes.size() - 8 - footer_size;
+ uint32_t length = footer_size;
+ tparquet::FileMetaData metadata;
+ ASSERT_TRUE(
+ deserialize_thrift_msg(bytes.data() + footer_offset, &length, true, &metadata).ok());
+ auto& column = metadata.row_groups[0].columns[0];
+ tparquet::OffsetIndex offsets;
+ length = column.offset_index_length;
+ ASSERT_TRUE(deserialize_thrift_msg(bytes.data() + column.offset_index_offset, &length, true,
+ &offsets)
+ .ok());
+ tparquet::ColumnIndex statistics;
+ length = column.column_index_length;
+ ASSERT_TRUE(deserialize_thrift_msg(bytes.data() + column.column_index_offset, &length, true,
+ &statistics)
+ .ok());
+ ASSERT_GE(offsets.page_locations.size(), 3);
+ const size_t page = omitted < 0 ? offsets.page_locations.size() - 1 : omitted;
+ ASSERT_LT(page, offsets.page_locations.size());
+ ASSERT_EQ(statistics.min_values[page].size(), sizeof(int32_t));
+ *matching_value = cast_set<int32_t>(decode_fixed32_le(
+ reinterpret_cast<const uint8_t*>(statistics.min_values[page].data())));
+ // Keep the physical page containing the sole predicate match intact, but omit it from both
+ // indexes. Equal index cardinalities must not make an incomplete page list safe for pruning.
+ offsets.page_locations.erase(offsets.page_locations.begin() + page);
+ statistics.null_pages.erase(statistics.null_pages.begin() + page);
+ statistics.min_values.erase(statistics.min_values.begin() + page);
+ statistics.max_values.erase(statistics.max_values.begin() + page);
+ if (statistics.__isset.null_counts) {
+ statistics.null_counts.erase(statistics.null_counts.begin() + page);
+ }
+ bytes.resize(footer_offset);
+ ThriftSerializer serializer(true, 1024);
+ std::vector<uint8_t> serialized;
+ ASSERT_TRUE(serializer.serialize(&offsets, &serialized).ok());
+ column.__set_offset_index_offset(bytes.size());
+ column.__set_offset_index_length(serialized.size());
+ bytes.insert(bytes.end(), serialized.begin(), serialized.end());
+ ASSERT_TRUE(serializer.serialize(&statistics, &serialized).ok());
+ column.__set_column_index_offset(bytes.size());
+ column.__set_column_index_length(serialized.size());
+ bytes.insert(bytes.end(), serialized.begin(), serialized.end());
+ ASSERT_TRUE(serializer.serialize(&metadata, &serialized).ok());
+ bytes.insert(bytes.end(), serialized.begin(), serialized.end());
+ std::array<uint8_t, 4> footer_length {};
+ encode_fixed32_le(footer_length.data(), cast_set<uint32_t>(serialized.size()));
+ bytes.insert(bytes.end(), footer_length.begin(), footer_length.end());
+ bytes.insert(bytes.end(), {'P', 'A', 'R', '1'});
+ std::ofstream output(file_path, std::ios::binary | std::ios::trunc);
+ output.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+ ASSERT_TRUE(output.good());
+}
+
void write_long_prefix_parquet_file(const std::string& file_path, size_t rows) {
std::vector<int32_t> ids(rows);
std::vector<int32_t> first(rows);
@@ -4623,6 +4731,124 @@ TEST_F(ParquetScanTest, PredicateOnlyUint32UsesConvertedDecoderDirectPath) {
EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0);
}
+TEST_F(ParquetScanTest, IncompletePageIndexesRetainTheOnlyPredicateMatch) {
+ const bool original = config::enable_parquet_page_index;
+ DEFER({ config::enable_parquet_page_index = original; });
+ for (const bool dictionary : {false, true}) {
+ for (const int omitted : {0, 1, -1}) {
+ std::vector<int32_t> ids(128);
+ std::iota(ids.begin(), ids.end(), 0);
+ auto table = arrow::Table::Make(
+ arrow::schema({arrow::field("value", arrow::int32(), false)}),
+ {build_int32_array(ids)});
+ write_table(_file_path, table, ids.size(), dictionary, true);
+ int32_t expected = -1;
+ omit_page_from_indexes(_file_path, omitted, &expected);
+ for (const bool enabled : {false, true}) {
+ SCOPED_TRACE(testing::Message() << "dictionary=" << dictionary << ", omitted="
+ << omitted << ", index=" << enabled);
+ config::enable_parquet_page_index = enabled;
+ RuntimeProfile profile("incomplete_index");
+ auto reader = create_reader(0, -1, &profile);
+ reader->set_batch_size(1);
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ format::FileScanRequestBuilder builder(request.get());
+ ASSERT_TRUE(builder.add_predicate_column(format::LocalColumnId(0)).ok());
+ auto conjunct =
+ create_int32_function_conjunct(0, "eq", TExprOpcode::EQ, expected, false);
+ ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor()).ok());
+ ASSERT_TRUE(conjunct->open(&state).ok());
+ DEFER({ conjunct->close(); });
+ request->conjuncts.push_back(conjunct);
+ ASSERT_TRUE(reader->open(request).ok());
+ std::vector<int32_t> actual;
+ bool eof = false;
+ while (!eof) {
+ Block block = build_file_block(schema);
+ size_t rows = 0;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ for (size_t row = 0; row < rows; ++row) {
+ actual.push_back(int32_data_column(*block.get_by_position(0).column)
+ .get_element(row));
+ }
+ }
+ EXPECT_EQ(actual, std::vector<int32_t> {expected});
+ EXPECT_EQ(counter_value(profile, "FilteredRowsByPage"), 0);
+ EXPECT_EQ(counter_value(profile, "PageIndexReadCalls"), 0);
+ }
+ }
+ }
+}
+
+TEST_F(ParquetScanTest, OffsetIndexFallbackPreservesFilteredRowsAcrossPages) {
+ std::vector<int32_t> values(24);
+ std::iota(values.begin(), values.end(), 0);
+ std::vector<int32_t> payloads(24);
+ std::iota(payloads.begin(), payloads.end(), 100);
+ auto arrow_schema = arrow::schema({arrow::field("value", arrow::int32(), false),
+ arrow::field("payload", arrow::int32(), false)});
+ auto table = arrow::Table::Make(arrow_schema,
+ {build_int32_array(values), build_int32_array(payloads)});
+ const bool original_page_index = config::enable_parquet_page_index;
+ DEFER({ config::enable_parquet_page_index = original_page_index; });
+ for (const bool dictionary : {false, true}) {
+ write_table(_file_path, table, 24, dictionary, true);
+ shorten_first_offset_index_page(_file_path);
+ for (const bool enable_index : {false, true}) {
+ for (const int batch_size : {5, 64}) {
+ SCOPED_TRACE(testing::Message() << "dictionary=" << dictionary << ", index="
+ << enable_index << ", batch=" << batch_size);
+ config::enable_parquet_page_index = enable_index;
+ RuntimeProfile profile("index_fallback");
+ auto reader = create_reader(0, -1, &profile);
+ reader->set_batch_size(batch_size);
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ format::FileScanRequestBuilder builder(request.get());
+ ASSERT_TRUE(builder.add_predicate_column(format::LocalColumnId(0)).ok());
+ ASSERT_TRUE(builder.add_non_predicate_column(format::LocalColumnId(1)).ok());
+ auto conjunct = create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 0, false);
+ ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor()).ok());
+ ASSERT_TRUE(conjunct->open(&state).ok());
+ DEFER({ conjunct->close(); });
+ ASSERT_TRUE(conjunct->root()->can_evaluate_zonemap_filter());
+ request->conjuncts.push_back(conjunct);
+ ASSERT_TRUE(reader->open(request).ok());
+ std::vector<int32_t> actual_values;
+ std::vector<int32_t> actual_payloads;
+ bool eof = false;
+ while (!eof) {
+ Block block = build_file_block(schema);
+ size_t rows = 0;
+ const auto status = reader->get_block(&block, &rows, &eof);
+ ASSERT_TRUE(status.ok()) << status;
+ for (size_t row = 0; row < rows; ++row) {
+ actual_values.push_back(int32_data_column(*block.get_by_position(0).column)
+ .get_element(row));
+ actual_payloads.push_back(
+ int32_data_column(*block.get_by_position(1).column)
+ .get_element(row));
+ }
+ }
+ EXPECT_EQ(actual_values, (std::vector<int32_t>(values.begin() + 1, values.end())));
+ EXPECT_EQ(actual_payloads,
+ (std::vector<int32_t>(payloads.begin() + 1, payloads.end())));
+ EXPECT_GT(counter_value(profile, dictionary ? "DictionaryPredicateDirectBatches"
+ : "FixedWidthPredicateDirectBatches"),
+ 0);
+ EXPECT_EQ(counter_value(profile, "PageIndexReadCalls") > 0, enable_index);
+ }
+ }
+ }
+}
+
TEST_F(ParquetScanTest, FixedWidthPredicateReportsUnsupportedEncodingAfterProgress) {
write_misdeclared_two_page_parquet_file(_file_path);
RuntimeProfile profile("profile");
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 26/28: branch-4.1: [refactor](lance) Remove the durable Lance index job framework (#66497) (#68781)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8b59ed1ec22ceb87a1d559d5bf1230b3eb253495
Author: kid <lu...@outlook.com>
AuthorDate: Sat Oct 10 09:34:21 2026 +0800
branch-4.1: [refactor](lance) Remove the durable Lance index job framework (#66497) (#68781)
Per the updated direction in [#66497
(comment)](https://github.com/apache/doris/issues/66497#issuecomment-6053021847),
Lance index mutations move to synchronous SQL with bounded worker
execution. A durable asynchronous job framework is no longer part of the
initial implementation, so this change reverts it explicitly rather than
leaving it disabled or maintaining two execution paths.
### Removed
- the durable Lance index job state machine, manager, same-name fences,
unresolved-job quotas, and replay infrastructure introduced by #67235
- the job-specific parts of #67630: durable job creation and JobId
results, `SHOW LANCE INDEX JOB(S)` with its grammar and plan wiring, and
the catalog DDL guards tied to unresolved jobs
- the Flight SQL JobId result-schema special case for Lance index DDL,
so Flight advertises the same generic DDL status schema the statement
now returns
- the three `lance_index_job_max_unresolved_*` quota configuration keys,
the job-specific tests, the now-unreferenced job-locator resolver pair,
and the orphaned dataset-locator helper
### Preserved
- read-only `SHOW INDEX` (#66637) and the `lance_index_entries`
inspection TVF (#66671)
- the statically validated DDL surface (#67201);
`enable_lance_index_mutation` remains the mutation gate
- the privilege, schema, parameter, snapshot, and IF-condition
validation from #67630, rewired as pure admission preflight; a mutation
that passes the whole preflight now ends in the shared not-supported
rejection until the synchronous implementation lands
- the schema-contract and name-normalization helpers, relocated out of
the removed `job` package next to their remaining callers; the
replay-path target guard keeps slimmed direct coverage
### Journal and image compatibility
`enable_lance_index_mutation` has defaulted to `false` since #67630
merged, and no released build exposes it enabled, so no deployment can
hold persisted job records unless someone manually enabled an
undocumented, gated feature on an unreleased build. The edit-log
operation `OP_LANCE_INDEX_JOB_UPSERT` (= 500) keeps its number as a
tombstone and is never reused; its old Text-delimited JSON payload is
consumed and ignored by a minimal discard branch in the journal reader,
so edit logs from such a build stay readable without restoring any
framework code. The image chunk the old manager wrote is skipped through
the existing deprecated-module list, so images from previous builds
still load.
### Evidence
Focused FE suites (`mvn test -pl fe-common,fe-core -am
-Dtest='*Lance*,DorisFlightSqlSchemaTest'`): 453 tests run, 0 failures,
0 errors, 6 skipped (known environment skips), checkstyle included,
BUILD SUCCESS — the Flight schema suite is now in the filter (33 tests
green, with the Lance DDL cases rewritten against the generic
`StatusResult` schema) and a slimmed direct test covers the retained
replay-path target guard (`hasLanceIdentityKeyChange` and its
admission-epoch advance). Net change: 62 files, +400/−8922.
Co-authored-by: u70b3 <u7...@users.noreply.github.com>
---
.../main/java/org/apache/doris/common/Config.java | 31 +-
.../doris/common/LanceIndexConfigValidator.java | 31 +-
.../common/LanceIndexConfigValidatorTest.java | 65 +-
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 -
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 3 -
.../main/java/org/apache/doris/catalog/Env.java | 25 -
.../java/org/apache/doris/common/ErrorCode.java | 3 +-
.../org/apache/doris/datasource/CatalogMgr.java | 48 +-
.../doris/datasource/lance/LanceCatalogClient.java | 6 -
.../datasource/lance/LanceExternalCatalog.java | 26 -
.../datasource/lance/LanceIndexAdmission.java | 221 +-----
.../doris/datasource/lance/LanceIndexFamilies.java | 1 -
.../lance/LanceIndexMutationValidator.java | 9 +-
.../lance/{job => }/LanceIndexNameNormalizer.java | 16 +-
.../lance/{job => }/LanceIndexSchemaContract.java | 2 +-
.../lance/LanceSchemaContractBuilder.java | 1 -
.../lance/job/LanceIndexDatasetLocator.java | 143 ----
.../datasource/lance/job/LanceIndexFenceKey.java | 92 ---
.../doris/datasource/lance/job/LanceIndexJob.java | 699 ------------------
.../lance/job/LanceIndexJobCompletionReason.java | 28 -
.../datasource/lance/job/LanceIndexJobManager.java | 790 --------------------
.../lance/job/LanceIndexJobMutationState.java | 47 --
.../lance/job/LanceIndexJobMutationType.java | 28 -
.../datasource/lance/job/LanceIndexJobQuota.java | 189 -----
.../lance/job/LanceIndexJobRefreshState.java | 41 -
.../datasource/lance/job/LanceIndexJobResult.java | 112 ---
.../lance/job/LanceIndexJobResultCode.java | 158 ----
.../lance/job/LanceIndexTerminationProof.java | 33 -
.../org/apache/doris/journal/JournalEntity.java | 14 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 20 -
.../apache/doris/nereids/trees/plans/PlanType.java | 2 -
.../trees/plans/commands/AlterTableCommand.java | 35 +-
.../plans/commands/ShowLanceIndexJobCommand.java | 174 -----
.../plans/commands/ShowLanceIndexJobsCommand.java | 378 ----------
.../trees/plans/visitor/CommandVisitor.java | 10 -
.../java/org/apache/doris/persist/EditLog.java | 15 +-
.../org/apache/doris/persist/OperationType.java | 6 +-
.../doris/persist/meta/MetaPersistMethod.java | 6 -
.../doris/persist/meta/PersistMetaModules.java | 7 +-
.../arrowflight/DorisFlightSqlProducer.java | 15 +
.../service/arrowflight/FlightSqlQuerySchema.java | 15 -
.../doris/datasource/CatalogMgrLanceGuardTest.java | 505 ++-----------
.../lance/LanceCatalogLifecycleTest.java | 20 -
.../datasource/lance/LanceIndexAdmissionTest.java | 400 ++--------
.../{job => }/LanceIndexNameNormalizerTest.java | 2 +-
.../LanceSchemaContractBuilderRealDatasetTest.java | 1 -
.../lance/LanceSchemaContractBuilderTest.java | 1 -
.../lance/job/LanceIndexDatasetLocatorTest.java | 183 -----
.../lance/job/LanceIndexJobManagerPersistTest.java | 457 ------------
.../lance/job/LanceIndexJobManagerQueryTest.java | 131 ----
.../lance/job/LanceIndexJobManagerReplayTest.java | 628 ----------------
.../lance/job/LanceIndexJobQuotaTest.java | 208 ------
.../lance/job/LanceIndexJobResultClassifyTest.java | 239 ------
.../lance/job/LanceIndexJobStateMachineTest.java | 640 ----------------
.../datasource/lance/job/LanceIndexJobTest.java | 303 --------
.../lance/job/LanceIndexJobWiringTest.java | 78 --
.../parser/ShowLanceIndexJobsParserTest.java | 183 -----
.../AlterTableCommandLanceAdmissionTest.java | 159 ++--
.../commands/ShowLanceIndexJobCommandTest.java | 565 --------------
.../commands/ShowLanceIndexJobsCommandTest.java | 821 ---------------------
.../arrowflight/DorisFlightSqlSchemaTest.java | 83 ++-
.../lance/test_lance_index_admission.groovy | 147 +---
.../lance/test_lance_index_ddl.groovy | 3 +-
63 files changed, 438 insertions(+), 8865 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 963ce97a1a8..e5b7ca1b114 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -4237,33 +4237,14 @@ public class Config extends ConfigBase {
public static long lance_catalog_arrow_memory_limit_bytes = 256L * 1024 * 1024;
@ConfField(mutable = true, masterOnly = true, varType = VariableAnnotation.EXPERIMENTAL, description = {
- "是否启用 Lance 外表索引变更(CREATE/CREATE OR REPLACE/DROP INDEX)的 admission。默认关闭;"
- + "启用前需确认未决 job 配额均为正值。注意:在 dispatch(后续版本)与 FORCE_RELEASE(后续版本)就绪前"
- + "开启本开关会产生不可回收的 PENDING job 并冻结对应 catalog 的身份属性变更与 DROP CATALOG。",
- "Enable admission of Lance index mutations (CREATE/CREATE OR REPLACE/DROP INDEX). "
- + "Disabled by default; unresolved-job quotas must be positive before enabling. "
- + "WARNING: enabling before dispatch and FORCE_RELEASE land in a later release creates "
- + "PENDING jobs that cannot be resolved and freezes catalog identity changes and DROP CATALOG."})
+ "是否启用 Lance 外表索引变更(CREATE/CREATE OR REPLACE/DROP INDEX)的权威校验路径。默认关闭;"
+ + "在同步执行路径就绪前,开启本开关只会让通过校验的语句收到暂不支持的错误。",
+ "Enable the authoritative validation path of Lance index mutations "
+ + "(CREATE/CREATE OR REPLACE/DROP INDEX). Disabled by default; until the synchronous "
+ + "execution path lands, enabling it only makes validated statements fail with the "
+ + "not-supported error."})
public static boolean enable_lance_index_mutation = false;
- @ConfField(mutable = true, masterOnly = true,
- callback = LanceIndexConfigValidator.PositiveLongConfigHandler.class,
- description = {"单个 Lance 数据表(locator 身份)允许的最大未决索引 job 数。",
- "Max unresolved Lance index jobs per table (locator identity)."})
- public static long lance_index_job_max_unresolved_per_table = 8;
-
- @ConfField(mutable = true, masterOnly = true,
- callback = LanceIndexConfigValidator.PositiveLongConfigHandler.class,
- description = {"单个 Lance catalog 允许的最大未决索引 job 数。",
- "Max unresolved Lance index jobs per catalog."})
- public static long lance_index_job_max_unresolved_per_catalog = 64;
-
- @ConfField(mutable = true, masterOnly = true,
- callback = LanceIndexConfigValidator.PositiveLongConfigHandler.class,
- description = {"全部 catalog 合计允许的最大未决 Lance 索引 job 数。",
- "Max unresolved Lance index jobs across all catalogs (global)."})
- public static long lance_index_job_max_unresolved_global = 256;
-
@ConfField(mutable = true, masterOnly = true,
callback = LanceIndexConfigValidator.PositiveIntConfigHandler.class,
description = {"Lance IVF_PQ 索引 num_partitions 的静态上限。",
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/LanceIndexConfigValidator.java b/fe/fe-common/src/main/java/org/apache/doris/common/LanceIndexConfigValidator.java
index 3f14c8bb885..40e061f4817 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/LanceIndexConfigValidator.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/LanceIndexConfigValidator.java
@@ -20,40 +20,19 @@ package org.apache.doris.common;
import java.lang.reflect.Field;
/**
- * Validators for the Lance index admission configuration items in {@link Config}. The
- * unresolved-job quotas and the static bounds must stay positive (section 9.7 enabling
- * precondition), so every handler rejects a zero, negative, or unparsable value before
- * assigning. The handlers extend {@link ConfigBase.DefaultConfHandler} and delegate to
+ * Validators for the Lance index configuration items in {@link Config}. The static bounds
+ * must stay positive, so the handler rejects a zero, negative, or unparsable value before
+ * assigning. The handler extends {@link ConfigBase.DefaultConfHandler} and delegates to
* {@code super.handle} after validation: a bare {@code ConfHandler} implementation would
* validate without ever assigning, letting ADMIN SET FRONTEND CONFIG pass with no effect.
* Note the callback only runs on the ADMIN SET path; values loaded from fe.conf bypass it
- * ({@code ConfigBase.setFields} assigns directly), so the admission path re-asserts the
- * positive invariant where the values are consumed.
+ * ({@code ConfigBase.setFields} assigns directly), so the consuming validation path
+ * re-asserts the positive invariant where the values are used.
*/
public final class LanceIndexConfigValidator {
private LanceIndexConfigValidator() {
}
- /**
- * Accepts only a positive long: validates first, then assigns via the default handler.
- */
- public static class PositiveLongConfigHandler extends ConfigBase.DefaultConfHandler {
- @Override
- public void handle(Field field, String confVal) throws Exception {
- String trimmedVal = confVal == null ? "" : confVal.trim();
- final long value;
- try {
- value = Long.parseLong(trimmedVal);
- } catch (NumberFormatException e) {
- throw new ConfigException(field.getName() + " must be a positive long, but got: " + confVal, e);
- }
- if (value <= 0) {
- throw new ConfigException(field.getName() + " must be a positive long, but got: " + confVal);
- }
- super.handle(field, trimmedVal);
- }
- }
-
/**
* Accepts only a positive int: validates first, then assigns via the default handler.
*/
diff --git a/fe/fe-common/src/test/java/org/apache/doris/common/LanceIndexConfigValidatorTest.java b/fe/fe-common/src/test/java/org/apache/doris/common/LanceIndexConfigValidatorTest.java
index c9e0de341ac..6088abe2fc0 100644
--- a/fe/fe-common/src/test/java/org/apache/doris/common/LanceIndexConfigValidatorTest.java
+++ b/fe/fe-common/src/test/java/org/apache/doris/common/LanceIndexConfigValidatorTest.java
@@ -25,8 +25,8 @@ import java.nio.file.Files;
import java.nio.file.Path;
/**
- * Coverage for the Lance index admission configuration items in {@link Config} and the
- * positive-value validators in {@link LanceIndexConfigValidator}: reviewed defaults, the
+ * Coverage for the Lance index configuration items in {@link Config} and the positive-value
+ * validator in {@link LanceIndexConfigValidator}: reviewed defaults, the
* {@link ConfigBase.ConfField} wiring (mutable/masterOnly/callback/varType), rejection of
* zero, negative and non-numeric values, and that an accepted value is really assigned to
* the field (a bare validating handler that never assigns would let ADMIN SET pass without
@@ -37,21 +37,12 @@ public class LanceIndexConfigValidatorTest {
@Test
public void testReviewedDefaults() {
Assertions.assertFalse(Config.enable_lance_index_mutation);
- Assertions.assertEquals(8L, Config.lance_index_job_max_unresolved_per_table);
- Assertions.assertEquals(64L, Config.lance_index_job_max_unresolved_per_catalog);
- Assertions.assertEquals(256L, Config.lance_index_job_max_unresolved_global);
Assertions.assertEquals(4096, Config.lance_index_max_num_partitions);
Assertions.assertEquals(256, Config.lance_index_max_num_sub_vectors);
}
@Test
public void testConfFieldWiring() throws Exception {
- assertCallbackWiring("lance_index_job_max_unresolved_per_table", true,
- LanceIndexConfigValidator.PositiveLongConfigHandler.class);
- assertCallbackWiring("lance_index_job_max_unresolved_per_catalog", true,
- LanceIndexConfigValidator.PositiveLongConfigHandler.class);
- assertCallbackWiring("lance_index_job_max_unresolved_global", true,
- LanceIndexConfigValidator.PositiveLongConfigHandler.class);
assertCallbackWiring("lance_index_max_num_partitions", true,
LanceIndexConfigValidator.PositiveIntConfigHandler.class);
assertCallbackWiring("lance_index_max_num_sub_vectors", true,
@@ -74,48 +65,6 @@ public class LanceIndexConfigValidatorTest {
Assertions.assertEquals(callback, anno.callback(), fieldName);
}
- @Test
- public void testPositiveLongHandlerAssignsAcceptedValue() throws Exception {
- assertLongAssigns("lance_index_job_max_unresolved_per_table");
- assertLongAssigns("lance_index_job_max_unresolved_per_catalog");
- assertLongAssigns("lance_index_job_max_unresolved_global");
- }
-
- private static void assertLongAssigns(String fieldName) throws Exception {
- Field field = Config.class.getField(fieldName);
- long original = field.getLong(null);
- try {
- new LanceIndexConfigValidator.PositiveLongConfigHandler().handle(field, " 12345 ");
- Assertions.assertEquals(12345L, field.getLong(null), fieldName);
- } finally {
- field.setLong(null, original);
- }
- }
-
- @Test
- public void testPositiveLongHandlerRejectsInvalidValues() throws Exception {
- assertLongRejected("lance_index_job_max_unresolved_per_table", "0");
- assertLongRejected("lance_index_job_max_unresolved_per_catalog", "0");
- assertLongRejected("lance_index_job_max_unresolved_global", "0");
- assertLongRejected("lance_index_job_max_unresolved_per_table", "-8");
- assertLongRejected("lance_index_job_max_unresolved_per_table", "not-a-number");
- assertLongRejected("lance_index_job_max_unresolved_per_table", "");
- }
-
- private static void assertLongRejected(String fieldName, String value) throws Exception {
- Field field = Config.class.getField(fieldName);
- long original = field.getLong(null);
- try {
- ConfigException e = Assertions.assertThrows(ConfigException.class,
- () -> new LanceIndexConfigValidator.PositiveLongConfigHandler().handle(field, value),
- fieldName + " <- \"" + value + "\"");
- Assertions.assertTrue(e.getMessage().contains(fieldName), e.getMessage());
- Assertions.assertEquals(original, field.getLong(null), fieldName + " must remain unchanged");
- } finally {
- field.setLong(null, original);
- }
- }
-
@Test
public void testPositiveIntHandlerAssignsAcceptedValue() throws Exception {
assertIntAssigns("lance_index_max_num_partitions");
@@ -167,18 +116,9 @@ public class LanceIndexConfigValidatorTest {
tempFile.toFile().deleteOnExit();
config.init(tempFile.toAbsolutePath().toString());
- long originalQuota = Config.lance_index_job_max_unresolved_per_catalog;
int originalBound = Config.lance_index_max_num_partitions;
boolean originalGate = Config.enable_lance_index_mutation;
try {
- ConfigBase.setMutableConfig("lance_index_job_max_unresolved_per_catalog", "96");
- Assertions.assertEquals(96L, Config.lance_index_job_max_unresolved_per_catalog);
- Assertions.assertThrows(ConfigException.class,
- () -> ConfigBase.setMutableConfig("lance_index_job_max_unresolved_per_catalog", "0"));
- Assertions.assertThrows(ConfigException.class,
- () -> ConfigBase.setMutableConfig("lance_index_job_max_unresolved_per_catalog", "xyz"));
- Assertions.assertEquals(96L, Config.lance_index_job_max_unresolved_per_catalog);
-
ConfigBase.setMutableConfig("lance_index_max_num_partitions", "8192");
Assertions.assertEquals(8192, Config.lance_index_max_num_partitions);
Assertions.assertThrows(ConfigException.class,
@@ -188,7 +128,6 @@ public class LanceIndexConfigValidatorTest {
ConfigBase.setMutableConfig("enable_lance_index_mutation", "true");
Assertions.assertTrue(Config.enable_lance_index_mutation);
} finally {
- Config.lance_index_job_max_unresolved_per_catalog = originalQuota;
Config.lance_index_max_num_partitions = originalBound;
Config.enable_lance_index_mutation = originalGate;
}
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index 4334e001e38..4aabc00ca9c 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -334,7 +334,6 @@ KEY: 'KEY';
KEYS: 'KEYS';
KILL: 'KILL';
LABEL: 'LABEL';
-LANCE: 'LANCE';
LARGEINT: 'LARGEINT';
LAYOUT: 'LAYOUT';
LAST: 'LAST';
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 372429744dc..c44ebefb158 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -497,8 +497,6 @@ supportedShowStatement
| SHOW WARM UP JOB wildWhere? #showWarmUpJob
| SHOW PYTHON VERSIONS #showPythonVersions
| SHOW PYTHON PACKAGES IN STRING_LITERAL #showPythonPackages
- | SHOW LANCE INDEX JOBS ((FROM | IN) db=multipartIdentifier)? (WHERE expression)? #showLanceIndexJobs
- | SHOW LANCE INDEX JOB jobId=INTEGER_VALUE #showLanceIndexJob
;
supportedLoadStatement
@@ -2206,7 +2204,6 @@ nonReserved
| JSON
| JSONB
| LABEL
- | LANCE
| LAST
| LDAP
| LDAP_ADMIN_PASSWORD
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 9eccdb1374b..05b1967bc1e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -111,7 +111,6 @@ import org.apache.doris.datasource.hive.event.MetastoreEventsProcessor;
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.datasource.iceberg.IcebergSysExternalTable;
import org.apache.doris.datasource.jdbc.JdbcExternalTable;
-import org.apache.doris.datasource.lance.job.LanceIndexJobManager;
import org.apache.doris.datasource.paimon.PaimonExternalTable;
import org.apache.doris.datasource.paimon.PaimonSysExternalTable;
import org.apache.doris.deploy.DeployManager;
@@ -572,8 +571,6 @@ public class Env {
private InsertOverwriteManager insertOverwriteManager;
- private LanceIndexJobManager lanceIndexJobManager;
-
private DNSCache dnsCache;
private final NereidsSqlCacheManager sqlCacheManager;
@@ -859,7 +856,6 @@ public class Env {
this.mtmvService = new MTMVService();
this.eventProcessor = new EventProcessor(mtmvService);
this.insertOverwriteManager = new InsertOverwriteManager();
- this.lanceIndexJobManager = new LanceIndexJobManager();
this.dnsCache = new DNSCache();
this.sqlCacheManager = new NereidsSqlCacheManager();
this.sortedPartitionsCacheManager = new NereidsSortedPartitionsCacheManager();
@@ -985,10 +981,6 @@ public class Env {
return insertOverwriteManager;
}
- public LanceIndexJobManager getLanceIndexJobManager() {
- return lanceIndexJobManager;
- }
-
public TabletScheduler getTabletScheduler() {
return tabletScheduler;
}
@@ -1830,11 +1822,6 @@ public class Env {
insertOverwriteManager.allTaskFail();
- // A durable RUNNING Lance index job at this point may have lost its result with the
- // old master: sweep it to UNKNOWN (and refresh RUNNING back to REQUIRED) before any
- // master-only dispatcher could start.
- lanceIndexJobManager.onTransferToMaster();
-
toMasterProgress = "start daemon threads";
// coz current fe was not master fe and didn't get all fes' alive session report before, which cause
@@ -2685,18 +2672,6 @@ public class Env {
return checksum;
}
- public long loadLanceIndexJobManager(DataInputStream in, long checksum) throws IOException {
- this.lanceIndexJobManager = LanceIndexJobManager.read(in);
- LOG.info("finished replay lance index job manager from image");
- return checksum;
- }
-
- public long saveLanceIndexJobManager(CountingDataOutputStream out, long checksum) throws IOException {
- this.lanceIndexJobManager.write(out);
- LOG.info("finished save lance index job manager to image");
- return checksum;
- }
-
// Only called by checkpoint thread
// return the latest image file's absolute path
public String saveImage() throws IOException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
index 59455962bb3..b0748cef9a3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
@@ -1242,7 +1242,8 @@ public enum ErrorCode {
ERR_LANCE_INDEX_MUTATION_DISABLED(5102, new byte[]{'4', '2', '0', '0', '0'},
"%s is disabled for Lance catalog tables (enable_lance_index_mutation = false)"),
- ERR_LANCE_INDEX_JOB_NOT_FOUND(5103, new byte[]{'4', '2', '0', '0', '0'}, "Lance index job not found: %s"),
+ // 5103 (ERR_LANCE_INDEX_JOB_NOT_FOUND) was removed with the durable Lance index job
+ // framework; the number stays unused so old client-visible codes keep their meaning.
ERR_NOT_CLOUD_MODE(6000, new byte[]{'4', '2', '0', '0', '0'},
"Command only support in cloud mode.");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 918dac65e40..bede3d4d6a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -92,18 +92,18 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
/**
* The Lance catalog properties whose change moves the persisted target identity
- * (provider, stable locator, or namespace mapping) out from under unresolved index
- * jobs. Credentials stay unguarded and may be rotated freely: an access key, secret
- * key or session token authenticates to the same target. Storage-routing properties
- * are part of the identity instead, because a job persists only a URI and a different
- * endpoint or region can land that same URI on a different storage service. The
- * routing entries are the canonical endpoint/region spellings of every S3-compatible
- * property family the Lance storage chain consumes: an s3 dataset reads whichever
- * S3-compatible configuration the catalog carries (LanceS3StorageProvider prefers a
- * concrete family over the generic s3 keys), while an oss dataset reads the oss keys
- * only. Other properties are not target-changing and stay unguarded. Keys match
- * case-insensitively because the catalog property chain performs no key
- * normalization.
+ * (provider, stable locator, or namespace mapping) out from under an index
+ * statement that captured that identity. Credentials stay unguarded and may be
+ * rotated freely: an access key, secret key or session token authenticates to the
+ * same target. Storage-routing properties are part of the identity instead, because
+ * the target is identified by a URI and a different endpoint or region can land that
+ * same URI on a different storage service. The routing entries are the canonical
+ * endpoint/region spellings of every S3-compatible property family the Lance storage
+ * chain consumes: an s3 dataset reads whichever S3-compatible configuration the
+ * catalog carries (LanceS3StorageProvider prefers a concrete family over the generic
+ * s3 keys), while an oss dataset reads the oss keys only. Other properties are not
+ * target-changing and stay unguarded. Keys match case-insensitively because the
+ * catalog property chain performs no key normalization.
*/
private static final Set<String> LANCE_TARGET_IDENTITY_KEYS = ImmutableSet.of(
"lance.catalog.type", "warehouse",
@@ -337,12 +337,6 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
if (catalog == null) {
throw new DdlException("No catalog found with name: " + catalogName);
}
- if (catalog instanceof LanceExternalCatalog
- && Env.getCurrentEnv().getLanceIndexJobManager().hasUnresolvedJobsForCatalog(catalog.getId())) {
- throw new DdlException("catalog '" + catalogName + "' has unresolved Lance index jobs; "
- + "they must be released via FORCE_RELEASE (available in a later release) "
- + "before dropping the catalog");
- }
CatalogLog log = new CatalogLog();
log.setCatalogId(catalog.getId());
removedCatalog = removeCatalog(log.getCatalogId());
@@ -458,13 +452,6 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
.equalsIgnoreCase(newProperties.get("type"))) {
throw new DdlException("Can't modify the type of catalog property with name: " + catalogName);
}
- if (catalog instanceof LanceExternalCatalog
- && hasLanceIdentityKeyChange(oldProperties, newProperties)
- && Env.getCurrentEnv().getLanceIndexJobManager().hasUnresolvedJobsForCatalog(catalog.getId())) {
- throw new DdlException("catalog '" + catalogName + "' has unresolved Lance index jobs; "
- + "they must be released via FORCE_RELEASE (available in a later release) "
- + "before changing target identity properties");
- }
CatalogLog log = new CatalogLog();
log.setCatalogId(catalog.getId());
Map<String, String> loggedProperties = Maps.newHashMap(newProperties);
@@ -518,10 +505,9 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
}
/**
- * Revalidate the target and transfer a prepared admission to the job manager atomically
- * with DROP CATALOG and identity ALTER. The action must contain only local job creation or a no-op;
- * all metadata loading must finish before entering this short critical section.
- * Lock order is CatalogMgr then LanceIndexJobManager, as on the catalog DDL path.
+ * Revalidate the target and run the prepared admission action atomically with DROP CATALOG
+ * and identity ALTER. The action must contain only local work or a no-op; all metadata
+ * loading must finish before entering this short critical section.
*/
public <T> T withLanceIndexAdmission(LanceExternalCatalog catalog, LanceIndexTarget expectedTarget,
LanceIndexAdmissionAction<T> action) throws Exception {
@@ -561,9 +547,9 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
* Whether the supplied properties change any Lance target identity key relative to the
* currently persisted values. A same-value rewrite is an idempotent no-op and is not a
* change; a newly supplied identity key counts as a change when the persisted value
- * differs (including when it was never set).
+ * differs (including when it was never set). Package-private for the guard test.
*/
- private static boolean hasLanceIdentityKeyChange(Map<String, String> oldProperties,
+ static boolean hasLanceIdentityKeyChange(Map<String, String> oldProperties,
Map<String, String> newProperties) {
for (Map.Entry<String, String> entry : newProperties.entrySet()) {
String key = entry.getKey();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java
index 4cd3ed64329..563b1778e42 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java
@@ -24,7 +24,6 @@ import org.apache.doris.datasource.lance.index.LanceIndexInspection;
import org.apache.doris.datasource.lance.index.LanceIndexInspectionExecutor;
import org.apache.doris.datasource.lance.index.LancePhysicalIndexEntry;
import org.apache.doris.datasource.lance.index.LanceShowIndexInfo;
-import org.apache.doris.datasource.lance.job.LanceIndexDatasetLocator;
import org.apache.doris.datasource.lance.metadata.LanceMetadataLoader;
import org.apache.doris.datasource.lance.metadata.LanceReadOptions;
import org.apache.doris.datasource.lance.metadata.LanceRefSelector;
@@ -842,11 +841,6 @@ final class LanceCatalogClient implements AutoCloseable {
(dataset, uri) -> LanceIndexInspection.readPhysicalEntries(dataset));
}
- String resolveCurrentIndexJobLocator(String dbName, String tableName) {
- return LanceIndexDatasetLocator.normalize(
- namespaceClient.resolveTableAccessUncached(dbName, tableName).getDatasetUri());
- }
-
public LanceIndexAdmissionSnapshot loadTableIndexAdmissionSnapshot(String dbName, String tableName) {
return inspectTableIndexes(dbName, tableName, LanceIndexInspection::readAdmissionSnapshot);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java
index dc62cb7ff63..d1080e4e942 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java
@@ -27,7 +27,6 @@ import org.apache.doris.datasource.InitCatalogLog;
import org.apache.doris.datasource.SessionContext;
import org.apache.doris.datasource.lance.index.LancePhysicalIndexEntry;
import org.apache.doris.datasource.lance.index.LanceShowIndexInfo;
-import org.apache.doris.datasource.lance.job.LanceIndexDatasetLocator;
import org.apache.doris.datasource.lance.metadata.LanceMetadataLoader;
import org.apache.doris.datasource.lance.metadata.LanceRefSelector;
import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
@@ -80,31 +79,6 @@ public class LanceExternalCatalog extends ExternalCatalog {
indexTargetVersion++;
}
- /**
- * Resolves the dataset the given (db, table) names currently point at, through the
- * same table-access resolution the readers use, and returns its durable locator form
- * ({@link LanceIndexDatasetLocator#normalize}).
- *
- * <p>This exists for SHOW-authorization revalidation of persisted Lance index jobs:
- * once a job reaches a terminal state and releases its guard, a legitimate catalog
- * ALTER can repoint the same db.table names at a different dataset, and the job must
- * stop being readable through table-level SHOW on the new target. Any failure - the
- * catalog is not initialized, the provider is unreachable, credentials expired, or
- * the names no longer resolve - yields {@code null}; callers must treat null as
- * "not resolved" (the orphan, ADMIN-only visibility rule), never as an authorization
- * grant.
- *
- * @return the normalized locator of the dataset the names currently point at, or
- * null when it cannot be resolved
- */
- public String resolveCurrentIndexJobLocator(String dbName, String tableName) {
- try {
- return withClient(current -> current.resolveCurrentIndexJobLocator(dbName, tableName));
- } catch (Exception e) {
- return null;
- }
- }
-
public LanceExternalCatalog(long catalogId, String name, String resource, Map<String, String> props,
String comment) {
super(catalogId, name, InitCatalogLog.Type.LANCE, comment);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexAdmission.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexAdmission.java
index b408af0dbcb..6d755fb63f2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexAdmission.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexAdmission.java
@@ -20,21 +20,12 @@ package org.apache.doris.datasource.lance;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.datasource.CatalogMgr;
import org.apache.doris.datasource.lance.index.LanceIndexInspection;
import org.apache.doris.datasource.lance.index.LanceShowIndexInfo;
-import org.apache.doris.datasource.lance.job.LanceIndexDatasetLocator;
-import org.apache.doris.datasource.lance.job.LanceIndexFenceKey;
-import org.apache.doris.datasource.lance.job.LanceIndexJob;
-import org.apache.doris.datasource.lance.job.LanceIndexJobMutationType;
-import org.apache.doris.datasource.lance.job.LanceIndexNameNormalizer;
-import org.apache.doris.datasource.lance.job.LanceIndexSchemaContract;
import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition;
-import org.apache.doris.persist.gson.GsonUtils;
-import org.apache.doris.qe.ConnectContext;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
@@ -46,27 +37,25 @@ import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.TreeMap;
import javax.annotation.Nullable;
/**
* Lance index admission (design sections 2.2 and 4.1): the single place where a statically
* validated top-level CREATE [OR REPLACE]/DROP INDEX statement against a Lance catalog table is
- * turned into a durable job. The whole flow runs against one pinned admission snapshot — the
- * IF preflight never takes a second metadata read, and no catalog/db/table metadata lock is held
- * while the snapshot loader does its JNI work (design section 5.1).
+ * checked against authoritative metadata. The whole flow runs against one pinned admission
+ * snapshot — the IF preflight never takes a second metadata read, and no catalog/db/table
+ * metadata lock is held while the snapshot loader does its JNI work (design section 5.1).
*
* <p>The step order is the correctness contract: name normalization and reserved prefix come
* first, before target capture and the snapshot read (fail cheap-first — a reserved name
* rejected at admission depth costs no remote read), then case-only collision analysis, IF
* preflight (including the two-stage {@code matches}: requested-algorithm equality plus
* physical-family corroboration), column-lookup collision analysis, schema contract from
- * the stored column name, locator
- * normalization, deterministic properties JSON, positive-quota assertion, and only then exactly
- * one id allocation and the durable {@code createJob} transfer.
- * Every rejection before {@code createJob} leaves no job, no fence, no quota charge, no journal
- * record, and no id allocation; the manager's own fence/quota rejections pass through verbatim
- * (an id burned by them is accepted — ids are never required to be contiguous).
+ * the stored column name, and deterministic properties JSON. The validation collected here is
+ * the request-validation stage the synchronous execution path builds on; until that path
+ * lands, an admitted mutation terminates with the shared not-supported rejection, while the
+ * IF no-op cases complete as genuine no-ops (nothing is created or dropped). Every rejection
+ * leaves no durable state behind.
*/
public final class LanceIndexAdmission {
@@ -79,24 +68,6 @@ public final class LanceIndexAdmission {
throws Exception;
}
- /** The admission result: the durable job id, or null for an IF no-op (no job created). */
- public static final class Outcome {
- private final Long jobId;
-
- private Outcome(Long jobId) {
- this.jobId = jobId;
- }
-
- /**
- * The admitted job id, or null when the IF preflight made the statement an immediate
- * no-op (design section 2.2: "returns an immediate no-op, without creating a job").
- */
- @Nullable
- public Long getJobId() {
- return jobId;
- }
- }
-
private static final SnapshotLoader DEFAULT_LOADER = new SnapshotLoader() {
@Override
public LanceIndexAdmissionSnapshot load(LanceExternalCatalog catalog, String dbName,
@@ -113,12 +84,12 @@ public final class LanceIndexAdmission {
* ({@link LanceIndexMutationValidator#validateCreateIndex}) must already have passed for
* {@code def}.
*/
- public static Outcome admitCreate(LanceExternalCatalog catalog, LanceExternalDatabase db,
+ public static void admitCreate(LanceExternalCatalog catalog, LanceExternalDatabase db,
LanceExternalTable table, IndexDefinition def, boolean ifNotExists) throws Exception {
- return admitCreate(DEFAULT_LOADER, catalog, db, table, def, ifNotExists);
+ admitCreate(DEFAULT_LOADER, catalog, db, table, def, ifNotExists);
}
- static Outcome admitCreate(SnapshotLoader loader, LanceExternalCatalog catalog,
+ static void admitCreate(SnapshotLoader loader, LanceExternalCatalog catalog,
LanceExternalDatabase db, LanceExternalTable table, IndexDefinition def, boolean ifNotExists)
throws Exception {
// 1. Display/normalized names and the reserved system prefix, checked before any metadata
@@ -141,57 +112,30 @@ public final class LanceIndexAdmission {
}
String storedName = LanceIndexFamilies.uniqueMatch(storedNames, normalizedName);
// 4. IF preflight (design section 2.2).
- boolean orReplace = def.isOrReplace();
- if (!orReplace && storedName != null) {
+ if (!def.isOrReplace() && storedName != null) {
if (!ifNotExists) {
rejectInvalid("index '" + displayName + "' already exists");
}
if (!matchesExistingDefinition(snapshot, storedName, def)) {
rejectInvalid("index '" + displayName + "' already exists with a different definition");
}
- return catalogMgr.withLanceIndexAdmission(catalog, target, () -> new Outcome(null));
+ catalogMgr.withLanceIndexAdmission(catalog, target, () -> null);
+ return;
}
// 5. Fail closed when the table lookup relation cannot resolve the request column
// uniquely: a dataset can hold top-level fields that differ only by case (V versus v),
// and ExternalTable.getColumn returns the first hit, so admitting would journal an
// arbitrary one of them. The pinned snapshot decides, never the cached table schema.
rejectIfAmbiguousLookupColumn(snapshot, def.getCols().get(0));
- // 6. Schema contract v1 from the stored column name (never the raw user spelling).
- String storedColumnName = storedColumnName(table, def.getCols().get(0));
- LanceIndexSchemaContract contract =
- LanceSchemaContractBuilder.build(snapshot.getTopLevelFields(), storedColumnName);
- // 7. The fence locator is the normalized dataset uri of the same pinned snapshot.
- String locator = normalizeLocator(snapshot);
- // 8. Deterministic normalized properties JSON for ANN; scalar families persist null.
- boolean ann = def.getLanceIndexType() == null;
- String indexType = ann ? annIndexType(def) : def.getLanceIndexType();
- String propertiesJson = ann ? buildAnnPropertiesJson(def) : null;
- // D7 backstop: quota values from fe.conf bypass the ADMIN SET callback, so admission
- // re-asserts positivity before any id allocation or durable transfer.
- assertPositiveQuotas();
- return catalogMgr.withLanceIndexAdmission(catalog, target, () -> {
- // 9. Exactly one id allocation, after every preflight above has passed.
- long jobId = Env.getCurrentEnv().getNextId();
- String creator = ConnectContext.get().getQualifiedUser();
- // 10. REPLACE on an existing name persists the stored display name (section 4.1) so the
- // worker locates the case-sensitive target; a fresh REPLACE keeps the user's spelling.
- String persistedDisplayName = (orReplace && storedName != null) ? storedName : displayName;
- LanceIndexJob job;
- try {
- job = new LanceIndexJob(jobId, creator, catalog.getId(), db.getFullName(), table.getName(),
- LanceIndexFenceKey.PROVIDER_DIRECTORY, locator, persistedDisplayName, normalizedName,
- orReplace ? LanceIndexJobMutationType.REPLACE : LanceIndexJobMutationType.CREATE,
- ifNotExists, false, indexType, storedColumnName, propertiesJson,
- snapshot.getDatasetVersion(), contract);
- } catch (IllegalArgumentException e) {
- throw invalidAdmission(e.getMessage());
- }
- Env.getCurrentEnv().getLanceIndexJobManager().createJob(job,
- Config.lance_index_job_max_unresolved_per_table,
- Config.lance_index_job_max_unresolved_per_catalog,
- Config.lance_index_job_max_unresolved_global);
- // 11. The job and its fence are durable once createJob returns.
- return new Outcome(jobId);
+ // 6. Schema contract v1 from the stored column name (never the raw user spelling). The
+ // build itself is the admission-depth type validation against the pinned snapshot.
+ LanceSchemaContractBuilder.build(snapshot.getTopLevelFields(), storedColumnName(table, def.getCols().get(0)));
+ // Every validation above is preserved for the synchronous execution path; until that
+ // path lands, a mutation that would be admitted terminates with the shared rejection.
+ catalogMgr.withLanceIndexAdmission(catalog, target, () -> {
+ LanceIndexMutationValidator.rejectUnsupportedOperation(
+ def.isOrReplace() ? "CREATE OR REPLACE INDEX" : "CREATE INDEX", "catalog tables");
+ return null;
});
}
@@ -199,12 +143,12 @@ public final class LanceIndexAdmission {
* Admits a top-level DROP INDEX. The static name bounds
* ({@link LanceIndexMutationValidator#validateDropIndex}) must already have passed.
*/
- public static Outcome admitDrop(LanceExternalCatalog catalog, LanceExternalDatabase db,
+ public static void admitDrop(LanceExternalCatalog catalog, LanceExternalDatabase db,
LanceExternalTable table, String indexName, boolean ifExists) throws Exception {
- return admitDrop(DEFAULT_LOADER, catalog, db, table, indexName, ifExists);
+ admitDrop(DEFAULT_LOADER, catalog, db, table, indexName, ifExists);
}
- static Outcome admitDrop(SnapshotLoader loader, LanceExternalCatalog catalog,
+ static void admitDrop(SnapshotLoader loader, LanceExternalCatalog catalog,
LanceExternalDatabase db, LanceExternalTable table, String indexName, boolean ifExists)
throws Exception {
// Fail cheap-first: the reserved prefix is rejected before target capture and the
@@ -222,31 +166,16 @@ public final class LanceIndexAdmission {
String storedName = LanceIndexFamilies.uniqueMatch(storedNames, normalizedName);
if (storedName == null) {
if (ifExists) {
- return catalogMgr.withLanceIndexAdmission(catalog, target, () -> new Outcome(null));
+ catalogMgr.withLanceIndexAdmission(catalog, target, () -> null);
+ return;
}
rejectInvalid("index '" + indexName + "' not found");
}
- String locator = normalizeLocator(snapshot);
- assertPositiveQuotas();
- return catalogMgr.withLanceIndexAdmission(catalog, target, () -> {
- long jobId = Env.getCurrentEnv().getNextId();
- String creator = ConnectContext.get().getQualifiedUser();
- // DROP only runs past the preflight with a unique match, so the stored display name is
- // always persisted (section 4.1); definition fields stay null on a DROP job record.
- LanceIndexJob job;
- try {
- job = new LanceIndexJob(jobId, creator, catalog.getId(), db.getFullName(), table.getName(),
- LanceIndexFenceKey.PROVIDER_DIRECTORY, locator, storedName, normalizedName,
- LanceIndexJobMutationType.DROP, false, ifExists, null, null, null,
- snapshot.getDatasetVersion(), null);
- } catch (IllegalArgumentException e) {
- throw invalidAdmission(e.getMessage());
- }
- Env.getCurrentEnv().getLanceIndexJobManager().createJob(job,
- Config.lance_index_job_max_unresolved_per_table,
- Config.lance_index_job_max_unresolved_per_catalog,
- Config.lance_index_job_max_unresolved_global);
- return new Outcome(jobId);
+ // The preflight above is preserved for the synchronous execution path; until that path
+ // lands, a mutation that would be admitted terminates with the shared rejection.
+ catalogMgr.withLanceIndexAdmission(catalog, target, () -> {
+ LanceIndexMutationValidator.rejectUnsupportedOperation("DROP INDEX", "catalog tables");
+ return null;
});
}
@@ -344,7 +273,7 @@ public final class LanceIndexAdmission {
JsonObject compression = exposed == null || !exposed.has("compression")
? null : exposed.getAsJsonObject("compression");
// The request side of num_bits is never "unset": the validator accepts an omitted value
- // and the persisted job pins it to 8 (section 2.4), so the preflight compares an
+ // and the effective default is 8 (section 2.4), so the preflight compares an
// effective 8 against any exposed compression.num_bits instead of skipping.
String requestNumBits = request.get("num_bits") == null ? "8" : request.get("num_bits");
return numericPropertyMatches(request.get("num_sub_vectors"), compression, "num_sub_vectors")
@@ -405,47 +334,6 @@ public final class LanceIndexAdmission {
return normalizedAnnProperties(def.getProperties()).get("index_type");
}
- /** The persisted job index type: the uppercased validated ANN index_type, else the literal. */
- private static String annIndexType(IndexDefinition def) {
- String indexType = normalizedAnnProperties(def.getProperties()).get("index_type");
- return indexType == null ? null : indexType.toUpperCase(Locale.ROOT);
- }
-
- /**
- * The deterministic params JSON for an admitted ANN job: lowercased keys in TreeMap order,
- * index_type as the uppercased original value, metric lowercased, numeric values as their
- * original strings, and num_bits pinned to 8 (design section 2.4). Bounded well under
- * {@link LanceIndexJob#MAX_PROPERTIES_JSON_BYTES} by the five-key whitelist.
- */
- private static String buildAnnPropertiesJson(IndexDefinition def) throws AnalysisException {
- Map<String, String> request = normalizedAnnProperties(def.getProperties());
- TreeMap<String, String> persisted = new TreeMap<>();
- String indexType = request.get("index_type");
- if (indexType != null) {
- persisted.put("index_type", indexType.toUpperCase(Locale.ROOT));
- }
- String metric = request.get("metric");
- if (metric != null) {
- persisted.put("metric", metric.toLowerCase(Locale.ROOT));
- }
- putIfPresent(persisted, request, "num_partitions");
- putIfPresent(persisted, request, "num_sub_vectors");
- persisted.put("num_bits", "8");
- try {
- return GsonUtils.GSON.toJson(persisted);
- } catch (IllegalArgumentException e) {
- throw invalidAdmission(e.getMessage());
- }
- }
-
- private static void putIfPresent(TreeMap<String, String> target, Map<String, String> source,
- String key) {
- String value = source.get(key);
- if (value != null) {
- target.put(key, value);
- }
- }
-
/**
* The request properties with case-folded keys. Static validation has already rejected
* unknown and duplicate (case-insensitively) keys, so a plain last-wins fold is exact here.
@@ -491,15 +379,6 @@ public final class LanceIndexAdmission {
return column.getName();
}
- private static String normalizeLocator(LanceIndexAdmissionSnapshot snapshot)
- throws AnalysisException {
- try {
- return LanceIndexDatasetLocator.normalize(snapshot.getDatasetUri());
- } catch (IllegalArgumentException e) {
- throw invalidAdmission(e.getMessage());
- }
- }
-
private static List<String> logicalIndexNames(LanceIndexAdmissionSnapshot snapshot) {
List<String> names = new ArrayList<>(snapshot.getLogicalIndexes().size());
for (LanceShowIndexInfo index : snapshot.getLogicalIndexes()) {
@@ -508,39 +387,7 @@ public final class LanceIndexAdmission {
return names;
}
- /**
- * D7: the unresolved-job quotas are a section 9.7 enablement precondition. The ADMIN SET
- * callback validates them, but fe.conf loading bypasses callbacks, so admission asserts them
- * again before allocating an id. The manager independently rejects non-positive limits
- * at the durable-transfer boundary.
- */
- private static void assertPositiveQuotas() throws AnalysisException {
- if (Config.lance_index_job_max_unresolved_per_table <= 0) {
- rejectNonPositiveQuota("lance_index_job_max_unresolved_per_table",
- Config.lance_index_job_max_unresolved_per_table);
- }
- if (Config.lance_index_job_max_unresolved_per_catalog <= 0) {
- rejectNonPositiveQuota("lance_index_job_max_unresolved_per_catalog",
- Config.lance_index_job_max_unresolved_per_catalog);
- }
- if (Config.lance_index_job_max_unresolved_global <= 0) {
- rejectNonPositiveQuota("lance_index_job_max_unresolved_global",
- Config.lance_index_job_max_unresolved_global);
- }
- }
-
- private static void rejectNonPositiveQuota(String configItem, long value) throws AnalysisException {
- ErrorReport.reportAnalysisException("%s", ErrorCode.ERR_LANCE_INDEX_MUTATION_DISABLED,
- "Lance index admission requires positive unresolved-job quotas, but " + configItem
- + " = " + value + "; fix the FE configuration before enabling "
- + "enable_lance_index_mutation");
- }
-
private static void rejectInvalid(String detail) throws AnalysisException {
ErrorReport.reportAnalysisException(ErrorCode.ERR_LANCE_INDEX_INVALID, detail);
}
-
- private static AnalysisException invalidAdmission(String detail) {
- return new AnalysisException(detail, ErrorCode.ERR_LANCE_INDEX_INVALID);
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexFamilies.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexFamilies.java
index 50dd6119511..88a0b8ae5f0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexFamilies.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexFamilies.java
@@ -17,7 +17,6 @@
package org.apache.doris.datasource.lance;
-import org.apache.doris.datasource.lance.job.LanceIndexNameNormalizer;
import com.google.common.collect.ImmutableSet;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMutationValidator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMutationValidator.java
index 0d2a340d710..f21e2053994 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMutationValidator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMutationValidator.java
@@ -26,7 +26,6 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
-import org.apache.doris.datasource.lance.job.LanceIndexNameNormalizer;
import org.apache.doris.datasource.lance.metadata.LanceTypeConverter;
import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition;
@@ -130,9 +129,9 @@ public final class LanceIndexMutationValidator {
}
/**
- * Shared Lance index-name bounds for the CREATE and DROP paths: the name becomes the durable
- * logical identity that an admitted job and its same-name fence key are built on, so
- * null/empty names are rejected here instead of being masked as unsupported operations.
+ * Shared Lance index-name bounds for the CREATE and DROP paths: the name is the durable
+ * logical identity the admission preflight matches against authoritative Lance metadata,
+ * so null/empty names are rejected here instead of being masked as unsupported operations.
*/
private static void validateIndexName(String indexName) throws AnalysisException {
if (indexName == null || indexName.isEmpty()) {
@@ -168,7 +167,7 @@ public final class LanceIndexMutationValidator {
rejectInvalidDefinition("ANN index column item type must be float type");
}
// Keys match case-insensitively; the normalized view below is validation-local only.
- // Persisting normalized keys/values into the admitted job spec is owned by admission.
+ // Comparing the normalized request against authoritative metadata is owned by admission.
Map<String, String> lowerCaseProperties = new HashMap<>();
for (Map.Entry<String, String> entry : properties.entrySet()) {
String key = entry.getKey().toLowerCase(Locale.ROOT);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizer.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexNameNormalizer.java
similarity index 80%
rename from fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizer.java
rename to fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexNameNormalizer.java
index b467aa287f7..5635792c3e4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexNameNormalizer.java
@@ -15,28 +15,28 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.datasource.lance.job;
+package org.apache.doris.datasource.lance;
import java.nio.charset.StandardCharsets;
import java.util.Locale;
/**
* Logical index name normalization v1, the only definition: the UTF-8 result
- * of Java {@code toLowerCase(Locale.ROOT)}. Both the display name and the
- * normalized bytes are persisted on the job; Doris preserves display case,
- * rejects new case-only duplicates, and fails mutation on ambiguous external
- * case-only collisions. No normalization migration or mixed-version fence
- * protocol exists.
+ * of Java {@code toLowerCase(Locale.ROOT)}. Doris preserves display case,
+ * rejects new case-only duplicates against authoritative metadata, and fails
+ * mutation on ambiguous external case-only collisions. No normalization
+ * migration exists.
*/
public final class LanceIndexNameNormalizer {
- /** Bound on the persisted logical index name, aligned with the external string bound. */
+ /** Bound on the logical index name, aligned with the external string bound. */
public static final int MAX_INDEX_NAME_BYTES = 1024;
private LanceIndexNameNormalizer() {
}
/**
- * Normalization v1. The result is the identity bytes of the same-name fence key.
+ * Normalization v1. The result is the matching key for authoritative-metadata
+ * comparison and case-only duplicate rejection during the admission preflight.
*/
public static String normalize(String displayName) {
if (displayName == null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexSchemaContract.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexSchemaContract.java
similarity index 99%
rename from fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexSchemaContract.java
rename to fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexSchemaContract.java
index 1175de7223d..7b1c7e6a986 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexSchemaContract.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexSchemaContract.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.datasource.lance.job;
+package org.apache.doris.datasource.lance;
import com.google.gson.annotations.SerializedName;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceSchemaContractBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceSchemaContractBuilder.java
index 4581a6c0efa..8b59f6525a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceSchemaContractBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceSchemaContractBuilder.java
@@ -20,7 +20,6 @@ package org.apache.doris.datasource.lance;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
-import org.apache.doris.datasource.lance.job.LanceIndexSchemaContract;
import org.apache.doris.datasource.lance.metadata.LanceTypeConverter;
import org.apache.arrow.vector.types.DateUnit;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java
deleted file mode 100644
index bed1ae696d8..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java
+++ /dev/null
@@ -1,143 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
-import java.util.Locale;
-
-/**
- * Dataset locator normalization v1 for the durable fence key. The rules, in
- * order:
- *
- * <ol>
- * <li>trim surrounding whitespace;</li>
- * <li>if a {@code scheme://} prefix is present, lowercase the scheme
- * (aligned with the {@code LanceStorageProvider.schemeOf} precedent);</li>
- * <li>a URL whose authority carries userinfo is rejected: credential-bearing
- * URLs are never identity;</li>
- * <li>a locator with a scheme but neither an authority nor a path (for
- * example {@code "s3://"}) carries no identity and is rejected; an empty
- * authority with a non-empty path ({@code "file:///x"}) stays legal;</li>
- * <li>trailing {@code '/'} characters are removed, keeping the root
- * (a scheme-less {@code "/"} stays {@code "/"});</li>
- * <li>without a scheme the locator must be an absolute path (start with
- * {@code '/'}), otherwise it is rejected.</li>
- * </ol>
- *
- * <p>The authority (host/bucket) and path keep their original case: bucket and
- * path components are case-sensitive on the providers Doris supports, and
- * normalization v1 deliberately does not define cross-alias equivalence. URI
- * aliases and external writers replacing the dataset at the same URI are
- * outside Doris serialization.
- */
-public final class LanceIndexDatasetLocator {
- private static final String SCHEME_SEPARATOR = "://";
- /** Finite bound for the stable locator persisted in every durable job record. */
- public static final int MAX_LOCATOR_BYTES = 4096;
-
- private LanceIndexDatasetLocator() {
- }
-
- /**
- * Normalize a raw dataset locator into its durable identity form.
- *
- * @throws IllegalArgumentException if the locator is null/empty/oversized,
- * is not a valid hierarchical URI or absolute path, carries
- * userinfo/query/fragment data, has neither an authority nor a path,
- * or is a scheme-less relative path
- */
- public static String normalize(String rawLocator) {
- if (rawLocator == null) {
- throw new IllegalArgumentException("dataset locator must not be null");
- }
- String locator = rawLocator.trim();
- if (locator.isEmpty()) {
- throw new IllegalArgumentException("dataset locator must not be empty");
- }
- if (locator.getBytes(StandardCharsets.UTF_8).length > MAX_LOCATOR_BYTES) {
- throw new IllegalArgumentException(
- "dataset locator exceeds " + MAX_LOCATOR_BYTES + " UTF-8 bytes");
- }
-
- // Preserve the established spelling of the scheme-less filesystem root.
- // java.net.URI rejects "//" as a network-path reference without an
- // authority, while it is a valid absolute filesystem path here.
- if (containsOnlySlashes(locator)) {
- return "/";
- }
-
- URI uri;
- try {
- uri = new URI(locator);
- } catch (URISyntaxException e) {
- // Do not include the raw locator: it may contain credentials.
- throw new IllegalArgumentException("dataset locator is not a valid URI or absolute path");
- }
- if (uri.getRawUserInfo() != null) {
- throw new IllegalArgumentException(
- "credential-bearing dataset locators are never identity (userinfo is not allowed)");
- }
- if (uri.getRawQuery() != null) {
- // Presigned object-store URLs and SAS URLs carry credentials here.
- throw new IllegalArgumentException("dataset locator query parameters are not allowed");
- }
- if (uri.getRawFragment() != null) {
- throw new IllegalArgumentException("dataset locator fragments are not allowed");
- }
-
- String scheme = uri.getScheme();
- if (scheme == null) {
- if (!locator.startsWith("/")) {
- throw new IllegalArgumentException("dataset locator without a scheme must be an absolute path");
- }
- return stripTrailingSlashes(locator, 1);
- }
- if (uri.isOpaque() || !locator.regionMatches(scheme.length(), SCHEME_SEPARATOR, 0,
- SCHEME_SEPARATOR.length())) {
- throw new IllegalArgumentException("dataset locator scheme must use hierarchical '://' syntax");
- }
- String authority = uri.getRawAuthority() == null ? "" : uri.getRawAuthority();
- String rawPath = uri.getRawPath() == null ? "" : uri.getRawPath();
- String path = stripTrailingSlashes(rawPath, 0);
- if (authority.isEmpty() && path.isEmpty()) {
- // "s3://" / "file://" carry no identity at all; "file:///x" (empty
- // authority, non-empty path) is legal and does not reach this.
- throw new IllegalArgumentException("dataset locator has neither an authority nor a path");
- }
- return scheme.toLowerCase(Locale.ROOT) + SCHEME_SEPARATOR + authority + path;
- }
-
- private static boolean containsOnlySlashes(String value) {
- for (int index = 0; index < value.length(); index++) {
- if (value.charAt(index) != '/') {
- return false;
- }
- }
- return true;
- }
-
- private static String stripTrailingSlashes(String value, int minLength) {
- int end = value.length();
- while (end > minLength && value.charAt(end - 1) == '/') {
- end--;
- }
- return value.substring(0, end);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexFenceKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexFenceKey.java
deleted file mode 100644
index 19cc9efe80a..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexFenceKey.java
+++ /dev/null
@@ -1,92 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import java.util.Objects;
-
-/**
- * The durable same-name target/fence key:
- * <pre>
- * ( persisted catalog identity, provider = DIRECTORY,
- * normalized stable dataset locator, persisted normalized logical-index-name bytes )
- * </pre>
- * The display name is not part of the key; it is persisted on the job itself.
- * This class is a derived in-memory index key and is not persisted directly.
- */
-public final class LanceIndexFenceKey {
- /** Provider of every job in this delivery slice, mapped from a filesystem (Directory) Lance catalog. */
- public static final String PROVIDER_DIRECTORY = "DIRECTORY";
-
- private final long catalogId;
- private final String provider;
- private final String normalizedLocator;
- private final String normalizedIndexName;
-
- public LanceIndexFenceKey(long catalogId, String provider, String normalizedLocator, String normalizedIndexName) {
- this.catalogId = catalogId;
- this.provider = Objects.requireNonNull(provider, "provider");
- this.normalizedLocator = Objects.requireNonNull(normalizedLocator, "normalizedLocator");
- this.normalizedIndexName = Objects.requireNonNull(normalizedIndexName, "normalizedIndexName");
- }
-
- public long getCatalogId() {
- return catalogId;
- }
-
- public String getProvider() {
- return provider;
- }
-
- public String getNormalizedLocator() {
- return normalizedLocator;
- }
-
- public String getNormalizedIndexName() {
- return normalizedIndexName;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (!(o instanceof LanceIndexFenceKey)) {
- return false;
- }
- LanceIndexFenceKey that = (LanceIndexFenceKey) o;
- return catalogId == that.catalogId
- && provider.equals(that.provider)
- && normalizedLocator.equals(that.normalizedLocator)
- && normalizedIndexName.equals(that.normalizedIndexName);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(catalogId, provider, normalizedLocator, normalizedIndexName);
- }
-
- /**
- * Deliberately omits the locator: fence-conflict messages may surface to
- * users without target privileges and must not disclose it.
- */
- @Override
- public String toString() {
- return "LanceIndexFenceKey{catalogId=" + catalogId + ", provider=" + provider
- + ", normalizedIndexName=" + normalizedIndexName + '}';
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java
deleted file mode 100644
index f92f734b18f..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java
+++ /dev/null
@@ -1,699 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import org.apache.doris.common.io.Text;
-import org.apache.doris.common.io.Writable;
-import org.apache.doris.persist.gson.GsonUtils;
-
-import com.google.gson.annotations.SerializedName;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Objects;
-
-/**
- * The minimal durable Lance index job record. It contains only what the
- * lifecycle invariants require: job identity/creator/revision and bounded
- * timestamps; the persisted target identity and normalized same-name fence
- * key; the mutation intent; the admitted dataset version and schema-contract
- * representation; the mutation outcome, independent refresh state, typed
- * result, and a bounded sanitized message; the dispatch identity (selected BE,
- * BE process epoch, immutable invocation ID, deadline), possible-live
- * ownership, and any matching termination proof; and the FORCE audit fields
- * (populated only by the FORCE_RELEASE slice; here they are carried for
- * durability and replay).
- *
- * <p>The record never carries secrets, raw/opaque provider data, or unbounded
- * values: name/message/note/properties fields are length-checked at
- * construction/mutation time. Credentials are resolved at execution time from
- * the current catalog and are never persisted here.
- *
- * <p>Serialization is the standard Gson stream: {@code Text.writeString} of
- * the JSON form. Every durable field carries a short {@link SerializedName};
- * fields missing in old data keep the safe in-class defaults, so replay stays
- * tolerant of additive evolution. An instance is effectively immutable while
- * published in the manager: every durable transition is applied to a private
- * copy that is logged and then swapped in, so the object written to the edit
- * log is never mutated afterwards.
- */
-public class LanceIndexJob implements Writable {
- /** Bound on the user/build properties JSON snapshot. */
- public static final int MAX_PROPERTIES_JSON_BYTES = 4096;
- /** Bound on the FORCE_RELEASE operator note and the late-commit warning text. */
- public static final int MAX_FORCE_TEXT_BYTES = 1024;
- /** Bound on persisted creator, target-name, mutation, and FORCE-actor text. */
- public static final int MAX_DURABLE_TEXT_BYTES = 1024;
- /** Dispatch identities are UUID-like tokens, not arbitrary worker output. */
- public static final int MAX_INVOCATION_ID_BYTES = 256;
-
- // ------------------------------------------------------------------
- // Identity
- // ------------------------------------------------------------------
- @SerializedName(value = "jid")
- private long jobId;
-
- @SerializedName(value = "cr")
- private String creator;
-
- /** Bumped by +1 on every durable transition; callbacks and replay are revision-checked. */
- @SerializedName(value = "rev")
- private long revision;
-
- @SerializedName(value = "ctm")
- private long createTimeMs;
-
- @SerializedName(value = "utm")
- private long updateTimeMs;
-
- // ------------------------------------------------------------------
- // Target / fence identity
- // ------------------------------------------------------------------
- @SerializedName(value = "cid")
- private long catalogId;
-
- /** Local database name, kept for privilege and FORCE resolution when the catalog still resolves. */
- @SerializedName(value = "dbn")
- private String dbName;
-
- /** Local table name, kept for privilege and FORCE resolution when the catalog still resolves. */
- @SerializedName(value = "tbn")
- private String tableName;
-
- @SerializedName(value = "prv")
- private String provider = LanceIndexFenceKey.PROVIDER_DIRECTORY;
-
- @SerializedName(value = "loc")
- private String normalizedLocator;
-
- @SerializedName(value = "din")
- private String displayIndexName;
-
- @SerializedName(value = "nin")
- private String normalizedIndexName;
-
- // ------------------------------------------------------------------
- // Mutation intent
- // ------------------------------------------------------------------
- @SerializedName(value = "mt")
- private LanceIndexJobMutationType mutationType = LanceIndexJobMutationType.CREATE;
-
- @SerializedName(value = "ine")
- private boolean ifNotExists;
-
- @SerializedName(value = "ie")
- private boolean ifExists;
-
- /** Logical Lance algorithm (IVF_PQ / BTREE / BITMAP); required for CREATE/REPLACE, nullable for DROP. */
- @SerializedName(value = "it")
- private String indexType;
-
- @SerializedName(value = "cn")
- private String columnName;
-
- @SerializedName(value = "pj")
- private String propertiesJson;
-
- // ------------------------------------------------------------------
- // Admission snapshot
- // ------------------------------------------------------------------
- @SerializedName(value = "adv")
- private long admittedDatasetVersion;
-
- /** Nullable only for DROP, which does not revalidate an indexed-field contract. */
- @SerializedName(value = "sc")
- private LanceIndexSchemaContract schemaContract;
-
- // ------------------------------------------------------------------
- // Dual state
- // ------------------------------------------------------------------
- @SerializedName(value = "ms")
- private LanceIndexJobMutationState mutationState = LanceIndexJobMutationState.UNKNOWN;
-
- /**
- * Safe default for a corrupt record missing the key: REQUIRED holds the fence on a
- * terminal record, mirroring the null fallback in {@link #isUnresolved()}. A legal
- * record always carries the state explicitly (admission sets NOT_REQUIRED).
- */
- @SerializedName(value = "rs")
- private LanceIndexJobRefreshState refreshState = LanceIndexJobRefreshState.REQUIRED;
-
- // ------------------------------------------------------------------
- // Result
- // ------------------------------------------------------------------
- @SerializedName(value = "res")
- private LanceIndexJobResult result;
-
- // ------------------------------------------------------------------
- // Dispatch
- // ------------------------------------------------------------------
- @SerializedName(value = "bid")
- private Long backendId;
-
- @SerializedName(value = "bpe")
- private Long beProcessEpoch;
-
- /**
- * Immutable revision established by PENDING -> RUNNING. Result and
- * termination-proof callbacks use this dispatch identity rather than racing
- * on the record's global revision. Null only before dispatch or in old data.
- */
- @SerializedName(value = "drv")
- private Long dispatchRevision;
-
- /** Immutable per-dispatch UUID; callbacks must present the matching identity. */
- @SerializedName(value = "iid")
- private String invocationId;
-
- /** Bounds wait/runtime only; never proves termination and never releases a possible-live slot. */
- @SerializedName(value = "dlm")
- private Long deadlineMs;
-
- @SerializedName(value = "plo")
- private boolean possibleLiveOwned;
-
- @SerializedName(value = "tp")
- private LanceIndexTerminationProof terminationProof = LanceIndexTerminationProof.NONE;
-
- // ------------------------------------------------------------------
- // FORCE_RELEASE audit (populated only by the FORCE slice; durable + replayable here)
- // ------------------------------------------------------------------
- @SerializedName(value = "fr")
- private boolean forceReleased;
-
- @SerializedName(value = "fa")
- private String forceActor;
-
- @SerializedName(value = "ftm")
- private Long forceTimeMs;
-
- @SerializedName(value = "fn")
- private String forceNote;
-
- @SerializedName(value = "fw")
- private String forceWarning;
-
- /**
- * No-arg constructor for Gson replay only; missing fields keep the safe
- * defaults declared above (UNKNOWN mutation state holds the fence, never
- * the redispatchable PENDING; REQUIRED refresh state owes a refresh rather
- * than silently releasing the fence).
- */
- public LanceIndexJob() {
- }
-
- /**
- * Admission constructor: identity, intent, and the admission snapshot. The
- * manager initializes the lifecycle fields when the job is admitted.
- */
- public LanceIndexJob(long jobId, String creator, long catalogId, String dbName, String tableName,
- String provider, String normalizedLocator, String displayIndexName, String normalizedIndexName,
- LanceIndexJobMutationType mutationType, boolean ifNotExists, boolean ifExists, String indexType,
- String columnName, String propertiesJson, long admittedDatasetVersion,
- LanceIndexSchemaContract schemaContract) {
- this.jobId = jobId;
- this.creator = checkRequiredBytes(creator, MAX_DURABLE_TEXT_BYTES, "creator");
- this.catalogId = catalogId;
- this.dbName = checkRequiredBytes(dbName, MAX_DURABLE_TEXT_BYTES, "dbName");
- this.tableName = checkRequiredBytes(tableName, MAX_DURABLE_TEXT_BYTES, "tableName");
- this.provider = Objects.requireNonNull(provider, "provider");
- this.normalizedLocator = Objects.requireNonNull(normalizedLocator, "normalizedLocator");
- setDisplayIndexName(displayIndexName);
- setNormalizedIndexName(normalizedIndexName);
- this.mutationType = Objects.requireNonNull(mutationType, "mutationType");
- this.ifNotExists = ifNotExists;
- this.ifExists = ifExists;
- setIndexType(indexType);
- setColumnName(columnName);
- setPropertiesJson(propertiesJson);
- this.admittedDatasetVersion = admittedDatasetVersion;
- this.schemaContract = schemaContract;
- validateForAdmission();
- }
-
- /**
- * Copy used by the manager to stage a durable transition: the copy is
- * mutated, written to the edit log, and then swapped in verbatim, so a
- * published instance is never mutated after being logged. Every field is
- * carried over; the {@code result} and {@code schemaContract} references
- * are shared, which is safe because both are immutable values that are
- * only ever replaced wholesale, never mutated in place.
- */
- public LanceIndexJob(LanceIndexJob other) {
- this.jobId = other.jobId;
- this.creator = other.creator;
- this.revision = other.revision;
- this.createTimeMs = other.createTimeMs;
- this.updateTimeMs = other.updateTimeMs;
- this.catalogId = other.catalogId;
- this.dbName = other.dbName;
- this.tableName = other.tableName;
- this.provider = other.provider;
- this.normalizedLocator = other.normalizedLocator;
- this.displayIndexName = other.displayIndexName;
- this.normalizedIndexName = other.normalizedIndexName;
- this.mutationType = other.mutationType;
- this.ifNotExists = other.ifNotExists;
- this.ifExists = other.ifExists;
- this.indexType = other.indexType;
- this.columnName = other.columnName;
- this.propertiesJson = other.propertiesJson;
- this.admittedDatasetVersion = other.admittedDatasetVersion;
- this.schemaContract = other.schemaContract;
- this.mutationState = other.mutationState;
- this.refreshState = other.refreshState;
- this.result = other.result;
- this.backendId = other.backendId;
- this.beProcessEpoch = other.beProcessEpoch;
- this.dispatchRevision = other.dispatchRevision;
- this.invocationId = other.invocationId;
- this.deadlineMs = other.deadlineMs;
- this.possibleLiveOwned = other.possibleLiveOwned;
- this.terminationProof = other.terminationProof;
- this.forceReleased = other.forceReleased;
- this.forceActor = other.forceActor;
- this.forceTimeMs = other.forceTimeMs;
- this.forceNote = other.forceNote;
- this.forceWarning = other.forceWarning;
- }
-
- // ------------------------------------------------------------------
- // Derived helpers
- // ------------------------------------------------------------------
-
- /**
- * The durable same-name fence key of this job.
- */
- public LanceIndexFenceKey fenceKey() {
- return new LanceIndexFenceKey(catalogId, provider, normalizedLocator, normalizedIndexName);
- }
-
- /**
- * The per persisted table/locator quota identity.
- */
- public LanceIndexJobQuota.TableQuotaKey getTableQuotaKey() {
- return new LanceIndexJobQuota.TableQuotaKey(catalogId, normalizedLocator);
- }
-
- /**
- * Whether this job is still unresolved, i.e. must keep holding its same-name
- * fence and unresolved quota. Fence and quota are released together:
- * PENDING/RUNNING always hold; a known terminal job holds until its required
- * refresh is DONE (FAILED still holds, refresh may retry); UNKNOWN holds until
- * a durable FORCE_RELEASE. A null state from a corrupt record is treated as
- * UNKNOWN, the safe direction (never redispatched). Holding the fence itself
- * requires the fence identity to survive: the manager keeps an identity-less
- * corrupt record out of the books (its fence key cannot be reconstructed) and
- * instead blocks all new admissions fail-closed until a later durable record
- * resolves it.
- */
- public boolean isUnresolved() {
- LanceIndexJobMutationState ms = mutationState == null ? LanceIndexJobMutationState.UNKNOWN : mutationState;
- switch (ms) {
- case PENDING:
- case RUNNING:
- return true;
- case UNKNOWN:
- return !forceReleased;
- case COMMITTED:
- case NOT_COMMITTED:
- default:
- LanceIndexJobRefreshState rs =
- refreshState == null ? LanceIndexJobRefreshState.REQUIRED : refreshState;
- return rs == LanceIndexJobRefreshState.REQUIRED
- || rs == LanceIndexJobRefreshState.RUNNING
- || rs == LanceIndexJobRefreshState.FAILED;
- }
- }
-
- /**
- * Whether this job still owns a possible-live worker slot: released only
- * by a matching termination proof or a durable FORCE_RELEASE, never by a
- * deadline.
- */
- public boolean holdsPossibleLiveSlot() {
- return possibleLiveOwned
- && (terminationProof == null || terminationProof == LanceIndexTerminationProof.NONE)
- && !forceReleased;
- }
-
- /**
- * Validate the invariant-bearing and bounded fields before this record is
- * admitted. This is intentionally separate from Gson replay, which must
- * remain tolerant of old or corrupt records in the safe direction.
- */
- public void validateForAdmission() {
- checkRequiredBytes(creator, MAX_DURABLE_TEXT_BYTES, "creator");
- checkRequiredBytes(dbName, MAX_DURABLE_TEXT_BYTES, "dbName");
- checkRequiredBytes(tableName, MAX_DURABLE_TEXT_BYTES, "tableName");
- if (provider == null) {
- throw new IllegalArgumentException("lance index job provider must not be null");
- }
- if (!LanceIndexFenceKey.PROVIDER_DIRECTORY.equals(provider)) {
- throw new IllegalArgumentException("lance index job provider must be DIRECTORY");
- }
- if (normalizedLocator == null) {
- throw new IllegalArgumentException("normalized dataset locator must not be null");
- }
- String canonicalLocator = LanceIndexDatasetLocator.normalize(normalizedLocator);
- if (!canonicalLocator.equals(normalizedLocator)) {
- throw new IllegalArgumentException("dataset locator is not in canonical identity form");
- }
- LanceIndexNameNormalizer.validateDisplayName(displayIndexName);
- validateNormalizedIndexName(normalizedIndexName);
- String expectedNormalizedName = LanceIndexNameNormalizer.normalize(displayIndexName);
- if (!expectedNormalizedName.equals(normalizedIndexName)) {
- throw new IllegalArgumentException("normalized index name does not match the display name");
- }
- if (mutationType == null) {
- throw new IllegalArgumentException("mutation type must not be null");
- }
- checkBytes(indexType, MAX_DURABLE_TEXT_BYTES, "indexType");
- checkBytes(columnName, MAX_DURABLE_TEXT_BYTES, "columnName");
- checkBytes(propertiesJson, MAX_PROPERTIES_JSON_BYTES, "propertiesJson");
- checkBytes(invocationId, MAX_INVOCATION_ID_BYTES, "invocationId");
- checkBytes(forceActor, MAX_DURABLE_TEXT_BYTES, "forceActor");
- checkBytes(forceNote, MAX_FORCE_TEXT_BYTES, "forceNote");
- checkBytes(forceWarning, MAX_FORCE_TEXT_BYTES, "forceWarning");
- if (result != null) {
- checkBytes(result.getSanitizedMessage(), LanceIndexJobResult.MAX_MESSAGE_BYTES, "sanitizedMessage");
- }
- if (schemaContract != null) {
- schemaContract.validateForAdmission();
- }
- }
-
- // ------------------------------------------------------------------
- // Accessors. Setters for bounded text fields re-validate the bound.
- // ------------------------------------------------------------------
-
- public long getJobId() {
- return jobId;
- }
-
- public void setJobId(long jobId) {
- this.jobId = jobId;
- }
-
- public String getCreator() {
- return creator;
- }
-
- public void setCreator(String creator) {
- this.creator = checkBytes(creator, MAX_DURABLE_TEXT_BYTES, "creator");
- }
-
- public long getRevision() {
- return revision;
- }
-
- public void setRevision(long revision) {
- this.revision = revision;
- }
-
- public long getCreateTimeMs() {
- return createTimeMs;
- }
-
- public void setCreateTimeMs(long createTimeMs) {
- this.createTimeMs = createTimeMs;
- }
-
- public long getUpdateTimeMs() {
- return updateTimeMs;
- }
-
- public void setUpdateTimeMs(long updateTimeMs) {
- this.updateTimeMs = updateTimeMs;
- }
-
- public long getCatalogId() {
- return catalogId;
- }
-
- public String getDbName() {
- return dbName;
- }
-
- public String getTableName() {
- return tableName;
- }
-
- public String getProvider() {
- return provider;
- }
-
- public String getNormalizedLocator() {
- return normalizedLocator;
- }
-
- public String getDisplayIndexName() {
- return displayIndexName;
- }
-
- public final void setDisplayIndexName(String displayIndexName) {
- LanceIndexNameNormalizer.validateDisplayName(displayIndexName);
- this.displayIndexName = displayIndexName;
- }
-
- public String getNormalizedIndexName() {
- return normalizedIndexName;
- }
-
- public final void setNormalizedIndexName(String normalizedIndexName) {
- validateNormalizedIndexName(normalizedIndexName);
- this.normalizedIndexName = normalizedIndexName;
- }
-
- private static void validateNormalizedIndexName(String normalizedIndexName) {
- if (normalizedIndexName == null || normalizedIndexName.isEmpty()) {
- throw new IllegalArgumentException("normalized index name must not be null or empty");
- }
- if (normalizedIndexName.getBytes(StandardCharsets.UTF_8).length
- > LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES) {
- throw new IllegalArgumentException(
- "normalized index name exceeds " + LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + " UTF-8 bytes");
- }
- }
-
- public LanceIndexJobMutationType getMutationType() {
- return mutationType;
- }
-
- public boolean isIfNotExists() {
- return ifNotExists;
- }
-
- public boolean isIfExists() {
- return ifExists;
- }
-
- public String getIndexType() {
- return indexType;
- }
-
- public void setIndexType(String indexType) {
- this.indexType = checkBytes(indexType, MAX_DURABLE_TEXT_BYTES, "indexType");
- }
-
- public String getColumnName() {
- return columnName;
- }
-
- public void setColumnName(String columnName) {
- this.columnName = checkBytes(columnName, MAX_DURABLE_TEXT_BYTES, "columnName");
- }
-
- public String getPropertiesJson() {
- return propertiesJson;
- }
-
- public final void setPropertiesJson(String propertiesJson) {
- this.propertiesJson = checkBytes(propertiesJson, MAX_PROPERTIES_JSON_BYTES, "propertiesJson");
- }
-
- public long getAdmittedDatasetVersion() {
- return admittedDatasetVersion;
- }
-
- public LanceIndexSchemaContract getSchemaContract() {
- return schemaContract;
- }
-
- public LanceIndexJobMutationState getMutationState() {
- return mutationState;
- }
-
- public void setMutationState(LanceIndexJobMutationState mutationState) {
- this.mutationState = Objects.requireNonNull(mutationState, "mutationState");
- }
-
- public LanceIndexJobRefreshState getRefreshState() {
- return refreshState;
- }
-
- public void setRefreshState(LanceIndexJobRefreshState refreshState) {
- this.refreshState = Objects.requireNonNull(refreshState, "refreshState");
- }
-
- public LanceIndexJobResult getResult() {
- return result;
- }
-
- public void setResult(LanceIndexJobResult result) {
- this.result = result;
- }
-
- public Long getBackendId() {
- return backendId;
- }
-
- public void setBackendId(Long backendId) {
- this.backendId = backendId;
- }
-
- public Long getBeProcessEpoch() {
- return beProcessEpoch;
- }
-
- public void setBeProcessEpoch(Long beProcessEpoch) {
- this.beProcessEpoch = beProcessEpoch;
- }
-
- public Long getDispatchRevision() {
- return dispatchRevision;
- }
-
- public void setDispatchRevision(Long dispatchRevision) {
- this.dispatchRevision = dispatchRevision;
- }
-
- public String getInvocationId() {
- return invocationId;
- }
-
- public void setInvocationId(String invocationId) {
- this.invocationId = checkBytes(invocationId, MAX_INVOCATION_ID_BYTES, "invocationId");
- }
-
- public Long getDeadlineMs() {
- return deadlineMs;
- }
-
- public void setDeadlineMs(Long deadlineMs) {
- this.deadlineMs = deadlineMs;
- }
-
- public boolean isPossibleLiveOwned() {
- return possibleLiveOwned;
- }
-
- public void setPossibleLiveOwned(boolean possibleLiveOwned) {
- this.possibleLiveOwned = possibleLiveOwned;
- }
-
- public LanceIndexTerminationProof getTerminationProof() {
- return terminationProof;
- }
-
- public void setTerminationProof(LanceIndexTerminationProof terminationProof) {
- this.terminationProof = Objects.requireNonNull(terminationProof, "terminationProof");
- }
-
- public boolean isForceReleased() {
- return forceReleased;
- }
-
- public void setForceReleased(boolean forceReleased) {
- this.forceReleased = forceReleased;
- }
-
- public String getForceActor() {
- return forceActor;
- }
-
- public void setForceActor(String forceActor) {
- this.forceActor = checkBytes(forceActor, MAX_DURABLE_TEXT_BYTES, "forceActor");
- }
-
- public Long getForceTimeMs() {
- return forceTimeMs;
- }
-
- public void setForceTimeMs(Long forceTimeMs) {
- this.forceTimeMs = forceTimeMs;
- }
-
- public String getForceNote() {
- return forceNote;
- }
-
- public void setForceNote(String forceNote) {
- this.forceNote = checkBytes(forceNote, MAX_FORCE_TEXT_BYTES, "forceNote");
- }
-
- public String getForceWarning() {
- return forceWarning;
- }
-
- public void setForceWarning(String forceWarning) {
- this.forceWarning = checkBytes(forceWarning, MAX_FORCE_TEXT_BYTES, "forceWarning");
- }
-
- private static String checkBytes(String value, int maxBytes, String fieldName) {
- if (value != null && value.getBytes(StandardCharsets.UTF_8).length > maxBytes) {
- throw new IllegalArgumentException(fieldName + " exceeds " + maxBytes + " UTF-8 bytes");
- }
- return value;
- }
-
- private static String checkRequiredBytes(String value, int maxBytes, String fieldName) {
- if (value == null || value.isEmpty()) {
- throw new IllegalArgumentException(fieldName + " must not be null or empty");
- }
- return checkBytes(value, maxBytes, fieldName);
- }
-
- // ------------------------------------------------------------------
- // Serialization (Gson stream style, see DropIndexPolicyLog)
- // ------------------------------------------------------------------
-
- @Override
- public void write(DataOutput out) throws IOException {
- Text.writeString(out, GsonUtils.GSON.toJson(this));
- }
-
- public static LanceIndexJob read(DataInput in) throws IOException {
- return GsonUtils.GSON.fromJson(Text.readString(in), LanceIndexJob.class);
- }
-
- /**
- * Deliberately omits the locator: job lookups must not disclose target
- * details to callers without privilege, and log lines reuse this form.
- */
- @Override
- public String toString() {
- return "LanceIndexJob{jobId=" + jobId + ", revision=" + revision + ", catalogId=" + catalogId
- + ", db=" + dbName + ", table=" + tableName + ", index=" + displayIndexName
- + ", mutationType=" + mutationType + ", mutationState=" + mutationState
- + ", refreshState=" + refreshState + ", possibleLiveSlot=" + holdsPossibleLiveSlot()
- + ", forceReleased=" + forceReleased + '}';
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobCompletionReason.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobCompletionReason.java
deleted file mode 100644
index 94bfcd32053..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobCompletionReason.java
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-/**
- * Why a job reached its terminal state. NOOP is not a state: a typed
- * post-dispatch DROP-not-found for DROP IF EXISTS is NOT_COMMITTED with
- * completion reason IF_CONDITION_NOOP.
- */
-public enum LanceIndexJobCompletionReason {
- NONE,
- IF_CONDITION_NOOP
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java
deleted file mode 100644
index a34c9c1a087..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java
+++ /dev/null
@@ -1,790 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import org.apache.doris.catalog.Env;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.io.Text;
-import org.apache.doris.common.io.Writable;
-import org.apache.doris.persist.gson.GsonPostProcessable;
-import org.apache.doris.persist.gson.GsonUtils;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Maps;
-import com.google.gson.annotations.SerializedName;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Objects;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * Master-owned manager of the durable Lance index job records, the same-name
- * fences, and the three-level unresolved-job quotas. It deliberately reuses
- * neither the generic JobManager scheduling framework nor the internal
- * IndexChangeJob machinery: the external one-shot CAS, no-redispatch rule,
- * same-name fence, and possible-live ownership required here are not provided
- * by either.
- *
- * <p>Every FE keeps the same in-memory image of {@link #jobs}: the master
- * writes each durable transition to the edit log and then applies the same
- * record locally; followers apply it from replay. All transitions share one
- * write-path shape:
- * <pre>
- * writeLock -> validate (state, revision CAS, callback identity)
- * -> writeEditLog(updated copy) // fails only by System.exit
- * -> applyToMemory(updated copy) // verbatim swap + fence/quota accounting
- * </pre>
- * A published {@link LanceIndexJob} is never mutated; a transition stages a
- * copy, logs it, and swaps it in. {@link #replayUpsertJob(LanceIndexJob)} is a
- * verbatim replace with a monotonic-revision guard and performs no state
- * transformation at all: a follower tailing a live master must keep a fresh
- * RUNNING record RUNNING. The only place a durable RUNNING without a complete
- * matching terminal result becomes UNKNOWN is {@link #onTransferToMaster()},
- * the master-election sweep that runs after metadata replay and before any
- * dispatcher could start.
- *
- * <p>Fence and quota live and die together ({@link LanceIndexJob#isUnresolved()}):
- * released when a known terminal job's refresh is NOT_REQUIRED or DONE, or by
- * a durable FORCE_RELEASE; an unresolved UNKNOWN holds both across failover,
- * timeout, termination proof, and metadata observations. A replayed unresolved
- * record that lacks fence identity (provider/locator/name) cannot join the
- * books — its fence key cannot be reconstructed — so it is tracked in
- * {@link #corruptUnresolvedJobIds} and blocks all new admissions fail-closed
- * until a later durable record resolves it.
- *
- * <p>The class starts no threads, so no checkpoint-thread guard is needed in
- * the constructor; the derived fence index and quota counters are rebuilt in
- * {@link #gsonPostProcess()} after image load.
- */
-public class LanceIndexJobManager implements Writable, GsonPostProcessable {
- private static final Logger LOG = LogManager.getLogger(LanceIndexJobManager.class);
-
- @SerializedName(value = "jobs")
- private ConcurrentMap<Long, LanceIndexJob> jobs = Maps.newConcurrentMap();
-
- /**
- * Derived: fence key -> all job ids holding that fence. Legal admission creates
- * exactly one owner; retaining every owner for a corrupt collision keeps the
- * fence fail-closed when one of those jobs later settles. Identity-less corrupt
- * records are kept out of the books and tracked in {@link #corruptUnresolvedJobIds}
- * instead. Rebuilt after replay/image load.
- */
- private final Map<LanceIndexFenceKey, NavigableSet<Long>> fenceIndex = Maps.newHashMap();
-
- /**
- * Derived: ids of replayed unresolved jobs that lack fence identity (a corrupt
- * journal/image record whose fence key cannot be reconstructed). While this set
- * is non-empty, {@link #createJob} rejects every admission fail-closed: such a
- * record may still track a live or already-committed mutation under an unknown
- * name, and no new job may race it. An id leaves the set when a later durable
- * record resolves the job (a known terminal state with no pending refresh, or
- * the durable force release of a follow-up PR). Rebuilt after replay/image load.
- */
- private final NavigableSet<Long> corruptUnresolvedJobIds = new TreeSet<>();
-
- /** Derived: three-level unresolved counters, rebuilt from the unresolved jobs. */
- private final LanceIndexJobQuota quota = new LanceIndexJobQuota();
-
- private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
-
- public LanceIndexJobManager() {
- }
-
- private void readLock() {
- lock.readLock().lock();
- }
-
- private void readUnlock() {
- lock.readLock().unlock();
- }
-
- private void writeLock() {
- lock.writeLock().lock();
- }
-
- private void writeUnlock() {
- lock.writeLock().unlock();
- }
-
- /**
- * Edit-log seam: the only place this manager writes the journal. Tests
- * subclass and override to capture or swallow the record.
- */
- protected void writeEditLog(LanceIndexJob job) {
- Env.getCurrentEnv().getEditLog().logLanceIndexJob(job);
- }
-
- // ------------------------------------------------------------------
- // Master-only write paths
- // ------------------------------------------------------------------
-
- /**
- * Admit a new job: same-name fence CAS and unresolved-quota admission
- * happen inside the write lock, before anything is logged. A rejection
- * leaves no job, no fence, no quota charge, and no edit-log record.
- *
- * @throws DdlException on a fence conflict, a duplicate job id, quota overload,
- * or while corrupt unresolved records keep admission fail-closed
- */
- public void createJob(LanceIndexJob job, long tableLimit, long catalogLimit, long globalLimit)
- throws DdlException {
- Objects.requireNonNull(job, "job");
- writeLock();
- try {
- try {
- job.validateForAdmission();
- } catch (IllegalArgumentException e) {
- throw new DdlException("invalid lance index job: " + e.getMessage(), e);
- }
- if (tableLimit <= 0 || catalogLimit <= 0 || globalLimit <= 0) {
- throw new DdlException("lance index job quota limits must all be positive");
- }
- if (jobs.containsKey(job.getJobId())) {
- throw new DdlException("lance index job id already exists: " + job.getJobId());
- }
- if (!corruptUnresolvedJobIds.isEmpty()) {
- // Fail closed: a corrupt unresolved record may track a live or already-committed
- // mutation whose fence key cannot be reconstructed, so no new admission may
- // race it. The message stays bounded and discloses no target identity.
- throw new DdlException("lance index admission is fail-closed: "
- + corruptUnresolvedJobIds.size() + " unresolved job(s) lack fence identity"
- + " (corrupt journal/image record), smallest job id " + corruptUnresolvedJobIds.first()
- + "; resolve them (FORCE_RELEASE) before admitting new jobs");
- }
- NavigableSet<Long> fencingJobIds = fenceIndex.get(job.fenceKey());
- if (fencingJobIds != null && !fencingJobIds.isEmpty()) {
- // Never disclose the locator in the rejection (the caller may lack target privilege).
- throw new DdlException("lance index '" + job.getDisplayIndexName()
- + "' is fenced by unresolved job " + fencingJobIds.first()
- + "; resolve that job (FORCE_RELEASE) before reusing the name");
- }
- // Pure admission check; the charge itself happens in applyToMemory together with the fence,
- // after the record is durable. An edit-log write failure exits the process, so no rollback exists.
- if (!quota.hasCapacity(job, tableLimit, catalogLimit, globalLimit)) {
- throw new DdlException("unresolved lance index job quota exceeded for index '"
- + job.getDisplayIndexName() + "'; resolve or finish existing jobs first");
- }
- long now = System.currentTimeMillis();
- // Stage a private copy and reset every lifecycle field: the caller's object is
- // never published, and admission always starts from the same PENDING record no
- // matter what the caller left in the lifecycle fields.
- LanceIndexJob admitted = new LanceIndexJob(job);
- admitted.setMutationState(LanceIndexJobMutationState.PENDING);
- admitted.setRefreshState(LanceIndexJobRefreshState.NOT_REQUIRED);
- admitted.setRevision(0);
- admitted.setCreateTimeMs(now);
- admitted.setUpdateTimeMs(now);
- admitted.setResult(null);
- admitted.setBackendId(null);
- admitted.setBeProcessEpoch(null);
- admitted.setInvocationId(null);
- admitted.setDispatchRevision(null);
- admitted.setDeadlineMs(null);
- admitted.setPossibleLiveOwned(false);
- admitted.setTerminationProof(LanceIndexTerminationProof.NONE);
- admitted.setForceReleased(false);
- admitted.setForceActor(null);
- admitted.setForceTimeMs(null);
- admitted.setForceNote(null);
- admitted.setForceWarning(null);
- writeEditLog(admitted);
- applyToMemory(admitted);
- } finally {
- writeUnlock();
- }
- }
-
- /**
- * PENDING -> RUNNING, the durable dispatch boundary. Compare-and-set on
- * (jobId, revision): only a PENDING job at the expected revision may be
- * dispatched, which is what makes redispatch after replay impossible.
- * Records the dispatch identity (backend, BE process epoch, immutable
- * invocation id, deadline) and takes the possible-live slot.
- *
- * @return false (with a warning) on any mismatch; the caller must not send
- */
- public boolean markRunning(long jobId, long expectedRevision, long backendId, long beProcessEpoch,
- String invocationId, long deadlineMs) {
- if (StringUtils.isBlank(invocationId)) {
- // A null/blank invocation identity would silently match a null field under
- // Objects.equals in completeWithResult and defeat the stale-callback guard.
- LOG.warn("reject markRunning for lance index job {}: invocation id is null or blank", jobId);
- return false;
- }
- writeLock();
- try {
- LanceIndexJob current = jobs.get(jobId);
- if (current == null || current.getRevision() != expectedRevision
- || current.getMutationState() != LanceIndexJobMutationState.PENDING) {
- LOG.warn("reject markRunning for lance index job {}: expected revision {}, current {}",
- jobId, expectedRevision, current);
- return false;
- }
- try {
- current.validateForAdmission();
- } catch (IllegalArgumentException e) {
- LOG.warn("reject markRunning for invalid lance index job {}: {}", jobId, e.getMessage());
- return false;
- }
- if (!hasFenceIdentity(current)) {
- LOG.warn("reject markRunning for lance index job {} without a valid fence identity", jobId);
- return false;
- }
- LanceIndexJob updated = new LanceIndexJob(current);
- updated.setMutationState(LanceIndexJobMutationState.RUNNING);
- updated.setBackendId(backendId);
- updated.setBeProcessEpoch(beProcessEpoch);
- updated.setInvocationId(invocationId);
- updated.setDeadlineMs(deadlineMs);
- updated.setPossibleLiveOwned(true);
- long dispatchRevision = current.getRevision() + 1;
- updated.setRevision(dispatchRevision);
- updated.setDispatchRevision(dispatchRevision);
- updated.setUpdateTimeMs(System.currentTimeMillis());
- writeEditLog(updated);
- applyToMemory(updated);
- return true;
- } finally {
- writeUnlock();
- }
- }
-
- /**
- * RUNNING -> terminal, from a worker/supervisor result. A callback must
- * match the durable dispatch identity exactly (immutable dispatch revision, invocation
- * id, and BE process epoch); a stale callback only logs a warning and
- * changes nothing. The typed result is classified into (mutation state,
- * refresh obligation, completion reason); message text is never inspected.
- * A known terminal job whose refresh is NOT_REQUIRED releases fence and
- * quota immediately; an UNKNOWN keeps both until a durable FORCE_RELEASE.
- *
- * <p>This is also the channel for marking a job UNKNOWN on an ambiguous
- * result (result code NO_TRUSTED_RESULT), including the master-transfer
- * sweep; no separate markUnknown API exists.
- *
- * @return false (with a warning) when the callback is stale or the job is not RUNNING
- */
- public boolean completeWithResult(long jobId, long expectedDispatchRevision, String invocationId,
- Long beProcessEpoch,
- LanceIndexJobResult result) {
- Objects.requireNonNull(result, "result");
- writeLock();
- try {
- LanceIndexJob current = jobs.get(jobId);
- if (current == null || dispatchRevisionOf(current) != expectedDispatchRevision) {
- LOG.warn("reject stale lance index job callback for job {}: expected dispatch revision {}, current {}",
- jobId, expectedDispatchRevision, current);
- return false;
- }
- if (current.getMutationState() != LanceIndexJobMutationState.RUNNING) {
- LOG.warn("reject lance index job callback for job {} in state {}: only RUNNING accepts a result",
- jobId, current.getMutationState());
- return false;
- }
- if (!Objects.equals(current.getInvocationId(), invocationId)
- || !Objects.equals(current.getBeProcessEpoch(), beProcessEpoch)) {
- LOG.warn("reject stale lance index job callback for job {}: invocation/epoch mismatch, current {}",
- jobId, current);
- return false;
- }
- LanceIndexJobResultCode.Classification classification = LanceIndexJobResultCode.classify(
- current.getMutationType(), result.getResultCode(), current.isIfExists(),
- result.isExternalMetadataAdvanced());
- LanceIndexJob updated = new LanceIndexJob(current);
- if (updated.getDispatchRevision() == null) {
- // Backfill old RUNNING records before the global revision advances so
- // an independent termination proof can still identify this dispatch.
- updated.setDispatchRevision(expectedDispatchRevision);
- }
- updated.setMutationState(classification.getMutationState());
- updated.setRefreshState(classification.getRefreshState());
- updated.setResult(new LanceIndexJobResult(result.getResultCode(), classification.getCompletionReason(),
- result.getSanitizedMessage(), result.isExternalMetadataAdvanced()));
- updated.setRevision(current.getRevision() + 1);
- updated.setUpdateTimeMs(System.currentTimeMillis());
- writeEditLog(updated);
- applyToMemory(updated);
- return true;
- } finally {
- writeUnlock();
- }
- }
-
- /**
- * Refresh REQUIRED -> RUNNING, or FAILED -> RUNNING for a retry
- * through the idempotent external-table refresh path.
- */
- public boolean markRefreshRunning(long jobId, long expectedRevision) {
- return transitionRefresh(jobId, expectedRevision, LanceIndexJobRefreshState.RUNNING);
- }
-
- /**
- * Refresh RUNNING -> DONE. On a known terminal job this releases the
- * same-name fence and the unresolved quota.
- */
- public boolean markRefreshDone(long jobId, long expectedRevision) {
- return transitionRefresh(jobId, expectedRevision, LanceIndexJobRefreshState.DONE);
- }
-
- /**
- * Refresh RUNNING -> FAILED. The job keeps fence and quota; a later
- * {@link #markRefreshRunning} retries through the idempotent path.
- */
- public boolean markRefreshFailed(long jobId, long expectedRevision) {
- return transitionRefresh(jobId, expectedRevision, LanceIndexJobRefreshState.FAILED);
- }
-
- private boolean transitionRefresh(long jobId, long expectedRevision, LanceIndexJobRefreshState target) {
- writeLock();
- try {
- LanceIndexJob current = jobs.get(jobId);
- if (current == null || current.getRevision() != expectedRevision) {
- LOG.warn("reject refresh transition to {} for lance index job {}: expected revision {}, current {}",
- target, jobId, expectedRevision, current);
- return false;
- }
- if (current.getMutationState() == null || !current.getMutationState().isTerminal()) {
- LOG.warn("reject refresh transition to {} for non-terminal lance index job {} in mutation state {}",
- target, jobId, current.getMutationState());
- return false;
- }
- LanceIndexJobRefreshState from = current.getRefreshState();
- boolean legal = (target == LanceIndexJobRefreshState.RUNNING
- && (from == LanceIndexJobRefreshState.REQUIRED || from == LanceIndexJobRefreshState.FAILED))
- || (target == LanceIndexJobRefreshState.DONE && from == LanceIndexJobRefreshState.RUNNING)
- || (target == LanceIndexJobRefreshState.FAILED && from == LanceIndexJobRefreshState.RUNNING);
- if (!legal) {
- LOG.warn("reject illegal refresh transition {} -> {} for lance index job {}",
- from, target, jobId);
- return false;
- }
- LanceIndexJob updated = new LanceIndexJob(current);
- updated.setRefreshState(target);
- updated.setRevision(current.getRevision() + 1);
- updated.setUpdateTimeMs(System.currentTimeMillis());
- writeEditLog(updated);
- applyToMemory(updated);
- return true;
- } finally {
- writeUnlock();
- }
- }
-
- /**
- * Record a matching termination proof for a job that still owns a
- * possible-live slot. Backend, BE process epoch, invocation id, and immutable
- * dispatch revision must all match. This releases only the slot: it never
- * changes the mutation state and never releases the fence or quota.
- */
- public boolean recordTerminationProof(long jobId, long expectedDispatchRevision, long backendId,
- long beProcessEpoch, String invocationId, LanceIndexTerminationProof proof) {
- Objects.requireNonNull(proof, "proof");
- writeLock();
- try {
- LanceIndexJob current = jobs.get(jobId);
- if (current == null || dispatchRevisionOf(current) != expectedDispatchRevision) {
- LOG.warn("reject termination proof for lance index job {}: expected dispatch revision {}, current {}",
- jobId, expectedDispatchRevision, current);
- return false;
- }
- if (StringUtils.isBlank(invocationId)
- || !Objects.equals(current.getBackendId(), backendId)
- || !Objects.equals(current.getBeProcessEpoch(), beProcessEpoch)
- || !Objects.equals(current.getInvocationId(), invocationId)) {
- LOG.warn("reject stale termination proof for lance index job {}:"
- + " dispatch identity mismatch, current {}", jobId, current);
- return false;
- }
- if (proof == LanceIndexTerminationProof.NONE || !current.isPossibleLiveOwned()
- || current.getTerminationProof() != LanceIndexTerminationProof.NONE) {
- LOG.warn("reject termination proof {} for lance index job {}: no possible-live slot owned, current {}",
- proof, jobId, current);
- return false;
- }
- LanceIndexJob updated = new LanceIndexJob(current);
- if (updated.getDispatchRevision() == null) {
- updated.setDispatchRevision(expectedDispatchRevision);
- }
- updated.setTerminationProof(proof);
- updated.setPossibleLiveOwned(false);
- updated.setRevision(current.getRevision() + 1);
- updated.setUpdateTimeMs(System.currentTimeMillis());
- writeEditLog(updated);
- applyToMemory(updated);
- return true;
- } finally {
- writeUnlock();
- }
- }
-
- /**
- * Master-election sweep, hooked from {@code Env.transferToMaster()} after
- * metadata replay and before any master-only dispatcher could start. A
- * durable RUNNING at this point means the terminal result may have been
- * lost with the old master: the job becomes UNKNOWN through the same
- * completeWithResult channel (result code NO_TRUSTED_RESULT, fence/quota/
- * possible-live ownership retained, never redispatched), and an in-flight
- * refresh is downgraded to REQUIRED so the idempotent refresh retries.
- * Both transitions are written to the edit log so followers converge.
- */
- public void onTransferToMaster() {
- List<LanceIndexJob> snapshot;
- readLock();
- try {
- snapshot = new ArrayList<>(jobs.values());
- } finally {
- readUnlock();
- }
- for (LanceIndexJob job : snapshot) {
- if (job.getMutationState() == LanceIndexJobMutationState.RUNNING) {
- boolean completed = completeWithResult(job.getJobId(), dispatchRevisionOf(job), job.getInvocationId(),
- job.getBeProcessEpoch(),
- new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT,
- LanceIndexJobCompletionReason.NONE,
- "FE master transferred while the job was RUNNING; the result is not trusted", false));
- if (completed) {
- LOG.info("lance index job {} transitioned RUNNING -> UNKNOWN on master transfer",
- job.getJobId());
- }
- }
- if (job.getRefreshState() == LanceIndexJobRefreshState.RUNNING) {
- downgradeRunningRefresh(job);
- }
- }
- }
-
- private void downgradeRunningRefresh(LanceIndexJob sweepCandidate) {
- writeLock();
- try {
- LanceIndexJob current = jobs.get(sweepCandidate.getJobId());
- if (current == null || current.getRefreshState() != LanceIndexJobRefreshState.RUNNING) {
- return;
- }
- LanceIndexJob updated = new LanceIndexJob(current);
- updated.setRefreshState(LanceIndexJobRefreshState.REQUIRED);
- updated.setRevision(current.getRevision() + 1);
- updated.setUpdateTimeMs(System.currentTimeMillis());
- writeEditLog(updated);
- applyToMemory(updated);
- } finally {
- writeUnlock();
- }
- }
-
- // ------------------------------------------------------------------
- // Replay (all FEs)
- // ------------------------------------------------------------------
-
- /**
- * Apply one durable record from the journal: a verbatim replace of the
- * in-memory record plus derived-index accounting, with a monotonic
- * revision guard so a stale record can never roll the state back. No state
- * transformation happens here on purpose: a follower tailing a live master
- * must keep a fresh RUNNING record RUNNING. Idempotent and tolerant of
- * missing fields (Gson defaults); never throws on behalf of record content.
- */
- public void replayUpsertJob(LanceIndexJob job) {
- if (job == null) {
- LOG.warn("ignore null lance index job record");
- return;
- }
- writeLock();
- try {
- LanceIndexJob replayed = new LanceIndexJob(job);
- LanceIndexJob existing = jobs.get(replayed.getJobId());
- if (existing != null && replayed.getRevision() < existing.getRevision()) {
- LOG.warn("ignore stale lance index job record for job {}: replayed revision {} < current {}",
- replayed.getJobId(), replayed.getRevision(), existing.getRevision());
- return;
- }
- applyToMemory(replayed);
- } finally {
- writeUnlock();
- }
- }
-
- /**
- * Swap a staged record into memory and settle fence/quota accounting for
- * the replaced record. Fence and quota always move together: a record
- * holds both while {@link LanceIndexJob#isUnresolved()}, provided it
- * carries fence identity (a corrupt identity-less record stays queryable
- * but out of the books on both the charge and the release side, and is
- * tracked in {@link #corruptUnresolvedJobIds} instead). A private
- * copy is always stored so neither a replay input nor an edit-log seam
- * reference can mutate the published record. Caller holds the write lock.
- */
- private void applyToMemory(LanceIndexJob job) {
- LanceIndexJob stored = new LanceIndexJob(job);
- LanceIndexJob old = jobs.put(stored.getJobId(), stored);
- // A replaced id leaves the admission blocker no matter what replaces it: the
- // new record either resolves the job, or carries identity and joins the books,
- // or is re-added below as still identity-less and unresolved.
- corruptUnresolvedJobIds.remove(stored.getJobId());
- // Release only what the identity guard below booked: an identity-less corrupt
- // record was stored without fence/quota accounting, so keying on it would throw.
- if (old != null && old.isUnresolved() && hasFenceIdentity(old)) {
- removeFenceOwner(old);
- quota.release(old);
- }
- if (stored.isUnresolved()) {
- if (hasFenceIdentity(stored)) {
- NavigableSet<Long> owners = fenceIndex.computeIfAbsent(stored.fenceKey(), ignored -> new TreeSet<>());
- if (!owners.isEmpty() && !owners.contains(stored.getJobId())) {
- LOG.warn("fence key collision between unresolved lance index jobs {} and {};"
- + " keeping fence owner {}", owners.first(), stored.getJobId(),
- Math.min(owners.first(), stored.getJobId()));
- }
- owners.add(stored.getJobId());
- quota.charge(stored);
- } else {
- // Corrupt record tolerance: keep it queryable but out of the fence/quota
- // books (keying on it would throw), and block all new admissions
- // fail-closed until a later durable record resolves it.
- corruptUnresolvedJobIds.add(stored.getJobId());
- LOG.warn("lance index job {} lacks fence identity (provider/locator/name);"
- + " stored without fence/quota accounting and blocking all new"
- + " admissions fail-closed until resolved", stored.getJobId());
- }
- }
- }
-
- private void removeFenceOwner(LanceIndexJob job) {
- LanceIndexFenceKey fenceKey = job.fenceKey();
- NavigableSet<Long> owners = fenceIndex.get(fenceKey);
- if (owners == null) {
- return;
- }
- owners.remove(job.getJobId());
- if (owners.isEmpty()) {
- fenceIndex.remove(fenceKey);
- }
- }
-
- private static long dispatchRevisionOf(LanceIndexJob job) {
- return job.getDispatchRevision() == null ? job.getRevision() : job.getDispatchRevision();
- }
-
- private static boolean hasFenceIdentity(LanceIndexJob job) {
- return job.getProvider() != null && job.getNormalizedLocator() != null
- && job.getNormalizedIndexName() != null;
- }
-
- // ------------------------------------------------------------------
- // Queries
- // ------------------------------------------------------------------
-
- public LanceIndexJob getJob(long jobId) {
- readLock();
- try {
- LanceIndexJob job = jobs.get(jobId);
- return job == null ? null : new LanceIndexJob(job);
- } finally {
- readUnlock();
- }
- }
-
- /**
- * Returns a snapshot copy of every job, ordered by jobId. Used by SHOW LANCE INDEX JOBS.
- */
- public List<LanceIndexJob> getAllJobsSnapshot() {
- readLock();
- try {
- List<LanceIndexJob> result = new ArrayList<>();
- for (LanceIndexJob job : jobs.values()) {
- if (job != null) {
- result.add(new LanceIndexJob(job));
- }
- }
- result.sort(Comparator.comparingLong(LanceIndexJob::getJobId));
- return result;
- } finally {
- readUnlock();
- }
- }
-
- /**
- * Returns true if any unresolved job targets the given catalog. Used by catalog DDL guard (Section 4.4).
- */
- public boolean hasUnresolvedJobsForCatalog(long catalogId) {
- readLock();
- try {
- for (LanceIndexJob job : jobs.values()) {
- if (job != null && job.getCatalogId() == catalogId && job.isUnresolved()) {
- return true;
- }
- }
- return false;
- } finally {
- readUnlock();
- }
- }
-
- /**
- * All jobs still holding a fence and unresolved quota: PENDING/RUNNING,
- * unforced UNKNOWN, and known terminal jobs with unfinished refresh.
- * Unresolved jobs that lack fence identity (corrupt records) hold no fence
- * and are excluded here; they are visible through
- * {@link #getCorruptUnresolvedJobIds()} instead.
- */
- public List<LanceIndexJob> getUnresolvedJobs() {
- readLock();
- try {
- List<LanceIndexJob> result = new ArrayList<>();
- for (LanceIndexJob job : jobs.values()) {
- if (job != null && job.isUnresolved() && hasFenceIdentity(job)) {
- result.add(new LanceIndexJob(job));
- }
- }
- return result;
- } finally {
- readUnlock();
- }
- }
-
- /**
- * Terminal jobs the refresh driver must pick up: refresh REQUIRED (waiting to
- * run, including jobs downgraded by the master-transfer sweep) or FAILED
- * (waiting for a retry). Both resume through the idempotent refresh path via
- * {@link #markRefreshRunning}; a FAILED job invisible here would hold its
- * fence forever with no retry channel.
- */
- public List<LanceIndexJob> getJobsNeedingRefresh() {
- readLock();
- try {
- List<LanceIndexJob> result = new ArrayList<>();
- for (LanceIndexJob job : jobs.values()) {
- if (job != null && job.getMutationState() != null && job.getMutationState().isTerminal()
- && hasFenceIdentity(job)
- && (job.getRefreshState() == LanceIndexJobRefreshState.REQUIRED
- || job.getRefreshState() == LanceIndexJobRefreshState.FAILED)) {
- result.add(new LanceIndexJob(job));
- }
- }
- return result;
- } finally {
- readUnlock();
- }
- }
-
- public boolean isFenceHeld(LanceIndexFenceKey fenceKey) {
- readLock();
- try {
- return fenceIndex.containsKey(fenceKey);
- } finally {
- readUnlock();
- }
- }
-
- /**
- * Ids of unresolved jobs that lack fence identity (corrupt journal/image
- * records whose fence key cannot be reconstructed). While non-empty, every
- * {@link #createJob} admission is rejected fail-closed; the force-release
- * transition of a follow-up PR resolves such a job by id, without needing
- * its fence key.
- */
- public List<Long> getCorruptUnresolvedJobIds() {
- readLock();
- try {
- return new ArrayList<>(corruptUnresolvedJobIds);
- } finally {
- readUnlock();
- }
- }
-
- @VisibleForTesting
- LanceIndexJobQuota getQuota() {
- return quota;
- }
-
- @VisibleForTesting
- public int getJobCount() {
- readLock();
- try {
- return jobs.size();
- } finally {
- readUnlock();
- }
- }
-
- // ------------------------------------------------------------------
- // Image serialization (whole-object Gson, IndexPolicyMgr style)
- // ------------------------------------------------------------------
-
- @Override
- public void write(DataOutput out) throws IOException {
- // Serialize against the declared base type: GsonUtils.BLOCK_INACCESSIBLE_JAVA refuses
- // reflection on non-public runtime classes, so a subclassed manager (the test seam)
- // would otherwise serialize as "null".
- Text.writeString(out, GsonUtils.GSON.toJson(this, LanceIndexJobManager.class));
- }
-
- public static LanceIndexJobManager read(DataInput in) throws IOException {
- return GsonUtils.GSON.fromJson(Text.readString(in), LanceIndexJobManager.class);
- }
-
- /**
- * Rebuild the derived fence index and quota counters from the durable
- * jobs after Gson image load. A fence-key collision between unresolved
- * jobs (only possible on a corrupt image) retains every owner so the fence
- * remains held until all colliding jobs settle; conflict reporting still
- * uses the smaller job id. Unresolved jobs without fence identity are kept
- * out of the books and recorded in {@link #corruptUnresolvedJobIds}, which
- * keeps admission fail-closed on their behalf.
- */
- @Override
- public void gsonPostProcess() throws IOException {
- fenceIndex.clear();
- corruptUnresolvedJobIds.clear();
- List<LanceIndexJob> unresolvedJobs = new ArrayList<>();
- if (jobs == null) {
- jobs = Maps.newConcurrentMap();
- }
- for (LanceIndexJob job : jobs.values()) {
- if (job == null || !job.isUnresolved()) {
- continue;
- }
- if (!hasFenceIdentity(job)) {
- corruptUnresolvedJobIds.add(job.getJobId());
- LOG.warn("lance index job {} lacks fence identity (provider/locator/name);"
- + " excluded from fence/quota rebuild and blocking all new"
- + " admissions fail-closed until resolved", job.getJobId());
- continue;
- }
- unresolvedJobs.add(job);
- NavigableSet<Long> owners = fenceIndex.computeIfAbsent(job.fenceKey(), ignored -> new TreeSet<>());
- if (!owners.isEmpty() && !owners.contains(job.getJobId())) {
- LOG.warn("fence key collision between unresolved lance index jobs {} and {}; keeping fence owner {}",
- owners.first(), job.getJobId(), Math.min(owners.first(), job.getJobId()));
- }
- owners.add(job.getJobId());
- }
- quota.rebuild(unresolvedJobs);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationState.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationState.java
deleted file mode 100644
index 9add2d7c20f..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationState.java
+++ /dev/null
@@ -1,47 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-/**
- * Compact durable mutation lifecycle:
- * <pre>
- * PENDING -> RUNNING -> COMMITTED
- * -> NOT_COMMITTED
- * -> UNKNOWN
- * </pre>
- * All three outcomes are terminal. UNKNOWN has no outgoing transition:
- * metadata never changes it to a known outcome, and FORCE_RELEASE (a later
- * delivery slice) only releases the fence/quota/slot without rewriting the
- * outcome.
- */
-public enum LanceIndexJobMutationState {
- /** The request and fence are durable; no execute send may have occurred. */
- PENDING,
- /** The durable dispatch boundary has been crossed; the one-shot call may execute or may already have executed. */
- RUNNING,
- /** A complete identity-matched typed success proves this job committed. */
- COMMITTED,
- /** A complete trusted result proves this job did not commit. */
- NOT_COMMITTED,
- /** Doris cannot safely prove whether this job committed. */
- UNKNOWN;
-
- public boolean isTerminal() {
- return this == COMMITTED || this == NOT_COMMITTED || this == UNKNOWN;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationType.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationType.java
deleted file mode 100644
index 4d5d3e893ff..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationType.java
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-/**
- * The one-shot Lance index mutation a durable job intends to perform.
- * REPLACE is a full same-name rebuild (CREATE OR REPLACE INDEX).
- */
-public enum LanceIndexJobMutationType {
- CREATE,
- REPLACE,
- DROP
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java
deleted file mode 100644
index 5634dc6f935..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java
+++ /dev/null
@@ -1,189 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * Three-level unresolved-job quota counters: per persisted table/locator
- * identity, per catalog, and globally. An unresolved job is any job that still
- * holds its same-name fence (PENDING/RUNNING, an unforced UNKNOWN, or a known
- * terminal job whose required refresh is not DONE); active-plus-UNKNOWN counts
- * stay bounded at every level.
- *
- * <p>This class only counts. Callers hold the manager write lock, so no
- * internal synchronization exists. Config-gated limits are resolved by the
- * admission layer and passed in as positive finite values. The counters are rebuilt from the durable jobs after
- * replay/image load and are never persisted themselves.
- */
-public class LanceIndexJobQuota {
- private static final Logger LOG = LogManager.getLogger(LanceIndexJobQuota.class);
-
- private long globalCount;
- private final Map<Long, Long> catalogCounts = new HashMap<>();
- private final Map<TableQuotaKey, Long> tableCounts = new HashMap<>();
-
- /**
- * Check every level whose limit is positive, then increment all three
- * levels. Returns false (and increments nothing) when any enforced level
- * is full: "current < limit" must hold at every level.
- *
- * <p>The manager admission path uses {@link #hasCapacity} plus
- * {@link #charge} instead: the check and the charge deliberately straddle
- * the edit-log write. This self-contained variant is reserved for direct
- * use by the later admission slice and by tests.
- */
- public boolean tryAcquire(LanceIndexJob job, long tableLimit, long catalogLimit, long globalLimit) {
- if (!hasCapacity(job, tableLimit, catalogLimit, globalLimit)) {
- return false;
- }
- charge(job);
- return true;
- }
-
- /**
- * Pure check variant of {@link #tryAcquire}: true when incrementing would
- * not exceed any positive finite limit. A non-positive limit is rejected.
- */
- public boolean hasCapacity(LanceIndexJob job, long tableLimit, long catalogLimit, long globalLimit) {
- Objects.requireNonNull(job, "job");
- if (tableLimit <= 0 || catalogLimit <= 0 || globalLimit <= 0) {
- return false;
- }
- if (globalCount >= globalLimit) {
- return false;
- }
- if (getCatalogCount(job.getCatalogId()) >= catalogLimit) {
- return false;
- }
- return getTableCount(job.getTableQuotaKey()) < tableLimit;
- }
-
- /**
- * Increment all three levels unconditionally. Used when a durable record
- * (already admitted, or replayed) is applied to memory.
- */
- public void charge(LanceIndexJob job) {
- globalCount++;
- catalogCounts.merge(job.getCatalogId(), 1L, Long::sum);
- tableCounts.merge(job.getTableQuotaKey(), 1L, Long::sum);
- }
-
- /**
- * Decrement all three levels. Underflow is clamped at zero and warned
- * about: it indicates a bookkeeping bug, not a reason to fail replay.
- */
- public void release(LanceIndexJob job) {
- if (globalCount <= 0) {
- LOG.warn("lance index job quota global underflow on release of job {}", job.getJobId());
- } else {
- globalCount--;
- }
- decrement(catalogCounts, job.getCatalogId(), job.getJobId());
- decrement(tableCounts, job.getTableQuotaKey(), job.getJobId());
- }
-
- private static <K> void decrement(Map<K, Long> counts, K key, long jobId) {
- Long current = counts.get(key);
- if (current == null || current <= 0) {
- LOG.warn("lance index job quota underflow on release of job {} at key {}", jobId, key);
- counts.remove(key);
- return;
- }
- if (current == 1L) {
- counts.remove(key);
- } else {
- counts.put(key, current - 1);
- }
- }
-
- /**
- * Reset and recount from the unresolved durable jobs (replay / image load).
- */
- public void rebuild(Iterable<LanceIndexJob> unresolvedJobs) {
- globalCount = 0;
- catalogCounts.clear();
- tableCounts.clear();
- for (LanceIndexJob job : unresolvedJobs) {
- charge(job);
- }
- }
-
- public long getGlobalCount() {
- return globalCount;
- }
-
- public long getCatalogCount(long catalogId) {
- return catalogCounts.getOrDefault(catalogId, 0L);
- }
-
- public long getTableCount(TableQuotaKey key) {
- return tableCounts.getOrDefault(key, 0L);
- }
-
- /**
- * Per persisted table/locator identity: (catalogId, normalizedLocator).
- */
- public static final class TableQuotaKey {
- private final long catalogId;
- private final String normalizedLocator;
-
- public TableQuotaKey(long catalogId, String normalizedLocator) {
- this.catalogId = catalogId;
- this.normalizedLocator = Objects.requireNonNull(normalizedLocator, "normalizedLocator");
- }
-
- public long getCatalogId() {
- return catalogId;
- }
-
- public String getNormalizedLocator() {
- return normalizedLocator;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (!(o instanceof TableQuotaKey)) {
- return false;
- }
- TableQuotaKey that = (TableQuotaKey) o;
- return catalogId == that.catalogId && normalizedLocator.equals(that.normalizedLocator);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(catalogId, normalizedLocator);
- }
-
- /**
- * Deliberately omits the locator, like {@link LanceIndexFenceKey#toString()}.
- */
- @Override
- public String toString() {
- return "TableQuotaKey{catalogId=" + catalogId + '}';
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobRefreshState.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobRefreshState.java
deleted file mode 100644
index f6ece16ac93..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobRefreshState.java
+++ /dev/null
@@ -1,41 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-/**
- * Independent metadata-refresh state of a durable Lance index job. Stored and
- * transitioned independently from the mutation state: refresh success/failure
- * only sets DONE/FAILED and never changes the mutation outcome.
- * <pre>
- * NOT_REQUIRED | REQUIRED -> RUNNING -> DONE | FAILED, FAILED -> RUNNING (retry)
- * </pre>
- * Refresh replays through the existing idempotent external-table refresh path;
- * a failed refresh may be retried and still holds the same-name fence.
- */
-public enum LanceIndexJobRefreshState {
- /** No refresh is owed (e.g. a proven pre-invocation failure with no relevant metadata change). */
- NOT_REQUIRED,
- /** A refresh of the authoritative metadata is owed before the fence may be released. */
- REQUIRED,
- /** A refresh attempt is in flight. */
- RUNNING,
- /** The required refresh finished; the fence may be released. */
- DONE,
- /** The refresh attempt failed; retry through the idempotent path is allowed. */
- FAILED
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResult.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResult.java
deleted file mode 100644
index 49782f275b3..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResult.java
+++ /dev/null
@@ -1,112 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import com.google.gson.annotations.SerializedName;
-
-import java.nio.charset.StandardCharsets;
-import java.util.Objects;
-
-/**
- * The immutable typed result of the one-shot invocation, persisted on the job.
- * Contains only what is needed to classify the invocation: the saved typed
- * code, the classified completion reason, a bounded sanitized message, and
- * whether trusted pre-invocation revalidation observed relevant external
- * metadata advancement. Never carries secrets, raw provider responses, or
- * unbounded text.
- */
-public class LanceIndexJobResult {
- public static final int MAX_MESSAGE_BYTES = 1024;
-
- @SerializedName(value = "rc")
- private LanceIndexJobResultCode resultCode = LanceIndexJobResultCode.NO_TRUSTED_RESULT;
-
- @SerializedName(value = "cr")
- private LanceIndexJobCompletionReason completionReason = LanceIndexJobCompletionReason.NONE;
-
- @SerializedName(value = "msg")
- private String sanitizedMessage;
-
- @SerializedName(value = "ema")
- private boolean externalMetadataAdvanced;
-
- /**
- * No-arg constructor for Gson replay only; missing fields keep the safe
- * defaults declared above.
- */
- public LanceIndexJobResult() {
- }
-
- public LanceIndexJobResult(LanceIndexJobResultCode resultCode, LanceIndexJobCompletionReason completionReason,
- String sanitizedMessage, boolean externalMetadataAdvanced) {
- this.resultCode = Objects.requireNonNull(resultCode, "resultCode");
- this.completionReason = completionReason == null ? LanceIndexJobCompletionReason.NONE : completionReason;
- this.sanitizedMessage = checkMessageBytes(sanitizedMessage);
- this.externalMetadataAdvanced = externalMetadataAdvanced;
- }
-
- private static String checkMessageBytes(String message) {
- if (message != null && message.getBytes(StandardCharsets.UTF_8).length > MAX_MESSAGE_BYTES) {
- throw new IllegalArgumentException(
- "sanitized message exceeds " + MAX_MESSAGE_BYTES + " UTF-8 bytes");
- }
- return message;
- }
-
- public LanceIndexJobResultCode getResultCode() {
- return resultCode;
- }
-
- public LanceIndexJobCompletionReason getCompletionReason() {
- return completionReason;
- }
-
- public String getSanitizedMessage() {
- return sanitizedMessage;
- }
-
- public boolean isExternalMetadataAdvanced() {
- return externalMetadataAdvanced;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (!(o instanceof LanceIndexJobResult)) {
- return false;
- }
- LanceIndexJobResult that = (LanceIndexJobResult) o;
- return externalMetadataAdvanced == that.externalMetadataAdvanced
- && resultCode == that.resultCode
- && completionReason == that.completionReason
- && Objects.equals(sanitizedMessage, that.sanitizedMessage);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(resultCode, completionReason, sanitizedMessage, externalMetadataAdvanced);
- }
-
- @Override
- public String toString() {
- return "LanceIndexJobResult{resultCode=" + resultCode + ", completionReason=" + completionReason
- + ", externalMetadataAdvanced=" + externalMetadataAdvanced + '}';
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultCode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultCode.java
deleted file mode 100644
index fad096b5087..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultCode.java
+++ /dev/null
@@ -1,158 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-/**
- * Typed result codes of the one-shot Lance index invocation. Only saved typed
- * codes are ever classified; message text is never inspected to infer an
- * outcome. Codes fall into three groups:
- *
- * <ul>
- * <li>PRE_INVOCATION_*: a complete trusted rejection before the native call
- * (dataset version / schema contract / credential / resource revalidation).
- * These always prove NOT_COMMITTED.</li>
- * <li>NATIVE_*: the saved LanceErrorCode of the single native invocation,
- * read before the consuming error message.</li>
- * <li>NO_TRUSTED_RESULT: any ambiguity after the send may have occurred
- * (EOF, signal, OOM, panic, timeout, BE loss, malformed/partial protocol,
- * identity mismatch after acceptance, or a master-transfer sweep that found
- * a durable RUNNING without a complete matching terminal result).</li>
- * </ul>
- */
-public enum LanceIndexJobResultCode {
- /** Pre-invocation: admitted dataset version / schema contract no longer matches. */
- PRE_INVOCATION_STALE_ADMISSION,
- /** Pre-invocation: the recomputed contract is not a supported contract v1. */
- PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT,
- /** Pre-invocation: credentials are known to be expired. */
- PRE_INVOCATION_CREDENTIAL_EXPIRED,
- /** Pre-invocation: trusted busy / pre-FFI resource rejection. */
- PRE_INVOCATION_RESOURCE_REJECTED,
- /** LANCE_OK from the one native invocation. */
- NATIVE_OK,
- /** LANCE_ERR_COMMIT_CONFLICT: the commit lost a race; the external dataset advanced. */
- NATIVE_COMMIT_CONFLICT,
- /** LANCE_ERR_NOT_FOUND after invocation. */
- NATIVE_NOT_FOUND,
- /** LANCE_ERR_INVALID_ARGUMENT after invocation. */
- NATIVE_INVALID_ARGUMENT,
- /** LANCE_ERR_NOT_SUPPORTED after invocation. */
- NATIVE_NOT_SUPPORTED,
- /** LANCE_ERR_INDEX after invocation (also the coarse duplicate-CREATE error). */
- NATIVE_INDEX,
- /** LANCE_ERR_IO after invocation. */
- NATIVE_IO,
- /** LANCE_ERR_INTERNAL after invocation. */
- NATIVE_INTERNAL,
- /** No complete trusted result exists; commitment cannot be proven either way. */
- NO_TRUSTED_RESULT;
-
- public boolean isPreInvocation() {
- return this == PRE_INVOCATION_STALE_ADMISSION
- || this == PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT
- || this == PRE_INVOCATION_CREDENTIAL_EXPIRED
- || this == PRE_INVOCATION_RESOURCE_REJECTED;
- }
-
- /**
- * Classify a complete saved result into the durable (mutationState, refreshState,
- * completionReason) triple, following the provider-result classification table:
- *
- * <ul>
- * <li>PRE_INVOCATION_* -> NOT_COMMITTED; refresh REQUIRED only when the trusted
- * pre-invocation revalidation observed relevant external metadata advancement.</li>
- * <li>NATIVE_OK -> COMMITTED + REQUIRED.</li>
- * <li>NATIVE_COMMIT_CONFLICT -> NOT_COMMITTED + REQUIRED (the external dataset
- * advanced).</li>
- * <li>NATIVE_NOT_FOUND -> CREATE/REPLACE: UNKNOWN + NOT_REQUIRED (no attribution);
- * DROP: NOT_COMMITTED + REQUIRED, with IF_CONDITION_NOOP for DROP IF EXISTS.</li>
- * <li>NATIVE_INVALID_ARGUMENT / NOT_SUPPORTED / INDEX / IO / INTERNAL ->
- * UNKNOWN + NOT_REQUIRED (no outcome inference).</li>
- * <li>NO_TRUSTED_RESULT -> UNKNOWN + NOT_REQUIRED.</li>
- * </ul>
- */
- public static Classification classify(LanceIndexJobMutationType mutationType, LanceIndexJobResultCode resultCode,
- boolean ifExists, boolean externalMetadataAdvanced) {
- if (resultCode == null) {
- resultCode = NO_TRUSTED_RESULT;
- }
- switch (resultCode) {
- case PRE_INVOCATION_STALE_ADMISSION:
- case PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT:
- case PRE_INVOCATION_CREDENTIAL_EXPIRED:
- case PRE_INVOCATION_RESOURCE_REJECTED:
- return new Classification(LanceIndexJobMutationState.NOT_COMMITTED,
- externalMetadataAdvanced
- ? LanceIndexJobRefreshState.REQUIRED : LanceIndexJobRefreshState.NOT_REQUIRED,
- LanceIndexJobCompletionReason.NONE);
- case NATIVE_OK:
- return new Classification(LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.REQUIRED,
- LanceIndexJobCompletionReason.NONE);
- case NATIVE_COMMIT_CONFLICT:
- return new Classification(LanceIndexJobMutationState.NOT_COMMITTED, LanceIndexJobRefreshState.REQUIRED,
- LanceIndexJobCompletionReason.NONE);
- case NATIVE_NOT_FOUND:
- if (mutationType == LanceIndexJobMutationType.DROP) {
- return new Classification(LanceIndexJobMutationState.NOT_COMMITTED,
- LanceIndexJobRefreshState.REQUIRED,
- ifExists ? LanceIndexJobCompletionReason.IF_CONDITION_NOOP
- : LanceIndexJobCompletionReason.NONE);
- }
- return new Classification(LanceIndexJobMutationState.UNKNOWN, LanceIndexJobRefreshState.NOT_REQUIRED,
- LanceIndexJobCompletionReason.NONE);
- case NATIVE_INVALID_ARGUMENT:
- case NATIVE_NOT_SUPPORTED:
- case NATIVE_INDEX:
- case NATIVE_IO:
- case NATIVE_INTERNAL:
- case NO_TRUSTED_RESULT:
- default:
- return new Classification(LanceIndexJobMutationState.UNKNOWN, LanceIndexJobRefreshState.NOT_REQUIRED,
- LanceIndexJobCompletionReason.NONE);
- }
- }
-
- /**
- * The immutable outcome of {@link #classify}: the durable mutation state, the
- * independent refresh obligation, and the completion reason.
- */
- public static final class Classification {
- private final LanceIndexJobMutationState mutationState;
- private final LanceIndexJobRefreshState refreshState;
- private final LanceIndexJobCompletionReason completionReason;
-
- private Classification(LanceIndexJobMutationState mutationState, LanceIndexJobRefreshState refreshState,
- LanceIndexJobCompletionReason completionReason) {
- this.mutationState = mutationState;
- this.refreshState = refreshState;
- this.completionReason = completionReason;
- }
-
- public LanceIndexJobMutationState getMutationState() {
- return mutationState;
- }
-
- public LanceIndexJobRefreshState getRefreshState() {
- return refreshState;
- }
-
- public LanceIndexJobCompletionReason getCompletionReason() {
- return completionReason;
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexTerminationProof.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexTerminationProof.java
deleted file mode 100644
index d5de9b5cbdb..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexTerminationProof.java
+++ /dev/null
@@ -1,33 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-/**
- * Evidence that releases a possible-live worker slot. Termination proof
- * releases only that slot: it neither changes an UNKNOWN outcome nor releases
- * the same-name fence. Deadlines bound wait/runtime but never prove
- * termination.
- */
-public enum LanceIndexTerminationProof {
- /** No proof; the worker may still be running. */
- NONE,
- /** The supervisor reaped the exact matching child process. */
- CHILD_REAPED,
- /** The recorded BE process epoch no longer exists (the BE process was replaced). */
- BE_PROCESS_EPOCH_GONE
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
index d2c1952211d..10420d1230a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
@@ -49,7 +49,6 @@ import org.apache.doris.datasource.ExternalObjectLog;
import org.apache.doris.datasource.InitCatalogLog;
import org.apache.doris.datasource.InitDatabaseLog;
import org.apache.doris.datasource.MetaIdMappingsLog;
-import org.apache.doris.datasource.lance.job.LanceIndexJob;
import org.apache.doris.ha.MasterInfo;
import org.apache.doris.indexpolicy.DropIndexPolicyLog;
import org.apache.doris.indexpolicy.IndexPolicy;
@@ -1014,11 +1013,6 @@ public class JournalEntity implements Writable {
isRead = true;
break;
}
- case OperationType.OP_LANCE_INDEX_JOB_UPSERT: {
- data = LanceIndexJob.read(in);
- isRead = true;
- break;
- }
case OperationType.OP_BEGIN_SNAPSHOT: {
data = SnapshotState.read(in);
isRead = true;
@@ -1029,6 +1023,14 @@ public class JournalEntity implements Writable {
isRead = true;
break;
}
+ case OperationType.OP_LANCE_INDEX_JOB_UPSERT: {
+ // Payload of the removed durable Lance index job framework (Text-delimited
+ // JSON). Consumed and ignored so journals from builds that manually enabled
+ // the gate stay readable; the operation code is never reused.
+ Text.readString(in);
+ isRead = true;
+ break;
+ }
default: {
IOException e = new IOException();
LOG.error("UNKNOWN Operation Type {}", opCode, e);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index a027bc4d307..68cd2da1898 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -414,8 +414,6 @@ import org.apache.doris.nereids.DorisParser.ShowIndexCharFilterContext;
import org.apache.doris.nereids.DorisParser.ShowIndexNormalizerContext;
import org.apache.doris.nereids.DorisParser.ShowIndexTokenFilterContext;
import org.apache.doris.nereids.DorisParser.ShowIndexTokenizerContext;
-import org.apache.doris.nereids.DorisParser.ShowLanceIndexJobContext;
-import org.apache.doris.nereids.DorisParser.ShowLanceIndexJobsContext;
import org.apache.doris.nereids.DorisParser.ShowLastInsertContext;
import org.apache.doris.nereids.DorisParser.ShowLoadContext;
import org.apache.doris.nereids.DorisParser.ShowLoadProfileContext;
@@ -855,8 +853,6 @@ import org.apache.doris.nereids.trees.plans.commands.ShowIndexNormalizerCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowIndexStatsCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowIndexTokenFilterCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowIndexTokenizerCommand;
-import org.apache.doris.nereids.trees.plans.commands.ShowLanceIndexJobCommand;
-import org.apache.doris.nereids.trees.plans.commands.ShowLanceIndexJobsCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowLastInsertCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowLoadCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowLoadProfileCommand;
@@ -7038,22 +7034,6 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
return new ShowCatalogRecycleBinCommand(whereClause);
}
- @Override
- public LogicalPlan visitShowLanceIndexJobs(ShowLanceIndexJobsContext ctx) {
- List<String> nameParts = ctx.db == null ? null : visitMultipartIdentifier(ctx.db);
- Expression whereClause = null;
- if (ctx.WHERE() != null) {
- whereClause = getExpression(ctx.expression());
- }
- return new ShowLanceIndexJobsCommand(nameParts, whereClause);
- }
-
- @Override
- public LogicalPlan visitShowLanceIndexJob(ShowLanceIndexJobContext ctx) {
- long jobId = Long.parseLong(ctx.jobId.getText());
- return new ShowLanceIndexJobCommand(jobId);
- }
-
@Override
public LogicalPlan visitShowStorageEngines(ShowStorageEnginesContext ctx) {
return new ShowStorageEnginesCommand();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 776d18d1dad..683939e70c8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -320,8 +320,6 @@ public enum PlanType {
SHOW_GRANTS_COMMAND,
SHOW_INDEX_COMMAND,
SHOW_INDEX_STATS_COMMAND,
- SHOW_LANCE_INDEX_JOB_COMMAND,
- SHOW_LANCE_INDEX_JOBS_COMMAND,
SHOW_LAST_INSERT_COMMAND,
SHOW_LOAD_COMMAND,
SHOW_LOAD_PROFILE_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java
index d7dba9958af..b2b97ed633c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java
@@ -28,7 +28,6 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
@@ -77,15 +76,11 @@ import org.apache.doris.nereids.trees.plans.commands.info.ReplacePartitionFieldO
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.ShowResultSet;
-import org.apache.doris.qe.ShowResultSetMetaData;
import org.apache.doris.qe.StmtExecutor;
import com.google.common.base.Preconditions;
-import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -554,43 +549,21 @@ public class AlterTableCommand extends Command implements ForwardWithSync {
Preconditions.checkState(lanceIndexOps.size() == 1,
"a top-level Lance index statement must carry exactly one operation");
AlterTableOp op = lanceIndexOps.get(0);
- LanceIndexAdmission.Outcome outcome;
if (op instanceof CreateIndexOp) {
IndexDefinition indexDef = ((CreateIndexOp) op).getIndexDef();
- outcome = LanceIndexAdmission.admitCreate((LanceExternalCatalog) catalog,
+ LanceIndexAdmission.admitCreate((LanceExternalCatalog) catalog,
(LanceExternalDatabase) dbIf, (LanceExternalTable) tableIf, indexDef,
indexDef.isIfNotExists());
} else {
DropIndexOp dropIndexOp = (DropIndexOp) op;
- outcome = LanceIndexAdmission.admitDrop((LanceExternalCatalog) catalog,
+ LanceIndexAdmission.admitDrop((LanceExternalCatalog) catalog,
(LanceExternalDatabase) dbIf, (LanceExternalTable) tableIf,
dropIndexOp.getIndexName(), dropIndexOp.isSetIfExists());
}
- sendJobIdResult(executor, outcome);
+ // An IF preflight no-op returns normally and completes with the default OK packet;
+ // any mutation that would be admitted rejects inside admission.
return;
}
ctx.getEnv().alterTable(this);
}
-
- /**
- * Answers the admission with a single-column JobId result set (WarmUpClusterCommand
- * pattern): one row carrying the durable job id, or zero rows for an IF no-op — "no job was
- * created" is self-evident from the empty set, and sendResultSet's trailing setEof()
- * suppresses the default OK packet on both the direct and the forwarded (proxy) link.
- */
- private void sendJobIdResult(StmtExecutor executor, LanceIndexAdmission.Outcome outcome)
- throws IOException {
- ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
- builder.addColumn(new Column("JobId", ScalarType.createVarchar(30)));
- List<List<String>> rows = new ArrayList<>();
- if (outcome.getJobId() != null) {
- rows.add(Collections.singletonList(String.valueOf(outcome.getJobId())));
- }
- ShowResultSet resultSet = new ShowResultSet(builder.build(), rows);
- if (executor.isProxy()) {
- executor.setProxyShowResultSet(resultSet);
- return;
- }
- executor.sendResultSet(resultSet);
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobCommand.java
deleted file mode 100644
index fe74170c116..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobCommand.java
+++ /dev/null
@@ -1,174 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.trees.plans.commands;
-
-import org.apache.doris.analysis.RedirectStatus;
-import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.DatabaseIf;
-import org.apache.doris.catalog.Env;
-import org.apache.doris.catalog.ScalarType;
-import org.apache.doris.catalog.TableIf;
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.ErrorCode;
-import org.apache.doris.common.util.TimeUtils;
-import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.datasource.lance.job.LanceIndexJob;
-import org.apache.doris.datasource.lance.job.LanceIndexJobResult;
-import org.apache.doris.nereids.trees.plans.PlanType;
-import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.ShowResultSet;
-import org.apache.doris.qe.ShowResultSetMetaData;
-import org.apache.doris.qe.StmtExecutor;
-
-import com.google.common.collect.ImmutableList;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * SHOW LANCE INDEX JOB <jobId>.
- *
- * <p>Non-disclosing job detail (v5.1 section 8): the record is loaded first and authorized
- * against its persisted target before any field is returned. Orphan and half-orphan targets
- * (catalog gone, persisted db/table no longer resolvable, the resolution failing outright
- * because the provider is unreachable, or the table repointed at a different dataset
- * locator) require global ADMIN; a resolvable, still-matching target requires table-level
- * SHOW. A missing job and an unauthorized job share
- * the same fixed ERR_LANCE_INDEX_JOB_NOT_FOUND response that names only the job id, so the
- * existence and target of a job are never disclosed. The job locator, provider, normalized
- * names, propertiesJson and schema contract contents are never shown.
- */
-public class ShowLanceIndexJobCommand extends ShowCommand {
- public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
- .addAll(ShowLanceIndexJobsCommand.TITLE_NAMES)
- .add("Creator")
- .add("ResultCode")
- .add("CompletionReason")
- .add("ExternalMetadataAdvanced")
- .add("BackendId")
- .add("BeProcessEpoch")
- .add("InvocationId")
- .add("DeadlineMs")
- .add("TerminationProof")
- .add("AdmittedDatasetVersion")
- .add("SchemaContractVersion")
- .add("Revision")
- .add("ForceNote")
- .add("ForceWarning")
- .add("IfNotExists")
- .add("IfExists")
- .add("IndexType")
- .add("ColumnName")
- .build();
-
- private final long jobId;
-
- public ShowLanceIndexJobCommand(long jobId) {
- super(PlanType.SHOW_LANCE_INDEX_JOB_COMMAND);
- this.jobId = jobId;
- }
-
- public long getJobId() {
- return jobId;
- }
-
- @Override
- public ShowResultSetMetaData getMetaData() {
- ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
- for (String title : TITLE_NAMES) {
- builder.addColumn(new Column(title, ScalarType.createVarchar(30)));
- }
- return builder.build();
- }
-
- @Override
- public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
- LanceIndexJob job = Env.getCurrentEnv().getLanceIndexJobManager().getJob(jobId);
- if (job == null) {
- throw notFound();
- }
- CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog =
- Env.getCurrentEnv().getCatalogMgr().getCatalog(job.getCatalogId());
- // Missing and unauthorized jobs share the same fixed response; authorize before
- // returning any field.
- if (!ShowLanceIndexJobsCommand.isAuthorized(ctx, catalog, job)) {
- throw notFound();
- }
- return new ShowResultSet(getMetaData(), Collections.singletonList(renderRow(job, catalog)));
- }
-
- private AnalysisException notFound() {
- return new AnalysisException(ErrorCode.ERR_LANCE_INDEX_JOB_NOT_FOUND.formatErrorMsg(jobId),
- ErrorCode.ERR_LANCE_INDEX_JOB_NOT_FOUND);
- }
-
- private static List<String> renderRow(LanceIndexJob job,
- CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog) {
- List<String> row = new ArrayList<>(TITLE_NAMES.size());
- row.add(String.valueOf(job.getJobId()));
- // See ShowLanceIndexJobsCommand: the catalog name is not persisted on the job record,
- // so an orphan row renders it empty rather than fabricating one from the internal id.
- row.add(catalog == null ? "" : catalog.getName());
- row.add(job.getDbName());
- row.add(job.getTableName());
- row.add(job.getDisplayIndexName());
- row.add(job.getMutationType() == null ? "" : job.getMutationType().name());
- row.add(job.getMutationState() == null ? "" : job.getMutationState().name());
- row.add(job.getRefreshState() == null ? "" : job.getRefreshState().name());
- row.add(job.holdsPossibleLiveSlot() ? "YES" : "NO");
- row.add(TimeUtils.longToTimeString(job.getCreateTimeMs()));
- row.add(TimeUtils.longToTimeString(job.getUpdateTimeMs()));
- LanceIndexJobResult result = job.getResult();
- // The result is always null until a worker reports (no worker exists yet); the same
- // holds for every dispatch field. All of them render as empty strings, never NPE.
- row.add(result == null || result.getSanitizedMessage() == null ? "" : result.getSanitizedMessage());
- row.addAll(ShowLanceIndexJobsCommand.renderForceAudit(job));
- row.add(job.getCreator() == null ? "" : job.getCreator());
- row.add(result == null ? "" : result.getResultCode().name());
- row.add(result == null || result.getCompletionReason() == null ? "" : result.getCompletionReason().name());
- row.add(result == null ? "" : (result.isExternalMetadataAdvanced() ? "YES" : "NO"));
- row.add(job.getBackendId() == null ? "" : String.valueOf(job.getBackendId()));
- row.add(job.getBeProcessEpoch() == null ? "" : String.valueOf(job.getBeProcessEpoch()));
- row.add(job.getInvocationId() == null ? "" : job.getInvocationId());
- row.add(job.getDeadlineMs() == null ? "" : String.valueOf(job.getDeadlineMs()));
- row.add(job.getTerminationProof() == null ? "" : job.getTerminationProof().name());
- row.add(String.valueOf(job.getAdmittedDatasetVersion()));
- row.add(job.getSchemaContract() == null
- ? "" : String.valueOf(job.getSchemaContract().getSchemaContractVersion()));
- row.add(String.valueOf(job.getRevision()));
- row.add(job.isForceReleased() && job.getForceNote() != null ? job.getForceNote() : "");
- row.add(job.isForceReleased() && job.getForceWarning() != null ? job.getForceWarning() : "");
- row.add(job.isIfNotExists() ? "YES" : "NO");
- row.add(job.isIfExists() ? "YES" : "NO");
- row.add(job.getIndexType() == null ? "" : job.getIndexType());
- row.add(job.getColumnName() == null ? "" : job.getColumnName());
- return row;
- }
-
- @Override
- public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
- return visitor.visitShowLanceIndexJobCommand(this, context);
- }
-
- @Override
- public RedirectStatus toRedirectStatus() {
- return RedirectStatus.FORWARD_NO_SYNC;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobsCommand.java
deleted file mode 100644
index 825c3e17f0a..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobsCommand.java
+++ /dev/null
@@ -1,378 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.trees.plans.commands;
-
-import org.apache.doris.analysis.RedirectStatus;
-import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.DatabaseIf;
-import org.apache.doris.catalog.Env;
-import org.apache.doris.catalog.ScalarType;
-import org.apache.doris.catalog.TableIf;
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.util.TimeUtils;
-import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.datasource.lance.LanceExternalCatalog;
-import org.apache.doris.datasource.lance.job.LanceIndexJob;
-import org.apache.doris.datasource.lance.job.LanceIndexJobMutationState;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.nereids.analyzer.UnboundSlot;
-import org.apache.doris.nereids.trees.expressions.And;
-import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
-import org.apache.doris.nereids.trees.expressions.EqualTo;
-import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
-import org.apache.doris.nereids.trees.plans.PlanType;
-import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.ShowResultSet;
-import org.apache.doris.qe.ShowResultSetMetaData;
-import org.apache.doris.qe.StmtExecutor;
-
-import com.google.common.collect.ImmutableList;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * SHOW LANCE INDEX JOBS [FROM [catalog.]db] [WHERE TableName = "tbl" [AND State = "PENDING"]].
- *
- * <p>Lists the durable Lance index job records held by the master. Rows whose persisted
- * target no longer resolves (the catalog is gone, the catalog is there but the db or
- * table no longer resolves, or the resolution itself fails because the provider is
- * unreachable), or whose persisted dataset locator no longer matches the catalog's
- * current one, are visible to global ADMIN only; every other row requires table-level
- * SHOW on the persisted (catalog, db, table). Rows that fail the check are omitted
- * entirely, so non-ADMIN users see no orphan trace, not even a count. The job locator,
- * provider, normalized names, propertiesJson and schema contract are never shown.
- *
- * <p>The WHERE clause is deliberately narrowed to EqualTo predicates combined with AND
- * over the case-insensitive keys TableName and State (no Like, unlike the SHOW COPY
- * precedent).
- */
-public class ShowLanceIndexJobsCommand extends ShowCommand {
- public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
- .add("JobId")
- .add("CatalogName")
- .add("DbName")
- .add("TableName")
- .add("IndexName")
- .add("Operation")
- .add("State")
- .add("RefreshState")
- .add("PossibleLive")
- .add("CreateTime")
- .add("UpdateTime")
- .add("Message")
- .add("ForceReleased")
- .add("ForceActor")
- .add("ForceTime")
- .build();
-
- private static final String KEY_TABLE_NAME = "TableName";
- private static final String KEY_STATE = "State";
- private static final String WHERE_HINT = "Where clause should looks like: TableName = \"your_table_name\""
- + " or State = \"PENDING|RUNNING|COMMITTED|NOT_COMMITTED|UNKNOWN\","
- + " or compound predicate with operator AND";
-
- private final List<String> nameParts;
- private final Expression whereClause;
-
- private String ctlName;
- private String dbName;
- private String tableNameValue;
- private String stateValue;
-
- public ShowLanceIndexJobsCommand(List<String> nameParts, Expression whereClause) {
- super(PlanType.SHOW_LANCE_INDEX_JOBS_COMMAND);
- this.nameParts = nameParts;
- this.whereClause = whereClause;
- }
-
- public List<String> getNameParts() {
- return nameParts;
- }
-
- public Expression getWhereClause() {
- return whereClause;
- }
-
- @Override
- public ShowResultSetMetaData getMetaData() {
- ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
- for (String title : TITLE_NAMES) {
- builder.addColumn(new Column(title, ScalarType.createVarchar(30)));
- }
- return builder.build();
- }
-
- private void validate(ConnectContext ctx) throws AnalysisException {
- if (nameParts != null) {
- if (nameParts.size() == 1) {
- dbName = nameParts.get(0);
- CatalogIf currentCatalog = ctx.getCurrentCatalog();
- ctlName = currentCatalog == null ? null : currentCatalog.getName();
- normalizeDbName(currentCatalog);
- } else if (nameParts.size() == 2) {
- ctlName = nameParts.get(0);
- dbName = nameParts.get(1);
- normalizeDbName(Env.getCurrentEnv().getCatalogMgr().getCatalog(ctlName));
- } else {
- throw new AnalysisException(
- "Only support SHOW LANCE INDEX JOBS FROM [catalog.]database, but get: " + nameParts);
- }
- }
- analyzeWhereClause();
- }
-
- /**
- * Resolves the FROM database name through the target catalog's own naming rules
- * before it is compared against the persisted job records: with
- * lower_case_database_names = 1/2, {@code FROM DB1} must still match jobs stored
- * under the resolved full name {@code db1}. A name the catalog cannot resolve (or a
- * resolution that fails outright) keeps the raw requested string, so the filter just
- * stays an exact comparison; the resolution itself must never fail the statement.
- */
- private void normalizeDbName(CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog) {
- if (dbName == null || catalog == null) {
- return;
- }
- try {
- DatabaseIf<? extends TableIf> db = catalog.getDbNullable(dbName);
- if (db == null) {
- return;
- }
- String fullName = db.getFullName();
- if (fullName != null && !fullName.isEmpty()) {
- dbName = fullName;
- }
- } catch (Exception e) {
- // Keep the raw requested name; the filter remains an exact comparison.
- }
- }
-
- private void analyzeWhereClause() throws AnalysisException {
- if (whereClause == null) {
- return;
- }
- List<Expression> children = new ArrayList<>();
- splitCompoundPredicate(whereClause, children);
- Set<String> names = new HashSet<>();
- for (Expression child : children) {
- analyzeSubPredicate(child);
- String name = ((UnboundSlot) child.child(0)).getName().toLowerCase(Locale.ROOT);
- if (!names.add(name)) {
- throw new AnalysisException("column names on both sides of operator AND should be different");
- }
- }
- }
-
- private void splitCompoundPredicate(Expression expr, List<Expression> children) throws AnalysisException {
- if (expr instanceof CompoundPredicate) {
- if (!(expr instanceof And)) {
- throw new AnalysisException("Only allow compound predicate with operator AND");
- }
- splitCompoundPredicate(expr.child(0), children);
- splitCompoundPredicate(expr.child(1), children);
- } else {
- children.add(expr);
- }
- }
-
- private void analyzeSubPredicate(Expression expr) throws AnalysisException {
- if (!(expr instanceof EqualTo)
- || !(expr.child(0) instanceof UnboundSlot)
- || !(expr.child(1) instanceof StringLikeLiteral)) {
- throw new AnalysisException(WHERE_HINT);
- }
- String key = ((UnboundSlot) expr.child(0)).getName();
- String value = ((StringLikeLiteral) expr.child(1)).getStringValue();
- if (key.equalsIgnoreCase(KEY_TABLE_NAME)) {
- tableNameValue = value;
- } else if (key.equalsIgnoreCase(KEY_STATE)) {
- stateValue = value.toUpperCase(Locale.ROOT);
- try {
- LanceIndexJobMutationState.valueOf(stateValue);
- } catch (IllegalArgumentException e) {
- throw new AnalysisException("Unknown Lance index job state: " + value + "; " + WHERE_HINT);
- }
- } else {
- throw new AnalysisException(WHERE_HINT);
- }
- }
-
- @Override
- public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
- validate(ctx);
- List<List<String>> rows = new ArrayList<>();
- // One locator resolution per unique (catalogId, db, table) for the whole listing:
- // each resolution is a provider round trip, and failed (null) resolutions are
- // cached too so a broken target is probed once, not once per sibling job.
- Map<String, String> locatorCache = new HashMap<>();
- for (LanceIndexJob job : Env.getCurrentEnv().getLanceIndexJobManager().getAllJobsSnapshot()) {
- CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog =
- Env.getCurrentEnv().getCatalogMgr().getCatalog(job.getCatalogId());
- if (!matchesFilters(catalog, job)) {
- continue;
- }
- if (!isAuthorized(ctx, catalog, job, locatorCache)) {
- continue;
- }
- rows.add(renderRow(job, catalog));
- }
- return new ShowResultSet(getMetaData(), rows);
- }
-
- private boolean matchesFilters(CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog, LanceIndexJob job) {
- if (ctlName != null && (catalog == null || !ctlName.equals(catalog.getName()))) {
- return false;
- }
- if (dbName != null && !dbName.equals(job.getDbName())) {
- return false;
- }
- if (tableNameValue != null && !tableNameValue.equals(job.getTableName())) {
- return false;
- }
- return stateValue == null
- || job.getMutationState() != null && stateValue.equals(job.getMutationState().name());
- }
-
- /**
- * Orphan and half-orphan rows (catalog gone, persisted db/table no longer resolvable,
- * or a resolution that fails outright because the provider is unreachable) are
- * visible to global ADMIN only. A Lance catalog target is additionally revalidated
- * against the catalog's current durable dataset locator: after a job reaches a
- * terminal state and releases its guard, a legitimate ALTER can repoint the same
- * db.table names at a different dataset, and SHOW on the new target must not
- * disclose the old job. Every other row needs table-level SHOW on the persisted
- * target. The caller omits the row when this returns false, so non-ADMIN users see
- * no trace of orphaned jobs, not even a count.
- */
- static boolean isAuthorized(ConnectContext ctx, CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog,
- LanceIndexJob job) {
- return isAuthorized(ctx, catalog, job, null);
- }
-
- /**
- * Same authorization as {@link #isAuthorized(ConnectContext, CatalogIf, LanceIndexJob)},
- * with an optional (catalogId, db, table)-keyed locator cache so a listing resolves
- * each unique target's current locator once per run instead of once per job row.
- */
- static boolean isAuthorized(ConnectContext ctx, CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog,
- LanceIndexJob job, Map<String, String> locatorCache) {
- if (!targetResolves(catalog, job) || !locatorMatches(catalog, job, locatorCache)) {
- return Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.ADMIN);
- }
- return Env.getCurrentEnv().getAccessManager().checkTblPriv(ctx, catalog.getName(),
- job.getDbName(), job.getTableName(), PrivPredicate.SHOW);
- }
-
- static boolean targetResolves(CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog, LanceIndexJob job) {
- if (catalog == null) {
- return false;
- }
- try {
- DatabaseIf<? extends TableIf> db = catalog.getDbNullable(job.getDbName());
- return db != null && db.getTableNullable(job.getTableName()) != null;
- } catch (Exception e) {
- // External metadata resolution can fail outright (credentials expired,
- // provider down). That must not leak the provider error or abort the listing:
- // a failed resolution is handled exactly like an unresolvable (orphan) target.
- return false;
- }
- }
-
- /**
- * Whether the catalog's current durable dataset locator still points at the dataset
- * this job was admitted against. Only Lance catalogs carry a durable locator; for
- * any other catalog instance (theoretically unreachable, since jobs are only
- * admitted for Lance catalogs) the name resolution above is the whole rule. A
- * locator that cannot be resolved right now counts as a mismatch, so authorization
- * fails closed to the orphan rule instead of granting SHOW through stale names.
- */
- private static boolean locatorMatches(CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog,
- LanceIndexJob job, Map<String, String> locatorCache) {
- if (!(catalog instanceof LanceExternalCatalog)) {
- return true;
- }
- String key = job.getCatalogId() + ":" + job.getDbName() + ":" + job.getTableName();
- String currentLocator;
- if (locatorCache != null && locatorCache.containsKey(key)) {
- currentLocator = locatorCache.get(key);
- } else {
- currentLocator = ((LanceExternalCatalog) catalog).resolveCurrentIndexJobLocator(
- job.getDbName(), job.getTableName());
- if (locatorCache != null) {
- locatorCache.put(key, currentLocator);
- }
- }
- return currentLocator != null && currentLocator.equals(job.getNormalizedLocator());
- }
-
- private static List<String> renderRow(LanceIndexJob job,
- CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog) {
- List<String> row = new ArrayList<>(TITLE_NAMES.size());
- row.add(String.valueOf(job.getJobId()));
- // The catalog name is not persisted on the job record; for orphan rows (catalog gone)
- // there is nothing safe to render, so the column stays empty rather than fabricating
- // a placeholder from the internal catalog id.
- row.add(catalog == null ? "" : catalog.getName());
- row.add(job.getDbName());
- row.add(job.getTableName());
- row.add(job.getDisplayIndexName());
- row.add(job.getMutationType() == null ? "" : job.getMutationType().name());
- row.add(job.getMutationState() == null ? "" : job.getMutationState().name());
- row.add(job.getRefreshState() == null ? "" : job.getRefreshState().name());
- row.add(job.holdsPossibleLiveSlot() ? "YES" : "NO");
- row.add(TimeUtils.longToTimeString(job.getCreateTimeMs()));
- row.add(TimeUtils.longToTimeString(job.getUpdateTimeMs()));
- // job.getResult() is always null until a worker reports (no worker exists yet).
- row.add(job.getResult() == null || job.getResult().getSanitizedMessage() == null
- ? "" : job.getResult().getSanitizedMessage());
- row.addAll(renderForceAudit(job));
- return row;
- }
-
- static List<String> renderForceAudit(LanceIndexJob job) {
- List<String> forceColumns = new ArrayList<>(3);
- if (!job.isForceReleased()) {
- forceColumns.add("");
- forceColumns.add("");
- forceColumns.add("");
- return forceColumns;
- }
- forceColumns.add("YES");
- forceColumns.add(job.getForceActor() == null ? "" : job.getForceActor());
- forceColumns.add(job.getForceTimeMs() == null ? "" : TimeUtils.longToTimeString(job.getForceTimeMs()));
- return forceColumns;
- }
-
- @Override
- public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
- return visitor.visitShowLanceIndexJobsCommand(this, context);
- }
-
- @Override
- public RedirectStatus toRedirectStatus() {
- return RedirectStatus.FORWARD_NO_SYNC;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index e0e184ca760..61f5ae23df0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -229,8 +229,6 @@ import org.apache.doris.nereids.trees.plans.commands.ShowIndexNormalizerCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowIndexStatsCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowIndexTokenFilterCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowIndexTokenizerCommand;
-import org.apache.doris.nereids.trees.plans.commands.ShowLanceIndexJobCommand;
-import org.apache.doris.nereids.trees.plans.commands.ShowLanceIndexJobsCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowLastInsertCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowLoadCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowLoadProfileCommand;
@@ -685,14 +683,6 @@ public interface CommandVisitor<R, C> {
return visitCommand(showCreateRepositoryCommand, context);
}
- default R visitShowLanceIndexJobCommand(ShowLanceIndexJobCommand showLanceIndexJobCommand, C context) {
- return visitCommand(showLanceIndexJobCommand, context);
- }
-
- default R visitShowLanceIndexJobsCommand(ShowLanceIndexJobsCommand showLanceIndexJobsCommand, C context) {
- return visitCommand(showLanceIndexJobsCommand, context);
- }
-
default R visitShowLastInsertCommand(ShowLastInsertCommand showLastInsertCommand, C context) {
return visitCommand(showLastInsertCommand, context);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index ce7ab3219cf..ea93b941cf1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -64,7 +64,6 @@ import org.apache.doris.datasource.InitCatalogLog;
import org.apache.doris.datasource.InitDatabaseLog;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.MetaIdMappingsLog;
-import org.apache.doris.datasource.lance.job.LanceIndexJob;
import org.apache.doris.dictionary.Dictionary;
import org.apache.doris.ha.MasterInfo;
import org.apache.doris.indexpolicy.DropIndexPolicyLog;
@@ -1445,11 +1444,6 @@ public class EditLog {
env.getKeyManager().replayKeyOperation(info);
break;
}
- case OperationType.OP_LANCE_INDEX_JOB_UPSERT: {
- LanceIndexJob job = (LanceIndexJob) journal.getData();
- env.getLanceIndexJobManager().replayUpsertJob(job);
- break;
- }
case OperationType.OP_BEGIN_SNAPSHOT: {
// SnapshotState info = (SnapshotState) journal.getData();
// TODO: implement
@@ -1460,6 +1454,11 @@ public class EditLog {
// This log is only used to keep FE/MS cut point in journal timeline.
break;
}
+ case OperationType.OP_LANCE_INDEX_JOB_UPSERT: {
+ // Consumed and ignored by JournalEntity: payload of the removed durable
+ // Lance index job framework; nothing to replay.
+ break;
+ }
default: {
IOException e = new IOException();
LOG.error("UNKNOWN Operation Type {}, log id: {}", opCode, logId, e);
@@ -2369,10 +2368,6 @@ public class EditLog {
logEdit(OperationType.OP_DROP_INDEX_POLICY, policy);
}
- public void logLanceIndexJob(LanceIndexJob job) {
- logEdit(OperationType.OP_LANCE_INDEX_JOB_UPSERT, job);
- }
-
public void logCatalogLog(short id, CatalogLog log) {
logEdit(id, log);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
index e2fbc2131f7..f3ed185dab1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
@@ -422,7 +422,11 @@ public class OperationType {
public static final short OP_CREATE_ROLE_MAPPING = 496;
public static final short OP_DROP_ROLE_MAPPING = 497;
- // lance index job 500 ~ 509
+ // Payload of the removed durable Lance index job framework: kept only as a tombstone so
+ // old journals stay parseable (JournalEntity consumes and ignores the record). The code
+ // must never be reused: an edit log or image written by a build that still had the
+ // framework carries that opcode. No production deployment could hold such records: the
+ // mutation gate (enable_lance_index_mutation) never left its disabled default.
public static final short OP_LANCE_INDEX_JOB_UPSERT = 500;
// For cloud.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java
index 6ed7645d461..8f8425ec4a8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java
@@ -286,12 +286,6 @@ public class MetaPersistMethod {
metaPersistMethod.writeMethod = Env.class.getDeclaredMethod("saveKeyManagerStore",
CountingDataOutputStream.class, long.class);
break;
- case "lanceIndexJobManager":
- metaPersistMethod.readMethod = Env.class.getDeclaredMethod("loadLanceIndexJobManager",
- DataInputStream.class, long.class);
- metaPersistMethod.writeMethod = Env.class.getDeclaredMethod("saveLanceIndexJobManager",
- CountingDataOutputStream.class, long.class);
- break;
default:
break;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
index 8f8f00dd64e..ee4f93cc801 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
@@ -44,7 +44,7 @@ public class PersistMetaModules {
"globalFunction", "workloadGroups",
"binlogs", "resourceGroups", "AnalysisMgrV2", "AsyncJobManager", "workloadSchedPolicy",
"insertOverwrite", "plsql", "dictionaryManager", "indexPolicy", "KeyManagerStore",
- "authenticationIntegrations", "roleMappings", "lanceIndexJobManager"
+ "authenticationIntegrations", "roleMappings"
);
// The modules in `CloudEnv`.
@@ -52,7 +52,10 @@ public class PersistMetaModules {
// Modules in this list is deprecated and will not be saved in meta file. (also should not be in MODULE_NAMES)
public static final ImmutableList<String> DEPRECATED_MODULE_NAMES = ImmutableList.of(
- "loadJob", "cooldownJob", "AnalysisMgr", "mtmvJobManager", "JobTaskManager", "syncJob");
+ "loadJob", "cooldownJob", "AnalysisMgr", "mtmvJobManager", "JobTaskManager", "syncJob",
+ // Removed with the durable Lance index job framework; images written while it
+ // existed carry this (possibly empty) chunk and must stay readable.
+ "lanceIndexJobManager");
static {
MODULES_MAP = Maps.newHashMap();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java
index 13b3f506fb9..40940266e07 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java
@@ -238,6 +238,21 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
if (connectContext.getState().getStateType() == MysqlStateType.ERR) {
throw new RuntimeException("after executeQueryStatement handleQuery");
}
+ // A statement forwarded to the master that was rejected there keeps the local
+ // state OK: the MySQL protocol path returns the master's ERR packet verbatim in
+ // ConnectProcessor#finalizeCommand and the audit log reads the proxy status (see
+ // the multi-statement loop in ConnectProcessor#handleQuery). Arrow Flight has no
+ // channel for that packet, so the forwarded rejection must be surfaced here;
+ // otherwise the OK result synthesized below would report success for a statement
+ // the master refused to execute.
+ StmtExecutor stmtExecutor = connectContext.getExecutor();
+ if (stmtExecutor != null && stmtExecutor.hasForwardedToMaster()
+ && stmtExecutor.getProxyStatusCode() != 0) {
+ connectContext.getState().setError("forwarded statement failed on master FE, error code: "
+ + stmtExecutor.getProxyStatusCode() + ", error msg: "
+ + stmtExecutor.getProxyErrMsg());
+ throw new RuntimeException("after executeQueryStatement forwardToMaster");
+ }
if (connectContext.isReturnResultFromLocal()) {
// set/use etc. stmt returns an OK result by default.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java
index 0ed2b77e92f..b8a739c2796 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java
@@ -30,7 +30,6 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.Pair;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.es.EsExternalCatalog;
-import org.apache.doris.datasource.lance.LanceExternalCatalog;
import org.apache.doris.mysql.MysqlCommand;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.CascadesContext;
@@ -46,7 +45,6 @@ import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.plans.PlaceholderId;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PrepareCommandPlanner;
-import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand;
import org.apache.doris.nereids.trees.plans.commands.Command;
import org.apache.doris.nereids.trees.plans.commands.DeleteFromCommand;
import org.apache.doris.nereids.trees.plans.commands.DescribeCommand;
@@ -64,8 +62,6 @@ import org.apache.doris.nereids.trees.plans.commands.ShowSnapshotCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowTableCommand;
import org.apache.doris.nereids.trees.plans.commands.TransactionCommand;
import org.apache.doris.nereids.trees.plans.commands.UpdateCommand;
-import org.apache.doris.nereids.trees.plans.commands.info.CreateIndexOp;
-import org.apache.doris.nereids.trees.plans.commands.info.DropIndexOp;
import org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand;
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTVFCommand;
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
@@ -361,17 +357,6 @@ final class FlightSqlQuerySchema {
return stringMetadata("Explain String(Nereids Planner)");
} else if (command instanceof ReplayCommand) {
return stringMetadata("Plan Replayer dump url");
- } else if (command instanceof AlterTableCommand) {
- AlterTableCommand alter = (AlterTableCommand) command;
- String catalog = alter.getTbl().getCtl();
- // Lance index admission returns a JobId header even for an IF no-op. Do not run
- // validation/admission here: those paths can resolve remote tables or allocate IDs.
- if (context.getCatalog(catalog == null ? context.getDefaultCatalog() : catalog)
- instanceof LanceExternalCatalog && alter.getNereidsOps().stream().anyMatch(op ->
- (op instanceof CreateIndexOp && !((CreateIndexOp) op).isAlter())
- || (op instanceof DropIndexOp && !((DropIndexOp) op).isAlter()))) {
- return stringMetadata("JobId");
- }
}
ResultSetMetaData metadata = command.getResultSetMetaData();
// Concrete DML commands return OK, but subclasses such as WARM UP SELECT supply rows.
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrLanceGuardTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrLanceGuardTest.java
index 1b667faa05a..054687076b0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrLanceGuardTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrLanceGuardTest.java
@@ -19,20 +19,10 @@ package org.apache.doris.datasource;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
-import org.apache.doris.catalog.RefreshManager;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.DdlException;
import org.apache.doris.datasource.lance.LanceExternalCatalog;
-import org.apache.doris.datasource.lance.job.LanceIndexFenceKey;
-import org.apache.doris.datasource.lance.job.LanceIndexJob;
-import org.apache.doris.datasource.lance.job.LanceIndexJobManager;
-import org.apache.doris.datasource.lance.job.LanceIndexJobMutationType;
-import org.apache.doris.datasource.lance.job.LanceIndexNameNormalizer;
-import org.apache.doris.persist.EditLog;
-import org.apache.doris.persist.OperationType;
-import org.apache.doris.statistics.query.QueryStats;
-import com.google.common.collect.ImmutableMap;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
@@ -42,454 +32,95 @@ import java.lang.reflect.Field;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
/**
- * Coverage for the Lance catalog DDL guard in {@link CatalogMgr}: while the catalog has
- * unresolved Lance index jobs, ALTER CATALOG ... SET PROPERTIES must reject actual value
- * changes of the target identity keys - the namespace keys (lance.catalog.type, warehouse,
- * lance.namespace.parent, lance.namespace.delimiter, lance.namespace.root_database) and
- * the storage-routing keys (the s3.endpoint/s3.region family) that can land the same
- * persisted URI on a different storage service - and DROP CATALOG must be rejected;
- * credential-only/unrelated property changes, same-value rewrites, renames, and the
- * replay path stay unguarded. The guard message must stay neutral: no locator and no
- * release-statement syntax.
+ * Coverage for the surviving Lance target-identity guard in {@link CatalogMgr}:
+ * {@code hasLanceIdentityKeyChange} decides whether a property rewrite touches the
+ * namespace or storage-routing keys that can land the same table names on a different
+ * dataset target, and the ALTER-catalog replay path answers such a change by advancing
+ * the Lance admission epoch so in-flight admission snapshots fail revalidation.
+ * Same-value rewrites, credential-only changes, and unrelated keys stay no-ops.
*/
public class CatalogMgrLanceGuardTest {
- private static final long CATALOG_ID = 10L;
- private static final String CATALOG_NAME = "lance_catalog";
- private static final String LOCATOR = "s3://bucket/dataset";
- private static final String[] IDENTITY_KEYS = {
- "lance.catalog.type", "warehouse",
- "lance.namespace.parent", "lance.namespace.delimiter", "lance.namespace.root_database",
- "s3.endpoint", "s3.region"};
- private static final class Fixture {
- private final CatalogMgr catalogMgr = new CatalogMgr();
- private final LanceExternalCatalog catalog = Mockito.mock(LanceExternalCatalog.class);
- private final TestJobManager jobManager = new TestJobManager();
- private final Env env = Mockito.mock(Env.class);
- private final EditLog editLog = Mockito.mock(EditLog.class);
- private final ExternalMetaCacheMgr metaCacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
- private final Map<String, String> oldProperties = new HashMap<>();
+ @Test
+ public void testIdentityKeyChangeSemantics() {
+ Map<String, String> persisted = new HashMap<>();
+ persisted.put("s3.endpoint", "s3://a");
+ persisted.put("type", "hms");
+ persisted.put("s3.access_key", "old");
+
+ Assertions.assertTrue(CatalogMgr.hasLanceIdentityKeyChange(persisted, props("s3.endpoint", "s3://b")),
+ "a value change of an identity key is a target change");
+ Assertions.assertFalse(CatalogMgr.hasLanceIdentityKeyChange(persisted, props("s3.endpoint", "s3://a")),
+ "a same-value rewrite is an idempotent no-op");
+ Assertions.assertTrue(CatalogMgr.hasLanceIdentityKeyChange(persisted, props("s3.region", "us-west")),
+ "a newly supplied identity key the persisted properties never set is a change");
+ Assertions.assertTrue(CatalogMgr.hasLanceIdentityKeyChange(persisted, props("S3.ENDPOINT", "s3://b")),
+ "identity keys match case-insensitively");
+ Assertions.assertFalse(CatalogMgr.hasLanceIdentityKeyChange(persisted, props("s3.access_key", "new")),
+ "a credential-only change is not a target change");
+ Assertions.assertFalse(CatalogMgr.hasLanceIdentityKeyChange(persisted, props("type", "jdbc")),
+ "an unrelated property change is not a target change");
+ Map<String, String> withNullKey = new HashMap<>();
+ withNullKey.put(null, "x");
+ Assertions.assertFalse(CatalogMgr.hasLanceIdentityKeyChange(persisted, withNullKey),
+ "a null key is never an identity key");
+ }
+
+ @Test
+ public void testAlterReplayAdvancesAdmissionEpochOnlyOnTargetChange() throws Exception {
+ CatalogMgr catalogMgr = new CatalogMgr();
+ LanceExternalCatalog catalog = Mockito.mock(LanceExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(10L);
+ Mockito.when(catalog.getProperties()).thenReturn(props("s3.endpoint", "s3://a"));
+ registerCatalog(catalogMgr, catalog);
+
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr metaCacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(metaCacheMgr);
+ Mockito.when(metaCacheMgr.withCatalogLifecycleLock(Mockito.anyLong(), Mockito.any()))
+ .thenAnswer(invocation -> {
+ Supplier<?> action = invocation.getArgument(1);
+ return action.get();
+ });
- Fixture(String lanceCatalogType) throws Exception {
- oldProperties.put("type", "lance");
- oldProperties.put("lance.catalog.type", lanceCatalogType);
- oldProperties.put("warehouse", "s3://bucket/warehouse");
- oldProperties.put("lance.namespace.parent", "s3://bucket/ns");
- oldProperties.put("lance.namespace.delimiter", ".");
- oldProperties.put("lance.namespace.root_database", "root_db");
- oldProperties.put("s3.access_key", "ak");
- oldProperties.put("s3.endpoint", "https://s3.example.com");
- oldProperties.put("s3.region", "us-east-1");
+ try (MockedStatic<Env> currentEnv = Mockito.mockStatic(Env.class)) {
+ currentEnv.when(Env::getCurrentEnv).thenReturn(env);
- AtomicReference<String> catalogName = new AtomicReference<>(CATALOG_NAME);
- Mockito.when(catalog.getId()).thenReturn(CATALOG_ID);
- Mockito.when(catalog.getType()).thenReturn("lance");
- Mockito.when(catalog.getName()).thenAnswer(invocation -> catalogName.get());
- Mockito.doAnswer(invocation -> {
- catalogName.set(invocation.getArgument(0));
- return null;
- }).when(catalog).modifyCatalogName(Mockito.anyString());
- AtomicLong targetVersion = new AtomicLong();
- Mockito.when(catalog.getIndexTargetVersion()).thenAnswer(invocation -> targetVersion.get());
- Mockito.doAnswer(invocation -> {
- targetVersion.incrementAndGet();
- return null;
- }).when(catalog).advanceIndexTargetVersion();
- Mockito.when(catalog.getProperties()).thenReturn(oldProperties);
- Mockito.doAnswer(invocation -> {
- oldProperties.putAll(invocation.getArgument(0));
- return null;
- }).when(catalog).modifyCatalogProps(Mockito.anyMap());
- registerCatalog(catalogMgr, catalog);
+ replayAlter(catalogMgr, 10L, props("s3.access_key", "new"));
+ Mockito.verify(catalog, Mockito.never()).advanceIndexTargetVersion();
- Mockito.when(env.getLanceIndexJobManager()).thenReturn(jobManager);
- Mockito.when(env.getEditLog()).thenReturn(editLog);
- Mockito.when(env.getExtMetaCacheMgr()).thenReturn(metaCacheMgr);
- Mockito.when(env.getRefreshManager()).thenReturn(Mockito.mock(RefreshManager.class));
- Mockito.when(env.getQueryStats()).thenReturn(Mockito.mock(QueryStats.class));
- Mockito.when(metaCacheMgr.withCatalogLifecycleLock(Mockito.anyLong(), Mockito.any()))
- .thenAnswer(invocation -> {
- Supplier<?> action = invocation.getArgument(1);
- return action.get();
- });
- }
+ replayAlter(catalogMgr, 10L, props("s3.endpoint", "s3://a"));
+ Mockito.verify(catalog, Mockito.never()).advanceIndexTargetVersion();
- void admitUnresolvedJob() throws Exception {
- jobManager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+ replayAlter(catalogMgr, 10L, props("s3.endpoint", "s3://b"));
+ Mockito.verify(catalog, Mockito.times(1)).advanceIndexTargetVersion();
}
+ }
- MockedStatic<Env> mockCurrentEnv() {
- MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class);
- mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
- return mockedEnv;
- }
+ private static void replayAlter(CatalogMgr catalogMgr, long catalogId, Map<String, String> newProps)
+ throws DdlException {
+ CatalogLog log = new CatalogLog();
+ log.setCatalogId(catalogId);
+ log.setNewProps(newProps);
+ catalogMgr.replayAlterCatalogProps(log, null, true);
}
@SuppressWarnings("unchecked")
- private static void registerCatalog(CatalogMgr catalogMgr, CatalogIf catalog) throws Exception {
+ private static void registerCatalog(CatalogMgr catalogMgr, LanceExternalCatalog catalog) throws Exception {
Field idToCatalogField = CatalogMgr.class.getDeclaredField("idToCatalog");
idToCatalogField.setAccessible(true);
((ConcurrentMap<Long, CatalogIf<? extends DatabaseIf<? extends TableIf>>>)
idToCatalogField.get(catalogMgr)).put(catalog.getId(), catalog);
- Field nameToCatalogField = CatalogMgr.class.getDeclaredField("nameToCatalog");
- nameToCatalogField.setAccessible(true);
- ((Map<String, CatalogIf>) nameToCatalogField.get(catalogMgr)).put(catalog.getName(), catalog);
- }
-
- private static LanceIndexJob newCreateJob(long jobId, String displayName) {
- return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
- displayName, LanceIndexNameNormalizer.normalize(displayName),
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v",
- null, 7L, null);
- }
-
- private static void assertNeutralGuardMessage(DdlException e) {
- String message = e.getMessage();
- Assertions.assertNotNull(message);
- Assertions.assertTrue(message.contains("unresolved Lance index jobs"), message);
- Assertions.assertFalse(message.contains("RESOLVE"), message);
- Assertions.assertFalse(message.contains(LOCATOR), message);
- Assertions.assertFalse(message.contains("s3://"), message);
- }
-
- @Test
- public void testEveryIdentityKeyValueChangeIsBlocked() throws Exception {
- Fixture fixture = new Fixture("filesystem");
- fixture.admitUnresolvedJob();
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- for (String key : IDENTITY_KEYS) {
- Map<String, String> newProperties = ImmutableMap.of(key, "changed-value");
- DdlException e = Assertions.assertThrows(DdlException.class,
- () -> fixture.catalogMgr.alterCatalogProps(CATALOG_NAME, newProperties), key);
- assertNeutralGuardMessage(e);
- }
- Mockito.verify(fixture.editLog, Mockito.never())
- .logCatalogLog(Mockito.anyShort(), Mockito.any());
- Mockito.verify(fixture.catalog, Mockito.never()).modifyCatalogProps(Mockito.any());
- }
- }
-
- @Test
- public void testAddingAbsentIdentityKeyIsBlocked() throws Exception {
- Fixture fixture = new Fixture("filesystem");
- fixture.oldProperties.remove("lance.namespace.root_database");
- fixture.admitUnresolvedJob();
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- DdlException e = Assertions.assertThrows(DdlException.class,
- () -> fixture.catalogMgr.alterCatalogProps(CATALOG_NAME,
- ImmutableMap.of("lance.namespace.root_database", "other_db")));
- assertNeutralGuardMessage(e);
- }
- }
-
- @Test
- public void testSameValueRewriteIsAllowed() throws Exception {
- Fixture fixture = new Fixture("filesystem");
- fixture.admitUnresolvedJob();
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- // Re-setting every property (identity keys included) to its current value is
- // an idempotent no-op and must not be blocked.
- fixture.catalogMgr.alterCatalogProps(CATALOG_NAME, fixture.oldProperties);
- Mockito.verify(fixture.editLog, Mockito.times(1))
- .logCatalogLog(Mockito.eq(OperationType.OP_ALTER_CATALOG_PROPS), Mockito.any());
- }
- }
-
- @Test
- public void testCaseVariantIdentityKeyChangeIsBlocked() throws Exception {
- Fixture fixture = new Fixture("filesystem");
- fixture.admitUnresolvedJob();
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- // Catalog property keys are not normalized upstream; a case variant with a
- // changed value is still a target identity change.
- DdlException e = Assertions.assertThrows(DdlException.class,
- () -> fixture.catalogMgr.alterCatalogProps(CATALOG_NAME,
- ImmutableMap.of("WAREHOUSE", "s3://other/warehouse")));
- assertNeutralGuardMessage(e);
- Mockito.verify(fixture.catalog, Mockito.never()).modifyCatalogProps(Mockito.any());
- }
- }
-
- @Test
- public void testCredentialAndUnrelatedKeyChangesAreAllowed() throws Exception {
- Fixture fixture = new Fixture("filesystem");
- fixture.admitUnresolvedJob();
- Map<String, String> newProperties = new HashMap<>();
- // Credentials authenticate to the same target, so rotating them stays allowed;
- // storage routing (s3.endpoint and family) is identity and is covered below.
- newProperties.put("s3.access_key", "rotated-ak");
- newProperties.put("s3.secret_key", "rotated-sk");
- newProperties.put("custom.unrelated.property", "value");
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- fixture.catalogMgr.alterCatalogProps(CATALOG_NAME, newProperties);
- Mockito.verify(fixture.editLog, Mockito.times(1))
- .logCatalogLog(Mockito.eq(OperationType.OP_ALTER_CATALOG_PROPS), Mockito.any());
- }
- }
-
- @Test
- public void testStorageRoutingKeyChangeWithUnresolvedJobIsBlocked() throws Exception {
- // s3.endpoint and its family route the persisted URI to storage rather than
- // authenticate to it: a changed endpoint can be a different storage service, so
- // it is target identity, not a credential.
- Fixture fixture = new Fixture("filesystem");
- fixture.admitUnresolvedJob();
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- DdlException e = Assertions.assertThrows(DdlException.class,
- () -> fixture.catalogMgr.alterCatalogProps(CATALOG_NAME,
- ImmutableMap.of("s3.endpoint", "https://s3.other-example.com")));
- assertNeutralGuardMessage(e);
- // A routing key the catalog never carried still counts as a change from its
- // implicit absence, exactly like the namespace identity keys.
- DdlException absent = Assertions.assertThrows(DdlException.class,
- () -> fixture.catalogMgr.alterCatalogProps(CATALOG_NAME,
- ImmutableMap.of("oss.endpoint", "https://oss.example.com")));
- assertNeutralGuardMessage(absent);
- Mockito.verify(fixture.editLog, Mockito.never())
- .logCatalogLog(Mockito.anyShort(), Mockito.any());
- Mockito.verify(fixture.catalog, Mockito.never()).modifyCatalogProps(Mockito.any());
- }
- }
-
- @Test
- public void testDropCatalogIsBlockedWithUnresolvedJob() throws Exception {
- Fixture fixture = new Fixture("filesystem");
- fixture.admitUnresolvedJob();
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- DdlException e = Assertions.assertThrows(DdlException.class,
- () -> fixture.catalogMgr.dropCatalog(CATALOG_NAME, false));
- assertNeutralGuardMessage(e);
- Assertions.assertSame(fixture.catalog, fixture.catalogMgr.getCatalog(CATALOG_NAME));
- Mockito.verify(fixture.editLog, Mockito.never())
- .logCatalogLog(Mockito.anyShort(), Mockito.any());
- }
- }
-
- @Test
- public void testDropCatalogIsAllowedWithoutUnresolvedJobs() throws Exception {
- Fixture fixture = new Fixture("filesystem");
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- fixture.catalogMgr.dropCatalog(CATALOG_NAME, false);
- Assertions.assertNull(fixture.catalogMgr.getCatalog(CATALOG_NAME));
- Mockito.verify(fixture.editLog, Mockito.times(1))
- .logCatalogLog(Mockito.eq(OperationType.OP_DROP_CATALOG), Mockito.any());
- }
- }
-
- @Test
- public void testRenameIsAllowedWithUnresolvedJob() throws Exception {
- Fixture fixture = new Fixture("filesystem");
- fixture.admitUnresolvedJob();
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- // The fence identity is the persistent catalog id, so a rename is not target-changing.
- fixture.catalogMgr.alterCatalogName(CATALOG_NAME, "lance_catalog_renamed");
- Assertions.assertSame(fixture.catalog, fixture.catalogMgr.getCatalog("lance_catalog_renamed"));
- Assertions.assertNull(fixture.catalogMgr.getCatalog(CATALOG_NAME));
- Mockito.verify(fixture.editLog, Mockito.times(1))
- .logCatalogLog(Mockito.eq(OperationType.OP_ALTER_CATALOG_NAME), Mockito.any());
- }
- }
-
- @Test
- public void testIdentityKeyChangeIsAllowedWithoutUnresolvedJobs() throws Exception {
- Fixture fixture = new Fixture("filesystem");
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- fixture.catalogMgr.alterCatalogProps(CATALOG_NAME,
- ImmutableMap.of("warehouse", "s3://bucket/elsewhere"));
- Mockito.verify(fixture.editLog, Mockito.times(1))
- .logCatalogLog(Mockito.eq(OperationType.OP_ALTER_CATALOG_PROPS), Mockito.any());
- }
- }
-
- @Test
- public void testRestCatalogIsNotAffected() throws Exception {
- // Admission rejects REST catalogs, so a REST catalog can never hold a job and the
- // guard is a natural no-op for it.
- Fixture fixture = new Fixture("rest");
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- fixture.catalogMgr.alterCatalogProps(CATALOG_NAME,
- ImmutableMap.of("warehouse", "s3://bucket/elsewhere"));
- fixture.catalogMgr.dropCatalog(CATALOG_NAME, false);
- Assertions.assertNull(fixture.catalogMgr.getCatalog(CATALOG_NAME));
- }
- }
-
- @Test
- public void testReplayAlterCatalogPropsIsNotGuarded() throws Exception {
- Fixture fixture = new Fixture("filesystem");
- fixture.admitUnresolvedJob();
- CatalogLog log = new CatalogLog();
- log.setCatalogId(CATALOG_ID);
- log.setNewProps(ImmutableMap.of("warehouse", "s3://bucket/elsewhere"));
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- // Replay must apply verbatim: the guard lives only on the master DDL path.
- fixture.catalogMgr.replayAlterCatalogProps(log, null, true);
- Mockito.verify(fixture.catalog).advanceIndexTargetVersion();
- Mockito.verify(fixture.catalog).modifyCatalogProps(log.getNewProps());
- }
- }
-
- @Test
- public void targetChangeAfterCaptureRejectsAdmission() throws Exception {
- for (String key : IDENTITY_KEYS) {
- Fixture fixture = new Fixture("filesystem");
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- CatalogMgr.LanceIndexTarget target = fixture.catalogMgr.captureLanceIndexTarget(fixture.catalog);
- fixture.catalogMgr.alterCatalogProps(CATALOG_NAME,
- ImmutableMap.of(key, fixture.oldProperties.get(key) + "_changed"));
- DdlException error = Assertions.assertThrows(DdlException.class,
- () -> fixture.catalogMgr.withLanceIndexAdmission(fixture.catalog, target, () -> {
- fixture.admitUnresolvedJob();
- return null;
- }));
- Assertions.assertTrue(error.getMessage().contains("target changed"));
- Assertions.assertEquals(0, fixture.jobManager.getJobCount());
- }
- }
- }
-
- @Test
- public void targetChangeAndRestoreRejectsSnapshotOfIntermediateTarget() throws Exception {
- Fixture fixture = new Fixture("filesystem");
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- CatalogMgr.LanceIndexTarget target = fixture.catalogMgr.captureLanceIndexTarget(fixture.catalog);
- fixture.catalogMgr.alterCatalogProps(CATALOG_NAME, ImmutableMap.of("warehouse", "s3://bucket/B"));
- // A lock-free snapshot loader can read B before another ALTER restores A.
- fixture.catalogMgr.alterCatalogProps(CATALOG_NAME,
- ImmutableMap.of("warehouse", "s3://bucket/warehouse"));
- Assertions.assertThrows(DdlException.class,
- () -> fixture.catalogMgr.withLanceIndexAdmission(fixture.catalog, target, () -> {
- fixture.admitUnresolvedJob();
- return null;
- }));
- Assertions.assertEquals(0, fixture.jobManager.getJobCount());
- }
- }
-
- @Test
- public void failedIdentityAlterInvalidatesAdmissionEvenAfterRollback() throws Exception {
- Fixture fixture = new Fixture("filesystem");
- Map<String, String> tentative = new HashMap<>(fixture.oldProperties);
- tentative.put("warehouse", "s3://bucket/temporary");
- Mockito.doAnswer(invocation -> {
- Mockito.when(fixture.catalog.getProperties()).thenReturn(tentative);
- return null;
- }).when(fixture.catalog).tryModifyCatalogProps(Mockito.anyMap());
- Mockito.doThrow(new DdlException("invalid target")).when(fixture.catalog).checkProperties();
- Mockito.doAnswer(invocation -> {
- Mockito.when(fixture.catalog.getProperties()).thenReturn(fixture.oldProperties);
- return null;
- }).when(fixture.metaCacheMgr).rollbackCatalogProperties(Mockito.eq(fixture.catalog), Mockito.anyMap());
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- CatalogMgr.LanceIndexTarget target = fixture.catalogMgr.captureLanceIndexTarget(fixture.catalog);
- Assertions.assertThrows(DdlException.class, () -> fixture.catalogMgr.alterCatalogProps(CATALOG_NAME,
- ImmutableMap.of("warehouse", "s3://bucket/temporary")));
- Assertions.assertSame(fixture.oldProperties, fixture.catalog.getProperties());
- Assertions.assertThrows(DdlException.class,
- () -> fixture.catalogMgr.withLanceIndexAdmission(fixture.catalog, target, () -> {
- fixture.admitUnresolvedJob();
- return null;
- }));
- Assertions.assertEquals(0, fixture.jobManager.getJobCount());
- }
- }
-
- @Test
- public void sameTargetAndCredentialChangesPreserveAdmission() throws Exception {
- Fixture fixture = new Fixture("filesystem");
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- CatalogMgr.LanceIndexTarget target = fixture.catalogMgr.captureLanceIndexTarget(fixture.catalog);
- fixture.catalogMgr.alterCatalogProps(CATALOG_NAME,
- ImmutableMap.of("warehouse", "s3://bucket/warehouse", "s3.access_key", "new_key"));
- fixture.catalogMgr.withLanceIndexAdmission(fixture.catalog, target, () -> {
- fixture.admitUnresolvedJob();
- return null;
- });
- Assertions.assertEquals(1, fixture.jobManager.getJobCount());
- }
- }
-
- @Test
- public void dropAfterCaptureRejectsAdmission() throws Exception {
- Fixture fixture = new Fixture("filesystem");
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- CatalogMgr.LanceIndexTarget target = fixture.catalogMgr.captureLanceIndexTarget(fixture.catalog);
- fixture.catalogMgr.dropCatalog(CATALOG_NAME, false);
- Assertions.assertThrows(DdlException.class,
- () -> fixture.catalogMgr.withLanceIndexAdmission(fixture.catalog, target, () -> {
- fixture.admitUnresolvedJob();
- return null;
- }));
- Assertions.assertEquals(0, fixture.jobManager.getJobCount());
- }
- }
-
- @Test
- public void concurrentDdlWaitsForAdmissionAndSeesItsUnresolvedJob() throws Exception {
- // Both DDL paths must be excluded throughout the final admission transfer.
- for (boolean drop : new boolean[] {true, false}) {
- Fixture fixture = new Fixture("filesystem");
- CatalogMgr.LanceIndexTarget target = fixture.catalogMgr.captureLanceIndexTarget(fixture.catalog);
- CountDownLatch admissionEntered = new CountDownLatch(1);
- CountDownLatch finishAdmission = new CountDownLatch(1);
- CountDownLatch ddlStarted = new CountDownLatch(1);
- ExecutorService executor = Executors.newFixedThreadPool(2);
- try {
- Future<?> admission = executor.submit(() -> {
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- return fixture.catalogMgr.withLanceIndexAdmission(fixture.catalog, target, () -> {
- admissionEntered.countDown();
- Assertions.assertTrue(finishAdmission.await(10, TimeUnit.SECONDS));
- fixture.admitUnresolvedJob();
- return null;
- });
- }
- });
- Assertions.assertTrue(admissionEntered.await(10, TimeUnit.SECONDS));
- Future<DdlException> ddl = executor.submit(() -> {
- try (MockedStatic<Env> mockedEnv = fixture.mockCurrentEnv()) {
- ddlStarted.countDown();
- return Assertions.assertThrows(DdlException.class, () -> {
- if (drop) {
- fixture.catalogMgr.dropCatalog(CATALOG_NAME, false);
- } else {
- fixture.catalogMgr.alterCatalogProps(CATALOG_NAME,
- ImmutableMap.of("warehouse", "s3://bucket/new"));
- }
- });
- }
- });
- Assertions.assertTrue(ddlStarted.await(10, TimeUnit.SECONDS));
- Assertions.assertThrows(TimeoutException.class, () -> ddl.get(200, TimeUnit.MILLISECONDS));
- finishAdmission.countDown();
- admission.get(10, TimeUnit.SECONDS);
- assertNeutralGuardMessage(ddl.get(10, TimeUnit.SECONDS));
- Assertions.assertSame(fixture.catalog, fixture.catalogMgr.getCatalog(CATALOG_ID));
- Assertions.assertEquals("s3://bucket/warehouse", fixture.oldProperties.get("warehouse"));
- } finally {
- finishAdmission.countDown();
- executor.shutdownNow();
- Assertions.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS));
- }
- }
}
- private static class TestJobManager extends LanceIndexJobManager {
- @Override
- protected void writeEditLog(LanceIndexJob job) {
- // No journal in a pure DDL-guard unit test.
+ private static Map<String, String> props(String... keyValue) {
+ Map<String, String> properties = new HashMap<>();
+ for (int i = 0; i < keyValue.length; i += 2) {
+ properties.put(keyValue[i], keyValue[i + 1]);
}
+ return properties;
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java
index 8b4dbd73660..14d3f8de3f7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java
@@ -25,7 +25,6 @@ import org.apache.doris.datasource.ExternalDatabase;
import org.apache.doris.datasource.ExternalMetaCacheMgr;
import org.apache.doris.datasource.ExternalObjectLog;
import org.apache.doris.datasource.ExternalTable;
-import org.apache.doris.datasource.lance.job.LanceIndexDatasetLocator;
import org.apache.doris.persist.EditLog;
import org.apache.arrow.memory.BufferAllocator;
@@ -291,25 +290,6 @@ public class LanceCatalogLifecycleTest {
}
}
- @Test
- public void testIndexJobLocatorBypassesQueryAccessCache() throws Exception {
- LanceNamespace namespace = Mockito.mock(LanceNamespace.class);
- Mockito.when(namespace.describeTable(Mockito.any())).thenReturn(
- new DescribeTableResponse().tableUri("file:///warehouse/original.lance"),
- new DescribeTableResponse().tableUri("file:///warehouse/replacement.lance"));
- try (LanceCatalogClient client = new LanceCatalogClient(namespace, Mockito.mock(BufferAllocator.class),
- Mockito.mock(Session.class), "filesystem", "default", Collections.emptyList(),
- Collections.emptyList(), Collections.emptyMap(), Collections.emptyList())) {
- Field field = LanceCatalogClient.class.getDeclaredField("namespaceClient");
- field.setAccessible(true);
- LanceNamespaceClient namespaceClient = (LanceNamespaceClient) field.get(client);
- namespaceClient.resolveTableAccess("default", "items");
- Assertions.assertEquals(LanceIndexDatasetLocator.normalize("file:///warehouse/replacement.lance"),
- client.resolveCurrentIndexJobLocator("default", "items"));
- Mockito.verify(namespace, Mockito.times(2)).describeTable(Mockito.any());
- }
- }
-
@Test
public void testRoutineDatabaseObjectCleanupPreservesHotAccess() throws Exception {
try (AccessFixture fixture = new AccessFixture()) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceIndexAdmissionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceIndexAdmissionTest.java
index 26ac7d69d7a..e85f971cc9f 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceIndexAdmissionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceIndexAdmissionTest.java
@@ -23,18 +23,12 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.CatalogMgr;
import org.apache.doris.datasource.lance.LanceIndexAdmissionSnapshot.PhysicalIndexInfo;
import org.apache.doris.datasource.lance.index.LanceShowIndexInfo;
-import org.apache.doris.datasource.lance.job.LanceIndexFenceKey;
-import org.apache.doris.datasource.lance.job.LanceIndexJob;
-import org.apache.doris.datasource.lance.job.LanceIndexJobManager;
-import org.apache.doris.datasource.lance.job.LanceIndexJobMutationType;
-import org.apache.doris.datasource.lance.job.LanceIndexSchemaContract;
import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition;
import org.apache.doris.qe.ConnectContext;
@@ -51,21 +45,19 @@ import org.lance.schema.LanceField;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
/**
* Section 2.2/4.1 admission coverage for {@link LanceIndexAdmission}: snapshots are injected
- * through the {@code SnapshotLoader} seam (no FE, no JNI), the job manager is the real 3B
- * implementation with its edit-log seam captured, and {@code Env} is mocked only for id
- * allocation and the manager getter. Every rejection path must leave no job, no fence, no
- * quota charge, no journal record, and no id allocation behind (the 3B ReplayTest pattern).
+ * through the {@code SnapshotLoader} seam (no FE, no JNI) and {@code Env} is mocked only for
+ * the catalog manager. Every rejection path must leave no durable state and no id allocation
+ * behind. A mutation that passes the whole preflight terminates with the shared
+ * not-supported rejection (the synchronous execution path has not landed), while an IF
+ * preflight no-op returns normally.
*/
public class LanceIndexAdmissionTest {
private static final long CATALOG_ID = 10L;
@@ -77,9 +69,6 @@ public class LanceIndexAdmissionTest {
private static final long DATASET_VERSION = 7L;
private static final String MATCHING_ANN_PROPERTIES_JSON =
"{\"compression\":{\"num_bits\":8,\"num_sub_vectors\":16},\"metric_type\":\"l2\"}";
- private static final String NORMALIZED_ANN_PROPERTIES_JSON =
- "{\"index_type\":\"IVF_PQ\",\"metric\":\"l2\",\"num_bits\":\"8\","
- + "\"num_partitions\":\"256\",\"num_sub_vectors\":\"16\"}";
/**
* The KELVIN SIGN (U+212A) and the A-with-diaeresis pair: {@code String.equalsIgnoreCase}
* folds both pairs case-insensitively (the Kelvin sign through its per-character
@@ -91,26 +80,17 @@ public class LanceIndexAdmissionTest {
private MockedStatic<Env> mockedEnv;
private Env env;
- private TestManager manager;
- private AtomicLong idAllocator;
private LanceExternalCatalog catalog;
private LanceExternalDatabase database;
private LanceExternalTable table;
private ConnectContext connectContext;
private final Map<String, Column> tableColumns = new HashMap<>();
- private long originalTableQuota;
- private long originalCatalogQuota;
- private long originalGlobalQuota;
@BeforeEach
public void setUp() throws Exception {
mockedEnv = Mockito.mockStatic(Env.class);
env = Mockito.mock(Env.class);
- manager = new TestManager();
- idAllocator = new AtomicLong(100L);
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
- Mockito.when(env.getNextId()).thenAnswer(invocation -> idAllocator.incrementAndGet());
- Mockito.when(env.getLanceIndexJobManager()).thenReturn(manager);
catalog = Mockito.mock(LanceExternalCatalog.class);
Mockito.when(catalog.getId()).thenReturn(CATALOG_ID);
@@ -146,17 +126,10 @@ public class LanceIndexAdmissionTest {
connectContext = new ConnectContext();
connectContext.setThreadLocalInfo();
connectContext.setCurrentUserIdentity(UserIdentity.createAnalyzedUserIdentWithIp("tester", "%"));
-
- originalTableQuota = Config.lance_index_job_max_unresolved_per_table;
- originalCatalogQuota = Config.lance_index_job_max_unresolved_per_catalog;
- originalGlobalQuota = Config.lance_index_job_max_unresolved_global;
}
@AfterEach
public void tearDown() {
- Config.lance_index_job_max_unresolved_per_table = originalTableQuota;
- Config.lance_index_job_max_unresolved_per_catalog = originalCatalogQuota;
- Config.lance_index_job_max_unresolved_global = originalGlobalQuota;
ConnectContext.remove();
mockedEnv.close();
}
@@ -165,16 +138,6 @@ public class LanceIndexAdmissionTest {
// Fixtures
// ------------------------------------------------------------------
- /** Edit-log seam: captures durable records instead of writing the journal (3B pattern). */
- private static class TestManager extends LanceIndexJobManager {
- private final List<LanceIndexJob> editLog = new ArrayList<>();
-
- @Override
- protected void writeEditLog(LanceIndexJob job) {
- editLog.add(job);
- }
- }
-
private static Column notNullColumn(String name, Type type) {
return new Column(name, type, false, null, false, null, "");
}
@@ -268,92 +231,46 @@ public class LanceIndexAdmissionTest {
new HashMap<>(), "", orReplace);
}
- private LanceIndexAdmission.Outcome admitCreate(LanceIndexAdmissionSnapshot snapshot,
+ private void admitCreate(LanceIndexAdmissionSnapshot snapshot,
IndexDefinition def, boolean ifNotExists) throws Exception {
- return LanceIndexAdmission.admitCreate((cat, dbName, tblName) -> snapshot, catalog, database,
+ LanceIndexAdmission.admitCreate((cat, dbName, tblName) -> snapshot, catalog, database,
table, def, ifNotExists);
}
- private LanceIndexAdmission.Outcome admitDrop(LanceIndexAdmissionSnapshot snapshot,
+ private void admitDrop(LanceIndexAdmissionSnapshot snapshot,
String indexName, boolean ifExists) throws Exception {
- return LanceIndexAdmission.admitDrop((cat, dbName, tblName) -> snapshot, catalog, database,
+ LanceIndexAdmission.admitDrop((cat, dbName, tblName) -> snapshot, catalog, database,
table, indexName, ifExists);
}
private void assertNothingPersisted() {
- Assertions.assertEquals(0, manager.getJobCount());
- Assertions.assertTrue(manager.editLog.isEmpty());
- Assertions.assertTrue(manager.getUnresolvedJobs().isEmpty());
Mockito.verify(env, Mockito.never()).getNextId();
}
- /** Unresolved-job count on this dataset locator — the public view of the table quota. */
- private int unresolvedJobsOnDataset() {
- int count = 0;
- for (LanceIndexJob job : manager.getUnresolvedJobs()) {
- if (job.getCatalogId() == CATALOG_ID && DATASET_URI.equals(job.getNormalizedLocator())) {
- count++;
- }
- }
- return count;
- }
-
private static void assertInvalid(Executable call, String expectedMessage) {
AnalysisException exception = Assertions.assertThrows(AnalysisException.class, call);
Assertions.assertEquals(expectedMessage, exception.getDetailMessage());
Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_INVALID, exception.getMysqlErrorCode());
}
+ /** A mutation whose whole preflight passed terminates with the shared not-supported error. */
+ private static void assertUnsupported(Executable call, String operation) {
+ AnalysisException exception = Assertions.assertThrows(AnalysisException.class, call);
+ Assertions.assertEquals(operation + " is not supported for Lance catalog tables",
+ exception.getDetailMessage());
+ Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_OPERATION_NOT_SUPPORTED,
+ exception.getMysqlErrorCode());
+ }
+
// ------------------------------------------------------------------
// CREATE admission
// ------------------------------------------------------------------
@Test
- public void plainCreateOnEmptySnapshotIsAdmitted() throws Exception {
- LanceIndexAdmission.Outcome outcome = admitCreate(emptySnapshot(), annDef("MyIdx", false, false),
- false);
-
- Assertions.assertNotNull(outcome.getJobId());
- LanceIndexJob job = manager.getJob(outcome.getJobId());
- Assertions.assertNotNull(job);
- Assertions.assertEquals("tester", job.getCreator());
- Assertions.assertEquals(CATALOG_ID, job.getCatalogId());
- // Local names are persisted for privilege/FORCE resolution, not the remote ones.
- Assertions.assertEquals(DB, job.getDbName());
- Assertions.assertEquals(TBL, job.getTableName());
- Assertions.assertEquals(LanceIndexFenceKey.PROVIDER_DIRECTORY, job.getProvider());
- Assertions.assertEquals(DATASET_URI, job.getNormalizedLocator());
- Assertions.assertEquals("MyIdx", job.getDisplayIndexName());
- Assertions.assertEquals("myidx", job.getNormalizedIndexName());
- Assertions.assertEquals(LanceIndexJobMutationType.CREATE, job.getMutationType());
- Assertions.assertFalse(job.isIfNotExists());
- Assertions.assertFalse(job.isIfExists());
- Assertions.assertEquals("IVF_PQ", job.getIndexType());
- Assertions.assertEquals("v", job.getColumnName());
- Assertions.assertEquals(NORMALIZED_ANN_PROPERTIES_JSON, job.getPropertiesJson());
- Assertions.assertEquals(DATASET_VERSION, job.getAdmittedDatasetVersion());
-
- LanceIndexSchemaContract contract = job.getSchemaContract();
- Assertions.assertNotNull(contract);
- Assertions.assertEquals(LanceIndexSchemaContract.SCHEMA_CONTRACT_VERSION_V1,
- contract.getSchemaContractVersion());
- Assertions.assertEquals(1, contract.getFields().size());
- LanceIndexSchemaContract.IndexedField field = contract.getFields().get(0);
- Assertions.assertEquals(1L, field.getFieldId());
- Assertions.assertEquals("v", field.getNormalizedName());
- Assertions.assertEquals("fixed_size_list", field.getNormalizedType());
- Assertions.assertFalse(field.isNullable());
- Assertions.assertEquals(4, field.getFixedSizeListDimension());
- Assertions.assertEquals("float32", field.getVectorElementType());
- // The synthesized element child is always nullable — the manifest has no slot for
- // element nullability — so the contract records the reconstructed-schema fact.
- Assertions.assertEquals(Boolean.TRUE, field.getVectorElementNullable());
-
- Assertions.assertEquals(1, manager.getJobCount());
- Assertions.assertEquals(1, manager.editLog.size());
- Assertions.assertTrue(manager.isFenceHeld(job.fenceKey()));
- Assertions.assertEquals(1, manager.getUnresolvedJobs().size());
- Mockito.verify(env, Mockito.times(1)).getNextId();
+ public void plainCreateOnEmptySnapshotValidatesThenRejectsAsUnsupported() {
+ assertUnsupported(() -> admitCreate(emptySnapshot(), annDef("MyIdx", false, false), false),
+ "CREATE INDEX");
+ assertNothingPersisted();
}
@Test
@@ -364,10 +281,8 @@ public class LanceIndexAdmissionTest {
Mockito.when(catalog.loadTableIndexAdmissionSnapshot(REMOTE_DB, REMOTE_TBL))
.thenReturn(prepared);
- LanceIndexAdmission.Outcome outcome = LanceIndexAdmission.admitCreate(catalog, database, table,
- scalarDef("Idx", "BTREE", "c", false, false), false);
-
- Assertions.assertNotNull(outcome.getJobId());
+ assertUnsupported(() -> LanceIndexAdmission.admitCreate(catalog, database, table,
+ scalarDef("Idx", "BTREE", "c", false, false), false), "CREATE INDEX");
Mockito.verify(catalog, Mockito.times(1))
.loadTableIndexAdmissionSnapshot(REMOTE_DB, REMOTE_TBL);
}
@@ -392,9 +307,7 @@ public class LanceIndexAdmissionTest {
Collections.singletonList(logicalIndex("IdxA", "v", "IVF_PQ", MATCHING_ANN_PROPERTIES_JSON)),
Collections.singletonList(physicalIndex("IdxA", "VECTOR")));
- LanceIndexAdmission.Outcome outcome = admitCreate(snapshot, annDef("idxa", true, false), true);
-
- Assertions.assertNull(outcome.getJobId());
+ Assertions.assertDoesNotThrow(() -> admitCreate(snapshot, annDef("idxa", true, false), true));
assertNothingPersisted();
}
@@ -408,10 +321,8 @@ public class LanceIndexAdmissionTest {
Collections.singletonList(logicalIndex("idx", "c", "BTREE", "[]")),
Collections.singletonList(physicalIndex("idx", "SCALAR")));
- LanceIndexAdmission.Outcome outcome = admitCreate(snapshot,
- scalarDef("idx", "BTREE", "c", true, false), true);
-
- Assertions.assertNull(outcome.getJobId());
+ Assertions.assertDoesNotThrow(() -> admitCreate(snapshot,
+ scalarDef("idx", "BTREE", "c", true, false), true));
assertNothingPersisted();
}
@@ -442,8 +353,8 @@ public class LanceIndexAdmissionTest {
MATCHING_ANN_PROPERTIES_JSON)),
Collections.singletonList(physicalIndex("idx", "VECTOR")),
Collections.singletonList(vectorField(fieldName, 1)));
- Assertions.assertNull(admitCreate(snapshot,
- annDef("idx", fieldName, true, false), true).getJobId(), fieldName);
+ Assertions.assertDoesNotThrow(() -> admitCreate(snapshot,
+ annDef("idx", fieldName, true, false), true), fieldName);
}
assertNothingPersisted();
}
@@ -519,7 +430,7 @@ public class LanceIndexAdmissionTest {
Collections.singletonList(logicalIndex("idx", "v", "IVF_PQ", "{}")),
Collections.singletonList(physicalIndex("idx", "VECTOR")));
- Assertions.assertNull(admitCreate(snapshot, annDef("idx", true, false), true).getJobId());
+ Assertions.assertDoesNotThrow(() -> admitCreate(snapshot, annDef("idx", true, false), true));
assertNothingPersisted();
}
@@ -545,7 +456,8 @@ public class LanceIndexAdmissionTest {
Collections.singletonList(logicalIndex("idx", "v", "IVF_PQ",
"{\"compression\":{\"num_bits\":8}}")),
Collections.singletonList(physicalIndex("idx", "VECTOR")));
- Assertions.assertNull(admitCreate(partialExposure, annDef("idx", true, false), true).getJobId());
+ Assertions.assertDoesNotThrow(() -> admitCreate(partialExposure,
+ annDef("idx", true, false), true));
// An exposed but non-numeric value is malformed data: fail closed, never skip.
LanceIndexAdmissionSnapshot nonNumeric = snapshot(
@@ -559,7 +471,7 @@ public class LanceIndexAdmissionTest {
@Test
public void omittedNumBitsComparesAsThePersistedEight() throws Exception {
- // The validator accepts an omitted num_bits and admission always persists 8, so the
+ // The validator accepts an omitted num_bits and the persisted value is always 8, so the
// preflight compares an effective 8 — never skips the property — against any exposed
// compression.num_bits.
LanceIndexAdmissionSnapshot fourBits = snapshot(
@@ -573,7 +485,7 @@ public class LanceIndexAdmissionTest {
Collections.singletonList(logicalIndex("idx", "v", "IVF_PQ",
"{\"compression\":{\"num_bits\":8}}")),
Collections.singletonList(physicalIndex("idx", "VECTOR")));
- Assertions.assertNull(admitCreate(eightBits, annDef("idx", true, false), true).getJobId());
+ Assertions.assertDoesNotThrow(() -> admitCreate(eightBits, annDef("idx", true, false), true));
assertNothingPersisted();
}
@@ -584,8 +496,8 @@ public class LanceIndexAdmissionTest {
Collections.singletonList(logicalIndex("idx", "v", "IVF_PQ",
"{\"compression\":{\"num_sub_vectors\":" + exposedValue + "}}")),
Collections.singletonList(physicalIndex("idx", "VECTOR")));
- Assertions.assertNull(admitCreate(snapshot, annDef("idx", true, false), true).getJobId(),
- exposedValue);
+ Assertions.assertDoesNotThrow(() -> admitCreate(snapshot,
+ annDef("idx", true, false), true), exposedValue);
}
assertNothingPersisted();
}
@@ -658,52 +570,38 @@ public class LanceIndexAdmissionTest {
"{\"metric_type\":\"L2\"}")),
Collections.singletonList(physicalIndex("idx", "VECTOR")));
- Assertions.assertNull(admitCreate(snapshot, annDef("idx", true, false), true).getJobId());
+ Assertions.assertDoesNotThrow(() -> admitCreate(snapshot, annDef("idx", true, false), true));
assertNothingPersisted();
}
@Test
- public void replaceOnAbsentNameCreatesWithUserSpelling() throws Exception {
- LanceIndexAdmission.Outcome outcome = admitCreate(emptySnapshot(),
- annDef("Fresh", false, true), false);
-
- LanceIndexJob job = manager.getJob(outcome.getJobId());
- Assertions.assertEquals(LanceIndexJobMutationType.REPLACE, job.getMutationType());
- Assertions.assertEquals("Fresh", job.getDisplayIndexName());
- Assertions.assertEquals("fresh", job.getNormalizedIndexName());
+ public void replaceOnAbsentNameValidatesThenRejectsAsUnsupported() {
+ assertUnsupported(() -> admitCreate(emptySnapshot(), annDef("Fresh", false, true), false),
+ "CREATE OR REPLACE INDEX");
+ assertNothingPersisted();
}
@Test
- public void replaceWithCaseVariantPersistsTheStoredName() throws Exception {
- // M2: the worker locates the target case-sensitively, so the stored display name is
- // persisted instead of the user's spelling.
+ public void replaceWithCaseVariantResolvesThenRejectsAsUnsupported() {
+ // M2: a unique case-insensitive match resolves to the stored display name before the
+ // terminal rejection.
LanceIndexAdmissionSnapshot snapshot = snapshot(
Collections.singletonList(logicalIndex("IdxA", "v", "IVF_PQ", MATCHING_ANN_PROPERTIES_JSON)),
Collections.singletonList(physicalIndex("IdxA", "VECTOR")));
- LanceIndexAdmission.Outcome outcome = admitCreate(snapshot, annDef("IDXA", false, true), false);
-
- LanceIndexJob job = manager.getJob(outcome.getJobId());
- Assertions.assertEquals(LanceIndexJobMutationType.REPLACE, job.getMutationType());
- Assertions.assertEquals("IdxA", job.getDisplayIndexName());
- Assertions.assertEquals("idxa", job.getNormalizedIndexName());
+ assertUnsupported(() -> admitCreate(snapshot, annDef("IDXA", false, true), false),
+ "CREATE OR REPLACE INDEX");
+ assertNothingPersisted();
}
@Test
- public void mixedCaseColumnResolvesToTheStoredColumnName() throws Exception {
- // N5: static validation is case-insensitive; the contract and the persisted column name
- // use the stored Lance field name.
- LanceIndexAdmission.Outcome outcome = admitCreate(emptySnapshot(),
+ public void mixedCaseColumnResolvesUniquelyThroughThePreflight() {
+ // N5: static validation is case-insensitive; the schema-contract build keys on the
+ // stored Lance field name, so a mixed-case request resolves, passes the preflight,
+ // and ends in the shared not-supported rejection.
+ assertUnsupported(() -> admitCreate(emptySnapshot(),
new IndexDefinition("idx", false, Collections.singletonList("embedding"), "ANN",
- annProperties(), "", false),
- false);
-
- LanceIndexJob job = manager.getJob(outcome.getJobId());
- Assertions.assertEquals("Embedding", job.getColumnName());
- LanceIndexSchemaContract.IndexedField field = job.getSchemaContract().getFields().get(0);
- Assertions.assertEquals(4L, field.getFieldId());
- Assertions.assertEquals("embedding", field.getNormalizedName());
- Assertions.assertEquals("fixed_size_list", field.getNormalizedType());
+ annProperties(), "", false), false), "CREATE INDEX");
}
@Test
@@ -731,7 +629,7 @@ public class LanceIndexAdmissionTest {
// relation folds 'Ä' onto 'ä' and, through its Character.toLowerCase fallback, the KELVIN
// SIGN (U+212A) onto ASCII 'k', so a dataset holding such a pair is genuinely
// unresolvable by the lookup and fails closed; a lone KELVIN SIGN field still resolves
- // a 'k' request uniquely and admits without a false ambiguity.
+ // a 'k' request uniquely and passes the preflight without a false ambiguity.
LanceIndexAdmissionSnapshot kelvinPair = snapshotWithFields(
vectorField(KELVIN_SIGN, 1), vectorField("k", 2));
assertInvalid(() -> admitCreate(kelvinPair, annDef("idx", "k", false, false), false),
@@ -746,49 +644,19 @@ public class LanceIndexAdmissionTest {
+ "multiple Lance fields differ only by case");
assertNothingPersisted();
- // Unique resolution: exactly one lookup hit, and the stored field name is journaled.
+ // Unique resolution: exactly one lookup hit, so the request passes the preflight and
+ // ends in the shared not-supported rejection instead of the ambiguity error.
tableColumns.put(KELVIN_SIGN, notNullColumn(KELVIN_SIGN, new ArrayType(Type.FLOAT)));
- LanceIndexAdmission.Outcome outcome = admitCreate(snapshotWithFields(
- vectorField(KELVIN_SIGN, 1)), annDef("idx", "k", false, false), false);
- Assertions.assertNotNull(outcome.getJobId());
- Assertions.assertEquals(KELVIN_SIGN, manager.getJob(outcome.getJobId()).getColumnName());
+ assertUnsupported(() -> admitCreate(snapshotWithFields(
+ vectorField(KELVIN_SIGN, 1)), annDef("idx", "k", false, false), false),
+ "CREATE INDEX");
}
@Test
- public void btreeCreatePersistsNullPropertiesAndScalarContract() throws Exception {
- LanceIndexAdmission.Outcome outcome = admitCreate(emptySnapshot(),
- scalarDef("Idx", "BTREE", "c", false, false), false);
-
- LanceIndexJob job = manager.getJob(outcome.getJobId());
- Assertions.assertEquals("BTREE", job.getIndexType());
- Assertions.assertEquals("c", job.getColumnName());
- Assertions.assertNull(job.getPropertiesJson());
- LanceIndexSchemaContract.IndexedField field = job.getSchemaContract().getFields().get(0);
- Assertions.assertEquals("c", field.getNormalizedName());
- Assertions.assertEquals("int<32>", field.getNormalizedType());
- Assertions.assertNull(field.getFixedSizeListDimension());
- Assertions.assertNull(field.getVectorElementType());
- Assertions.assertNull(field.getVectorElementNullable());
- }
-
- @Test
- public void annPropertiesJsonIsNormalizedAndDeterministic() throws Exception {
- Map<String, String> mixedCase = new HashMap<>();
- mixedCase.put("Index_Type", "ivf_pq");
- mixedCase.put("METRIC", "L2");
- mixedCase.put("num_partitions", "256");
- mixedCase.put("num_sub_vectors", "16");
- mixedCase.put("num_bits", "8");
-
- LanceIndexAdmission.Outcome outcome = admitCreate(emptySnapshot(),
- annDef("idx", false, false, mixedCase), false);
-
- LanceIndexJob job = manager.getJob(outcome.getJobId());
- // Keys are lowercase, index_type is the uppercased original value, metric is lowercased,
- // numerics stay as their original strings, num_bits is pinned to 8, and the key order is
- // the TreeMap order — byte-for-byte deterministic.
- Assertions.assertEquals(NORMALIZED_ANN_PROPERTIES_JSON, job.getPropertiesJson());
- Assertions.assertEquals("IVF_PQ", job.getIndexType());
+ public void btreeCreateValidatesTheScalarContractThenRejectsAsUnsupported() {
+ assertUnsupported(() -> admitCreate(emptySnapshot(),
+ scalarDef("Idx", "BTREE", "c", false, false), false), "CREATE INDEX");
+ assertNothingPersisted();
}
@Test
@@ -853,130 +721,34 @@ public class LanceIndexAdmissionTest {
@Test
public void dropIfExistsWithAbsentNameIsNoOp() throws Exception {
- LanceIndexAdmission.Outcome outcome = admitDrop(emptySnapshot(), "nope", true);
-
- Assertions.assertNull(outcome.getJobId());
+ Assertions.assertDoesNotThrow(() -> admitDrop(emptySnapshot(), "nope", true));
assertNothingPersisted();
}
@Test
- public void dropExistingNameIsAdmittedWithDropShape() throws Exception {
+ public void dropExistingNameValidatesThenRejectsAsUnsupported() {
LanceIndexAdmissionSnapshot snapshot = snapshot(
Collections.singletonList(logicalIndex("IdxA", "v", "IVF_PQ", MATCHING_ANN_PROPERTIES_JSON)),
Collections.singletonList(physicalIndex("IdxA", "VECTOR")));
- LanceIndexAdmission.Outcome outcome = admitDrop(snapshot, "IdxA", true);
-
- LanceIndexJob job = manager.getJob(outcome.getJobId());
- Assertions.assertEquals(LanceIndexJobMutationType.DROP, job.getMutationType());
- Assertions.assertEquals("IdxA", job.getDisplayIndexName());
- Assertions.assertEquals("idxa", job.getNormalizedIndexName());
- Assertions.assertTrue(job.isIfExists());
- Assertions.assertFalse(job.isIfNotExists());
- Assertions.assertNull(job.getIndexType());
- Assertions.assertNull(job.getColumnName());
- Assertions.assertNull(job.getPropertiesJson());
- Assertions.assertNull(job.getSchemaContract());
- Assertions.assertEquals(DATASET_VERSION, job.getAdmittedDatasetVersion());
- Assertions.assertEquals(DATASET_URI, job.getNormalizedLocator());
+ assertUnsupported(() -> admitDrop(snapshot, "IdxA", true), "DROP INDEX");
+ assertNothingPersisted();
}
@Test
- public void dropWithCaseVariantPersistsTheStoredName() throws Exception {
+ public void dropWithCaseVariantResolvesThenRejectsAsUnsupported() {
LanceIndexAdmissionSnapshot snapshot = snapshot(
Collections.singletonList(logicalIndex("IdxA", "v", "IVF_PQ", MATCHING_ANN_PROPERTIES_JSON)),
Collections.singletonList(physicalIndex("IdxA", "VECTOR")));
- LanceIndexAdmission.Outcome outcome = admitDrop(snapshot, "idxa", false);
-
- LanceIndexJob job = manager.getJob(outcome.getJobId());
- Assertions.assertEquals("IdxA", job.getDisplayIndexName());
- Assertions.assertEquals("idxa", job.getNormalizedIndexName());
+ assertUnsupported(() -> admitDrop(snapshot, "idxa", false), "DROP INDEX");
+ assertNothingPersisted();
}
// ------------------------------------------------------------------
- // Fence, quota, and id discipline
+ // Target and snapshot discipline
// ------------------------------------------------------------------
- @Test
- public void fenceConflictPassesThroughTheManagerRejection() throws Exception {
- manager.createJob(newCreateJob(1L, "idxa"), 100, 100, 100);
-
- DdlException exception = Assertions.assertThrows(DdlException.class,
- () -> admitCreate(emptySnapshot(), annDef("idxA", false, false), false));
- Assertions.assertTrue(exception.getMessage().contains("fenced by unresolved job 1"),
- exception.getMessage());
- // The rejection never discloses the locator.
- Assertions.assertFalse(exception.getMessage().contains("bucket"), exception.getMessage());
- Assertions.assertFalse(exception.getMessage().contains(DATASET_URI), exception.getMessage());
-
- // One id was burned before the durable transfer rejected the job (accepted, ids are not
- // required to be contiguous); nothing else changed.
- Assertions.assertEquals(1, manager.getJobCount());
- Assertions.assertEquals(1, manager.editLog.size());
- Assertions.assertEquals(1, manager.getUnresolvedJobs().size());
- Mockito.verify(env, Mockito.times(1)).getNextId();
- }
-
- @Test
- public void quotaOverloadPassesThroughTheManagerRejection() throws Exception {
- Config.lance_index_job_max_unresolved_per_table = 1;
- manager.createJob(newCreateJob(1L, "idxz"), 100, 100, 100);
-
- DdlException exception = Assertions.assertThrows(DdlException.class,
- () -> admitCreate(emptySnapshot(), annDef("idxA", false, false), false));
- Assertions.assertTrue(exception.getMessage().contains("quota exceeded"), exception.getMessage());
-
- Assertions.assertEquals(1, manager.getJobCount());
- Assertions.assertEquals(1, manager.editLog.size());
- Assertions.assertEquals(1, manager.getUnresolvedJobs().size());
- Mockito.verify(env, Mockito.times(1)).getNextId();
- }
-
- @Test
- public void quotaBoundaryAdmitsAtTheLimitAndRejectsBeyondIt() throws Exception {
- Config.lance_index_job_max_unresolved_per_table = 2;
- manager.createJob(newCreateJob(1L, "idxz"), 100, 100, 100);
-
- // count + 1 == limit passes.
- LanceIndexAdmission.Outcome outcome = admitCreate(emptySnapshot(),
- annDef("idxA", false, false), false);
- Assertions.assertNotNull(outcome.getJobId());
- Assertions.assertEquals(2, manager.getJobCount());
-
- // count + 1 > limit is rejected.
- Assertions.assertThrows(DdlException.class,
- () -> admitCreate(emptySnapshot(), annDef("idxB", false, false), false));
- Assertions.assertEquals(2, manager.getJobCount());
- Assertions.assertEquals(2, unresolvedJobsOnDataset());
- }
-
- @Test
- public void nonPositiveQuotaConfigIsAssertedBeforeAnyIdAllocation() {
- // D7: a quota that reached fe.conf with a non-positive value (bypassing the ADMIN SET
- // callback) must fail admission closed with 5102 naming the config item.
- Config.lance_index_job_max_unresolved_per_table = 0;
- AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
- () -> admitCreate(emptySnapshot(), annDef("idxA", false, false), false));
- Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_MUTATION_DISABLED, exception.getMysqlErrorCode());
- Assertions.assertTrue(exception.getDetailMessage().contains("lance_index_job_max_unresolved_per_table"),
- exception.getDetailMessage());
- assertNothingPersisted();
-
- Config.lance_index_job_max_unresolved_per_table = originalTableQuota;
- Config.lance_index_job_max_unresolved_global = -1;
- exception = Assertions.assertThrows(AnalysisException.class,
- () -> admitDrop(snapshot(
- Collections.singletonList(
- logicalIndex("idxA", "v", "IVF_PQ", MATCHING_ANN_PROPERTIES_JSON)),
- Collections.singletonList(physicalIndex("idxA", "VECTOR"))),
- "idxA", false));
- Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_MUTATION_DISABLED, exception.getMysqlErrorCode());
- Assertions.assertTrue(exception.getDetailMessage().contains("lance_index_job_max_unresolved_global"),
- exception.getDetailMessage());
- assertNothingPersisted();
- }
-
@Test
public void identityChangeDuringSnapshotRejectsCreateAndDropBeforeAllocatingId() throws Exception {
LanceIndexAdmission.SnapshotLoader changingLoader = (cat, dbName, tblName) -> {
@@ -1010,34 +782,4 @@ public class LanceIndexAdmissionTest {
() -> LanceIndexAdmission.admitDrop(failingLoader, catalog, database, table, "idx", false));
assertNothingPersisted();
}
-
- @Test
- public void mixedPopulationAdmissionsChargeFenceAndQuotaCorrectly() throws Exception {
- LanceIndexAdmissionSnapshot withExisting = snapshot(
- Collections.singletonList(logicalIndex("IdxC", "c", "BTREE", "{}")),
- Collections.singletonList(physicalIndex("IdxC", "SCALAR")));
-
- LanceIndexAdmission.Outcome createA = admitCreate(emptySnapshot(),
- annDef("idxA", false, false), false);
- LanceIndexAdmission.Outcome createB = admitCreate(emptySnapshot(),
- scalarDef("idxB", "BITMAP", "s", false, false), false);
- LanceIndexAdmission.Outcome dropC = admitDrop(withExisting, "idxc", false);
-
- Assertions.assertEquals(3, manager.getJobCount());
- Assertions.assertEquals(3, manager.editLog.size());
- Assertions.assertEquals(3, manager.getUnresolvedJobs().size());
- Assertions.assertEquals(3, unresolvedJobsOnDataset());
- Assertions.assertTrue(manager.isFenceHeld(manager.getJob(createA.getJobId()).fenceKey()));
- Assertions.assertTrue(manager.isFenceHeld(manager.getJob(createB.getJobId()).fenceKey()));
- Assertions.assertTrue(manager.isFenceHeld(manager.getJob(dropC.getJobId()).fenceKey()));
- Assertions.assertEquals("IdxC", manager.getJob(dropC.getJobId()).getDisplayIndexName());
- }
-
- private static LanceIndexJob newCreateJob(long jobId, String displayName) {
- return new LanceIndexJob(jobId, "seeder", CATALOG_ID, DB, TBL,
- LanceIndexFenceKey.PROVIDER_DIRECTORY, DATASET_URI,
- displayName, displayName.toLowerCase(Locale.ROOT),
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v",
- null, DATASET_VERSION, null);
- }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceIndexNameNormalizerTest.java
similarity index 99%
rename from fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizerTest.java
rename to fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceIndexNameNormalizerTest.java
index d2ad765697c..c6b666e055d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceIndexNameNormalizerTest.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.datasource.lance.job;
+package org.apache.doris.datasource.lance;
import org.apache.commons.lang3.StringUtils;
import org.junit.jupiter.api.Assertions;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSchemaContractBuilderRealDatasetTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSchemaContractBuilderRealDatasetTest.java
index 816875a0d11..713657d92fb 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSchemaContractBuilderRealDatasetTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSchemaContractBuilderRealDatasetTest.java
@@ -17,7 +17,6 @@
package org.apache.doris.datasource.lance;
-import org.apache.doris.datasource.lance.job.LanceIndexSchemaContract;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSchemaContractBuilderTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSchemaContractBuilderTest.java
index 1e14b43771f..b537c400a18 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSchemaContractBuilderTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSchemaContractBuilderTest.java
@@ -18,7 +18,6 @@
package org.apache.doris.datasource.lance;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.datasource.lance.job.LanceIndexSchemaContract;
import org.apache.arrow.vector.types.DateUnit;
import org.apache.arrow.vector.types.FloatingPointPrecision;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java
deleted file mode 100644
index 2e46203b815..00000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java
+++ /dev/null
@@ -1,183 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import org.apache.commons.lang3.StringUtils;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-/**
- * Unit coverage for dataset-locator normalization v1, the fence-key identity for the
- * target dataset: trim, lowercase scheme, userinfo rejection (credential-bearing URLs
- * are never identity), rejection of locators with neither authority nor path,
- * trailing-slash stripping with a kept root, and the absolute-path requirement for
- * scheme-less locators.
- */
-public class LanceIndexDatasetLocatorTest {
-
- @Test
- public void lowercasesSchemeOnly() {
- Assertions.assertEquals("s3://bucket/path", LanceIndexDatasetLocator.normalize("S3://bucket/path"));
- Assertions.assertEquals("file:///data/x", LanceIndexDatasetLocator.normalize("FILE:///data/x"));
- Assertions.assertEquals("hdfs://nn:8020/data", LanceIndexDatasetLocator.normalize("HDFS://nn:8020/data"));
- }
-
- @Test
- public void preservesAuthorityAndPathCase() {
- // Bucket and path components are case-sensitive on the supported providers.
- Assertions.assertEquals("s3://MyBucket/MyPath", LanceIndexDatasetLocator.normalize("s3://MyBucket/MyPath"));
- }
-
- @Test
- public void trimsSurroundingWhitespace() {
- Assertions.assertEquals("/data/x", LanceIndexDatasetLocator.normalize(" /data/x \t\n"));
- Assertions.assertEquals("s3://bucket/p", LanceIndexDatasetLocator.normalize(" s3://bucket/p "));
- }
-
- @Test
- public void stripsTrailingSlashesFromPath() {
- Assertions.assertEquals("s3://b/p", LanceIndexDatasetLocator.normalize("s3://b/p///"));
- Assertions.assertEquals("/data/lance", LanceIndexDatasetLocator.normalize("/data/lance/"));
- }
-
- @Test
- public void keepsSchemeLessRoot() {
- Assertions.assertEquals("/", LanceIndexDatasetLocator.normalize("/"));
- Assertions.assertEquals("/", LanceIndexDatasetLocator.normalize("//"));
- Assertions.assertEquals("/", LanceIndexDatasetLocator.normalize("///"));
- }
-
- @Test
- public void stripsTrailingSlashBehindAuthority() {
- Assertions.assertEquals("s3://bucket", LanceIndexDatasetLocator.normalize("s3://bucket/"));
- Assertions.assertEquals("s3://bucket", LanceIndexDatasetLocator.normalize("s3://bucket//"));
- Assertions.assertEquals("s3://bucket", LanceIndexDatasetLocator.normalize("s3://bucket"));
- }
-
- @Test
- public void acceptsFileUrlWithEmptyAuthority() {
- Assertions.assertEquals("file:///data/x", LanceIndexDatasetLocator.normalize("file:///data/x"));
- }
-
- @Test
- public void keepsAuthorityPortAndQueryFreePathVerbatim() {
- Assertions.assertEquals("hdfs://nn:8020/base/table.lance",
- LanceIndexDatasetLocator.normalize("hdfs://nn:8020/base/table.lance"));
- }
-
- @Test
- public void allowsAtSignInPathButNotInAuthority() {
- Assertions.assertEquals("s3://bucket/p@th", LanceIndexDatasetLocator.normalize("s3://bucket/p@th"));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize("s3://user@bucket/path"));
- }
-
- @Test
- public void rejectsCredentialBearingUserinfo() {
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize("s3://user:secret@bucket/path"));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize("https://user:secret@example.com/ds"));
- }
-
- @Test
- public void rejectsQueriesAndFragmentsThatCouldCarryCredentials() {
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize(
- "https://bucket.example/ds?X-Amz-Credential=AKIA_TEST&X-Amz-Signature=secret"));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize("https://account.blob.core.windows.net/ds?sig=secret"));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize("s3://bucket/ds#credential-fragment"));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize("/data/ds?token=secret"));
- }
-
- @Test
- public void malformedLocatorErrorsDoNotEchoPotentialSecrets() {
- String malformed = "s 3://user:top-secret@bucket/path";
- IllegalArgumentException malformedError = Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize(malformed));
- Assertions.assertFalse(malformedError.getMessage().contains(malformed));
- Assertions.assertFalse(malformedError.getMessage().contains("top-secret"));
-
- String relative = "access-key:top-secret@bucket/path";
- IllegalArgumentException relativeError = Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize(relative));
- Assertions.assertFalse(relativeError.getMessage().contains(relative));
- Assertions.assertFalse(relativeError.getMessage().contains("top-secret"));
- }
-
- @Test
- public void rejectsMalformedOrNonHierarchicalSchemes() {
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize("1s3://bucket/path"));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize("s3:/bucket/path"));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize("s3:bucket/path"));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize("https://[malformed/path"));
- }
-
- @Test
- public void locatorBoundCountsUtf8Bytes() {
- String prefix = "s3://bucket/";
- String atLimit = prefix + StringUtils.repeat(
- "a", LanceIndexDatasetLocator.MAX_LOCATOR_BYTES - prefix.length());
- Assertions.assertEquals(atLimit, LanceIndexDatasetLocator.normalize(atLimit));
-
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize(atLimit + "a"));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize(prefix + StringUtils.repeat("é",
- (LanceIndexDatasetLocator.MAX_LOCATOR_BYTES - prefix.length()) / 2 + 1)));
- }
-
- @Test
- public void rejectsSchemeLessRelativePath() {
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize("data/x"));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize("./x"));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> LanceIndexDatasetLocator.normalize("bucket/path"));
- }
-
- @Test
- public void rejectsNullEmptyAndBlank() {
- Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize(null));
- Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize(""));
- Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize(" "));
- }
-
- @Test
- public void rejectsEmptyScheme() {
- Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize("://path"));
- }
-
- @Test
- public void rejectsLocatorWithNeitherAuthorityNorPath() {
- Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize("s3://"));
- Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize("file://"));
- // Trailing-slash stripping still leaves nothing behind the scheme.
- Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize("s3:///"));
- // An empty authority with a real path stays legal.
- Assertions.assertEquals("file:///x", LanceIndexDatasetLocator.normalize("file:///x"));
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java
deleted file mode 100644
index d29cb4c3a8b..00000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java
+++ /dev/null
@@ -1,457 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import org.apache.doris.common.DdlException;
-import org.apache.doris.journal.JournalEntity;
-import org.apache.doris.persist.OperationType;
-import org.apache.doris.persist.gson.GsonUtils;
-
-import org.apache.commons.lang3.StringUtils;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Persistence coverage for the Lance index job infrastructure: the manager image
- * write/read round trip (including the fence-index and quota rebuild in
- * gsonPostProcess), the journal mounting point for op code 500
- * ({@link OperationType#OP_LANCE_INDEX_JOB_UPSERT}) through
- * {@link JournalEntity#write}/{@link JournalEntity#readFields}, the single-record Gson
- * round trip field by field, and the bounded-text rejection at construction time.
- */
-public class LanceIndexJobManagerPersistTest {
- private static final long CATALOG_ID = 10L;
- private static final String LOCATOR = "s3://bucket/dataset";
- private static final long BACKEND_ID = 1001L;
- private static final long BE_EPOCH = 55L;
- private static final String INVOCATION_ID = "invocation-1";
-
- @Test
- public void managerImageRoundtripRebuildsDerivedFenceAndQuota() throws Exception {
- TestManager source = new TestManager();
- source.createJob(newCreateJob(1L, "IdxPending"), 100, 100, 100);
- source.createJob(newCreateJob(2L, "IdxRunning"), 100, 100, 100);
- source.markRunning(2L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, 9999L);
- source.createJob(newCreateJob(3L, "IdxCommitted"), 100, 100, 100);
- source.markRunning(3L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, 9999L);
- source.completeWithResult(3L, 1L, INVOCATION_ID, BE_EPOCH,
- new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
- LanceIndexJobCompletionReason.NONE, "ok", false));
- source.createJob(newCreateJob(4L, "IdxUnknown"), 100, 100, 100);
- source.markRunning(4L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, 9999L);
- source.completeWithResult(4L, 1L, INVOCATION_ID, BE_EPOCH,
- new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT,
- LanceIndexJobCompletionReason.NONE, "lost", false));
- source.replayUpsertJob(forceReleasedUnknownJob(5L, "idxforced"));
-
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- source.write(new DataOutputStream(byteStream));
- LanceIndexJobManager loaded =
- LanceIndexJobManager.read(new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray())));
-
- Assertions.assertEquals(source.getJobCount(), loaded.getJobCount());
- for (long jobId = 1L; jobId <= 5L; jobId++) {
- Assertions.assertEquals(GsonUtils.GSON.toJson(source.getJob(jobId)),
- GsonUtils.GSON.toJson(loaded.getJob(jobId)), "job " + jobId);
- }
-
- // The derived fence index survived: unresolved jobs still fence their names.
- for (long jobId = 1L; jobId <= 4L; jobId++) {
- Assertions.assertTrue(loaded.isFenceHeld(loaded.getJob(jobId).fenceKey()), "fence of job " + jobId);
- }
- Assertions.assertFalse(loaded.isFenceHeld(loaded.getJob(5L).fenceKey()));
- Assertions.assertEquals(4L, loaded.getQuota().getGlobalCount());
- Assertions.assertEquals(4L, loaded.getQuota().getCatalogCount(CATALOG_ID));
- Assertions.assertEquals(4, loaded.getUnresolvedJobs().size());
- Assertions.assertEquals(1, loaded.getJobsNeedingRefresh().size());
-
- // The fence is enforceable after the image load: a conflict is rejected by the fence
- // CAS, which precedes any journal write.
- Assertions.assertThrows(DdlException.class,
- () -> loaded.createJob(newCreateJob(9L, "idxunknown"), 100, 100, 100));
-
- // And a FORCE-released name is free again once the loaded image is rebuilt through replay.
- TestManager rebuilt = new TestManager();
- for (long jobId = 1L; jobId <= 5L; jobId++) {
- rebuilt.replayUpsertJob(loaded.getJob(jobId));
- }
- rebuilt.createJob(newCreateJob(9L, "IdxForced"), 100, 100, 100);
- Assertions.assertEquals(5L, rebuilt.getQuota().getGlobalCount());
- }
-
- @Test
- public void emptyManagerRoundtripStaysEmpty() throws Exception {
- TestManager source = new TestManager();
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- source.write(new DataOutputStream(byteStream));
- LanceIndexJobManager loaded =
- LanceIndexJobManager.read(new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray())));
-
- Assertions.assertEquals(0, loaded.getJobCount());
- Assertions.assertEquals(0L, loaded.getQuota().getGlobalCount());
- Assertions.assertTrue(loaded.getUnresolvedJobs().isEmpty());
- }
-
- @Test
- public void imageLoadKeepsTheSmallerJobIdOnAFenceCollision() {
- // Only a corrupt image can hold two unresolved jobs on the same fence key;
- // gsonPostProcess keeps the smaller job id (Gson auto-runs it after fromJson).
- String jobJson = "\"jid\":%d,\"rev\":0,\"cid\":" + CATALOG_ID
- + ",\"prv\":\"" + LanceIndexFenceKey.PROVIDER_DIRECTORY + "\",\"loc\":\"" + LOCATOR
- + "\",\"din\":\"IdxA\",\"nin\":\"idxa\",\"ms\":\"PENDING\",\"rs\":\"NOT_REQUIRED\"";
- String json = "{\"jobs\":{\"2\":{" + String.format(jobJson, 2L) + "},"
- + "\"1\":{" + String.format(jobJson, 1L) + "}}}";
- LanceIndexJobManager loaded = GsonUtils.GSON.fromJson(json, LanceIndexJobManager.class);
-
- Assertions.assertNotNull(loaded.getJob(1L));
- Assertions.assertTrue(loaded.isFenceHeld(loaded.getJob(1L).fenceKey()));
- // Both unresolved jobs still charge the quota.
- Assertions.assertEquals(2L, loaded.getQuota().getGlobalCount());
- // The fence conflict message names the surviving smaller job id.
- DdlException exception = Assertions.assertThrows(DdlException.class,
- () -> loaded.createJob(newCreateJob(9L, "idxa"), 100, 100, 100));
- Assertions.assertTrue(exception.getMessage().contains("unresolved job 1"));
- }
-
- @Test
- public void imageLoadKeepsAnIdentityLessRecordBlockingAdmission() {
- // A corrupt image record without fence identity cannot join the books; it must
- // still block new admissions fail-closed after gsonPostProcess rebuilds state
- // (Gson auto-runs it after fromJson).
- String json = "{\"jobs\":{\"5\":{\"jid\":5,\"rev\":0,\"ms\":\"PENDING\"}}}";
- LanceIndexJobManager loaded = GsonUtils.GSON.fromJson(json, LanceIndexJobManager.class);
-
- Assertions.assertNotNull(loaded.getJob(5L));
- Assertions.assertEquals(Collections.singletonList(5L), loaded.getCorruptUnresolvedJobIds());
- Assertions.assertEquals(0L, loaded.getQuota().getGlobalCount());
- Assertions.assertTrue(loaded.getUnresolvedJobs().isEmpty());
-
- // The rejection precedes any journal write, so the base-class edit-log seam
- // (which needs a live Env) never runs.
- DdlException exception = Assertions.assertThrows(DdlException.class,
- () -> loaded.createJob(newCreateJob(9L, "IdxB"), 100, 100, 100));
- Assertions.assertTrue(exception.getMessage().contains("smallest job id 5"));
- }
-
- @Test
- public void schemaContractEqualityIsOrderSensitive() {
- LanceIndexSchemaContract ordered = contract();
- LanceIndexSchemaContract sameOrder = contract();
- List<LanceIndexSchemaContract.IndexedField> reversed = new ArrayList<>();
- reversed.add(ordered.getFields().get(1));
- reversed.add(ordered.getFields().get(0));
- LanceIndexSchemaContract reordered = new LanceIndexSchemaContract(reversed);
-
- Assertions.assertEquals(ordered, sameOrder);
- Assertions.assertEquals(ordered.hashCode(), sameOrder.hashCode());
- // The same fields in a different order are a different contract.
- Assertions.assertNotEquals(ordered, reordered);
- }
-
- @Test
- public void schemaContractIsBoundedNullFreeAndImmutable() {
- LanceIndexSchemaContract.IndexedField field = new LanceIndexSchemaContract.IndexedField(
- 1L, "v", "fixed_size_list[float;192]", false, 192, "float", false);
- List<LanceIndexSchemaContract.IndexedField> atLimit = new ArrayList<>();
- for (int index = 0; index < LanceIndexSchemaContract.MAX_INDEXED_FIELDS; index++) {
- atLimit.add(field);
- }
- LanceIndexSchemaContract contractAtLimit = new LanceIndexSchemaContract(atLimit);
- Assertions.assertEquals(LanceIndexSchemaContract.MAX_INDEXED_FIELDS, contractAtLimit.getFields().size());
-
- atLimit.add(field);
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> new LanceIndexSchemaContract(atLimit));
- List<LanceIndexSchemaContract.IndexedField> withNull = new ArrayList<>();
- withNull.add(null);
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> new LanceIndexSchemaContract(withNull));
-
- List<LanceIndexSchemaContract.IndexedField> source = new ArrayList<>();
- source.add(field);
- LanceIndexSchemaContract copied = new LanceIndexSchemaContract(source);
- source.clear();
- Assertions.assertEquals(1, copied.getFields().size());
- Assertions.assertThrows(UnsupportedOperationException.class,
- () -> copied.getFields().add(field));
- }
-
- @Test
- public void schemaContractStringBoundsCountUtf8Bytes() {
- String atLimit = StringUtils.repeat("é", LanceIndexSchemaContract.MAX_FIELD_STRING_BYTES / 2);
- new LanceIndexSchemaContract.IndexedField(1L, atLimit, atLimit, false, null, atLimit, null);
-
- String overLimit = atLimit + "é";
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> new LanceIndexSchemaContract.IndexedField(
- 1L, overLimit, "type", false, null, null, null));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> new LanceIndexSchemaContract.IndexedField(
- 1L, "name", overLimit, false, null, null, null));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> new LanceIndexSchemaContract.IndexedField(
- 1L, "name", "type", false, null, overLimit, null));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> new LanceIndexSchemaContract.IndexedField(
- 1L, null, "type", false, null, null, null));
- }
-
- @Test
- public void journalEntityRoundtripUsesOpCode500() throws Exception {
- LanceIndexJob job = fullyPopulatedJob();
-
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- DataOutputStream output = new DataOutputStream(byteStream);
- JournalEntity journalEntity = new JournalEntity();
- journalEntity.setData(job);
- journalEntity.setOpCode(OperationType.OP_LANCE_INDEX_JOB_UPSERT);
- journalEntity.write(output);
- output.flush();
-
- JournalEntity replayed = new JournalEntity();
- replayed.readFields(new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray())));
-
- Assertions.assertEquals(OperationType.OP_LANCE_INDEX_JOB_UPSERT, replayed.getOpCode());
- Assertions.assertEquals(500, replayed.getOpCode());
- Assertions.assertTrue(replayed.getData() instanceof LanceIndexJob);
- Assertions.assertEquals(GsonUtils.GSON.toJson(job), GsonUtils.GSON.toJson(replayed.getData()));
- }
-
- @Test
- public void journalEntityStreamPreservesRecordOrder() throws Exception {
- List<LanceIndexJob> jobs = new ArrayList<>();
- jobs.add(fullyPopulatedJob());
- LanceIndexJob second = newCreateJob(77L, "IdxSecond");
- second.setMutationState(LanceIndexJobMutationState.PENDING);
- jobs.add(second);
-
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- DataOutputStream output = new DataOutputStream(byteStream);
- for (LanceIndexJob job : jobs) {
- JournalEntity journalEntity = new JournalEntity();
- journalEntity.setData(job);
- journalEntity.setOpCode(OperationType.OP_LANCE_INDEX_JOB_UPSERT);
- journalEntity.write(output);
- }
- output.flush();
-
- DataInputStream input = new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray()));
- for (LanceIndexJob expected : jobs) {
- JournalEntity replayed = new JournalEntity();
- replayed.readFields(input);
- Assertions.assertEquals(OperationType.OP_LANCE_INDEX_JOB_UPSERT, replayed.getOpCode());
- Assertions.assertEquals(GsonUtils.GSON.toJson(expected), GsonUtils.GSON.toJson(replayed.getData()));
- }
- Assertions.assertEquals(0, input.available());
- }
-
- @Test
- public void jobStreamRoundtripPreservesAllFields() throws Exception {
- LanceIndexJob job = fullyPopulatedJob();
-
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- job.write(new DataOutputStream(byteStream));
- LanceIndexJob restored =
- LanceIndexJob.read(new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray())));
-
- assertSameJobFields(job, restored);
- }
-
- @Test
- public void jobGsonRoundtripPreservesAllFields() {
- LanceIndexJob job = fullyPopulatedJob();
- LanceIndexJob restored = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(job), LanceIndexJob.class);
- assertSameJobFields(job, restored);
- }
-
- @Test
- public void boundedTextFieldsRejectOverflow() {
- String overMessage = StringUtils.repeat("m", LanceIndexJobResult.MAX_MESSAGE_BYTES + 1);
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
- LanceIndexJobCompletionReason.NONE, overMessage, false));
- // Multibyte characters count as UTF-8 bytes, not chars.
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
- LanceIndexJobCompletionReason.NONE, StringUtils.repeat("é", 513), false));
- new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK, LanceIndexJobCompletionReason.NONE,
- StringUtils.repeat("m", LanceIndexJobResult.MAX_MESSAGE_BYTES), false);
-
- LanceIndexJob job = newCreateJob(1L, "IdxA");
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> job.setCreator(StringUtils.repeat("c", LanceIndexJob.MAX_DURABLE_TEXT_BYTES + 1)));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> job.setIndexType(StringUtils.repeat("t", LanceIndexJob.MAX_DURABLE_TEXT_BYTES + 1)));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> job.setColumnName(StringUtils.repeat("c", LanceIndexJob.MAX_DURABLE_TEXT_BYTES + 1)));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> job.setInvocationId(StringUtils.repeat("i", LanceIndexJob.MAX_INVOCATION_ID_BYTES + 1)));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> job.setForceActor(StringUtils.repeat("a", LanceIndexJob.MAX_DURABLE_TEXT_BYTES + 1)));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> job.setPropertiesJson(StringUtils.repeat("p", LanceIndexJob.MAX_PROPERTIES_JSON_BYTES + 1)));
- job.setPropertiesJson(StringUtils.repeat("p", LanceIndexJob.MAX_PROPERTIES_JSON_BYTES));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> job.setForceNote(StringUtils.repeat("n", LanceIndexJob.MAX_FORCE_TEXT_BYTES + 1)));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> job.setForceWarning(StringUtils.repeat("w", LanceIndexJob.MAX_FORCE_TEXT_BYTES + 1)));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> job.setNormalizedIndexName(
- StringUtils.repeat("n", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + 1)));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> job.setDisplayIndexName(
- StringUtils.repeat("d", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + 1)));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> newCreateJob(2L, StringUtils.repeat("d", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + 1)));
-
- // Optional durable text remains nullable; required creator identity does not.
- job.setCreator(null);
- Assertions.assertThrows(IllegalArgumentException.class, job::validateForAdmission);
- job.setCreator("tester");
- job.setIndexType(null);
- job.setColumnName(null);
- job.setInvocationId(null);
- job.setForceActor(null);
- job.setForceNote(null);
- job.setForceWarning(null);
- job.setPropertiesJson(null);
- job.validateForAdmission();
- }
-
- @Test
- public void resultRejectsNullResultCode() {
- Assertions.assertThrows(NullPointerException.class,
- () -> new LanceIndexJobResult(null, LanceIndexJobCompletionReason.NONE, "msg", false));
- }
-
- private static LanceIndexJob newCreateJob(long jobId, String displayName) {
- return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
- displayName, LanceIndexNameNormalizer.normalize(displayName),
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v",
- null, 7L, null);
- }
-
- private static LanceIndexSchemaContract contract() {
- List<LanceIndexSchemaContract.IndexedField> fields = new ArrayList<>();
- fields.add(new LanceIndexSchemaContract.IndexedField(1L, "v", "fixed_size_list[float;192]",
- false, 192, "float", false));
- fields.add(new LanceIndexSchemaContract.IndexedField(2L, "s", "decimal(10,2)", true, null, null, null));
- return new LanceIndexSchemaContract(fields);
- }
-
- /**
- * A job with every durable field populated, including result, schema contract,
- * dispatch identity, and the FORCE audit fields.
- */
- private static LanceIndexJob fullyPopulatedJob() {
- LanceIndexJob job = new LanceIndexJob(42L, "creator", CATALOG_ID, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, "IdxΩ", "idxω",
- LanceIndexJobMutationType.REPLACE, true, false, "IVF_PQ", "v",
- "{\"num_partitions\":\"256\"}", 99L, contract());
- job.setRevision(7L);
- job.setCreateTimeMs(111L);
- job.setUpdateTimeMs(222L);
- job.setMutationState(LanceIndexJobMutationState.RUNNING);
- job.setRefreshState(LanceIndexJobRefreshState.RUNNING);
- job.setResult(new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_IO,
- LanceIndexJobCompletionReason.NONE, "io error", true));
- job.setBackendId(BACKEND_ID);
- job.setBeProcessEpoch(BE_EPOCH);
- job.setDispatchRevision(7L);
- job.setInvocationId(INVOCATION_ID);
- job.setDeadlineMs(123456L);
- job.setPossibleLiveOwned(true);
- job.setTerminationProof(LanceIndexTerminationProof.NONE);
- job.setForceActor("admin");
- job.setForceTimeMs(777L);
- job.setForceNote("note");
- job.setForceWarning("warning");
- return job;
- }
-
- private static LanceIndexJob forceReleasedUnknownJob(long jobId, String normalizedName) {
- String json = "{\"jid\":" + jobId + ",\"cr\":\"tester\",\"rev\":2,\"cid\":" + CATALOG_ID
- + ",\"dbn\":\"db1\",\"tbn\":\"tbl1\",\"prv\":\"" + LanceIndexFenceKey.PROVIDER_DIRECTORY
- + "\",\"loc\":\"" + LOCATOR + "\",\"din\":\"" + normalizedName + "\",\"nin\":\"" + normalizedName
- + "\",\"mt\":\"CREATE\",\"ms\":\"UNKNOWN\",\"rs\":\"NOT_REQUIRED\",\"fr\":true}";
- return GsonUtils.GSON.fromJson(json, LanceIndexJob.class);
- }
-
- private static void assertSameJobFields(LanceIndexJob expected, LanceIndexJob actual) {
- Assertions.assertEquals(expected.getJobId(), actual.getJobId());
- Assertions.assertEquals(expected.getCreator(), actual.getCreator());
- Assertions.assertEquals(expected.getRevision(), actual.getRevision());
- Assertions.assertEquals(expected.getCreateTimeMs(), actual.getCreateTimeMs());
- Assertions.assertEquals(expected.getUpdateTimeMs(), actual.getUpdateTimeMs());
- Assertions.assertEquals(expected.getCatalogId(), actual.getCatalogId());
- Assertions.assertEquals(expected.getDbName(), actual.getDbName());
- Assertions.assertEquals(expected.getTableName(), actual.getTableName());
- Assertions.assertEquals(expected.getProvider(), actual.getProvider());
- Assertions.assertEquals(expected.getNormalizedLocator(), actual.getNormalizedLocator());
- Assertions.assertEquals(expected.getDisplayIndexName(), actual.getDisplayIndexName());
- Assertions.assertEquals(expected.getNormalizedIndexName(), actual.getNormalizedIndexName());
- Assertions.assertEquals(expected.getMutationType(), actual.getMutationType());
- Assertions.assertEquals(expected.isIfNotExists(), actual.isIfNotExists());
- Assertions.assertEquals(expected.isIfExists(), actual.isIfExists());
- Assertions.assertEquals(expected.getIndexType(), actual.getIndexType());
- Assertions.assertEquals(expected.getColumnName(), actual.getColumnName());
- Assertions.assertEquals(expected.getPropertiesJson(), actual.getPropertiesJson());
- Assertions.assertEquals(expected.getAdmittedDatasetVersion(), actual.getAdmittedDatasetVersion());
- Assertions.assertEquals(expected.getSchemaContract(), actual.getSchemaContract());
- Assertions.assertEquals(expected.getMutationState(), actual.getMutationState());
- Assertions.assertEquals(expected.getRefreshState(), actual.getRefreshState());
- Assertions.assertEquals(expected.getResult().getResultCode(), actual.getResult().getResultCode());
- Assertions.assertEquals(expected.getResult().getCompletionReason(), actual.getResult().getCompletionReason());
- Assertions.assertEquals(expected.getResult().getSanitizedMessage(), actual.getResult().getSanitizedMessage());
- Assertions.assertEquals(expected.getResult().isExternalMetadataAdvanced(),
- actual.getResult().isExternalMetadataAdvanced());
- Assertions.assertEquals(expected.getBackendId(), actual.getBackendId());
- Assertions.assertEquals(expected.getBeProcessEpoch(), actual.getBeProcessEpoch());
- Assertions.assertEquals(expected.getDispatchRevision(), actual.getDispatchRevision());
- Assertions.assertEquals(expected.getInvocationId(), actual.getInvocationId());
- Assertions.assertEquals(expected.getDeadlineMs(), actual.getDeadlineMs());
- Assertions.assertEquals(expected.isPossibleLiveOwned(), actual.isPossibleLiveOwned());
- Assertions.assertEquals(expected.getTerminationProof(), actual.getTerminationProof());
- Assertions.assertEquals(expected.isForceReleased(), actual.isForceReleased());
- Assertions.assertEquals(expected.getForceActor(), actual.getForceActor());
- Assertions.assertEquals(expected.getForceTimeMs(), actual.getForceTimeMs());
- Assertions.assertEquals(expected.getForceNote(), actual.getForceNote());
- Assertions.assertEquals(expected.getForceWarning(), actual.getForceWarning());
- Assertions.assertEquals(expected.fenceKey(), actual.fenceKey());
- Assertions.assertEquals(expected.getTableQuotaKey(), actual.getTableQuotaKey());
- }
-
- /**
- * Edit-log seam: captures every durable record instead of writing the journal.
- */
- private static class TestManager extends LanceIndexJobManager {
- @Override
- protected void writeEditLog(LanceIndexJob job) {
- // No journal in a pure persistence unit test.
- }
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerQueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerQueryTest.java
deleted file mode 100644
index 7675d05ab18..00000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerQueryTest.java
+++ /dev/null
@@ -1,131 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.util.List;
-
-/**
- * Coverage for the two read-side queries added to {@link LanceIndexJobManager} for the
- * admission slice: {@link LanceIndexJobManager#getAllJobsSnapshot()} (the data source of
- * SHOW LANCE INDEX JOBS: every job, copies only, ordered by job id) and
- * {@link LanceIndexJobManager#hasUnresolvedJobsForCatalog(long)} (the catalog DDL guard
- * probe: any job still holding its fence and unresolved quota for the catalog).
- */
-public class LanceIndexJobManagerQueryTest {
- private static final long CATALOG_ID = 10L;
- private static final long OTHER_CATALOG_ID = 20L;
- private static final String LOCATOR = "s3://bucket/dataset";
- private static final long BACKEND_ID = 1001L;
- private static final long BE_EPOCH = 55L;
- private static final String INVOCATION_ID = "invocation-1";
- private static final long DEADLINE_MS = 9999L;
-
- @Test
- public void snapshotReturnsEveryJobOrderedByJobId() throws Exception {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(3L, "IdxC", CATALOG_ID), 100, 100, 100);
- manager.createJob(newCreateJob(1L, "IdxA", CATALOG_ID), 100, 100, 100);
- manager.createJob(newCreateJob(2L, "IdxB", OTHER_CATALOG_ID), 100, 100, 100);
-
- List<LanceIndexJob> snapshot = manager.getAllJobsSnapshot();
- Assertions.assertEquals(3, snapshot.size());
- Assertions.assertEquals(1L, snapshot.get(0).getJobId());
- Assertions.assertEquals(2L, snapshot.get(1).getJobId());
- Assertions.assertEquals(3L, snapshot.get(2).getJobId());
- }
-
- @Test
- public void snapshotCopiesAreIsolatedFromStoredRecords() throws Exception {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxA", CATALOG_ID), 100, 100, 100);
-
- LanceIndexJob copy = manager.getAllJobsSnapshot().get(0);
- copy.setMutationState(LanceIndexJobMutationState.UNKNOWN);
- copy.setDisplayIndexName("mutated");
- copy.setRevision(99L);
-
- LanceIndexJob stored = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.PENDING, stored.getMutationState());
- Assertions.assertEquals("IdxA", stored.getDisplayIndexName());
- Assertions.assertEquals(0L, stored.getRevision());
-
- // A later snapshot is unaffected by mutations of an earlier one.
- LanceIndexJob fresh = manager.getAllJobsSnapshot().get(0);
- Assertions.assertEquals(LanceIndexJobMutationState.PENDING, fresh.getMutationState());
- Assertions.assertEquals("IdxA", fresh.getDisplayIndexName());
- Assertions.assertEquals(0L, fresh.getRevision());
- }
-
- @Test
- public void hasUnresolvedJobsForCatalogScopesByCatalogId() throws Exception {
- TestManager manager = new TestManager();
- Assertions.assertFalse(manager.hasUnresolvedJobsForCatalog(CATALOG_ID));
-
- manager.createJob(newCreateJob(1L, "IdxA", CATALOG_ID), 100, 100, 100);
- Assertions.assertTrue(manager.hasUnresolvedJobsForCatalog(CATALOG_ID));
- Assertions.assertFalse(manager.hasUnresolvedJobsForCatalog(OTHER_CATALOG_ID));
- Assertions.assertFalse(manager.hasUnresolvedJobsForCatalog(999L));
- }
-
- @Test
- public void releasedTerminalJobDoesNotCountAsUnresolved() throws Exception {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxA", CATALOG_ID), 100, 100, 100);
- Assertions.assertTrue(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
- LanceIndexJobCompletionReason.NONE, "ok", false)));
- // COMMITTED with refresh REQUIRED still holds the fence and quota.
- Assertions.assertTrue(manager.hasUnresolvedJobsForCatalog(CATALOG_ID));
-
- Assertions.assertTrue(manager.markRefreshRunning(1L, 2L));
- Assertions.assertTrue(manager.markRefreshDone(1L, 3L));
- Assertions.assertFalse(manager.hasUnresolvedJobsForCatalog(CATALOG_ID));
-
- // A released terminal job remains visible in the full snapshot.
- Assertions.assertEquals(1, manager.getAllJobsSnapshot().size());
- Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED,
- manager.getAllJobsSnapshot().get(0).getMutationState());
- }
-
- @Test
- public void unresolvedJobInAnotherCatalogDoesNotLeak() throws Exception {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxA", OTHER_CATALOG_ID), 100, 100, 100);
- Assertions.assertFalse(manager.hasUnresolvedJobsForCatalog(CATALOG_ID));
- Assertions.assertTrue(manager.hasUnresolvedJobsForCatalog(OTHER_CATALOG_ID));
- }
-
- private static LanceIndexJob newCreateJob(long jobId, String displayName, long catalogId) {
- return new LanceIndexJob(jobId, "tester", catalogId, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
- displayName, LanceIndexNameNormalizer.normalize(displayName),
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v",
- null, 7L, null);
- }
-
- private static class TestManager extends LanceIndexJobManager {
- @Override
- protected void writeEditLog(LanceIndexJob job) {
- // No journal in a pure query unit test.
- }
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java
deleted file mode 100644
index 0a395bee6a0..00000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java
+++ /dev/null
@@ -1,628 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import org.apache.doris.common.DdlException;
-import org.apache.doris.persist.gson.GsonUtils;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Replay and master-transfer coverage for {@link LanceIndexJobManager}. The journal
- * records of a scenario are captured from a source manager through the edit-log seam
- * and replayed verbatim into a fresh target manager, exactly like a follower tailing
- * the edit log. The pinned invariants: replay is a verbatim replace with a monotonic
- * revision guard (no state transformation on followers); a replayed PENDING permits
- * exactly one dispatch; only the master-election sweep turns a durable RUNNING into
- * UNKNOWN, after which redispatch is permanently refused; an unresolved UNKNOWN keeps
- * its fence and quota across replay while a FORCE-released UNKNOWN frees both; stale
- * callbacks (revision / invocation id / BE epoch) never change state; and a rejected
- * admission leaves no job, fence, quota charge, or journal record behind. A replayed
- * unresolved record that lacks fence identity stays out of the books but blocks all
- * new admissions fail-closed until a later record resolves it.
- */
-public class LanceIndexJobManagerReplayTest {
- private static final long CATALOG_ID = 10L;
- private static final String LOCATOR = "s3://bucket/dataset";
- private static final long BACKEND_ID = 1001L;
- private static final long BE_EPOCH = 55L;
- private static final String INVOCATION_ID = "invocation-1";
- private static final long DEADLINE_MS = 9999L;
-
- @Test
- public void replayedPendingPermitsExactlyOneDispatch() throws DdlException {
- List<LanceIndexJob> records = runningRecords(1L, "IdxA");
- TestManager target = new TestManager();
- target.replayUpsertJob(records.get(0));
-
- Assertions.assertEquals(LanceIndexJobMutationState.PENDING, target.getJob(1L).getMutationState());
- Assertions.assertTrue(target.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertFalse(target.markRunning(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertFalse(target.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, target.getJob(1L).getMutationState());
- }
-
- @Test
- public void replayedRunningStaysRunningWithoutSweep() throws DdlException {
- List<LanceIndexJob> records = runningRecords(1L, "IdxA");
- TestManager target = new TestManager();
- target.replayUpsertJob(records.get(0));
- target.replayUpsertJob(records.get(1));
-
- LanceIndexJob stored = target.getJob(1L);
- // A follower tailing a live master must not transform a fresh RUNNING record.
- Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState());
- Assertions.assertEquals(1L, stored.getRevision());
- Assertions.assertTrue(stored.holdsPossibleLiveSlot());
- Assertions.assertTrue(target.isFenceHeld(stored.fenceKey()));
- Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
- // Replay itself never writes the journal.
- Assertions.assertTrue(target.editLog.isEmpty());
- }
-
- @Test
- public void transferToMasterConvertsRunningToUnknownAndNeverRedispatches() throws DdlException {
- List<LanceIndexJob> records = runningRecords(1L, "IdxA");
- TestManager target = new TestManager();
- target.replayUpsertJob(records.get(0));
- target.replayUpsertJob(records.get(1));
- LanceIndexFenceKey fenceKey = target.getJob(1L).fenceKey();
-
- target.onTransferToMaster();
-
- LanceIndexJob swept = target.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, swept.getMutationState());
- Assertions.assertEquals(2L, swept.getRevision());
- Assertions.assertEquals(LanceIndexJobResultCode.NO_TRUSTED_RESULT, swept.getResult().getResultCode());
- // Fence, quota, and the possible-live slot survive the sweep: only FORCE releases them.
- Assertions.assertTrue(target.isFenceHeld(fenceKey));
- Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
- Assertions.assertTrue(swept.holdsPossibleLiveSlot());
- Assertions.assertTrue(containsJob(target.getUnresolvedJobs(), swept.getJobId()));
- Assertions.assertEquals(1, target.editLog.size());
-
- Assertions.assertFalse(target.markRunning(1L, 2L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertFalse(target.markRunning(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertFalse(target.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, target.getJob(1L).getMutationState());
- }
-
- @Test
- public void transferToMasterDowngradesRunningRefreshToRequired() throws DdlException {
- TestManager source = new TestManager();
- source.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
- source.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
- source.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, okResult());
- source.markRefreshRunning(1L, 2L);
-
- TestManager target = new TestManager();
- for (LanceIndexJob record : source.editLog) {
- target.replayUpsertJob(record);
- }
- Assertions.assertEquals(LanceIndexJobRefreshState.RUNNING, target.getJob(1L).getRefreshState());
-
- target.onTransferToMaster();
-
- LanceIndexJob swept = target.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, swept.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, swept.getRefreshState());
- Assertions.assertEquals(4L, swept.getRevision());
- Assertions.assertTrue(containsJob(target.getJobsNeedingRefresh(), swept.getJobId()));
- Assertions.assertTrue(target.isFenceHeld(swept.fenceKey()));
- }
-
- @Test
- public void replayedTerminalWithRefreshRequiredOnlyAllowsRefreshPath() throws DdlException {
- TestManager source = new TestManager();
- source.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
- source.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
- source.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, okResult());
-
- TestManager target = new TestManager();
- for (LanceIndexJob record : source.editLog) {
- target.replayUpsertJob(record);
- }
- LanceIndexJob stored = target.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, stored.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, stored.getRefreshState());
- LanceIndexFenceKey fenceKey = stored.fenceKey();
-
- // The mutation lifecycle is closed; only the refresh transitions remain.
- Assertions.assertFalse(target.markRunning(1L, 2L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertFalse(target.completeWithResult(1L, 2L, INVOCATION_ID, BE_EPOCH, okResult()));
- Assertions.assertTrue(containsJob(target.getJobsNeedingRefresh(), stored.getJobId()));
- Assertions.assertTrue(target.isFenceHeld(fenceKey));
- Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
-
- Assertions.assertTrue(target.markRefreshRunning(1L, 2L));
- Assertions.assertTrue(target.markRefreshDone(1L, 3L));
- Assertions.assertFalse(target.isFenceHeld(fenceKey));
- Assertions.assertEquals(0L, target.getQuota().getGlobalCount());
- Assertions.assertTrue(target.getUnresolvedJobs().isEmpty());
- }
-
- @Test
- public void replayedUnresolvedUnknownFencesTheSameName() throws DdlException {
- TestManager source = new TestManager();
- source.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
- source.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
- source.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT,
- LanceIndexJobCompletionReason.NONE, "ambiguous", false));
-
- TestManager target = new TestManager();
- for (LanceIndexJob record : source.editLog) {
- target.replayUpsertJob(record);
- }
- Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, target.getJob(1L).getMutationState());
-
- // The unforced UNKNOWN still holds the same-name fence and the quota.
- Assertions.assertThrows(DdlException.class,
- () -> target.createJob(newCreateJob(9L, "IdxA"), 100, 100, 100));
- Assertions.assertThrows(DdlException.class,
- () -> target.createJob(newCreateJob(9L, "idxa"), 100, 100, 100));
- target.createJob(newCreateJob(9L, "IdxB"), 100, 100, 100);
- Assertions.assertEquals(2L, target.getQuota().getGlobalCount());
- }
-
- @Test
- public void replayedForceReleasedUnknownFreesNameAndQuota() {
- TestManager target = new TestManager();
- target.replayUpsertJob(forceReleasedUnknownJob(7L, "idxforce"));
-
- Assertions.assertEquals(0L, target.getQuota().getGlobalCount());
- Assertions.assertFalse(target.isFenceHeld(target.getJob(7L).fenceKey()));
- Assertions.assertTrue(target.getUnresolvedJobs().isEmpty());
-
- Assertions.assertDoesNotThrow(() -> target.createJob(newCreateJob(8L, "IdxForce"), 100, 100, 100));
- Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
- Assertions.assertEquals(1, target.editLog.size());
- }
-
- @Test
- public void staleCallbacksAreRejectedWithoutStateChange() throws DdlException {
- List<LanceIndexJob> records = runningRecords(1L, "IdxA");
- TestManager target = new TestManager();
- target.replayUpsertJob(records.get(0));
- target.replayUpsertJob(records.get(1));
-
- Assertions.assertFalse(target.completeWithResult(1L, 0L, INVOCATION_ID, BE_EPOCH, okResult()));
- Assertions.assertFalse(target.completeWithResult(1L, 1L, "invocation-x", BE_EPOCH, okResult()));
- Assertions.assertFalse(target.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH + 1, okResult()));
- Assertions.assertFalse(target.completeWithResult(1L, 1L, INVOCATION_ID, null, okResult()));
- Assertions.assertFalse(target.completeWithResult(404L, 1L, INVOCATION_ID, BE_EPOCH, okResult()));
-
- LanceIndexJob stored = target.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState());
- Assertions.assertEquals(1L, stored.getRevision());
- Assertions.assertNull(stored.getResult());
- Assertions.assertTrue(target.editLog.isEmpty());
-
- Assertions.assertTrue(target.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, okResult()));
- Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, target.getJob(1L).getMutationState());
- }
-
- @Test
- public void replayIsIdempotentForTheSameRecord() throws DdlException {
- List<LanceIndexJob> records = runningRecords(1L, "IdxA");
- TestManager target = new TestManager();
- target.replayUpsertJob(records.get(0));
- target.replayUpsertJob(records.get(0));
- target.replayUpsertJob(records.get(1));
- target.replayUpsertJob(records.get(1));
-
- Assertions.assertEquals(1, target.getJobCount());
- Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
- Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, target.getJob(1L).getMutationState());
- Assertions.assertEquals(1L, target.getJob(1L).getRevision());
- }
-
- @Test
- public void lowerRevisionRecordNeverOverwritesHigherRevision() throws DdlException {
- TestManager source = new TestManager();
- source.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
- source.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
- source.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, okResult());
- // Records: PENDING rev0, RUNNING rev1, COMMITTED+REQUIRED rev2.
-
- TestManager target = new TestManager();
- target.replayUpsertJob(source.editLog.get(2));
- target.replayUpsertJob(source.editLog.get(0));
- target.replayUpsertJob(source.editLog.get(1));
-
- LanceIndexJob stored = target.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, stored.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, stored.getRefreshState());
- Assertions.assertEquals(2L, stored.getRevision());
- Assertions.assertTrue(target.isFenceHeld(stored.fenceKey()));
- Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
- Assertions.assertTrue(target.editLog.isEmpty());
-
- // An equal revision replaces verbatim (idempotent re-delivery).
- target.replayUpsertJob(source.editLog.get(2));
- Assertions.assertEquals(2L, target.getJob(1L).getRevision());
- Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
- }
-
- @Test
- public void replayToleratesNullAndIdentityLessRecords() {
- TestManager target = new TestManager();
- target.replayUpsertJob(null);
- Assertions.assertEquals(0, target.getJobCount());
-
- // A corrupt record without fence identity stays queryable but out of the books.
- LanceIndexJob sparse = GsonUtils.GSON.fromJson(
- "{\"jid\":5,\"rev\":0,\"ms\":\"PENDING\"}", LanceIndexJob.class);
- target.replayUpsertJob(sparse);
- Assertions.assertEquals(1, target.getJobCount());
- Assertions.assertEquals(0L, target.getQuota().getGlobalCount());
- Assertions.assertEquals(LanceIndexJobMutationState.PENDING, target.getJob(5L).getMutationState());
- Assertions.assertTrue(target.getUnresolvedJobs().isEmpty());
- Assertions.assertFalse(target.markRunning(5L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
-
- // Out of the books does not mean invisible: it blocks all new admissions
- // fail-closed, and the rejection leaves nothing behind.
- Assertions.assertEquals(Collections.singletonList(5L), target.getCorruptUnresolvedJobIds());
- DdlException blocked = Assertions.assertThrows(DdlException.class,
- () -> target.createJob(newCreateJob(9L, "IdxB"), 100, 100, 100));
- Assertions.assertTrue(blocked.getMessage().contains("smallest job id 5"));
- Assertions.assertEquals(1, target.getJobCount());
- Assertions.assertTrue(target.editLog.isEmpty());
- Assertions.assertEquals(0L, target.getQuota().getGlobalCount());
- }
-
- @Test
- public void resolvingTheSmallestCollidingJobLeavesTheOtherFenceOwner() throws DdlException {
- TestManager target = new TestManager();
- target.replayUpsertJob(pendingRecord(1L, "IdxA"));
- target.replayUpsertJob(pendingRecord(2L, "IdxA"));
- LanceIndexFenceKey fenceKey = target.getJob(1L).fenceKey();
-
- Assertions.assertEquals(2L, target.getQuota().getGlobalCount());
- DdlException initialConflict = Assertions.assertThrows(DdlException.class,
- () -> target.createJob(newCreateJob(9L, "IdxA"), 100, 100, 100));
- Assertions.assertTrue(initialConflict.getMessage().contains("unresolved job 1"));
-
- Assertions.assertTrue(target.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertTrue(target.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- new LanceIndexJobResult(LanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED,
- LanceIndexJobCompletionReason.NONE, "rejected before invocation", false)));
-
- Assertions.assertTrue(target.isFenceHeld(fenceKey));
- Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
- DdlException remainingConflict = Assertions.assertThrows(DdlException.class,
- () -> target.createJob(newCreateJob(9L, "IdxA"), 100, 100, 100));
- Assertions.assertTrue(remainingConflict.getMessage().contains("unresolved job 2"));
- }
-
- @Test
- public void replayAndQueryBoundariesReturnDefensiveCopies() throws DdlException {
- TestManager target = new TestManager();
- LanceIndexJob replayed = pendingRecord(1L, "IdxA");
- target.replayUpsertJob(replayed);
-
- replayed.setMutationState(LanceIndexJobMutationState.COMMITTED);
- replayed.setRefreshState(LanceIndexJobRefreshState.DONE);
- replayed.setRevision(99L);
- Assertions.assertEquals(LanceIndexJobMutationState.PENDING, target.getJob(1L).getMutationState());
- Assertions.assertEquals(0L, target.getJob(1L).getRevision());
-
- LanceIndexJob queried = target.getJob(1L);
- queried.setForceReleased(true);
- queried.setRevision(88L);
- Assertions.assertFalse(target.getJob(1L).isForceReleased());
- Assertions.assertEquals(0L, target.getJob(1L).getRevision());
-
- LanceIndexJob listed = target.getUnresolvedJobs().get(0);
- listed.setForceReleased(true);
- Assertions.assertFalse(target.getJob(1L).isForceReleased());
-
- target.createJob(newCreateJob(2L, "IdxB"), 100, 100, 100);
- Assertions.assertTrue(target.markRunning(2L, 0L, BACKEND_ID, BE_EPOCH, "invocation-2", DEADLINE_MS));
- Assertions.assertTrue(target.completeWithResult(2L, 1L, "invocation-2", BE_EPOCH, okResult()));
- LanceIndexJob refreshCandidate = target.getJobsNeedingRefresh().get(0);
- refreshCandidate.setRefreshState(LanceIndexJobRefreshState.DONE);
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, target.getJob(2L).getRefreshState());
-
- target.replayUpsertJob(GsonUtils.GSON.fromJson(
- "{\"jid\":7,\"rev\":0,\"ms\":\"PENDING\"}", LanceIndexJob.class));
- List<Long> blockingIds = target.getCorruptUnresolvedJobIds();
- blockingIds.clear();
- Assertions.assertEquals(Collections.singletonList(7L), target.getCorruptUnresolvedJobIds());
- }
-
- @Test
- public void replayingOverAnIdentityLessRecordNeverThrows() {
- TestManager target = new TestManager();
- LanceIndexJob sparse = GsonUtils.GSON.fromJson(
- "{\"jid\":5,\"rev\":0,\"ms\":\"PENDING\"}", LanceIndexJob.class);
- target.replayUpsertJob(sparse);
- Assertions.assertEquals(0L, target.getQuota().getGlobalCount());
-
- // Re-delivering the same identity-less record must not key the release side on it.
- Assertions.assertDoesNotThrow(() -> target.replayUpsertJob(sparse));
- Assertions.assertEquals(1, target.getJobCount());
- Assertions.assertEquals(0L, target.getQuota().getGlobalCount());
-
- // Neither must a higher-revision identity-less upsert of the same job id.
- LanceIndexJob unknown = GsonUtils.GSON.fromJson(
- "{\"jid\":5,\"rev\":1,\"ms\":\"UNKNOWN\"}", LanceIndexJob.class);
- Assertions.assertDoesNotThrow(() -> target.replayUpsertJob(unknown));
- LanceIndexJob stored = target.getJob(5L);
- Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState());
- Assertions.assertEquals(1L, stored.getRevision());
- Assertions.assertEquals(1, target.getJobCount());
- Assertions.assertEquals(0L, target.getQuota().getGlobalCount());
-
- // Both deliveries kept the admission blocker pinned...
- Assertions.assertEquals(Collections.singletonList(5L), target.getCorruptUnresolvedJobIds());
- // ...and a resolving upsert lifts it without ever keying the books on the record.
- Assertions.assertDoesNotThrow(() -> target.replayUpsertJob(GsonUtils.GSON.fromJson(
- "{\"jid\":5,\"rev\":2,\"ms\":\"UNKNOWN\",\"fr\":true}", LanceIndexJob.class)));
- Assertions.assertTrue(target.getCorruptUnresolvedJobIds().isEmpty());
- Assertions.assertEquals(0L, target.getQuota().getGlobalCount());
- }
-
- @Test
- public void identityLessPendingBlocksAdmissionUntilResolved() throws DdlException {
- TestManager target = new TestManager();
- target.replayUpsertJob(GsonUtils.GSON.fromJson(
- "{\"jid\":5,\"rev\":0,\"ms\":\"PENDING\"}", LanceIndexJob.class));
- Assertions.assertThrows(DdlException.class,
- () -> target.createJob(newCreateJob(9L, "IdxB"), 100, 100, 100));
-
- // A later durable record resolving the job lifts the blockade.
- target.replayUpsertJob(GsonUtils.GSON.fromJson(
- "{\"jid\":5,\"rev\":1,\"ms\":\"COMMITTED\",\"rs\":\"DONE\"}", LanceIndexJob.class));
- Assertions.assertTrue(target.getCorruptUnresolvedJobIds().isEmpty());
- target.createJob(newCreateJob(9L, "IdxB"), 100, 100, 100);
- Assertions.assertEquals(2, target.getJobCount());
- }
-
- @Test
- public void identityLessUnknownBlocksAdmissionUntilForceReleased() throws DdlException {
- TestManager target = new TestManager();
- target.replayUpsertJob(GsonUtils.GSON.fromJson(
- "{\"jid\":5,\"rev\":0,\"ms\":\"UNKNOWN\"}", LanceIndexJob.class));
- Assertions.assertEquals(Collections.singletonList(5L), target.getCorruptUnresolvedJobIds());
- Assertions.assertThrows(DdlException.class,
- () -> target.createJob(newCreateJob(9L, "IdxB"), 100, 100, 100));
-
- // The force-release record of a follow-up PR resolves the job by id,
- // without needing its fence key.
- target.replayUpsertJob(GsonUtils.GSON.fromJson(
- "{\"jid\":5,\"rev\":1,\"ms\":\"UNKNOWN\",\"fr\":true}", LanceIndexJob.class));
- Assertions.assertTrue(target.getCorruptUnresolvedJobIds().isEmpty());
- target.createJob(newCreateJob(9L, "IdxB"), 100, 100, 100);
- Assertions.assertEquals(2, target.getJobCount());
- }
-
- @Test
- public void sweptIdentityLessRunningKeepsBlockingAdmission() {
- TestManager target = new TestManager();
- target.replayUpsertJob(GsonUtils.GSON.fromJson(
- "{\"jid\":5,\"rev\":1,\"ms\":\"RUNNING\"}", LanceIndexJob.class));
- Assertions.assertEquals(Collections.singletonList(5L), target.getCorruptUnresolvedJobIds());
-
- // The master-election sweep turns the ambiguous RUNNING into UNKNOWN; the
- // blockade survives because the record still lacks fence identity.
- target.onTransferToMaster();
- Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, target.getJob(5L).getMutationState());
- Assertions.assertEquals(Collections.singletonList(5L), target.getCorruptUnresolvedJobIds());
- Assertions.assertThrows(DdlException.class,
- () -> target.createJob(newCreateJob(9L, "IdxB"), 100, 100, 100));
- }
-
- @Test
- public void healthyJobLifecycleProceedsWhileAdmissionIsBlocked() throws DdlException {
- TestManager target = new TestManager();
- target.createJob(newCreateJob(9L, "IdxB"), 100, 100, 100);
- // Corruption can surface after a legal admission (a follower replays records in
- // journal order); only new admissions are blocked, never in-flight lifecycle steps.
- target.replayUpsertJob(GsonUtils.GSON.fromJson(
- "{\"jid\":5,\"rev\":0,\"ms\":\"PENDING\"}", LanceIndexJob.class));
- Assertions.assertThrows(DdlException.class,
- () -> target.createJob(newCreateJob(10L, "IdxC"), 100, 100, 100));
-
- Assertions.assertTrue(target.markRunning(9L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertTrue(target.completeWithResult(9L, 1L, INVOCATION_ID, BE_EPOCH, okResult()));
- Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, target.getJob(9L).getMutationState());
- }
-
- @Test
- public void admissionBlockMessageStaysBoundedWithMultipleCorruptRecords() {
- TestManager target = new TestManager();
- for (long jobId = 5L; jobId <= 7L; jobId++) {
- target.replayUpsertJob(GsonUtils.GSON.fromJson(
- "{\"jid\":" + jobId + ",\"rev\":0,\"ms\":\"PENDING\"}", LanceIndexJob.class));
- }
- Assertions.assertEquals(Arrays.asList(5L, 6L, 7L), target.getCorruptUnresolvedJobIds());
-
- DdlException exception = Assertions.assertThrows(DdlException.class,
- () -> target.createJob(newCreateJob(9L, "IdxB"), 100, 100, 100));
- // Bounded and non-disclosing: the count plus the smallest job id, never a list.
- Assertions.assertTrue(exception.getMessage().contains("3 unresolved"));
- Assertions.assertTrue(exception.getMessage().contains("smallest job id 5"));
- Assertions.assertFalse(exception.getMessage().contains("6"));
- Assertions.assertFalse(exception.getMessage().contains("7"));
- }
-
- @Test
- public void identityFullUpsertHealsAnIdentityLessRecord() throws DdlException {
- TestManager target = new TestManager();
- target.replayUpsertJob(GsonUtils.GSON.fromJson(
- "{\"jid\":5,\"rev\":0,\"ms\":\"PENDING\"}", LanceIndexJob.class));
- Assertions.assertEquals(Collections.singletonList(5L), target.getCorruptUnresolvedJobIds());
-
- // A record carrying fence identity heals the books at the same revision: the job
- // joins the fence/quota accounting and the admission blockade lifts.
- target.replayUpsertJob(pendingRecord(5L, "IdxA"));
- Assertions.assertTrue(target.getCorruptUnresolvedJobIds().isEmpty());
- Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
- Assertions.assertTrue(target.isFenceHeld(target.getJob(5L).fenceKey()));
- DdlException fenced = Assertions.assertThrows(DdlException.class,
- () -> target.createJob(newCreateJob(9L, "idxa"), 100, 100, 100));
- Assertions.assertTrue(fenced.getMessage().contains("unresolved job 5"));
- }
-
- @Test
- public void transferToMasterSweepsOnlyTheRunningJobInAMixedPopulation() throws DdlException {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxPending"), 100, 100, 100);
- manager.createJob(newCreateJob(2L, "IdxCommitted"), 100, 100, 100);
- manager.markRunning(2L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
- manager.completeWithResult(2L, 1L, INVOCATION_ID, BE_EPOCH, okResult());
- manager.markRefreshRunning(2L, 2L);
- manager.markRefreshDone(2L, 3L);
- manager.createJob(newCreateJob(3L, "IdxRunning"), 100, 100, 100);
- manager.markRunning(3L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
- // Journal so far: create(1), create+run+complete+refreshRun+refreshDone(2), create+run(3).
- Assertions.assertEquals(8, manager.editLog.size());
-
- manager.onTransferToMaster();
-
- // PENDING and the settled terminal job are replay-faithful: untouched.
- LanceIndexJob pending = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.PENDING, pending.getMutationState());
- Assertions.assertEquals(0L, pending.getRevision());
- LanceIndexJob committed = manager.getJob(2L);
- Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, committed.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.DONE, committed.getRefreshState());
- Assertions.assertEquals(4L, committed.getRevision());
-
- // Only the durable RUNNING becomes UNKNOWN, through exactly one journal record.
- LanceIndexJob swept = manager.getJob(3L);
- Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, swept.getMutationState());
- Assertions.assertEquals(2L, swept.getRevision());
- Assertions.assertEquals(9, manager.editLog.size());
- }
-
- @Test
- public void fenceRejectionLeavesNoJobNoQuotaAndNoJournalRecord() throws DdlException {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
-
- DdlException exception = Assertions.assertThrows(DdlException.class,
- () -> manager.createJob(newCreateJob(2L, "idxa"), 100, 100, 100));
- // The rejection must not disclose the dataset locator to an unauthorized caller.
- Assertions.assertFalse(exception.getMessage().contains("bucket"));
- Assertions.assertFalse(exception.getMessage().contains(LOCATOR));
-
- Assertions.assertEquals(1, manager.getJobCount());
- Assertions.assertEquals(1, manager.editLog.size());
- Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
- // Only the original job's fence exists; nothing new was registered for the rejection.
- Assertions.assertFalse(manager.isFenceHeld(newCreateJob(3L, "IdxB").fenceKey()));
- }
-
- @Test
- public void quotaRejectionLeavesNoJobNoFenceAndNoJournalRecord() throws DdlException {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxA"), 1, 1, 1);
-
- Assertions.assertThrows(DdlException.class,
- () -> manager.createJob(newCreateJob(2L, "IdxB"), 1, 1, 1));
-
- Assertions.assertEquals(1, manager.getJobCount());
- Assertions.assertEquals(1, manager.editLog.size());
- Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
- Assertions.assertFalse(manager.isFenceHeld(newCreateJob(2L, "IdxB").fenceKey()));
- }
-
- @Test
- public void duplicateJobIdIsRejectedBeforeAnythingElse() throws DdlException {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
-
- Assertions.assertThrows(DdlException.class,
- () -> manager.createJob(newCreateJob(1L, "IdxB"), 100, 100, 100));
- Assertions.assertEquals(1, manager.getJobCount());
- Assertions.assertEquals(1, manager.editLog.size());
- Assertions.assertFalse(manager.isFenceHeld(newCreateJob(1L, "IdxB").fenceKey()));
- }
-
- private static LanceIndexJob newCreateJob(long jobId, String displayName) {
- return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
- displayName, LanceIndexNameNormalizer.normalize(displayName),
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v",
- null, 7L, null);
- }
-
- private static LanceIndexJob pendingRecord(long jobId, String displayName) {
- LanceIndexJob job = newCreateJob(jobId, displayName);
- job.setMutationState(LanceIndexJobMutationState.PENDING);
- job.setRefreshState(LanceIndexJobRefreshState.NOT_REQUIRED);
- return job;
- }
-
- private static boolean containsJob(List<LanceIndexJob> jobs, long jobId) {
- return jobs.stream().anyMatch(job -> job.getJobId() == jobId);
- }
-
- private static LanceIndexJobResult okResult() {
- return new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
- LanceIndexJobCompletionReason.NONE, "ok", false);
- }
-
- /**
- * Runs create + dispatch on a throwaway source manager and returns its journal
- * records: [PENDING rev0, RUNNING rev1].
- */
- private static List<LanceIndexJob> runningRecords(long jobId, String displayName) throws DdlException {
- TestManager source = new TestManager();
- source.createJob(newCreateJob(jobId, displayName), 100, 100, 100);
- source.markRunning(jobId, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
- return source.editLog;
- }
-
- /**
- * Builds the durable form of a FORCE-released UNKNOWN job. The FORCE slice owns the
- * transition itself; here the record only exists as replay input, so it is built
- * from its JSON journal form.
- */
- private static LanceIndexJob forceReleasedUnknownJob(long jobId, String normalizedName) {
- String json = "{\"jid\":" + jobId + ",\"cr\":\"tester\",\"rev\":2,\"cid\":" + CATALOG_ID
- + ",\"dbn\":\"db1\",\"tbn\":\"tbl1\",\"prv\":\"" + LanceIndexFenceKey.PROVIDER_DIRECTORY
- + "\",\"loc\":\"" + LOCATOR + "\",\"din\":\"" + normalizedName + "\",\"nin\":\"" + normalizedName
- + "\",\"mt\":\"CREATE\",\"ms\":\"UNKNOWN\",\"rs\":\"NOT_REQUIRED\",\"fr\":true,"
- + "\"fa\":\"admin\",\"ftm\":12345,\"fn\":\"operator note\"}";
- LanceIndexJob job = GsonUtils.GSON.fromJson(json, LanceIndexJob.class);
- Assertions.assertTrue(job.isForceReleased());
- Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, job.getMutationState());
- return job;
- }
-
- /**
- * Edit-log seam: captures every durable record instead of writing the journal.
- */
- private static class TestManager extends LanceIndexJobManager {
- private final List<LanceIndexJob> editLog = new ArrayList<>();
-
- @Override
- protected void writeEditLog(LanceIndexJob job) {
- editLog.add(job);
- }
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java
deleted file mode 100644
index 9511f7f1bc0..00000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java
+++ /dev/null
@@ -1,208 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Unit coverage for the three-level unresolved-job quota counters (table/locator,
- * catalog, global): the "current < limit" boundary at each level, rejection of
- * non-positive limits, release recovery with underflow clamping, and rebuild
- * equivalence with live counting. The "which jobs count" semantics are
- * {@link LanceIndexJob#isUnresolved()}; the rebuild-side composition is pinned here too.
- */
-public class LanceIndexJobQuotaTest {
- private static final long CATALOG_ID = 10L;
- private static final String LOCATOR_A = "s3://bucket/table-a";
- private static final String LOCATOR_B = "s3://bucket/table-b";
-
- @Test
- public void tryAcquireChargesAllThreeLevels() {
- LanceIndexJobQuota quota = new LanceIndexJobQuota();
- LanceIndexJob job = newJob(1L, CATALOG_ID, LOCATOR_A);
-
- Assertions.assertTrue(quota.tryAcquire(job, 5, 5, 5));
- Assertions.assertEquals(1L, quota.getGlobalCount());
- Assertions.assertEquals(1L, quota.getCatalogCount(CATALOG_ID));
- Assertions.assertEquals(1L, quota.getTableCount(job.getTableQuotaKey()));
- }
-
- @Test
- public void tableLimitRejectsTheNextJobExactlyAtLimit() {
- LanceIndexJobQuota quota = new LanceIndexJobQuota();
- Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 2, 100, 100));
- Assertions.assertTrue(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_A), 2, 100, 100));
-
- LanceIndexJob third = newJob(3L, CATALOG_ID, LOCATOR_A);
- Assertions.assertFalse(quota.tryAcquire(third, 2, 100, 100));
- // A rejected acquire charges nothing at any level.
- Assertions.assertEquals(2L, quota.getGlobalCount());
- Assertions.assertEquals(2L, quota.getTableCount(third.getTableQuotaKey()));
-
- // The limit is per table/locator identity: another table still has room.
- Assertions.assertTrue(quota.tryAcquire(newJob(4L, CATALOG_ID, LOCATOR_B), 2, 100, 100));
- }
-
- @Test
- public void catalogLimitRejectsAcrossTables() {
- LanceIndexJobQuota quota = new LanceIndexJobQuota();
- Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 100, 2, 100));
- Assertions.assertTrue(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_B), 100, 2, 100));
- Assertions.assertFalse(quota.tryAcquire(newJob(3L, CATALOG_ID, LOCATOR_A), 100, 2, 100));
-
- // Another catalog is a separate level.
- Assertions.assertTrue(quota.tryAcquire(newJob(4L, 20L, LOCATOR_A), 100, 2, 100));
- }
-
- @Test
- public void globalLimitRejectsAcrossCatalogs() {
- LanceIndexJobQuota quota = new LanceIndexJobQuota();
- Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 100, 100, 2));
- Assertions.assertTrue(quota.tryAcquire(newJob(2L, 20L, LOCATOR_A), 100, 100, 2));
- Assertions.assertFalse(quota.tryAcquire(newJob(3L, 30L, LOCATOR_B), 100, 100, 2));
- Assertions.assertEquals(2L, quota.getGlobalCount());
- }
-
- @Test
- public void nonPositiveLimitIsRejectedWithoutACharge() {
- LanceIndexJobQuota quota = new LanceIndexJobQuota();
- LanceIndexJob job = newJob(1L, CATALOG_ID, LOCATOR_A);
- Assertions.assertFalse(quota.tryAcquire(job, 0, 1, 1));
- Assertions.assertFalse(quota.tryAcquire(job, 1, 0, 1));
- Assertions.assertFalse(quota.tryAcquire(job, 1, 1, 0));
- Assertions.assertFalse(quota.tryAcquire(job, -1, 1, 1));
- Assertions.assertEquals(0L, quota.getGlobalCount());
- }
-
- @Test
- public void releaseRecoversCapacityAtAllLevels() {
- LanceIndexJobQuota quota = new LanceIndexJobQuota();
- LanceIndexJob first = newJob(1L, CATALOG_ID, LOCATOR_A);
- Assertions.assertTrue(quota.tryAcquire(first, 1, 1, 1));
- Assertions.assertFalse(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_A), 1, 1, 1));
-
- quota.release(first);
- Assertions.assertEquals(0L, quota.getGlobalCount());
- Assertions.assertEquals(0L, quota.getCatalogCount(CATALOG_ID));
- Assertions.assertEquals(0L, quota.getTableCount(first.getTableQuotaKey()));
-
- Assertions.assertTrue(quota.tryAcquire(newJob(3L, CATALOG_ID, LOCATOR_A), 1, 1, 1));
- }
-
- @Test
- public void releaseUnderflowClampsAtZero() {
- LanceIndexJobQuota quota = new LanceIndexJobQuota();
- // Releasing a job that was never charged must not fail and must not go negative.
- quota.release(newJob(1L, CATALOG_ID, LOCATOR_A));
- Assertions.assertEquals(0L, quota.getGlobalCount());
- Assertions.assertEquals(0L, quota.getCatalogCount(CATALOG_ID));
- Assertions.assertTrue(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_A), 1, 1, 1));
- }
-
- @Test
- public void rebuildMatchesIncrementalCounting() {
- List<LanceIndexJob> unresolved = new ArrayList<>();
- unresolved.add(newJob(1L, CATALOG_ID, LOCATOR_A));
- unresolved.add(newJob(2L, CATALOG_ID, LOCATOR_A));
- unresolved.add(newJob(3L, CATALOG_ID, LOCATOR_B));
- unresolved.add(newJob(4L, 20L, LOCATOR_A));
-
- LanceIndexJobQuota incremental = new LanceIndexJobQuota();
- for (LanceIndexJob job : unresolved) {
- Assertions.assertTrue(incremental.tryAcquire(job, 100, 100, 100));
- }
- LanceIndexJobQuota rebuilt = new LanceIndexJobQuota();
- rebuilt.rebuild(unresolved);
-
- Assertions.assertEquals(incremental.getGlobalCount(), rebuilt.getGlobalCount());
- Assertions.assertEquals(incremental.getCatalogCount(CATALOG_ID), rebuilt.getCatalogCount(CATALOG_ID));
- Assertions.assertEquals(incremental.getCatalogCount(20L), rebuilt.getCatalogCount(20L));
- Assertions.assertEquals(incremental.getTableCount(unresolved.get(0).getTableQuotaKey()),
- rebuilt.getTableCount(unresolved.get(0).getTableQuotaKey()));
- Assertions.assertEquals(incremental.getTableCount(unresolved.get(2).getTableQuotaKey()),
- rebuilt.getTableCount(unresolved.get(2).getTableQuotaKey()));
-
- rebuilt.rebuild(Collections.<LanceIndexJob>emptyList());
- Assertions.assertEquals(0L, rebuilt.getGlobalCount());
- Assertions.assertEquals(0L, rebuilt.getCatalogCount(CATALOG_ID));
- }
-
- @Test
- public void rebuildCountsUnresolvedJobsOnly() {
- List<LanceIndexJob> jobs = new ArrayList<>();
- // Active and unforced-unknown jobs count.
- jobs.add(jobInState(1L, LanceIndexJobMutationState.PENDING, LanceIndexJobRefreshState.NOT_REQUIRED, false));
- jobs.add(jobInState(2L, LanceIndexJobMutationState.RUNNING, LanceIndexJobRefreshState.NOT_REQUIRED, false));
- jobs.add(jobInState(3L, LanceIndexJobMutationState.UNKNOWN, LanceIndexJobRefreshState.NOT_REQUIRED, false));
- jobs.add(jobInState(4L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.REQUIRED, false));
- jobs.add(jobInState(5L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.RUNNING, false));
- jobs.add(jobInState(6L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.FAILED, false));
- jobs.add(jobInState(7L, LanceIndexJobMutationState.NOT_COMMITTED, LanceIndexJobRefreshState.FAILED, false));
- // Resolved jobs do not count: forced UNKNOWN and terminal jobs with a settled refresh.
- jobs.add(jobInState(8L, LanceIndexJobMutationState.UNKNOWN, LanceIndexJobRefreshState.NOT_REQUIRED, true));
- jobs.add(jobInState(9L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.DONE, false));
- jobs.add(jobInState(10L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.NOT_REQUIRED, false));
- jobs.add(jobInState(11L, LanceIndexJobMutationState.NOT_COMMITTED, LanceIndexJobRefreshState.NOT_REQUIRED,
- false));
-
- List<LanceIndexJob> unresolved = new ArrayList<>();
- for (LanceIndexJob job : jobs) {
- if (job.isUnresolved()) {
- unresolved.add(job);
- }
- }
- Assertions.assertEquals(7, unresolved.size());
-
- LanceIndexJobQuota quota = new LanceIndexJobQuota();
- quota.rebuild(unresolved);
- Assertions.assertEquals(7L, quota.getGlobalCount());
- Assertions.assertEquals(7L, quota.getCatalogCount(CATALOG_ID));
- Assertions.assertEquals(7L, quota.getTableCount(unresolved.get(0).getTableQuotaKey()));
- }
-
- @Test
- public void tableQuotaKeyIdentityAndLocatorHiding() {
- LanceIndexJobQuota.TableQuotaKey key = new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR_A);
- Assertions.assertEquals(new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR_A), key);
- Assertions.assertEquals(new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR_A).hashCode(), key.hashCode());
- Assertions.assertNotEquals(new LanceIndexJobQuota.TableQuotaKey(20L, LOCATOR_A), key);
- Assertions.assertNotEquals(new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR_B), key);
- Assertions.assertFalse(key.toString().contains(LOCATOR_A));
- }
-
- private static LanceIndexJob newJob(long jobId, long catalogId, String locator) {
- return new LanceIndexJob(jobId, "tester", catalogId, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, locator,
- "idx" + jobId, "idx" + jobId,
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 1L, null);
- }
-
- private static LanceIndexJob jobInState(long jobId, LanceIndexJobMutationState mutationState,
- LanceIndexJobRefreshState refreshState, boolean forceReleased) {
- LanceIndexJob job = newJob(jobId, CATALOG_ID, LOCATOR_A);
- job.setMutationState(mutationState);
- job.setRefreshState(refreshState);
- job.setForceReleased(forceReleased);
- return job;
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultClassifyTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultClassifyTest.java
deleted file mode 100644
index cbceb49caf1..00000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultClassifyTest.java
+++ /dev/null
@@ -1,239 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-/**
- * Exhaustive coverage of the provider-result classification table: every saved typed
- * result code is classified into the durable (mutationState, refreshState,
- * completionReason) triple for the full cross product of 13 codes x 3 mutation types
- * x ifExists x externalMetadataAdvanced (156 combinations). The expected values are an
- * independent restatement of the design table, so a row flipped on either side fails.
- *
- * <p>Key invariants pinned here: PRE_INVOCATION_* always prove NOT_COMMITTED and owe a
- * refresh only when trusted revalidation observed external metadata advancement;
- * NATIVE_NOT_FOUND has no attribution for CREATE/REPLACE (UNKNOWN) but is a clean
- * NOT_COMMITTED for DROP; IF_CONDITION_NOOP exists only for DROP IF EXISTS + NOT_FOUND;
- * every other ambiguous post-invocation outcome is UNKNOWN + NOT_REQUIRED.
- */
-public class LanceIndexJobResultClassifyTest {
-
- @Test
- public void fullClassificationProductMatchesDesignTable() {
- int combos = 0;
- for (LanceIndexJobResultCode code : LanceIndexJobResultCode.values()) {
- for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) {
- for (boolean ifExists : new boolean[]{false, true}) {
- for (boolean advanced : new boolean[]{false, true}) {
- LanceIndexJobResultCode.Classification classification =
- LanceIndexJobResultCode.classify(type, code, ifExists, advanced);
- String context = "code=" + code + ", type=" + type
- + ", ifExists=" + ifExists + ", advanced=" + advanced;
- Assertions.assertEquals(expectedMutationState(type, code),
- classification.getMutationState(), context);
- Assertions.assertEquals(expectedRefreshState(type, code, advanced),
- classification.getRefreshState(), context);
- Assertions.assertEquals(expectedCompletionReason(type, code, ifExists),
- classification.getCompletionReason(), context);
- combos++;
- }
- }
- }
- }
- Assertions.assertEquals(13 * 3 * 2 * 2, combos);
- }
-
- @Test
- public void ifConditionNoopExistsOnlyForDropIfExistsNotFound() {
- int noopCombos = 0;
- for (LanceIndexJobResultCode code : LanceIndexJobResultCode.values()) {
- for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) {
- for (boolean ifExists : new boolean[]{false, true}) {
- for (boolean advanced : new boolean[]{false, true}) {
- LanceIndexJobResultCode.Classification classification =
- LanceIndexJobResultCode.classify(type, code, ifExists, advanced);
- if (classification.getCompletionReason() == LanceIndexJobCompletionReason.IF_CONDITION_NOOP) {
- noopCombos++;
- Assertions.assertEquals(LanceIndexJobMutationType.DROP, type);
- Assertions.assertEquals(LanceIndexJobResultCode.NATIVE_NOT_FOUND, code);
- Assertions.assertTrue(ifExists);
- }
- }
- }
- }
- }
- // Exactly the two advanced-flag variants of (DROP, NATIVE_NOT_FOUND, ifExists).
- Assertions.assertEquals(2, noopCombos);
- }
-
- @Test
- public void preInvocationRefreshOwesOnlyWhenExternalMetadataAdvanced() {
- for (LanceIndexJobResultCode code : LanceIndexJobResultCode.values()) {
- if (!code.isPreInvocation()) {
- continue;
- }
- for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) {
- Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED,
- LanceIndexJobResultCode.classify(type, code, false, false).getRefreshState());
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED,
- LanceIndexJobResultCode.classify(type, code, false, true).getRefreshState());
- Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED,
- LanceIndexJobResultCode.classify(type, code, true, true).getMutationState());
- }
- }
- }
-
- @Test
- public void nativeOkCommitsAndOwesRefreshForEveryMutationType() {
- for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) {
- LanceIndexJobResultCode.Classification classification =
- LanceIndexJobResultCode.classify(type, LanceIndexJobResultCode.NATIVE_OK, false, false);
- Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, classification.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, classification.getRefreshState());
- Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason());
- }
- }
-
- @Test
- public void commitConflictIsNotCommittedWithRefreshOwed() {
- for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) {
- LanceIndexJobResultCode.Classification classification = LanceIndexJobResultCode.classify(
- type, LanceIndexJobResultCode.NATIVE_COMMIT_CONFLICT, false, false);
- Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, classification.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, classification.getRefreshState());
- Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason());
- }
- }
-
- @Test
- public void nativeNotFoundAttributionDependsOnMutationType() {
- for (LanceIndexJobMutationType type : new LanceIndexJobMutationType[]{
- LanceIndexJobMutationType.CREATE, LanceIndexJobMutationType.REPLACE}) {
- LanceIndexJobResultCode.Classification classification = LanceIndexJobResultCode.classify(
- type, LanceIndexJobResultCode.NATIVE_NOT_FOUND, false, false);
- Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, classification.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, classification.getRefreshState());
- Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason());
- }
- LanceIndexJobResultCode.Classification dropClassification = LanceIndexJobResultCode.classify(
- LanceIndexJobMutationType.DROP, LanceIndexJobResultCode.NATIVE_NOT_FOUND, false, false);
- Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, dropClassification.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, dropClassification.getRefreshState());
- Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, dropClassification.getCompletionReason());
- }
-
- @Test
- public void ambiguousPostInvocationOutcomesAreUnknownWithoutRefresh() {
- for (LanceIndexJobResultCode code : new LanceIndexJobResultCode[]{
- LanceIndexJobResultCode.NATIVE_INVALID_ARGUMENT, LanceIndexJobResultCode.NATIVE_NOT_SUPPORTED,
- LanceIndexJobResultCode.NATIVE_INDEX, LanceIndexJobResultCode.NATIVE_IO,
- LanceIndexJobResultCode.NATIVE_INTERNAL, LanceIndexJobResultCode.NO_TRUSTED_RESULT}) {
- for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) {
- LanceIndexJobResultCode.Classification classification =
- LanceIndexJobResultCode.classify(type, code, true, true);
- Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, classification.getMutationState(),
- "code=" + code + ", type=" + type);
- Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, classification.getRefreshState(),
- "code=" + code + ", type=" + type);
- Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason(),
- "code=" + code + ", type=" + type);
- }
- }
- }
-
- @Test
- public void nullResultCodeFallsBackToNoTrustedResult() {
- LanceIndexJobResultCode.Classification classification =
- LanceIndexJobResultCode.classify(LanceIndexJobMutationType.CREATE, null, false, false);
- Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, classification.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, classification.getRefreshState());
- Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason());
- }
-
- @Test
- public void isPreInvocationCoversExactlyTheFourTrustedRejections() {
- int preInvocationCount = 0;
- for (LanceIndexJobResultCode code : LanceIndexJobResultCode.values()) {
- if (code.isPreInvocation()) {
- preInvocationCount++;
- }
- }
- Assertions.assertEquals(4, preInvocationCount);
- Assertions.assertEquals(13, LanceIndexJobResultCode.values().length);
- }
-
- /**
- * Independent restatement of the design table, mutation-state column.
- */
- private static LanceIndexJobMutationState expectedMutationState(
- LanceIndexJobMutationType type, LanceIndexJobResultCode code) {
- if (isPreInvocationSpec(code)) {
- return LanceIndexJobMutationState.NOT_COMMITTED;
- }
- switch (code) {
- case NATIVE_OK:
- return LanceIndexJobMutationState.COMMITTED;
- case NATIVE_COMMIT_CONFLICT:
- return LanceIndexJobMutationState.NOT_COMMITTED;
- case NATIVE_NOT_FOUND:
- return type == LanceIndexJobMutationType.DROP
- ? LanceIndexJobMutationState.NOT_COMMITTED : LanceIndexJobMutationState.UNKNOWN;
- default:
- return LanceIndexJobMutationState.UNKNOWN;
- }
- }
-
- /**
- * Independent restatement of the design table, refresh-obligation column.
- */
- private static LanceIndexJobRefreshState expectedRefreshState(
- LanceIndexJobMutationType type, LanceIndexJobResultCode code, boolean externalMetadataAdvanced) {
- if (isPreInvocationSpec(code)) {
- return externalMetadataAdvanced
- ? LanceIndexJobRefreshState.REQUIRED : LanceIndexJobRefreshState.NOT_REQUIRED;
- }
- switch (code) {
- case NATIVE_OK:
- case NATIVE_COMMIT_CONFLICT:
- return LanceIndexJobRefreshState.REQUIRED;
- case NATIVE_NOT_FOUND:
- return type == LanceIndexJobMutationType.DROP
- ? LanceIndexJobRefreshState.REQUIRED : LanceIndexJobRefreshState.NOT_REQUIRED;
- default:
- return LanceIndexJobRefreshState.NOT_REQUIRED;
- }
- }
-
- /**
- * Independent restatement of the design table, completion-reason column.
- */
- private static LanceIndexJobCompletionReason expectedCompletionReason(
- LanceIndexJobMutationType type, LanceIndexJobResultCode code, boolean ifExists) {
- return type == LanceIndexJobMutationType.DROP && code == LanceIndexJobResultCode.NATIVE_NOT_FOUND && ifExists
- ? LanceIndexJobCompletionReason.IF_CONDITION_NOOP : LanceIndexJobCompletionReason.NONE;
- }
-
- private static boolean isPreInvocationSpec(LanceIndexJobResultCode code) {
- return code == LanceIndexJobResultCode.PRE_INVOCATION_STALE_ADMISSION
- || code == LanceIndexJobResultCode.PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT
- || code == LanceIndexJobResultCode.PRE_INVOCATION_CREDENTIAL_EXPIRED
- || code == LanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED;
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java
deleted file mode 100644
index 54a5945eb79..00000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java
+++ /dev/null
@@ -1,640 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import org.apache.doris.common.DdlException;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Lifecycle state-machine coverage driven through the master write paths of
- * {@link LanceIndexJobManager} with the edit-log seam captured in memory. Covers the
- * legal mutation chain PENDING -> RUNNING -> (COMMITTED | NOT_COMMITTED |
- * UNKNOWN), the impossibility of leaving UNKNOWN, the independent refresh chain
- * REQUIRED -> RUNNING -> DONE / FAILED with FAILED -> RUNNING retry, the
- * revision compare-and-set on every transition, and the fence/quota release timing
- * (immediately on a terminal outcome with refresh NOT_REQUIRED, only at refresh DONE
- * otherwise, never for UNKNOWN without FORCE).
- */
-public class LanceIndexJobStateMachineTest {
- private static final long CATALOG_ID = 10L;
- private static final String LOCATOR = "s3://bucket/dataset";
- private static final long BACKEND_ID = 1001L;
- private static final long BE_EPOCH = 55L;
- private static final String INVOCATION_ID = "invocation-1";
- private static final long DEADLINE_MS = 9999L;
-
- @Test
- public void createInitializesPendingRecord() throws DdlException {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
-
- LanceIndexJob stored = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.PENDING, stored.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState());
- Assertions.assertEquals(0L, stored.getRevision());
- Assertions.assertFalse(stored.isPossibleLiveOwned());
- Assertions.assertEquals(LanceIndexTerminationProof.NONE, stored.getTerminationProof());
- Assertions.assertTrue(stored.getCreateTimeMs() > 0L);
- Assertions.assertTrue(manager.isFenceHeld(stored.fenceKey()));
- Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
- Assertions.assertEquals(1L, manager.getQuota().getCatalogCount(CATALOG_ID));
- Assertions.assertEquals(1L, manager.getQuota().getTableCount(stored.getTableQuotaKey()));
- Assertions.assertEquals(1, manager.getUnresolvedJobs().size());
- Assertions.assertEquals(1, manager.editLog.size());
- }
-
- @Test
- public void pendingToRunningToCommitted() throws DdlException {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
- Assertions.assertTrue(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
-
- LanceIndexJob running = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, running.getMutationState());
- Assertions.assertEquals(1L, running.getRevision());
- Assertions.assertEquals(BACKEND_ID, running.getBackendId().longValue());
- Assertions.assertEquals(BE_EPOCH, running.getBeProcessEpoch().longValue());
- Assertions.assertEquals(INVOCATION_ID, running.getInvocationId());
- Assertions.assertEquals(DEADLINE_MS, running.getDeadlineMs().longValue());
- Assertions.assertTrue(running.holdsPossibleLiveSlot());
-
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
- LanceIndexJob committed = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, committed.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, committed.getRefreshState());
- Assertions.assertEquals(2L, committed.getRevision());
- Assertions.assertEquals(LanceIndexJobResultCode.NATIVE_OK, committed.getResult().getResultCode());
- Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, committed.getResult().getCompletionReason());
- Assertions.assertTrue(containsJob(manager.getJobsNeedingRefresh(), committed.getJobId()));
- }
-
- @Test
- public void pendingToRunningToNotCommitted() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.PRE_INVOCATION_CREDENTIAL_EXPIRED)));
-
- LanceIndexJob stored = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, stored.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState());
- Assertions.assertEquals(2L, stored.getRevision());
- }
-
- @Test
- public void pendingToRunningToUnknown() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NO_TRUSTED_RESULT)));
-
- LanceIndexJob stored = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState());
- Assertions.assertEquals(2L, stored.getRevision());
- }
-
- @Test
- public void dropIfExistsNotFoundCompletesWithIfConditionNoop() throws DdlException {
- TestManager manager = new TestManager();
- manager.createJob(newDropJob(1L, "IdxA", true), 100, 100, 100);
- Assertions.assertTrue(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_NOT_FOUND)));
-
- LanceIndexJob stored = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, stored.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, stored.getRefreshState());
- Assertions.assertEquals(LanceIndexJobCompletionReason.IF_CONDITION_NOOP,
- stored.getResult().getCompletionReason());
- }
-
- @Test
- public void markRunningRejectsSecondDispatch() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
-
- Assertions.assertFalse(manager.markRunning(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertFalse(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(1L).getMutationState());
- Assertions.assertEquals(1L, manager.getJob(1L).getRevision());
- }
-
- @Test
- public void markRunningRejectsWrongRevisionAndUnknownJob() throws DdlException {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
-
- Assertions.assertFalse(manager.markRunning(1L, 5L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertFalse(manager.markRunning(404L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState());
- Assertions.assertEquals(1, manager.editLog.size());
- }
-
- @Test
- public void completeRejectsNonRunningJob() throws DdlException {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
-
- Assertions.assertFalse(manager.completeWithResult(1L, 0L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
- Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState());
-
- createAndRun(manager, 2L, "IdxB");
- Assertions.assertTrue(manager.completeWithResult(2L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
- Assertions.assertFalse(manager.completeWithResult(2L, 2L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
- Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, manager.getJob(2L).getMutationState());
- }
-
- @Test
- public void unknownHasNoOutgoingTransitions() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NO_TRUSTED_RESULT)));
- int loggedRecords = manager.editLog.size();
-
- Assertions.assertFalse(manager.markRunning(1L, 2L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertFalse(manager.completeWithResult(1L, 2L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
- Assertions.assertFalse(manager.markRefreshRunning(1L, 2L));
- Assertions.assertFalse(manager.markRefreshDone(1L, 2L));
- Assertions.assertFalse(manager.markRefreshFailed(1L, 2L));
-
- LanceIndexJob stored = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState());
- Assertions.assertEquals(2L, stored.getRevision());
- Assertions.assertEquals(loggedRecords, manager.editLog.size());
- }
-
- @Test
- public void refreshLifecycleReleasesFenceAndQuotaAtDone() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
-
- LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
- Assertions.assertTrue(manager.isFenceHeld(fenceKey));
- Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
-
- Assertions.assertTrue(manager.markRefreshRunning(1L, 2L));
- Assertions.assertEquals(LanceIndexJobRefreshState.RUNNING, manager.getJob(1L).getRefreshState());
- Assertions.assertTrue(manager.isFenceHeld(fenceKey));
-
- Assertions.assertTrue(manager.markRefreshDone(1L, 3L));
- LanceIndexJob stored = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobRefreshState.DONE, stored.getRefreshState());
- Assertions.assertEquals(4L, stored.getRevision());
- Assertions.assertFalse(manager.isFenceHeld(fenceKey));
- Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
- Assertions.assertTrue(manager.getUnresolvedJobs().isEmpty());
- Assertions.assertTrue(manager.getJobsNeedingRefresh().isEmpty());
- }
-
- @Test
- public void refreshFailureKeepsFenceAndRetriesThroughRunning() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
- LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
-
- Assertions.assertTrue(manager.markRefreshRunning(1L, 2L));
- Assertions.assertTrue(manager.markRefreshFailed(1L, 3L));
- LanceIndexJob failed = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobRefreshState.FAILED, failed.getRefreshState());
- Assertions.assertTrue(manager.isFenceHeld(fenceKey));
- Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
- Assertions.assertTrue(containsJob(manager.getUnresolvedJobs(), failed.getJobId()));
-
- // FAILED -> RUNNING is the retry entry through the idempotent refresh path.
- Assertions.assertTrue(manager.markRefreshRunning(1L, 4L));
- Assertions.assertEquals(LanceIndexJobRefreshState.RUNNING, manager.getJob(1L).getRefreshState());
- Assertions.assertTrue(manager.markRefreshDone(1L, 5L));
- Assertions.assertFalse(manager.isFenceHeld(fenceKey));
- Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
- }
-
- @Test
- public void refreshTransitionsRejectedFromNotRequired() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED)));
- Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, manager.getJob(1L).getRefreshState());
-
- Assertions.assertFalse(manager.markRefreshRunning(1L, 2L));
- Assertions.assertFalse(manager.markRefreshDone(1L, 2L));
- Assertions.assertFalse(manager.markRefreshFailed(1L, 2L));
- Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, manager.getJob(1L).getRefreshState());
- Assertions.assertEquals(2L, manager.getJob(1L).getRevision());
- }
-
- @Test
- public void refreshTransitionsRejectedFromDone() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
- Assertions.assertTrue(manager.markRefreshRunning(1L, 2L));
- Assertions.assertTrue(manager.markRefreshDone(1L, 3L));
-
- Assertions.assertFalse(manager.markRefreshRunning(1L, 4L));
- Assertions.assertFalse(manager.markRefreshDone(1L, 4L));
- Assertions.assertFalse(manager.markRefreshFailed(1L, 4L));
- Assertions.assertEquals(LanceIndexJobRefreshState.DONE, manager.getJob(1L).getRefreshState());
- }
-
- @Test
- public void refreshTransitionIsRevisionGuarded() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
-
- Assertions.assertFalse(manager.markRefreshRunning(1L, 99L));
- Assertions.assertFalse(manager.markRefreshRunning(404L, 2L));
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(1L).getRefreshState());
- Assertions.assertEquals(2L, manager.getJob(1L).getRevision());
- }
-
- @Test
- public void fenceAndQuotaReleasedImmediatelyWhenRefreshNotRequired() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
- LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
- Assertions.assertTrue(manager.isFenceHeld(fenceKey));
-
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.PRE_INVOCATION_STALE_ADMISSION)));
-
- Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, manager.getJob(1L).getMutationState());
- Assertions.assertFalse(manager.isFenceHeld(fenceKey));
- Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
- Assertions.assertEquals(0L, manager.getQuota().getCatalogCount(CATALOG_ID));
- Assertions.assertTrue(manager.getUnresolvedJobs().isEmpty());
- }
-
- @Test
- public void fenceAndQuotaSurviveUnknown() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
- LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
-
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_IO)));
-
- Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, manager.getJob(1L).getMutationState());
- Assertions.assertTrue(manager.isFenceHeld(fenceKey));
- Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
- Assertions.assertEquals(1, manager.getUnresolvedJobs().size());
- Assertions.assertTrue(manager.getJobsNeedingRefresh().isEmpty());
- }
-
- @Test
- public void terminationProofReleasesSlotOnly() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
- LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
- Assertions.assertTrue(manager.getJob(1L).holdsPossibleLiveSlot());
-
- Assertions.assertFalse(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
- LanceIndexTerminationProof.NONE));
- Assertions.assertFalse(manager.recordTerminationProof(1L, 99L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
- LanceIndexTerminationProof.CHILD_REAPED));
- Assertions.assertTrue(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
- LanceIndexTerminationProof.CHILD_REAPED));
-
- LanceIndexJob stored = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState());
- Assertions.assertEquals(LanceIndexTerminationProof.CHILD_REAPED, stored.getTerminationProof());
- Assertions.assertFalse(stored.holdsPossibleLiveSlot());
- Assertions.assertTrue(manager.isFenceHeld(fenceKey));
- Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
-
- // A slot may be proven exactly once.
- Assertions.assertFalse(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
- LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE));
- }
-
- @Test
- public void everyAcceptedTransitionWritesExactlyOneEditLogRecord() throws DdlException {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
- manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
- manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, result(LanceIndexJobResultCode.NATIVE_OK));
- manager.markRefreshRunning(1L, 2L);
- manager.markRefreshDone(1L, 3L);
- Assertions.assertEquals(5, manager.editLog.size());
-
- // Rejected transitions never reach the journal.
- Assertions.assertFalse(manager.markRefreshDone(1L, 4L));
- Assertions.assertFalse(manager.markRunning(1L, 4L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertEquals(5, manager.editLog.size());
- }
-
- @Test
- public void markRunningRejectsBlankInvocationId() throws DdlException {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
-
- // A null/blank invocation identity would match a null field under Objects.equals
- // in completeWithResult and silently defeat the stale-callback guard.
- Assertions.assertFalse(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, null, DEADLINE_MS));
- Assertions.assertFalse(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, "", DEADLINE_MS));
- Assertions.assertFalse(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, " \t\n", DEADLINE_MS));
-
- LanceIndexJob stored = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.PENDING, stored.getMutationState());
- Assertions.assertEquals(0L, stored.getRevision());
- Assertions.assertNull(stored.getInvocationId());
- Assertions.assertEquals(1, manager.editLog.size());
-
- // A well-formed dispatch is still accepted afterwards.
- Assertions.assertTrue(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(1L).getMutationState());
- }
-
- @Test
- public void failedRefreshJobStaysVisibleToTheRefreshDriver() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
- Assertions.assertTrue(manager.markRefreshRunning(1L, 2L));
- Assertions.assertTrue(manager.markRefreshFailed(1L, 3L));
-
- // FAILED still owes the idempotent retry: the driver must see the job.
- Assertions.assertTrue(containsJob(manager.getJobsNeedingRefresh(), 1L));
-
- Assertions.assertTrue(manager.markRefreshRunning(1L, 4L));
- Assertions.assertTrue(manager.markRefreshDone(1L, 5L));
- Assertions.assertTrue(manager.getJobsNeedingRefresh().isEmpty());
-
- // Terminal jobs with refresh DONE or NOT_REQUIRED never show up.
- createAndRun(manager, 2L, "IdxB");
- Assertions.assertTrue(manager.completeWithResult(2L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED)));
- Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, manager.getJob(2L).getRefreshState());
- Assertions.assertTrue(manager.getJobsNeedingRefresh().isEmpty());
- }
-
- @Test
- public void createJobResetsLifecycleFieldsAndPublishesAPrivateCopy() throws DdlException {
- LanceIndexJob dirty = newCreateJob(1L, "IdxA");
- dirty.setRevision(9L);
- dirty.setMutationState(LanceIndexJobMutationState.RUNNING);
- dirty.setRefreshState(LanceIndexJobRefreshState.RUNNING);
- dirty.setResult(result(LanceIndexJobResultCode.NATIVE_OK));
- dirty.setBackendId(BACKEND_ID);
- dirty.setBeProcessEpoch(BE_EPOCH);
- dirty.setInvocationId(INVOCATION_ID);
- dirty.setDeadlineMs(DEADLINE_MS);
- dirty.setPossibleLiveOwned(true);
- dirty.setTerminationProof(LanceIndexTerminationProof.CHILD_REAPED);
- dirty.setForceReleased(true);
- dirty.setForceActor("admin");
- dirty.setForceTimeMs(7L);
- dirty.setForceNote("note");
- dirty.setForceWarning("warning");
-
- TestManager manager = new TestManager();
- manager.createJob(dirty, 100, 100, 100);
-
- LanceIndexJob stored = manager.getJob(1L);
- Assertions.assertNotSame(dirty, stored);
- Assertions.assertEquals(LanceIndexJobMutationState.PENDING, stored.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState());
- Assertions.assertEquals(0L, stored.getRevision());
- Assertions.assertNull(stored.getResult());
- Assertions.assertNull(stored.getBackendId());
- Assertions.assertNull(stored.getBeProcessEpoch());
- Assertions.assertNull(stored.getInvocationId());
- Assertions.assertNull(stored.getDeadlineMs());
- Assertions.assertFalse(stored.isPossibleLiveOwned());
- Assertions.assertEquals(LanceIndexTerminationProof.NONE, stored.getTerminationProof());
- Assertions.assertFalse(stored.isForceReleased());
- Assertions.assertNull(stored.getForceActor());
- Assertions.assertNull(stored.getForceTimeMs());
- Assertions.assertNull(stored.getForceNote());
- Assertions.assertNull(stored.getForceWarning());
-
- // Mutating the caller's object after admission touches nothing inside the manager.
- dirty.setMutationState(LanceIndexJobMutationState.UNKNOWN);
- dirty.setNormalizedIndexName("idxb");
- Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState());
- Assertions.assertTrue(manager.isFenceHeld(newCreateJob(2L, "idxa").fenceKey()));
- Assertions.assertFalse(manager.isFenceHeld(newCreateJob(3L, "IdxB").fenceKey()));
- }
-
- @Test
- public void refreshDoneOrFailedRequiresAPrecedingRefreshRunning() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(1L).getRefreshState());
-
- // No direct REQUIRED -> DONE / FAILED: a refresh must actually run first.
- Assertions.assertFalse(manager.markRefreshDone(1L, 2L));
- Assertions.assertFalse(manager.markRefreshFailed(1L, 2L));
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(1L).getRefreshState());
- Assertions.assertEquals(2L, manager.getJob(1L).getRevision());
- Assertions.assertEquals(3, manager.editLog.size());
- }
-
- @Test
- public void terminationProofNeedsASlotAndStillLandsAfterTheTerminalResult() throws DdlException {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
-
- // PENDING owns no possible-live slot: a proof has nothing to release.
- Assertions.assertFalse(manager.recordTerminationProof(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
- LanceIndexTerminationProof.CHILD_REAPED));
- Assertions.assertEquals(0L, manager.getJob(1L).getRevision());
- Assertions.assertEquals(1, manager.editLog.size());
-
- Assertions.assertTrue(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
- // The terminal outcome does not release the slot; the proof still lands afterwards.
- LanceIndexJob committed = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, committed.getMutationState());
- Assertions.assertTrue(committed.holdsPossibleLiveSlot());
- LanceIndexFenceKey fenceKey = committed.fenceKey();
-
- Assertions.assertTrue(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
- LanceIndexTerminationProof.CHILD_REAPED));
- LanceIndexJob proven = manager.getJob(1L);
- Assertions.assertFalse(proven.holdsPossibleLiveSlot());
- // Fence and quota still follow the refresh rule, not the proof.
- Assertions.assertTrue(manager.isFenceHeld(fenceKey));
- Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
-
- Assertions.assertTrue(manager.markRefreshRunning(1L, 3L));
- Assertions.assertTrue(manager.markRefreshDone(1L, 4L));
- Assertions.assertFalse(manager.isFenceHeld(fenceKey));
- Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
- }
-
- @Test
- public void terminationProofAndResultUseTheImmutableDispatchRevisionInEitherOrder() throws DdlException {
- TestManager proofFirst = new TestManager();
- createAndRun(proofFirst, 1L, "IdxA");
- Assertions.assertEquals(1L, proofFirst.getJob(1L).getDispatchRevision());
-
- Assertions.assertFalse(proofFirst.recordTerminationProof(1L, 1L, BACKEND_ID + 1, BE_EPOCH,
- INVOCATION_ID, LanceIndexTerminationProof.CHILD_REAPED));
- Assertions.assertFalse(proofFirst.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH + 1,
- INVOCATION_ID, LanceIndexTerminationProof.CHILD_REAPED));
- Assertions.assertFalse(proofFirst.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH,
- "wrong-invocation", LanceIndexTerminationProof.CHILD_REAPED));
- Assertions.assertEquals(1L, proofFirst.getJob(1L).getRevision());
-
- Assertions.assertTrue(proofFirst.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
- LanceIndexTerminationProof.CHILD_REAPED));
- Assertions.assertEquals(2L, proofFirst.getJob(1L).getRevision());
- Assertions.assertEquals(1L, proofFirst.getJob(1L).getDispatchRevision());
- Assertions.assertTrue(proofFirst.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
- Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED,
- proofFirst.getJob(1L).getMutationState());
-
- TestManager resultFirst = new TestManager();
- createAndRun(resultFirst, 2L, "IdxB");
- Assertions.assertTrue(resultFirst.completeWithResult(2L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
- Assertions.assertTrue(resultFirst.recordTerminationProof(2L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
- LanceIndexTerminationProof.CHILD_REAPED));
- Assertions.assertFalse(resultFirst.getJob(2L).holdsPossibleLiveSlot());
- Assertions.assertEquals(1L, resultFirst.getJob(2L).getDispatchRevision());
-
- TestManager legacyResultFirst = new TestManager();
- createAndRun(legacyResultFirst, 3L, "IdxC");
- LanceIndexJob legacyRunning = legacyResultFirst.getJob(3L);
- legacyRunning.setDispatchRevision(null);
- legacyResultFirst.replayUpsertJob(legacyRunning);
- Assertions.assertTrue(legacyResultFirst.completeWithResult(3L, 1L, INVOCATION_ID, BE_EPOCH,
- result(LanceIndexJobResultCode.NATIVE_OK)));
- Assertions.assertEquals(1L, legacyResultFirst.getJob(3L).getDispatchRevision());
- Assertions.assertTrue(legacyResultFirst.recordTerminationProof(3L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
- LanceIndexTerminationProof.CHILD_REAPED));
- Assertions.assertFalse(legacyResultFirst.getJob(3L).holdsPossibleLiveSlot());
- }
-
- @Test
- public void refreshTransitionsRejectNonTerminalMutationState() {
- TestManager manager = new TestManager();
- LanceIndexJob corrupt = newCreateJob(1L, "IdxA");
- corrupt.setMutationState(LanceIndexJobMutationState.PENDING);
- corrupt.setRefreshState(LanceIndexJobRefreshState.REQUIRED);
- manager.replayUpsertJob(corrupt);
-
- Assertions.assertFalse(manager.markRefreshRunning(1L, 0L));
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(1L).getRefreshState());
- Assertions.assertTrue(manager.editLog.isEmpty());
- }
-
- @Test
- public void admissionRejectsNonPositiveQuotaLimitsBeforeDurableWrite() {
- TestManager manager = new TestManager();
- Assertions.assertThrows(DdlException.class,
- () -> manager.createJob(newCreateJob(1L, "IdxA"), 0, 1, 1));
- Assertions.assertThrows(DdlException.class,
- () -> manager.createJob(newCreateJob(1L, "IdxA"), 1, 0, 1));
- Assertions.assertThrows(DdlException.class,
- () -> manager.createJob(newCreateJob(1L, "IdxA"), 1, 1, -1));
- Assertions.assertEquals(0, manager.getJobCount());
- Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
- Assertions.assertTrue(manager.editLog.isEmpty());
- }
-
- @Test
- public void notCommittedWithRefreshRequiredReleasesFenceAtRefreshDone() throws DdlException {
- TestManager manager = new TestManager();
- createAndRun(manager, 1L, "IdxA");
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_COMMIT_CONFLICT,
- LanceIndexJobCompletionReason.NONE, "commit conflict", false)));
-
- LanceIndexJob stored = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, stored.getMutationState());
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, stored.getRefreshState());
- LanceIndexFenceKey fenceKey = stored.fenceKey();
- Assertions.assertTrue(manager.isFenceHeld(fenceKey));
- Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
- Assertions.assertTrue(containsJob(manager.getJobsNeedingRefresh(), stored.getJobId()));
-
- Assertions.assertTrue(manager.markRefreshRunning(1L, 2L));
- Assertions.assertTrue(manager.markRefreshDone(1L, 3L));
- Assertions.assertFalse(manager.isFenceHeld(fenceKey));
- Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
- Assertions.assertTrue(manager.getUnresolvedJobs().isEmpty());
- }
-
- private static LanceIndexJob newCreateJob(long jobId, String displayName) {
- return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
- displayName, LanceIndexNameNormalizer.normalize(displayName),
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v",
- null, 7L, null);
- }
-
- private static boolean containsJob(List<LanceIndexJob> jobs, long jobId) {
- return jobs.stream().anyMatch(job -> job.getJobId() == jobId);
- }
-
- private static LanceIndexJob newDropJob(long jobId, String displayName, boolean ifExists) {
- return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
- displayName, LanceIndexNameNormalizer.normalize(displayName),
- LanceIndexJobMutationType.DROP, false, ifExists, null, "v",
- null, 7L, null);
- }
-
- private static LanceIndexJobResult result(LanceIndexJobResultCode code) {
- return new LanceIndexJobResult(code, LanceIndexJobCompletionReason.NONE, "sanitized message", false);
- }
-
- private static LanceIndexJob createAndRun(TestManager manager, long jobId, String displayName) throws DdlException {
- manager.createJob(newCreateJob(jobId, displayName), 100, 100, 100);
- Assertions.assertTrue(manager.markRunning(jobId, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
- return manager.getJob(jobId);
- }
-
- /**
- * Edit-log seam: captures every durable record instead of writing the journal.
- */
- private static class TestManager extends LanceIndexJobManager {
- private final List<LanceIndexJob> editLog = new ArrayList<>();
-
- @Override
- protected void writeEditLog(LanceIndexJob job) {
- editLog.add(job);
- }
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java
deleted file mode 100644
index 1f61b184d19..00000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java
+++ /dev/null
@@ -1,303 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import org.apache.doris.persist.gson.GsonUtils;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Record-level coverage for {@link LanceIndexJob}: the derived fence and quota keys,
- * the full {@link LanceIndexJob#isUnresolved()} matrix that keeps fence and quota
- * alive together, the possible-live slot semantics (released only by a matching
- * termination proof or a durable FORCE_RELEASE, never by a deadline), corrupt-record
- * fallbacks toward the safe direction, and the deep-copy constructor.
- */
-public class LanceIndexJobTest {
- private static final long CATALOG_ID = 10L;
- private static final String LOCATOR = "s3://bucket/dataset";
- private static final long BACKEND_ID = 1001L;
- private static final long BE_EPOCH = 55L;
- private static final String INVOCATION_ID = "invocation-1";
-
- @Test
- public void fenceKeyCarriesIdentityAndHidesLocator() {
- LanceIndexJob job = newCreateJob(1L, "IdxA");
- LanceIndexFenceKey key = job.fenceKey();
-
- Assertions.assertEquals(new LanceIndexFenceKey(CATALOG_ID, LanceIndexFenceKey.PROVIDER_DIRECTORY,
- LOCATOR, "idxa"), key);
- Assertions.assertEquals(new LanceIndexFenceKey(CATALOG_ID, LanceIndexFenceKey.PROVIDER_DIRECTORY,
- LOCATOR, "idxa").hashCode(), key.hashCode());
- Assertions.assertNotEquals(new LanceIndexFenceKey(20L, LanceIndexFenceKey.PROVIDER_DIRECTORY,
- LOCATOR, "idxa"), key);
- Assertions.assertNotEquals(new LanceIndexFenceKey(CATALOG_ID, "REST", LOCATOR, "idxa"), key);
- Assertions.assertNotEquals(new LanceIndexFenceKey(CATALOG_ID, LanceIndexFenceKey.PROVIDER_DIRECTORY,
- "s3://bucket/other", "idxa"), key);
- Assertions.assertNotEquals(new LanceIndexFenceKey(CATALOG_ID, LanceIndexFenceKey.PROVIDER_DIRECTORY,
- LOCATOR, "idxb"), key);
-
- // Fence-conflict messages may surface to users without target privileges.
- Assertions.assertFalse(key.toString().contains(LOCATOR));
- Assertions.assertTrue(key.toString().contains("idxa"));
- }
-
- @Test
- public void fenceKeyIsStableAcrossJobsWithTheSameIdentity() {
- LanceIndexJob first = newCreateJob(1L, "IdxA");
- LanceIndexJob second = newCreateJob(2L, "idxa");
- // Display case differs, normalized identity does not.
- Assertions.assertEquals(first.fenceKey(), second.fenceKey());
- Assertions.assertEquals(first.fenceKey().hashCode(), second.fenceKey().hashCode());
- }
-
- @Test
- public void tableQuotaKeyDerivesFromCatalogAndLocator() {
- LanceIndexJob job = newCreateJob(1L, "IdxA");
- Assertions.assertEquals(new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR), job.getTableQuotaKey());
- }
-
- @Test
- public void activeStatesAreAlwaysUnresolved() {
- for (LanceIndexJobRefreshState refresh : LanceIndexJobRefreshState.values()) {
- Assertions.assertTrue(jobInState(LanceIndexJobMutationState.PENDING, refresh, false).isUnresolved());
- Assertions.assertTrue(jobInState(LanceIndexJobMutationState.RUNNING, refresh, false).isUnresolved());
- }
- }
-
- @Test
- public void knownTerminalStatesFollowTheRefreshState() {
- for (LanceIndexJobMutationState terminal : new LanceIndexJobMutationState[]{
- LanceIndexJobMutationState.COMMITTED, LanceIndexJobMutationState.NOT_COMMITTED}) {
- Assertions.assertFalse(jobInState(terminal, LanceIndexJobRefreshState.NOT_REQUIRED, false).isUnresolved());
- Assertions.assertFalse(jobInState(terminal, LanceIndexJobRefreshState.DONE, false).isUnresolved());
- Assertions.assertTrue(jobInState(terminal, LanceIndexJobRefreshState.REQUIRED, false).isUnresolved());
- Assertions.assertTrue(jobInState(terminal, LanceIndexJobRefreshState.RUNNING, false).isUnresolved());
- // A failed refresh still holds the fence: the retry goes through the idempotent path.
- Assertions.assertTrue(jobInState(terminal, LanceIndexJobRefreshState.FAILED, false).isUnresolved());
- }
- }
-
- @Test
- public void unknownIsUnresolvedUntilForceReleased() {
- for (LanceIndexJobRefreshState refresh : LanceIndexJobRefreshState.values()) {
- Assertions.assertTrue(jobInState(LanceIndexJobMutationState.UNKNOWN, refresh, false).isUnresolved());
- Assertions.assertFalse(jobInState(LanceIndexJobMutationState.UNKNOWN, refresh, true).isUnresolved());
- }
- }
-
- @Test
- public void nullStatesFallBackToTheSafeDirection() {
- // A corrupt record missing its states must keep the fence (treated as UNKNOWN)
- // and must never become redispatchable PENDING.
- LanceIndexJob job = GsonUtils.GSON.fromJson(
- "{\"jid\":1,\"ms\":null,\"rs\":null}", LanceIndexJob.class);
- Assertions.assertNull(job.getMutationState());
- Assertions.assertNull(job.getRefreshState());
- Assertions.assertTrue(job.isUnresolved());
-
- LanceIndexJob forced = GsonUtils.GSON.fromJson(
- "{\"jid\":1,\"ms\":null,\"rs\":null,\"fr\":true}", LanceIndexJob.class);
- Assertions.assertFalse(forced.isUnresolved());
-
- // A terminal record missing its refresh state owes one (treated as REQUIRED).
- LanceIndexJob committedNoRefresh = GsonUtils.GSON.fromJson(
- "{\"jid\":1,\"ms\":\"COMMITTED\",\"rs\":null}", LanceIndexJob.class);
- Assertions.assertTrue(committedNoRefresh.isUnresolved());
- }
-
- @Test
- public void missingRefreshStateKeyFallsBackToRequired() {
- // A corrupt terminal record without the "rs" key at all keeps the fence: the
- // field initial value is REQUIRED, the same safe direction as the null fallback.
- LanceIndexJob missingKey = GsonUtils.GSON.fromJson(
- "{\"jid\":1,\"ms\":\"COMMITTED\"}", LanceIndexJob.class);
- Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, missingKey.getRefreshState());
- Assertions.assertTrue(missingKey.isUnresolved());
-
- // A legal terminal record always carries the key explicitly.
- LanceIndexJob legal = GsonUtils.GSON.fromJson(
- "{\"jid\":1,\"ms\":\"COMMITTED\",\"rs\":\"NOT_REQUIRED\"}", LanceIndexJob.class);
- Assertions.assertFalse(legal.isUnresolved());
- }
-
- @Test
- public void possibleLiveSlotMatrix() {
- LanceIndexJob job = new LanceIndexJob();
- Assertions.assertFalse(job.holdsPossibleLiveSlot());
-
- job.setPossibleLiveOwned(true);
- Assertions.assertTrue(job.holdsPossibleLiveSlot());
-
- job.setTerminationProof(LanceIndexTerminationProof.CHILD_REAPED);
- Assertions.assertFalse(job.holdsPossibleLiveSlot());
-
- job.setTerminationProof(LanceIndexTerminationProof.NONE);
- job.setForceReleased(true);
- Assertions.assertFalse(job.holdsPossibleLiveSlot());
-
- // A corrupt record missing the proof is treated as NONE (slot still owned).
- LanceIndexJob nullProof = GsonUtils.GSON.fromJson(
- "{\"jid\":1,\"plo\":true,\"tp\":null}", LanceIndexJob.class);
- Assertions.assertNull(nullProof.getTerminationProof());
- Assertions.assertTrue(nullProof.holdsPossibleLiveSlot());
- }
-
- @Test
- public void terminationProofClearsSlotButKeepsFenceAndOutcome() throws Exception {
- TestManager manager = new TestManager();
- manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
- manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, 9999L);
- LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
-
- Assertions.assertTrue(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
- LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE));
- LanceIndexJob proven = manager.getJob(1L);
- Assertions.assertFalse(proven.holdsPossibleLiveSlot());
- Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, proven.getMutationState());
- Assertions.assertTrue(manager.isFenceHeld(fenceKey));
- Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
-
- // The ambiguous result still lands afterwards: UNKNOWN keeps the fence, and the
- // already-recorded proof keeps the slot released.
- Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
- new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT,
- LanceIndexJobCompletionReason.NONE, "ambiguous", false)));
- LanceIndexJob unknown = manager.getJob(1L);
- Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, unknown.getMutationState());
- Assertions.assertFalse(unknown.holdsPossibleLiveSlot());
- Assertions.assertTrue(unknown.isUnresolved());
- Assertions.assertTrue(manager.isFenceHeld(fenceKey));
- }
-
- @Test
- public void copyConstructorDuplicatesEveryFieldIndependently() {
- LanceIndexJob original = newCreateJob(1L, "IdxA");
- original.setRevision(3L);
- original.setMutationState(LanceIndexJobMutationState.RUNNING);
- original.setRefreshState(LanceIndexJobRefreshState.RUNNING);
- original.setResult(new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
- LanceIndexJobCompletionReason.NONE, "ok", false));
- original.setBackendId(BACKEND_ID);
- original.setBeProcessEpoch(BE_EPOCH);
- original.setDispatchRevision(1L);
- original.setInvocationId(INVOCATION_ID);
- original.setDeadlineMs(123L);
- original.setPossibleLiveOwned(true);
- original.setForceActor("admin");
- original.setForceTimeMs(9L);
- original.setForceNote("note");
- original.setForceWarning("warning");
-
- LanceIndexJob copy = new LanceIndexJob(original);
- Assertions.assertEquals(GsonUtils.GSON.toJson(original), GsonUtils.GSON.toJson(copy));
-
- copy.setRevision(99L);
- copy.setMutationState(LanceIndexJobMutationState.UNKNOWN);
- copy.setPossibleLiveOwned(false);
- Assertions.assertEquals(3L, original.getRevision());
- Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, original.getMutationState());
- Assertions.assertTrue(original.isPossibleLiveOwned());
- }
-
- @Test
- public void admissionConstructorRejectsNullIdentity() {
- Assertions.assertThrows(NullPointerException.class,
- () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
- null, LOCATOR, "IdxA", "idxa",
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
- Assertions.assertThrows(NullPointerException.class,
- () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, null, "IdxA", "idxa",
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
- Assertions.assertThrows(NullPointerException.class,
- () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, "IdxA", "idxa",
- null, false, false, "IVF_PQ", "v", null, 7L, null));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, null, "idxa",
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, "IdxA", null,
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
- }
-
- @Test
- public void admissionRejectsPseudoCanonicalFenceIdentity() {
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
- "directory", LOCATOR, "IdxA", "idxa",
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, "S3://bucket/dataset/", "IdxA", "idxa",
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, "IdxA", "IdxA",
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY,
- "https://bucket.example/ds?X-Amz-Signature=secret", "IdxA", "idxa",
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
- }
-
- @Test
- public void toStringNamesTheIndexButHidesTheLocator() {
- LanceIndexJob job = newCreateJob(1L, "IdxA");
- Assertions.assertTrue(job.toString().contains("IdxA"));
- Assertions.assertFalse(job.toString().contains("bucket"));
- Assertions.assertFalse(job.toString().contains(LOCATOR));
- }
-
- private static LanceIndexJob newCreateJob(long jobId, String displayName) {
- return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
- displayName, LanceIndexNameNormalizer.normalize(displayName),
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v",
- null, 7L, null);
- }
-
- private static LanceIndexJob jobInState(LanceIndexJobMutationState mutationState,
- LanceIndexJobRefreshState refreshState, boolean forceReleased) {
- LanceIndexJob job = new LanceIndexJob();
- job.setMutationState(mutationState);
- job.setRefreshState(refreshState);
- job.setForceReleased(forceReleased);
- return job;
- }
-
- /**
- * Edit-log seam: captures every durable record instead of writing the journal.
- */
- private static class TestManager extends LanceIndexJobManager {
- private final List<LanceIndexJob> editLog = new ArrayList<>();
-
- @Override
- protected void writeEditLog(LanceIndexJob job) {
- editLog.add(job);
- }
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java
deleted file mode 100644
index ecbd2b42799..00000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java
+++ /dev/null
@@ -1,78 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.lance.job;
-
-import org.apache.doris.catalog.Env;
-import org.apache.doris.common.io.CountingDataOutputStream;
-import org.apache.doris.persist.OperationType;
-import org.apache.doris.persist.meta.MetaPersistMethod;
-import org.apache.doris.persist.meta.PersistMetaModules;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.io.DataInputStream;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
-import java.lang.reflect.Modifier;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-public class LanceIndexJobWiringTest {
- private static final short LANCE_INDEX_JOB_OPCODE = 500;
- private static final String LANCE_INDEX_JOB_MODULE = "lanceIndexJobManager";
-
- @Test
- public void lanceIndexJobOpcodeIsUniquelyAssigned() throws IllegalAccessException {
- List<String> fieldsUsingOpcode = new ArrayList<>();
- for (Field field : OperationType.class.getDeclaredFields()) {
- if (Modifier.isStatic(field.getModifiers())
- && field.getType() == short.class
- && field.getShort(null) == LANCE_INDEX_JOB_OPCODE) {
- fieldsUsingOpcode.add(field.getName());
- }
- }
-
- Assertions.assertEquals(
- Collections.singletonList("OP_LANCE_INDEX_JOB_UPSERT"),
- fieldsUsingOpcode,
- "operation code 500 must remain uniquely assigned to Lance index job upserts");
- Assertions.assertEquals(LANCE_INDEX_JOB_OPCODE, OperationType.OP_LANCE_INDEX_JOB_UPSERT);
- }
-
- @Test
- public void lanceIndexJobManagerIsTheLastBaseImageModuleWithEnvBindings() throws Exception {
- Assertions.assertEquals(
- LANCE_INDEX_JOB_MODULE,
- PersistMetaModules.MODULE_NAMES.get(PersistMetaModules.MODULE_NAMES.size() - 1),
- "new image modules must be appended without reordering existing base modules");
-
- MetaPersistMethod persistMethod = PersistMetaModules.MODULES_MAP.get(LANCE_INDEX_JOB_MODULE);
- Assertions.assertNotNull(persistMethod);
-
- Method expectedReadMethod = Env.class.getDeclaredMethod(
- "loadLanceIndexJobManager", DataInputStream.class, long.class);
- Method expectedWriteMethod = Env.class.getDeclaredMethod(
- "saveLanceIndexJobManager", CountingDataOutputStream.class, long.class);
- Assertions.assertEquals(expectedReadMethod, persistMethod.readMethod);
- Assertions.assertEquals(expectedWriteMethod, persistMethod.writeMethod);
- Assertions.assertEquals(long.class, persistMethod.readMethod.getReturnType());
- Assertions.assertEquals(long.class, persistMethod.writeMethod.getReturnType());
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ShowLanceIndexJobsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ShowLanceIndexJobsParserTest.java
deleted file mode 100644
index ef98991e534..00000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ShowLanceIndexJobsParserTest.java
+++ /dev/null
@@ -1,183 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.parser;
-
-import org.apache.doris.nereids.analyzer.UnboundResultSink;
-import org.apache.doris.nereids.analyzer.UnboundSlot;
-import org.apache.doris.nereids.trees.expressions.And;
-import org.apache.doris.nereids.trees.expressions.EqualTo;
-import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.Like;
-import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
-import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.commands.ShowLanceIndexJobCommand;
-import org.apache.doris.nereids.trees.plans.commands.ShowLanceIndexJobsCommand;
-import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.util.Arrays;
-
-/**
- * Parser coverage for SHOW LANCE INDEX JOBS / SHOW LANCE INDEX JOB: all FROM and WHERE
- * forms, plus the guarantee that the new LANCE token stays non-reserved and can still be
- * used as an identifier. Illegal WHERE shapes that the grammar accepts as generic
- * expressions are rejected by the command, covered in ShowLanceIndexJobsCommandTest.
- */
-public class ShowLanceIndexJobsParserTest extends ParserTestBase {
- private final NereidsParser parser = new NereidsParser();
-
- private ShowLanceIndexJobsCommand parseJobs(String sql) {
- Plan plan = parser.parseSingle(sql);
- return Assertions.assertInstanceOf(ShowLanceIndexJobsCommand.class, plan);
- }
-
- private static void assertEqualToPredicate(Expression where, String slotName, String literalValue) {
- EqualTo equalTo = Assertions.assertInstanceOf(EqualTo.class, where);
- UnboundSlot slot = Assertions.assertInstanceOf(UnboundSlot.class, equalTo.child(0));
- Assertions.assertEquals(Arrays.asList(slotName), slot.getNameParts());
- // The parser hands out VarcharLiteral for ordinary-length strings; assert the common
- // string-literal base so the test states the grammar contract, not the length heuristic.
- StringLikeLiteral literal = Assertions.assertInstanceOf(StringLikeLiteral.class, equalTo.child(1));
- Assertions.assertEquals(literalValue, literal.getStringValue());
- }
-
- @Test
- public void testShowLanceIndexJobsBare() {
- ShowLanceIndexJobsCommand command = parseJobs("SHOW LANCE INDEX JOBS");
- Assertions.assertNull(command.getNameParts());
- Assertions.assertNull(command.getWhereClause());
- }
-
- @Test
- public void testShowLanceIndexJobsFromDb() {
- ShowLanceIndexJobsCommand command = parseJobs("SHOW LANCE INDEX JOBS FROM db1");
- Assertions.assertEquals(Arrays.asList("db1"), command.getNameParts());
-
- command = parseJobs("SHOW LANCE INDEX JOBS IN db1");
- Assertions.assertEquals(Arrays.asList("db1"), command.getNameParts());
- }
-
- @Test
- public void testShowLanceIndexJobsFromCatalogDb() {
- ShowLanceIndexJobsCommand command = parseJobs("SHOW LANCE INDEX JOBS FROM ctl1.db1");
- Assertions.assertEquals(Arrays.asList("ctl1", "db1"), command.getNameParts());
- }
-
- @Test
- public void testShowLanceIndexJobsWhere() {
- ShowLanceIndexJobsCommand command = parseJobs("SHOW LANCE INDEX JOBS WHERE TableName = \"tbl1\"");
- assertEqualToPredicate(command.getWhereClause(), "TableName", "tbl1");
-
- command = parseJobs("SHOW LANCE INDEX JOBS WHERE State = \"PENDING\"");
- assertEqualToPredicate(command.getWhereClause(), "State", "PENDING");
- }
-
- @Test
- public void testShowLanceIndexJobsWhereLikeParses() {
- // The LIKE syntax parses into a Like expression; rejecting it is a command-level
- // narrowing (ShowLanceIndexJobsCommandTest covers the rejection with hand-built
- // expressions), so this pins the grammar side of that split.
- ShowLanceIndexJobsCommand command = parseJobs("SHOW LANCE INDEX JOBS WHERE TableName LIKE \"t%\"");
- Like like = Assertions.assertInstanceOf(Like.class, command.getWhereClause());
- UnboundSlot slot = Assertions.assertInstanceOf(UnboundSlot.class, like.child(0));
- Assertions.assertEquals(Arrays.asList("TableName"), slot.getNameParts());
- StringLikeLiteral pattern = Assertions.assertInstanceOf(StringLikeLiteral.class, like.child(1));
- Assertions.assertEquals("t%", pattern.getStringValue());
- }
-
- @Test
- public void testShowLanceIndexJobsWhereReversedLiteralParses() {
- // A generic boolean expression: the grammar does not order slot vs literal, the command
- // does (the reversed shape is one of the WHERE forms the command rejects).
- ShowLanceIndexJobsCommand command = parseJobs("SHOW LANCE INDEX JOBS WHERE \"t\" = TableName");
- EqualTo equalTo = Assertions.assertInstanceOf(EqualTo.class, command.getWhereClause());
- StringLikeLiteral literal = Assertions.assertInstanceOf(StringLikeLiteral.class, equalTo.child(0));
- Assertions.assertEquals("t", literal.getStringValue());
- UnboundSlot slot = Assertions.assertInstanceOf(UnboundSlot.class, equalTo.child(1));
- Assertions.assertEquals(Arrays.asList("TableName"), slot.getNameParts());
- }
-
- @Test
- public void testShowLanceIndexJobsInCatalogDbWithCombinedWhere() {
- ShowLanceIndexJobsCommand command = parseJobs("SHOW LANCE INDEX JOBS IN ctl1.db1 "
- + "WHERE TableName = \"t\" AND State = \"PENDING\"");
- Assertions.assertEquals(Arrays.asList("ctl1", "db1"), command.getNameParts());
- And where = Assertions.assertInstanceOf(And.class, command.getWhereClause());
- assertEqualToPredicate(where.child(0), "TableName", "t");
- assertEqualToPredicate(where.child(1), "State", "PENDING");
- }
-
- @Test
- public void testShowLanceIndexJobsLowercaseParses() {
- // The lexer stream is case-insensitive; the new keywords must behave like the old ones.
- ShowLanceIndexJobsCommand command = parseJobs(
- "show lance index jobs from ctl1.db1 where TableName = \"t\" and State = \"pending\"");
- Assertions.assertEquals(Arrays.asList("ctl1", "db1"), command.getNameParts());
- Assertions.assertNotNull(command.getWhereClause());
- }
-
- @Test
- public void testShowLanceIndexJobParses() {
- Plan plan = parser.parseSingle("SHOW LANCE INDEX JOB 123");
- ShowLanceIndexJobCommand command = Assertions.assertInstanceOf(ShowLanceIndexJobCommand.class, plan);
- Assertions.assertEquals(123L, command.getJobId());
- }
-
- @Test
- public void testFromWithTooManyPartsStillParses() {
- // The grammar accepts any multipart identifier; the command rejects more than two parts.
- ShowLanceIndexJobsCommand command = parseJobs("SHOW LANCE INDEX JOBS FROM ctl1.db1.tbl1");
- Assertions.assertEquals(3, command.getNameParts().size());
- }
-
- @Test
- public void testIllegalWhereShapesStillParse() {
- // The grammar parses any boolean expression; shape validation lives in the command.
- Assertions.assertNotNull(parseJobs(
- "SHOW LANCE INDEX JOBS WHERE TableName = \"t\" OR State = \"PENDING\"").getWhereClause());
- Assertions.assertNotNull(parseJobs(
- "SHOW LANCE INDEX JOBS WHERE Foo = \"t\"").getWhereClause());
- Assertions.assertNotNull(parseJobs(
- "SHOW LANCE INDEX JOBS WHERE TableName = \"t\" AND tablename = \"t\"").getWhereClause());
- Assertions.assertNotNull(parseJobs(
- "SHOW LANCE INDEX JOBS WHERE State = \"BOGUS\"").getWhereClause());
- }
-
- @Test
- public void testLanceStaysUsableAsIdentifier() {
- // LANCE is added as a non-reserved keyword: it must still work as a column name.
- Plan plan = parser.parseSingle("SELECT lance FROM t");
- UnboundResultSink<?> sink = Assertions.assertInstanceOf(UnboundResultSink.class, plan);
- Assertions.assertInstanceOf(LogicalProject.class, sink.child());
-
- Assertions.assertNotNull(parser.parseSingle("SELECT lance FROM t WHERE lance > 0"));
- }
-
- @Test
- public void testLanceStaysUsableAsDatabaseAndCatalogName() {
- // Non-reserved in qualified names too: db and catalog parts named lance parse as such,
- // not as the LANCE token of the SHOW LANCE INDEX JOBS prefix.
- ShowLanceIndexJobsCommand command = parseJobs("SHOW LANCE INDEX JOBS FROM lance");
- Assertions.assertEquals(Arrays.asList("lance"), command.getNameParts());
-
- command = parseJobs("SHOW LANCE INDEX JOBS FROM lance.lance");
- Assertions.assertEquals(Arrays.asList("lance", "lance"), command.getNameParts());
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandLanceAdmissionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandLanceAdmissionTest.java
index 8ac5fc15cc7..a1736feccc1 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandLanceAdmissionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandLanceAdmissionTest.java
@@ -38,15 +38,10 @@ import org.apache.doris.datasource.lance.LanceExternalTable;
import org.apache.doris.datasource.lance.LanceIndexAdmissionSnapshot;
import org.apache.doris.datasource.lance.LanceIndexAdmissionSnapshot.PhysicalIndexInfo;
import org.apache.doris.datasource.lance.index.LanceShowIndexInfo;
-import org.apache.doris.datasource.lance.job.LanceIndexJob;
-import org.apache.doris.datasource.lance.job.LanceIndexJobManager;
-import org.apache.doris.datasource.lance.job.LanceIndexJobMutationType;
import org.apache.doris.mysql.privilege.AccessControllerManager;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.ResultSet;
-import org.apache.doris.qe.ShowResultSet;
import org.apache.doris.qe.StmtExecutor;
import org.apache.arrow.vector.types.FloatingPointPrecision;
@@ -58,7 +53,6 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.lance.schema.LanceField;
-import org.mockito.ArgumentCaptor;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
@@ -66,14 +60,14 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
/**
* Command rewiring coverage for Lance index admission: with the gate off the typed rejection is
* 5102 from {@code validate()} without any metadata read or id allocation; with the gate on the
* top-level CREATE/DROP INDEX op bypasses the generic op loop and {@code alterTable}, runs
- * admission in {@code run()}, and answers a single-column {@code JobId} result set (one row when
- * admitted, zero rows on an IF no-op). The 3A static-validation and REST/alter=true rejections
+ * admission in {@code run()}, completes an IF preflight no-op as a plain success, and rejects a
+ * mutation that would be admitted with the shared not-supported error (the synchronous
+ * execution path has not landed). The 3A static-validation and REST/alter=true rejections
* are unchanged in both modes.
*/
public class AlterTableCommandLanceAdmissionTest {
@@ -109,19 +103,9 @@ public class AlterTableCommandLanceAdmissionTest {
ConnectContext.remove();
}
- /** Edit-log seam: captures durable records instead of writing the journal (3B pattern). */
- private static class TestManager extends LanceIndexJobManager {
- private final List<LanceIndexJob> editLog = new ArrayList<>();
-
- @Override
- protected void writeEditLog(LanceIndexJob job) {
- editLog.add(job);
- }
- }
-
/**
* The 3A LanceFixture, extended for admission: remote/local names, the snapshot loader
- * entry point, real catalog/job managers behind the Env mock, and a mocked id allocator.
+ * entry point, and a real catalog manager behind the Env mock.
*/
private static class LanceFixture implements AutoCloseable {
private final MockedStatic<Env> mockedEnv;
@@ -130,8 +114,6 @@ public class AlterTableCommandLanceAdmissionTest {
private final LanceExternalCatalog catalog;
private final LanceExternalDatabase database;
private final LanceExternalTable table;
- private final TestManager manager;
- private final AtomicLong idAllocator;
private final StmtExecutor executor;
LanceFixture(boolean restCatalog) throws Exception {
@@ -142,8 +124,6 @@ public class AlterTableCommandLanceAdmissionTest {
catalog = Mockito.mock(LanceExternalCatalog.class);
database = Mockito.mock(LanceExternalDatabase.class);
table = Mockito.mock(LanceExternalTable.class);
- manager = new TestManager();
- idAllocator = new AtomicLong(100L);
executor = Mockito.mock(StmtExecutor.class);
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
@@ -162,8 +142,6 @@ public class AlterTableCommandLanceAdmissionTest {
Mockito.when(database.getFullName()).thenReturn(DB);
Mockito.when(table.getRemoteName()).thenReturn(REMOTE_TBL);
Mockito.when(table.getName()).thenReturn(TBL);
- Mockito.when(env.getNextId()).thenAnswer(invocation -> idAllocator.incrementAndGet());
- Mockito.when(env.getLanceIndexJobManager()).thenReturn(manager);
Mockito.when(table.getColumn(Mockito.anyString())).thenAnswer(invocation -> {
String name = invocation.getArgument(0);
if ("v".equalsIgnoreCase(name)) {
@@ -294,11 +272,6 @@ public class AlterTableCommandLanceAdmissionTest {
return command;
}
- private static void assertSingleJobIdColumn(ResultSet resultSet) {
- Assertions.assertEquals(1, resultSet.getMetaData().getColumnCount());
- Assertions.assertEquals("JobId", resultSet.getMetaData().getColumns().get(0).getName());
- }
-
// ------------------------------------------------------------------
// Gate off: 5102 from validate(), no metadata read, no id allocation
// ------------------------------------------------------------------
@@ -322,8 +295,6 @@ public class AlterTableCommandLanceAdmissionTest {
.loadTableIndexAdmissionSnapshot(Mockito.anyString(), Mockito.anyString());
Mockito.verify(fixture.env, Mockito.never()).getNextId();
Mockito.verify(fixture.executor, Mockito.never()).sendResultSet(Mockito.any());
- Assertions.assertEquals(0, fixture.manager.getJobCount());
- Assertions.assertTrue(fixture.manager.editLog.isEmpty());
}
}
@@ -337,8 +308,6 @@ public class AlterTableCommandLanceAdmissionTest {
"CREATE OR REPLACE INDEX idx ON " + CTL + "." + DB + "." + TBL
+ " (c) USING BTREE"));
Mockito.verify(fixture.env, Mockito.never()).getNextId();
- Assertions.assertEquals(0, fixture.manager.getJobCount());
- Assertions.assertTrue(fixture.manager.editLog.isEmpty());
}
}
@@ -353,47 +322,63 @@ public class AlterTableCommandLanceAdmissionTest {
runAndGetMessage(fixture.executor,
"DROP INDEX IF EXISTS idx ON " + CTL + "." + DB + "." + TBL));
Mockito.verify(fixture.env, Mockito.never()).getNextId();
- Assertions.assertEquals(0, fixture.manager.getJobCount());
- Assertions.assertTrue(fixture.manager.editLog.isEmpty());
}
}
// ------------------------------------------------------------------
- // Gate on: admission in run(), JobId result set, alterTable bypassed
+ // Gate on: admission in run(), IF no-ops complete, mutations reject as
+ // unsupported, alterTable bypassed
// ------------------------------------------------------------------
@Test
- public void gateOnCreateIndexAdmitsAndReturnsJobId() throws Exception {
+ public void gateOnCreateIndexValidatesThenRejectsAsUnsupported() throws Exception {
Config.enable_lance_index_mutation = true;
try (LanceFixture fixture = new LanceFixture(false)) {
fixture.respondWithSnapshot(Collections.emptyList(), Collections.emptyList());
- run(fixture.executor, "CREATE INDEX MyIdx ON " + CTL + "." + DB + "." + TBL
- + " (v) USING ANN " + VALID_ANN_PROPERTIES);
+ AnalysisException exception = runAndGetCommonAnalysisException(fixture.executor,
+ "CREATE INDEX MyIdx ON " + CTL + "." + DB + "." + TBL
+ + " (v) USING ANN " + VALID_ANN_PROPERTIES);
+ Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_OPERATION_NOT_SUPPORTED,
+ exception.getMysqlErrorCode());
+ Assertions.assertEquals("CREATE INDEX is not supported for Lance catalog tables",
+ exception.getDetailMessage());
- ArgumentCaptor<ResultSet> captor = ArgumentCaptor.forClass(ResultSet.class);
- Mockito.verify(fixture.executor, Mockito.times(1)).sendResultSet(captor.capture());
- assertSingleJobIdColumn(captor.getValue());
- Assertions.assertEquals(1, captor.getValue().getResultRows().size());
- long jobId = fixture.idAllocator.get();
- Assertions.assertEquals(String.valueOf(jobId),
- captor.getValue().getResultRows().get(0).get(0));
+ // The full preflight ran against the pinned snapshot, then rejected without any
+ // result set, id allocation, or generic alter path.
+ Mockito.verify(fixture.catalog, Mockito.times(1))
+ .loadTableIndexAdmissionSnapshot(REMOTE_DB, REMOTE_TBL);
+ Mockito.verify(fixture.executor, Mockito.never()).sendResultSet(Mockito.any());
+ Mockito.verify(fixture.env, Mockito.never()).getNextId();
+ Mockito.verify(fixture.env, Mockito.never()).alterTable(Mockito.any());
+ }
+ }
+
+ @Test
+ public void gateOnDropIndexValidatesThenRejectsAsUnsupported() throws Exception {
+ Config.enable_lance_index_mutation = true;
+ try (LanceFixture fixture = new LanceFixture(false)) {
+ fixture.respondWithSnapshot(
+ Collections.singletonList(new LanceShowIndexInfo("idx",
+ Collections.singletonList("c"), "BTREE", "{}")),
+ Collections.singletonList(
+ new PhysicalIndexInfo("idx", "uuid-1", DATASET_VERSION, "SCALAR")));
- // Admission short-circuits the generic alter path entirely.
+ AnalysisException exception = runAndGetCommonAnalysisException(fixture.executor,
+ "DROP INDEX idx ON " + CTL + "." + DB + "." + TBL);
+ Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_OPERATION_NOT_SUPPORTED,
+ exception.getMysqlErrorCode());
+ Assertions.assertEquals("DROP INDEX is not supported for Lance catalog tables",
+ exception.getDetailMessage());
+ Mockito.verify(fixture.catalog, Mockito.times(1))
+ .loadTableIndexAdmissionSnapshot(REMOTE_DB, REMOTE_TBL);
+ Mockito.verify(fixture.env, Mockito.never()).getNextId();
Mockito.verify(fixture.env, Mockito.never()).alterTable(Mockito.any());
- Mockito.verify(fixture.env, Mockito.times(1)).getNextId();
- Assertions.assertEquals(1, fixture.manager.getJobCount());
- Assertions.assertEquals(1, fixture.manager.editLog.size());
- LanceIndexJob job = fixture.manager.getJob(jobId);
- Assertions.assertEquals(LanceIndexJobMutationType.CREATE, job.getMutationType());
- Assertions.assertEquals("MyIdx", job.getDisplayIndexName());
- Assertions.assertEquals(DB, job.getDbName());
- Assertions.assertEquals(TBL, job.getTableName());
}
}
@Test
- public void gateOnCreateIfNotExistsMatchingReturnsZeroRows() throws Exception {
+ public void gateOnCreateIfNotExistsMatchingIsANoop() throws Exception {
Config.enable_lance_index_mutation = true;
try (LanceFixture fixture = new LanceFixture(false)) {
fixture.respondWithSnapshot(
@@ -405,32 +390,24 @@ public class AlterTableCommandLanceAdmissionTest {
run(fixture.executor, "CREATE INDEX IF NOT EXISTS idx ON " + CTL + "." + DB + "." + TBL
+ " (v) USING ANN " + VALID_ANN_PROPERTIES);
- ArgumentCaptor<ResultSet> captor = ArgumentCaptor.forClass(ResultSet.class);
- Mockito.verify(fixture.executor, Mockito.times(1)).sendResultSet(captor.capture());
- assertSingleJobIdColumn(captor.getValue());
- // The IF no-op creates no job: same schema, zero rows.
- Assertions.assertTrue(captor.getValue().getResultRows().isEmpty());
+ // The IF no-op completes as a plain success: no result set, no id, no alter path.
+ Mockito.verify(fixture.executor, Mockito.never()).sendResultSet(Mockito.any());
Mockito.verify(fixture.env, Mockito.never()).alterTable(Mockito.any());
Mockito.verify(fixture.env, Mockito.never()).getNextId();
- Assertions.assertEquals(0, fixture.manager.getJobCount());
- Assertions.assertTrue(fixture.manager.editLog.isEmpty());
}
}
@Test
- public void gateOnDropIfExistsAbsentReturnsZeroRows() throws Exception {
+ public void gateOnDropIfExistsAbsentIsANoop() throws Exception {
Config.enable_lance_index_mutation = true;
try (LanceFixture fixture = new LanceFixture(false)) {
fixture.respondWithSnapshot(Collections.emptyList(), Collections.emptyList());
run(fixture.executor, "DROP INDEX IF EXISTS idx ON " + CTL + "." + DB + "." + TBL);
- ArgumentCaptor<ResultSet> captor = ArgumentCaptor.forClass(ResultSet.class);
- Mockito.verify(fixture.executor, Mockito.times(1)).sendResultSet(captor.capture());
- assertSingleJobIdColumn(captor.getValue());
- Assertions.assertTrue(captor.getValue().getResultRows().isEmpty());
+ Mockito.verify(fixture.executor, Mockito.never()).sendResultSet(Mockito.any());
Mockito.verify(fixture.env, Mockito.never()).getNextId();
- Assertions.assertEquals(0, fixture.manager.getJobCount());
+ Mockito.verify(fixture.env, Mockito.never()).alterTable(Mockito.any());
}
}
@@ -451,31 +428,6 @@ public class AlterTableCommandLanceAdmissionTest {
Mockito.verify(fixture.executor, Mockito.never()).sendResultSet(Mockito.any());
Mockito.verify(fixture.env, Mockito.never()).getNextId();
Mockito.verify(fixture.env, Mockito.never()).alterTable(Mockito.any());
- Assertions.assertEquals(0, fixture.manager.getJobCount());
- Assertions.assertTrue(fixture.manager.editLog.isEmpty());
- }
- }
-
- @Test
- public void gateOnProxyExecutionUsesTheProxyResultSet() throws Exception {
- Config.enable_lance_index_mutation = true;
- try (LanceFixture fixture = new LanceFixture(false)) {
- fixture.respondWithSnapshot(
- Collections.singletonList(new LanceShowIndexInfo("idx",
- Collections.singletonList("c"), "BTREE", "{}")),
- Collections.singletonList(
- new PhysicalIndexInfo("idx", "uuid-1", DATASET_VERSION, "SCALAR")));
- Mockito.when(fixture.executor.isProxy()).thenReturn(true);
-
- run(fixture.executor, "DROP INDEX idx ON " + CTL + "." + DB + "." + TBL);
-
- ArgumentCaptor<ShowResultSet> captor = ArgumentCaptor.forClass(ShowResultSet.class);
- Mockito.verify(fixture.executor, Mockito.times(1)).setProxyShowResultSet(captor.capture());
- Mockito.verify(fixture.executor, Mockito.never()).sendResultSet(Mockito.any());
- assertSingleJobIdColumn(captor.getValue());
- Assertions.assertEquals(1, captor.getValue().getResultRows().size());
- Assertions.assertEquals(String.valueOf(fixture.idAllocator.get()),
- captor.getValue().getResultRows().get(0).get(0));
}
}
@@ -581,11 +533,20 @@ public class AlterTableCommandLanceAdmissionTest {
Config.enable_lance_index_mutation = true;
try (LanceFixture fixture = new LanceFixture(false)) {
fixture.respondWithSnapshot(Collections.emptyList(), Collections.emptyList());
- AlterTableCommand command = run(fixture.executor,
+ // The first run of a fresh command reaches admission (exactly one snapshot read)
+ // and ends in the shared not-supported rejection; a second run of the same
+ // command object re-validates, collects a second op, and trips the
+ // exactly-one-op precondition before any further metadata read.
+ AlterTableCommand command = (AlterTableCommand) parser.parseSingle(
"CREATE INDEX idx ON " + CTL + "." + DB + "." + TBL + " (c) USING BTREE");
+ AnalysisException first = Assertions.assertThrows(AnalysisException.class,
+ () -> command.run(connectContext, fixture.executor));
+ Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_OPERATION_NOT_SUPPORTED,
+ first.getMysqlErrorCode());
Assertions.assertThrows(IllegalStateException.class,
() -> command.run(connectContext, fixture.executor));
- Mockito.verify(fixture.env, Mockito.times(1)).getNextId();
+ Mockito.verify(fixture.catalog, Mockito.times(1))
+ .loadTableIndexAdmissionSnapshot(REMOTE_DB, REMOTE_TBL);
}
}
@@ -620,8 +581,6 @@ public class AlterTableCommandLanceAdmissionTest {
Mockito.verify(fixture.catalog, Mockito.never())
.loadTableIndexAdmissionSnapshot(Mockito.anyString(), Mockito.anyString());
Mockito.verify(fixture.executor, Mockito.never()).sendResultSet(Mockito.any());
- Assertions.assertEquals(0, fixture.manager.getJobCount());
- Assertions.assertTrue(fixture.manager.editLog.isEmpty());
}
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobCommandTest.java
deleted file mode 100644
index c49ad8ee69e..00000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobCommandTest.java
+++ /dev/null
@@ -1,565 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.trees.plans.commands;
-
-import org.apache.doris.analysis.RedirectStatus;
-import org.apache.doris.catalog.DatabaseIf;
-import org.apache.doris.catalog.Env;
-import org.apache.doris.catalog.TableIf;
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.ErrorCode;
-import org.apache.doris.common.util.TimeUtils;
-import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.datasource.CatalogMgr;
-import org.apache.doris.datasource.lance.LanceExternalCatalog;
-import org.apache.doris.datasource.lance.LanceExternalDatabase;
-import org.apache.doris.datasource.lance.LanceExternalTable;
-import org.apache.doris.datasource.lance.job.LanceIndexFenceKey;
-import org.apache.doris.datasource.lance.job.LanceIndexJob;
-import org.apache.doris.datasource.lance.job.LanceIndexJobCompletionReason;
-import org.apache.doris.datasource.lance.job.LanceIndexJobManager;
-import org.apache.doris.datasource.lance.job.LanceIndexJobMutationState;
-import org.apache.doris.datasource.lance.job.LanceIndexJobMutationType;
-import org.apache.doris.datasource.lance.job.LanceIndexJobRefreshState;
-import org.apache.doris.datasource.lance.job.LanceIndexJobResult;
-import org.apache.doris.datasource.lance.job.LanceIndexJobResultCode;
-import org.apache.doris.datasource.lance.job.LanceIndexNameNormalizer;
-import org.apache.doris.datasource.lance.job.LanceIndexSchemaContract;
-import org.apache.doris.datasource.lance.job.LanceIndexTerminationProof;
-import org.apache.doris.mysql.privilege.AccessControllerManager;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.SessionVariable;
-import org.apache.doris.qe.ShowResultSet;
-
-import mockit.Expectations;
-import mockit.Mocked;
-import mockit.Verifications;
-import mockit.VerificationsInOrder;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Covers SHOW LANCE INDEX JOB jobId: the record is loaded first and authorized against its
- * persisted target before any field is returned; a missing job, a job whose target the
- * user cannot see (orphan or half-orphan without ADMIN, a resolution that fails outright,
- * a repointed or unresolvable dataset locator, or a resolvable target without table
- * SHOW), all share the same fixed ERR_LANCE_INDEX_JOB_NOT_FOUND response naming only the
- * job id. Also covers full detail-column rendering with and without result/dispatch/force
- * fields.
- */
-public class ShowLanceIndexJobCommandTest {
- @Mocked
- private Env env;
- @Mocked
- private ConnectContext connectContext;
- @Mocked
- private AccessControllerManager accessControllerManager;
- @Mocked
- private LanceIndexJobManager lanceIndexJobManager;
- @Mocked
- private CatalogMgr catalogMgr;
- @Mocked
- private SessionVariable sessionVariable;
- @Mocked
- private CatalogIf catalog;
- @Mocked
- private DatabaseIf database;
- @Mocked
- private TableIf table;
- @Mocked
- private LanceExternalCatalog lanceCatalog;
- @Mocked
- private LanceExternalDatabase lanceDatabase;
- @Mocked
- private LanceExternalTable lanceTable;
-
- private static LanceIndexJob newJob(long jobId) {
- LanceIndexJob job = new LanceIndexJob(jobId, "creator", 10L, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, "s3://bucket/dataset",
- "idx1", LanceIndexNameNormalizer.normalize("idx1"),
- LanceIndexJobMutationType.CREATE, true, false, "IVF_PQ", "v", null, 7L, null);
- job.setMutationState(LanceIndexJobMutationState.PENDING);
- job.setRefreshState(LanceIndexJobRefreshState.NOT_REQUIRED);
- job.setCreateTimeMs(1000L);
- job.setUpdateTimeMs(2000L);
- return job;
- }
-
- private void expectEnv(LanceIndexJob job) {
- new Expectations() {
- {
- Env.getCurrentEnv();
- minTimes = 0;
- result = env;
-
- env.getLanceIndexJobManager();
- minTimes = 0;
- result = lanceIndexJobManager;
-
- env.getCatalogMgr();
- minTimes = 0;
- result = catalogMgr;
-
- env.getAccessManager();
- minTimes = 0;
- result = accessControllerManager;
-
- lanceIndexJobManager.getJob(anyLong);
- minTimes = 0;
- result = job;
-
- ConnectContext.get();
- minTimes = 0;
- result = connectContext;
-
- connectContext.getSessionVariable();
- minTimes = 0;
- result = sessionVariable;
-
- sessionVariable.getTimeZone();
- minTimes = 0;
- result = "Asia/Shanghai";
- }
- };
- }
-
- private void expectResolvableCatalog(boolean tableVisible) {
- new Expectations() {
- {
- catalogMgr.getCatalog(10L);
- minTimes = 0;
- result = catalog;
-
- catalog.getName();
- minTimes = 0;
- result = "lance_ctl";
-
- catalog.getDbNullable("db1");
- minTimes = 0;
- result = database;
-
- database.getTableNullable("tbl1");
- minTimes = 0;
- result = table;
-
- accessControllerManager.checkTblPriv(connectContext, "lance_ctl", "db1", "tbl1",
- PrivPredicate.SHOW);
- minTimes = 0;
- result = tableVisible;
- }
- };
- }
-
- private static int colIndex(ShowResultSet resultSet, String name) {
- for (int i = 0; i < resultSet.getMetaData().getColumns().size(); i++) {
- if (resultSet.getMetaData().getColumns().get(i).getName().equals(name)) {
- return i;
- }
- }
- throw new IllegalStateException("column not found: " + name);
- }
-
- /**
- * Catalog id 10 resolves as a Lance catalog whose db1.tbl1 names resolve and whose
- * current dataset locator is the given value: "s3://bucket/dataset" is what
- * {@link #newJob} persists, anything else simulates a table repointed at a new
- * dataset, and null a locator resolution that fails outright.
- */
- private void expectLanceCatalog(String currentLocator, boolean tableVisible) {
- new Expectations() {
- {
- catalogMgr.getCatalog(10L);
- minTimes = 0;
- result = lanceCatalog;
-
- lanceCatalog.getName();
- minTimes = 0;
- result = "lance_ctl";
-
- lanceCatalog.getDbNullable("db1");
- minTimes = 0;
- result = lanceDatabase;
-
- lanceDatabase.getTableNullable("tbl1");
- minTimes = 0;
- result = lanceTable;
-
- lanceCatalog.resolveCurrentIndexJobLocator("db1", "tbl1");
- minTimes = 0;
- result = currentLocator;
-
- accessControllerManager.checkTblPriv(connectContext, "lance_ctl", "db1", "tbl1",
- PrivPredicate.SHOW);
- minTimes = 0;
- result = tableVisible;
- }
- };
- }
-
- @Test
- public void testMissingJobAndUnauthorizedShareSameResponse() {
- // Missing job: 5103 naming only the job id.
- expectEnv(null);
- AnalysisException missing = Assertions.assertThrows(AnalysisException.class,
- () -> new ShowLanceIndexJobCommand(42L).doRun(connectContext, null));
- Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_JOB_NOT_FOUND, missing.getMysqlErrorCode());
- Assertions.assertTrue(missing.getMessage().contains("Lance index job not found: 42"));
-
- // Resolvable target but no table SHOW privilege: byte-identical response.
- expectEnv(newJob(42L));
- expectResolvableCatalog(false);
- AnalysisException denied = Assertions.assertThrows(AnalysisException.class,
- () -> new ShowLanceIndexJobCommand(42L).doRun(connectContext, null));
- Assertions.assertEquals(missing.getMessage(), denied.getMessage());
- Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_JOB_NOT_FOUND, denied.getMysqlErrorCode());
- }
-
- @Test
- public void testOrphanRequiresAdmin() {
- expectEnv(newJob(42L));
- new Expectations() {
- {
- catalogMgr.getCatalog(10L);
- minTimes = 0;
- result = null;
-
- accessControllerManager.checkGlobalPriv(connectContext, PrivPredicate.ADMIN);
- minTimes = 0;
- result = false;
- }
- };
- AnalysisException e = Assertions.assertThrows(AnalysisException.class,
- () -> new ShowLanceIndexJobCommand(42L).doRun(connectContext, null));
- Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_JOB_NOT_FOUND, e.getMysqlErrorCode());
- }
-
- @Test
- public void testOrphanNonAdminMatchesMissingResponse() {
- expectEnv(null);
- AnalysisException missing = Assertions.assertThrows(AnalysisException.class,
- () -> new ShowLanceIndexJobCommand(42L).doRun(connectContext, null));
- Assertions.assertTrue(missing.getMessage().contains("Lance index job not found: 42"));
-
- // Catalog deleted: a non-ADMIN caller gets the byte-identical response as for a
- // missing job, so the persisted catalog id behind a job is never disclosed.
- expectEnv(newJob(42L));
- new Expectations() {
- {
- catalogMgr.getCatalog(10L);
- minTimes = 0;
- result = null;
-
- accessControllerManager.checkGlobalPriv(connectContext, PrivPredicate.ADMIN);
- minTimes = 0;
- result = false;
- }
- };
- AnalysisException orphan = Assertions.assertThrows(AnalysisException.class,
- () -> new ShowLanceIndexJobCommand(42L).doRun(connectContext, null));
- Assertions.assertEquals(missing.getMessage(), orphan.getMessage());
- Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_JOB_NOT_FOUND, orphan.getMysqlErrorCode());
- // The job record was loaded and the ADMIN check actually ran, so the orphan branch
- // (not the missing branch) produced the response above.
- new Verifications() {
- {
- accessControllerManager.checkGlobalPriv(connectContext, PrivPredicate.ADMIN);
- times = 1;
- }
- };
- }
-
- @Test
- public void testHalfOrphanRequiresAdmin() {
- // Catalog resolves but the persisted db no longer does: same ADMIN-only rule.
- expectEnv(newJob(42L));
- new Expectations() {
- {
- catalogMgr.getCatalog(10L);
- minTimes = 0;
- result = catalog;
-
- catalog.getDbNullable("db1");
- minTimes = 0;
- result = null;
-
- accessControllerManager.checkGlobalPriv(connectContext, PrivPredicate.ADMIN);
- minTimes = 0;
- result = false;
- }
- };
- AnalysisException e = Assertions.assertThrows(AnalysisException.class,
- () -> new ShowLanceIndexJobCommand(42L).doRun(connectContext, null));
- Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_JOB_NOT_FOUND, e.getMysqlErrorCode());
- }
-
- @Test
- public void testProviderFailureResolvingTargetMatchesMissingResponse() {
- expectEnv(null);
- AnalysisException missing = Assertions.assertThrows(AnalysisException.class,
- () -> new ShowLanceIndexJobCommand(42L).doRun(connectContext, null));
- Assertions.assertTrue(missing.getMessage().contains("Lance index job not found: 42"));
-
- // A valid job whose target resolution fails outright (provider error, expired
- // credentials) gets the same fixed response for a non-ADMIN caller; the provider
- // error never surfaces.
- expectEnv(newJob(42L));
- new Expectations() {
- {
- catalogMgr.getCatalog(10L);
- minTimes = 0;
- result = catalog;
-
- catalog.getDbNullable("db1");
- minTimes = 0;
- result = new RuntimeException("provider exploded");
-
- accessControllerManager.checkGlobalPriv(connectContext, PrivPredicate.ADMIN);
- minTimes = 0;
- result = false;
- }
- };
- AnalysisException failed = Assertions.assertThrows(AnalysisException.class,
- () -> new ShowLanceIndexJobCommand(42L).doRun(connectContext, null));
- Assertions.assertEquals(missing.getMessage(), failed.getMessage());
- Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_JOB_NOT_FOUND, failed.getMysqlErrorCode());
- }
-
- @Test
- public void testLocatorMatchAuthorizedThroughTableShow() throws Exception {
- // The Lance catalog still points db1.tbl1 at the dataset the job was admitted
- // against, so ordinary table-level SHOW is enough to read the detail row.
- expectEnv(newJob(42L));
- expectLanceCatalog("s3://bucket/dataset", true);
-
- ShowResultSet resultSet = new ShowLanceIndexJobCommand(42L).doRun(connectContext, null);
- Assertions.assertEquals(1, resultSet.getResultRows().size());
- Assertions.assertEquals("42", resultSet.getResultRows().get(0).get(colIndex(resultSet, "JobId")));
- }
-
- @Test
- public void testLocatorMismatchMatchesMissingResponse() {
- expectEnv(null);
- AnalysisException missing = Assertions.assertThrows(AnalysisException.class,
- () -> new ShowLanceIndexJobCommand(42L).doRun(connectContext, null));
-
- // db1.tbl1 was repointed at a different dataset after the job terminated: even a
- // caller who would hold SHOW on the new target gets the fixed not-found response.
- expectEnv(newJob(42L));
- expectLanceCatalog("s3://bucket/repointed", true);
- new Expectations() {
- {
- accessControllerManager.checkGlobalPriv(connectContext, PrivPredicate.ADMIN);
- minTimes = 0;
- result = false;
- }
- };
- AnalysisException denied = Assertions.assertThrows(AnalysisException.class,
- () -> new ShowLanceIndexJobCommand(42L).doRun(connectContext, null));
- Assertions.assertEquals(missing.getMessage(), denied.getMessage());
- Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_JOB_NOT_FOUND, denied.getMysqlErrorCode());
- }
-
- @Test
- public void testLocatorResolutionFailureIsOrphan() {
- // The current locator cannot be resolved right now (provider failure): the target
- // is treated as not resolved - orphan semantics - instead of granting SHOW
- // through the stale names.
- expectEnv(newJob(42L));
- expectLanceCatalog(null, true);
- new Expectations() {
- {
- accessControllerManager.checkGlobalPriv(connectContext, PrivPredicate.ADMIN);
- minTimes = 0;
- result = false;
- }
- };
- AnalysisException e = Assertions.assertThrows(AnalysisException.class,
- () -> new ShowLanceIndexJobCommand(42L).doRun(connectContext, null));
- Assertions.assertEquals(ErrorCode.ERR_LANCE_INDEX_JOB_NOT_FOUND, e.getMysqlErrorCode());
- }
-
- @Test
- public void testOrphanVisibleToAdminWithEmptyCatalogName() throws Exception {
- LanceIndexJob job = newJob(42L);
- expectEnv(job);
- new Expectations() {
- {
- catalogMgr.getCatalog(10L);
- minTimes = 0;
- result = null;
-
- accessControllerManager.checkGlobalPriv(connectContext, PrivPredicate.ADMIN);
- minTimes = 0;
- result = true;
- }
- };
- ShowResultSet resultSet = new ShowLanceIndexJobCommand(42L).doRun(connectContext, null);
- Assertions.assertEquals(1, resultSet.getResultRows().size());
- List<String> row = resultSet.getResultRows().get(0);
- Assertions.assertEquals("42", row.get(colIndex(resultSet, "JobId")));
- Assertions.assertEquals("", row.get(colIndex(resultSet, "CatalogName")));
- Assertions.assertEquals("db1", row.get(colIndex(resultSet, "DbName")));
- }
-
- @Test
- public void testNullResultAndNullDispatchFieldsRenderEmpty() throws Exception {
- LanceIndexJob job = newJob(42L);
- Assertions.assertNull(job.getResult());
- expectEnv(job);
- expectResolvableCatalog(true);
-
- ShowResultSet resultSet = Assertions.assertDoesNotThrow(
- () -> new ShowLanceIndexJobCommand(42L).doRun(connectContext, null));
- Assertions.assertEquals(1, resultSet.getResultRows().size());
- List<String> row = resultSet.getResultRows().get(0);
- Assertions.assertEquals("creator", row.get(colIndex(resultSet, "Creator")));
- Assertions.assertEquals("", row.get(colIndex(resultSet, "Message")));
- Assertions.assertEquals("", row.get(colIndex(resultSet, "ResultCode")));
- Assertions.assertEquals("", row.get(colIndex(resultSet, "CompletionReason")));
- Assertions.assertEquals("", row.get(colIndex(resultSet, "ExternalMetadataAdvanced")));
- Assertions.assertEquals("", row.get(colIndex(resultSet, "BackendId")));
- Assertions.assertEquals("", row.get(colIndex(resultSet, "BeProcessEpoch")));
- Assertions.assertEquals("", row.get(colIndex(resultSet, "InvocationId")));
- Assertions.assertEquals("", row.get(colIndex(resultSet, "DeadlineMs")));
- Assertions.assertEquals("", row.get(colIndex(resultSet, "ForceNote")));
- Assertions.assertEquals("", row.get(colIndex(resultSet, "ForceWarning")));
- Assertions.assertEquals("7", row.get(colIndex(resultSet, "AdmittedDatasetVersion")));
- Assertions.assertEquals("", row.get(colIndex(resultSet, "SchemaContractVersion")));
- Assertions.assertEquals("0", row.get(colIndex(resultSet, "Revision")));
- Assertions.assertEquals("YES", row.get(colIndex(resultSet, "IfNotExists")));
- Assertions.assertEquals("NO", row.get(colIndex(resultSet, "IfExists")));
- Assertions.assertEquals("IVF_PQ", row.get(colIndex(resultSet, "IndexType")));
- Assertions.assertEquals("v", row.get(colIndex(resultSet, "ColumnName")));
- Assertions.assertEquals("NONE", row.get(colIndex(resultSet, "TerminationProof")));
- Assertions.assertEquals(TimeUtils.longToTimeString(1000L), row.get(colIndex(resultSet, "CreateTime")));
- }
-
- @Test
- public void testResultDispatchAndForceFieldsRender() throws Exception {
- LanceIndexSchemaContract contract = new LanceIndexSchemaContract(Collections.singletonList(
- new LanceIndexSchemaContract.IndexedField(1, "v", "fixed_size_list", false, 128,
- "float32", false)));
- LanceIndexJob job = new LanceIndexJob(42L, "creator", 10L, "db1", "tbl1",
- LanceIndexFenceKey.PROVIDER_DIRECTORY, "s3://bucket/dataset",
- "idx1", LanceIndexNameNormalizer.normalize("idx1"),
- LanceIndexJobMutationType.CREATE, true, false, "IVF_PQ", "v", null, 7L, contract);
- job.setMutationState(LanceIndexJobMutationState.PENDING);
- job.setRefreshState(LanceIndexJobRefreshState.NOT_REQUIRED);
- job.setCreateTimeMs(1000L);
- job.setUpdateTimeMs(2000L);
- job.setResult(new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
- LanceIndexJobCompletionReason.NONE, "ok", true));
- job.setBackendId(1001L);
- job.setBeProcessEpoch(55L);
- job.setInvocationId("inv-1");
- job.setDeadlineMs(9999L);
- job.setTerminationProof(LanceIndexTerminationProof.CHILD_REAPED);
- job.setRevision(3L);
- job.setForceReleased(true);
- job.setForceActor("admin");
- job.setForceTimeMs(3000L);
- job.setForceNote("note");
- job.setForceWarning("warn");
- expectEnv(job);
- expectResolvableCatalog(true);
-
- ShowResultSet resultSet = new ShowLanceIndexJobCommand(42L).doRun(connectContext, null);
- List<String> row = resultSet.getResultRows().get(0);
- Assertions.assertEquals("NATIVE_OK", row.get(colIndex(resultSet, "ResultCode")));
- Assertions.assertEquals("NONE", row.get(colIndex(resultSet, "CompletionReason")));
- Assertions.assertEquals("YES", row.get(colIndex(resultSet, "ExternalMetadataAdvanced")));
- Assertions.assertEquals("ok", row.get(colIndex(resultSet, "Message")));
- Assertions.assertEquals("1001", row.get(colIndex(resultSet, "BackendId")));
- Assertions.assertEquals("55", row.get(colIndex(resultSet, "BeProcessEpoch")));
- Assertions.assertEquals("inv-1", row.get(colIndex(resultSet, "InvocationId")));
- Assertions.assertEquals("9999", row.get(colIndex(resultSet, "DeadlineMs")));
- Assertions.assertEquals("CHILD_REAPED", row.get(colIndex(resultSet, "TerminationProof")));
- Assertions.assertEquals("3", row.get(colIndex(resultSet, "Revision")));
- Assertions.assertEquals("YES", row.get(colIndex(resultSet, "ForceReleased")));
- Assertions.assertEquals("admin", row.get(colIndex(resultSet, "ForceActor")));
- Assertions.assertEquals(TimeUtils.longToTimeString(3000L), row.get(colIndex(resultSet, "ForceTime")));
- Assertions.assertEquals("note", row.get(colIndex(resultSet, "ForceNote")));
- Assertions.assertEquals("warn", row.get(colIndex(resultSet, "ForceWarning")));
- Assertions.assertEquals("1", row.get(colIndex(resultSet, "SchemaContractVersion")));
- }
-
- @Test
- public void testRedirectStatus() {
- Assertions.assertEquals(RedirectStatus.FORWARD_NO_SYNC,
- new ShowLanceIndexJobCommand(1L).toRedirectStatus());
- }
-
- @Test
- public void testProxyContextCarriesIdentity() throws Exception {
- // V3-F16.8: the privilege check runs against the context handed to doRun.
- ConnectContext proxyCtx = new ConnectContext();
- expectEnv(newJob(42L));
- new Expectations() {
- {
- catalogMgr.getCatalog(10L);
- minTimes = 0;
- result = catalog;
-
- catalog.getName();
- minTimes = 0;
- result = "lance_ctl";
-
- catalog.getDbNullable("db1");
- minTimes = 0;
- result = database;
-
- database.getTableNullable("tbl1");
- minTimes = 0;
- result = table;
-
- accessControllerManager.checkTblPriv((ConnectContext) any, "lance_ctl", "db1", "tbl1",
- PrivPredicate.SHOW);
- minTimes = 0;
- result = true;
- }
- };
- new ShowLanceIndexJobCommand(42L).doRun(proxyCtx, null);
- new Verifications() {
- {
- accessControllerManager.checkTblPriv(proxyCtx, "lance_ctl", "db1", "tbl1", PrivPredicate.SHOW);
- times = 1;
- }
- };
- }
-
- @Test
- public void testJobLoadsBeforeTargetAuthorization() throws Exception {
- expectEnv(newJob(42L));
- expectResolvableCatalog(true);
-
- new ShowLanceIndexJobCommand(42L).doRun(connectContext, null);
-
- // The record is loaded first; authorization then runs against its persisted target.
- new VerificationsInOrder() {
- {
- lanceIndexJobManager.getJob(42L);
- accessControllerManager.checkTblPriv(connectContext, "lance_ctl", "db1", "tbl1",
- PrivPredicate.SHOW);
- }
- };
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobsCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobsCommandTest.java
deleted file mode 100644
index fb68c702893..00000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowLanceIndexJobsCommandTest.java
+++ /dev/null
@@ -1,821 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.trees.plans.commands;
-
-import org.apache.doris.analysis.RedirectStatus;
-import org.apache.doris.catalog.DatabaseIf;
-import org.apache.doris.catalog.Env;
-import org.apache.doris.catalog.TableIf;
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.util.TimeUtils;
-import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.datasource.CatalogMgr;
-import org.apache.doris.datasource.lance.LanceExternalCatalog;
-import org.apache.doris.datasource.lance.LanceExternalDatabase;
-import org.apache.doris.datasource.lance.LanceExternalTable;
-import org.apache.doris.datasource.lance.job.LanceIndexFenceKey;
-import org.apache.doris.datasource.lance.job.LanceIndexJob;
-import org.apache.doris.datasource.lance.job.LanceIndexJobManager;
-import org.apache.doris.datasource.lance.job.LanceIndexJobMutationState;
-import org.apache.doris.datasource.lance.job.LanceIndexJobMutationType;
-import org.apache.doris.datasource.lance.job.LanceIndexJobRefreshState;
-import org.apache.doris.datasource.lance.job.LanceIndexNameNormalizer;
-import org.apache.doris.mysql.privilege.AccessControllerManager;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.nereids.analyzer.UnboundSlot;
-import org.apache.doris.nereids.trees.expressions.And;
-import org.apache.doris.nereids.trees.expressions.EqualTo;
-import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.GreaterThan;
-import org.apache.doris.nereids.trees.expressions.Like;
-import org.apache.doris.nereids.trees.expressions.Or;
-import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
-import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
-import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.SessionVariable;
-import org.apache.doris.qe.ShowResultSet;
-
-import com.google.common.collect.Lists;
-import mockit.Expectations;
-import mockit.Mocked;
-import mockit.Verifications;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Covers SHOW LANCE INDEX JOBS: row-level SHOW privilege filtering, orphan and
- * half-orphan rows restricted to global ADMIN with no existence or count leakage,
- * provider failures during target resolution treated as orphans, dataset-locator
- * revalidation (matching, repointed and unresolvable locators, resolved once per
- * target within one listing), FROM db-name normalization through the target catalog's
- * own naming rules, FROM/WHERE in-memory filtering, null-result rendering, FORCE audit
- * columns, the deliberately narrowed WHERE grammar (EqualTo + AND only),
- * FORWARD_NO_SYNC redirect, and that authorization runs against the connection context
- * handed to the command (the forwarded original user identity in proxy mode).
- */
-public class ShowLanceIndexJobsCommandTest {
- @Mocked
- private Env env;
- @Mocked
- private ConnectContext connectContext;
- @Mocked
- private AccessControllerManager accessControllerManager;
- @Mocked
- private LanceIndexJobManager lanceIndexJobManager;
- @Mocked
- private CatalogMgr catalogMgr;
- @Mocked
- private SessionVariable sessionVariable;
- @Mocked
- private CatalogIf catalog;
- @Mocked
- private DatabaseIf database;
- @Mocked
- private TableIf table;
- @Mocked
- private LanceExternalCatalog lanceCatalog;
- @Mocked
- private LanceExternalDatabase lanceDatabase;
- @Mocked
- private LanceExternalTable lanceTable;
-
- private static LanceIndexJob newJob(long jobId, long catalogId, String dbName, String tableName,
- String indexName) {
- LanceIndexJob job = new LanceIndexJob(jobId, "creator", catalogId, dbName, tableName,
- LanceIndexFenceKey.PROVIDER_DIRECTORY, "s3://bucket/dataset",
- indexName, LanceIndexNameNormalizer.normalize(indexName),
- LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null);
- job.setMutationState(LanceIndexJobMutationState.PENDING);
- job.setRefreshState(LanceIndexJobRefreshState.NOT_REQUIRED);
- job.setCreateTimeMs(1000L);
- job.setUpdateTimeMs(2000L);
- return job;
- }
-
- private void expectEnv(List<LanceIndexJob> jobs) {
- new Expectations() {
- {
- Env.getCurrentEnv();
- minTimes = 0;
- result = env;
-
- env.getLanceIndexJobManager();
- minTimes = 0;
- result = lanceIndexJobManager;
-
- env.getCatalogMgr();
- minTimes = 0;
- result = catalogMgr;
-
- env.getAccessManager();
- minTimes = 0;
- result = accessControllerManager;
-
- lanceIndexJobManager.getAllJobsSnapshot();
- minTimes = 0;
- result = jobs;
-
- ConnectContext.get();
- minTimes = 0;
- result = connectContext;
-
- connectContext.getSessionVariable();
- minTimes = 0;
- result = sessionVariable;
-
- sessionVariable.getTimeZone();
- minTimes = 0;
- result = "Asia/Shanghai";
- }
- };
- }
-
- private void expectResolvableCatalog(boolean tableVisible) {
- new Expectations() {
- {
- catalogMgr.getCatalog(10L);
- minTimes = 0;
- result = catalog;
-
- catalog.getName();
- minTimes = 0;
- result = "lance_ctl";
-
- catalog.getDbNullable("db1");
- minTimes = 0;
- result = database;
-
- database.getFullName();
- minTimes = 0;
- result = "db1";
-
- database.getTableNullable("tbl1");
- minTimes = 0;
- result = table;
-
- accessControllerManager.checkTblPriv(connectContext, "lance_ctl", "db1", "tbl1",
- PrivPredicate.SHOW);
- minTimes = 0;
- result = tableVisible;
- }
- };
- }
-
- /**
- * Catalog id 10 resolves as a Lance catalog whose db1.tbl1 names resolve and whose
- * current dataset locator is the given value: "s3://bucket/dataset" is what
- * {@link #newJob} persists, anything else simulates a table repointed at a new
- * dataset, and null a locator resolution that fails outright.
- */
- private void expectLanceCatalog(String currentLocator, boolean tableVisible) {
- new Expectations() {
- {
- catalogMgr.getCatalog(10L);
- minTimes = 0;
- result = lanceCatalog;
-
- lanceCatalog.getName();
- minTimes = 0;
- result = "lance_ctl";
-
- lanceCatalog.getDbNullable("db1");
- minTimes = 0;
- result = lanceDatabase;
-
- lanceDatabase.getTableNullable("tbl1");
- minTimes = 0;
- result = lanceTable;
-
- lanceCatalog.resolveCurrentIndexJobLocator("db1", "tbl1");
- minTimes = 0;
- result = currentLocator;
-
- accessControllerManager.checkTblPriv(connectContext, "lance_ctl", "db1", "tbl1",
- PrivPredicate.SHOW);
- minTimes = 0;
- result = tableVisible;
- }
- };
- }
-
- private static int colIndex(ShowResultSet resultSet, String name) {
- for (int i = 0; i < resultSet.getMetaData().getColumns().size(); i++) {
- if (resultSet.getMetaData().getColumns().get(i).getName().equals(name)) {
- return i;
- }
- }
- throw new IllegalStateException("column not found: " + name);
- }
-
- @Test
- public void testMetadataColumns() throws Exception {
- expectEnv(Collections.emptyList());
- ShowLanceIndexJobsCommand command = new ShowLanceIndexJobsCommand(null, null);
- ShowResultSet resultSet = command.doRun(connectContext, null);
- List<String> expected = Arrays.asList("JobId", "CatalogName", "DbName", "TableName", "IndexName",
- "Operation", "State", "RefreshState", "PossibleLive", "CreateTime", "UpdateTime", "Message",
- "ForceReleased", "ForceActor", "ForceTime");
- Assertions.assertEquals(expected.size(), resultSet.getMetaData().getColumns().size());
- for (int i = 0; i < expected.size(); i++) {
- Assertions.assertEquals(expected.get(i), resultSet.getMetaData().getColumns().get(i).getName());
- }
- Assertions.assertTrue(resultSet.getResultRows().isEmpty());
- }
-
- @Test
- public void testRowLevelAuthFiltering() throws Exception {
- LanceIndexJob visible = newJob(1L, 10L, "db1", "tbl1", "idx1");
- LanceIndexJob hidden = newJob(2L, 10L, "db1", "tbl2", "idx2");
- expectEnv(Arrays.asList(visible, hidden));
- expectResolvableCatalog(true);
- new Expectations() {
- {
- database.getTableNullable("tbl2");
- minTimes = 0;
- result = table;
-
- accessControllerManager.checkTblPriv(connectContext, "lance_ctl", "db1", "tbl2",
- PrivPredicate.SHOW);
- minTimes = 0;
- result = false;
- }
- };
-
- ShowLanceIndexJobsCommand command = new ShowLanceIndexJobsCommand(null, null);
- ShowResultSet resultSet = command.doRun(connectContext, null);
- Assertions.assertEquals(1, resultSet.getResultRows().size());
- List<String> row = resultSet.getResultRows().get(0);
- Assertions.assertEquals("1", row.get(colIndex(resultSet, "JobId")));
- Assertions.assertEquals("lance_ctl", row.get(colIndex(resultSet, "CatalogName")));
- Assertions.assertEquals("db1", row.get(colIndex(resultSet, "DbName")));
- Assertions.assertEquals("tbl1", row.get(colIndex(resultSet, "TableName")));
- Assertions.assertEquals("idx1", row.get(colIndex(resultSet, "IndexName")));
- Assertions.assertEquals("CREATE", row.get(colIndex(resultSet, "Operation")));
- Assertions.assertEquals("PENDING", row.get(colIndex(resultSet, "State")));
- Assertions.assertEquals("NOT_REQUIRED", row.get(colIndex(resultSet, "RefreshState")));
- Assertions.assertEquals("NO", row.get(colIndex(resultSet, "PossibleLive")));
- Assertions.assertEquals(TimeUtils.longToTimeString(1000L), row.get(colIndex(resultSet, "CreateTime")));
- Assertions.assertEquals(TimeUtils.longToTimeString(2000L), row.get(colIndex(resultSet, "UpdateTime")));
- }
-
- @Test
- public void testOrphanRowHiddenFromNonAdmin() throws Exception {
- LanceIndexJob orphan = newJob(3L, 999L, "db1", "tbl1", "idx1");
- expectEnv(Collections.singletonList(orphan));
- new Expectations() {
... 898 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 17/28: branch-4.1: [fix](window) Recompute sliding frames for floating-point sum/avg #68113 (#68736)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3030d467957124ed2c65d2597a83cef5755429fd
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Fri Oct 9 17:59:14 2026 +0800
branch-4.1: [fix](window) Recompute sliding frames for floating-point sum/avg #68113 (#68736)
Cherry-picked from #68113
Co-authored-by: Jerry Hu <hu...@selectdb.com>
---
be/src/exprs/aggregate/aggregate_function_avg.h | 5 +-
be/src/exprs/aggregate/aggregate_function_sum.h | 8 +-
.../exec/operator/analytic_sink_operator_test.cpp | 79 +++++++++++
be/test/exprs/aggregate/agg_avg_test.cpp | 11 ++
.../{agg_avg_test.cpp => agg_sum_test.cpp} | 19 ++-
.../test_window_float_sliding_frame.out | 66 +++++++++
.../test_window_float_sliding_frame.groovy | 150 +++++++++++++++++++++
7 files changed, 332 insertions(+), 6 deletions(-)
diff --git a/be/src/exprs/aggregate/aggregate_function_avg.h b/be/src/exprs/aggregate/aggregate_function_avg.h
index ec8d3fa17fc..665757ab343 100644
--- a/be/src/exprs/aggregate/aggregate_function_avg.h
+++ b/be/src/exprs/aggregate/aggregate_function_avg.h
@@ -306,7 +306,10 @@ public:
return std::make_shared<DataTypeFixedLengthObject>();
}
- bool supported_incremental_mode() const override { return true; }
+ // Floating-point accumulation is not exactly invertible: subtracting an outgoing
+ // value cannot restore the rounding lost when it was added, so a value that has
+ // left the frame would still distort later results. Recompute such frames instead.
+ bool supported_incremental_mode() const override { return !std::is_floating_point_v<DataType>; }
void execute_function_with_incremental(int64_t partition_start, int64_t partition_end,
int64_t frame_start, int64_t frame_end,
diff --git a/be/src/exprs/aggregate/aggregate_function_sum.h b/be/src/exprs/aggregate/aggregate_function_sum.h
index b98a57b1ede..2943b0e1302 100644
--- a/be/src/exprs/aggregate/aggregate_function_sum.h
+++ b/be/src/exprs/aggregate/aggregate_function_sum.h
@@ -23,6 +23,7 @@
#include <stddef.h>
#include <memory>
+#include <type_traits>
#include <vector>
#include "core/assert_cast.h"
@@ -207,7 +208,12 @@ public:
return std::make_shared<DataTypeFixedLengthObject>();
}
- bool supported_incremental_mode() const override { return true; }
+ // Floating-point accumulation is not exactly invertible: subtracting an outgoing
+ // value cannot restore the rounding lost when it was added, so a value that has
+ // left the frame would still distort later results. Recompute such frames instead.
+ bool supported_incremental_mode() const override {
+ return !std::is_floating_point_v<typename PrimitiveTypeTraits<TResult>::CppType>;
+ }
NO_SANITIZE_UNDEFINED void execute_function_with_incremental(
int64_t partition_start, int64_t partition_end, int64_t frame_start, int64_t frame_end,
diff --git a/be/test/exec/operator/analytic_sink_operator_test.cpp b/be/test/exec/operator/analytic_sink_operator_test.cpp
index ec3934540c6..5a59a74ed4b 100644
--- a/be/test/exec/operator/analytic_sink_operator_test.cpp
+++ b/be/test/exec/operator/analytic_sink_operator_test.cpp
@@ -569,6 +569,85 @@ TEST_F(AnalyticSinkOperatorTest, SlidingRowsSumRetainsOutgoingRowDuringEviction)
std::cout << "######### sliding rows sum eviction test end #########" << std::endl;
}
+// Floating-point sum/avg cannot be rolled back exactly: once 2^54 + 1 rounds to 2^54,
+// removing 2^54 leaves 0 instead of 1. Sliding frames must be recomputed so that a value
+// which already left the frame cannot distort the current result.
+TEST_F(AnalyticSinkOperatorTest, SlidingRowsDoubleAvgIgnoresRoundingOfOutgoingRow) {
+ const std::vector<double> data_vals {18014398509481984.0, 1.0, 1.0};
+ const std::vector<double> expect_vals {18014398509481984.0, 9007199254740992.0, 1.0};
+ Initialize(data_vals.size());
+ create_operator(true, 1, "avg", {std::make_shared<DataTypeFloat64>()},
+ std::make_shared<DataTypeFloat64>());
+ sink->_agg_expr_ctxs.resize(1);
+ sink->_agg_expr_ctxs[0] =
+ MockSlotRef::create_mock_contexts(0, std::make_shared<DataTypeFloat64>());
+ TAnalyticWindow temp_window;
+ temp_window.type = TAnalyticWindowType::ROWS;
+ TAnalyticWindowBoundary window_start;
+ window_start.type = TAnalyticWindowBoundaryType::PRECEDING;
+ window_start.__set_rows_offset_value(1);
+ temp_window.__set_window_start(window_start);
+ TAnalyticWindowBoundary window_end;
+ window_end.type = TAnalyticWindowBoundaryType::CURRENT_ROW;
+ temp_window.__set_window_end(window_end);
+ create_window_type(true, true, temp_window);
+ create_local_state();
+ EXPECT_FALSE(sink_local_state->_support_incremental_calculate);
+
+ {
+ Block block = ColumnHelper::create_block<DataTypeFloat64>(data_vals);
+ auto st = sink->sink(state.get(), &block, true);
+ EXPECT_TRUE(st.ok()) << st.msg();
+ }
+ {
+ Block block = ColumnHelper::create_block<DataTypeFloat64>({});
+ bool eos = false;
+ auto st = source->get_block(state.get(), &block, &eos);
+ EXPECT_TRUE(st.ok()) << st.msg();
+ EXPECT_TRUE(ColumnHelper::block_equal(
+ block, ColumnHelper::create_block<DataTypeFloat64>(data_vals, expect_vals)))
+ << block.dump_data();
+ }
+}
+
+TEST_F(AnalyticSinkOperatorTest, SlidingRowsDoubleSumIgnoresRoundingOfOutgoingRow) {
+ const std::vector<double> data_vals {18014398509481984.0, 1.0, 1.0};
+ const std::vector<double> expect_vals {18014398509481984.0, 18014398509481984.0, 2.0};
+ Initialize(data_vals.size());
+ create_operator(true, 1, "sum", {std::make_shared<DataTypeFloat64>()},
+ std::make_shared<DataTypeFloat64>());
+ sink->_agg_expr_ctxs.resize(1);
+ sink->_agg_expr_ctxs[0] =
+ MockSlotRef::create_mock_contexts(0, std::make_shared<DataTypeFloat64>());
+ TAnalyticWindow temp_window;
+ temp_window.type = TAnalyticWindowType::ROWS;
+ TAnalyticWindowBoundary window_start;
+ window_start.type = TAnalyticWindowBoundaryType::PRECEDING;
+ window_start.__set_rows_offset_value(1);
+ temp_window.__set_window_start(window_start);
+ TAnalyticWindowBoundary window_end;
+ window_end.type = TAnalyticWindowBoundaryType::CURRENT_ROW;
+ temp_window.__set_window_end(window_end);
+ create_window_type(true, true, temp_window);
+ create_local_state();
+ EXPECT_FALSE(sink_local_state->_support_incremental_calculate);
+
+ {
+ Block block = ColumnHelper::create_block<DataTypeFloat64>(data_vals);
+ auto st = sink->sink(state.get(), &block, true);
+ EXPECT_TRUE(st.ok()) << st.msg();
+ }
+ {
+ Block block = ColumnHelper::create_block<DataTypeFloat64>({});
+ bool eos = false;
+ auto st = source->get_block(state.get(), &block, &eos);
+ EXPECT_TRUE(st.ok()) << st.msg();
+ EXPECT_TRUE(ColumnHelper::block_equal(
+ block, ColumnHelper::create_block<DataTypeFloat64>(data_vals, expect_vals)))
+ << block.dump_data();
+ }
+}
+
TEST_F(AnalyticSinkOperatorTest, AggFunction5) {
int batch_size = 2;
Initialize(batch_size);
diff --git a/be/test/exprs/aggregate/agg_avg_test.cpp b/be/test/exprs/aggregate/agg_avg_test.cpp
index 90847bda0aa..5c1099af8a6 100644
--- a/be/test/exprs/aggregate/agg_avg_test.cpp
+++ b/be/test/exprs/aggregate/agg_avg_test.cpp
@@ -31,4 +31,15 @@ TEST_F(AggregateFunctionAvgTest, test_int64) {
execute(Block({ColumnHelper::create_column_with_name<DataTypeInt64>({1, 2, 3})}),
ColumnHelper::create_column_with_name<DataTypeFloat64>({2}));
}
+
+TEST_F(AggregateFunctionAvgTest, test_incremental_mode_only_for_exact_sum) {
+ create_agg("avg", false, {std::make_shared<DataTypeInt64>()},
+ std::make_shared<DataTypeFloat64>());
+ EXPECT_TRUE(agg_fn->function()->supported_incremental_mode());
+
+ // Floating-point sums are not exactly invertible, so sliding frames must be recomputed.
+ create_agg("avg", false, {std::make_shared<DataTypeFloat64>()},
+ std::make_shared<DataTypeFloat64>());
+ EXPECT_FALSE(agg_fn->function()->supported_incremental_mode());
+}
} // namespace doris
diff --git a/be/test/exprs/aggregate/agg_avg_test.cpp b/be/test/exprs/aggregate/agg_sum_test.cpp
similarity index 58%
copy from be/test/exprs/aggregate/agg_avg_test.cpp
copy to be/test/exprs/aggregate/agg_sum_test.cpp
index 90847bda0aa..99ca936f2fe 100644
--- a/be/test/exprs/aggregate/agg_avg_test.cpp
+++ b/be/test/exprs/aggregate/agg_sum_test.cpp
@@ -22,13 +22,24 @@
namespace doris {
-struct AggregateFunctionAvgTest : public AggregateFunctiontest {};
+struct AggregateFunctionSumTest : public AggregateFunctiontest {};
-TEST_F(AggregateFunctionAvgTest, test_int64) {
- create_agg("avg", false, {std::make_shared<DataTypeInt64>()},
+TEST_F(AggregateFunctionSumTest, test_int64) {
+ create_agg("sum", false, {std::make_shared<DataTypeInt64>()},
std::make_shared<DataTypeInt64>());
execute(Block({ColumnHelper::create_column_with_name<DataTypeInt64>({1, 2, 3})}),
- ColumnHelper::create_column_with_name<DataTypeFloat64>({2}));
+ ColumnHelper::create_column_with_name<DataTypeInt64>({6}));
+}
+
+TEST_F(AggregateFunctionSumTest, test_incremental_mode_only_for_exact_sum) {
+ create_agg("sum", false, {std::make_shared<DataTypeInt64>()},
+ std::make_shared<DataTypeInt64>());
+ EXPECT_TRUE(agg_fn->function()->supported_incremental_mode());
+
+ // Floating-point sums are not exactly invertible, so sliding frames must be recomputed.
+ create_agg("sum", false, {std::make_shared<DataTypeFloat64>()},
+ std::make_shared<DataTypeFloat64>());
+ EXPECT_FALSE(agg_fn->function()->supported_incremental_mode());
}
} // namespace doris
diff --git a/regression-test/data/query_p0/sql_functions/window_functions/test_window_float_sliding_frame.out b/regression-test/data/query_p0/sql_functions/window_functions/test_window_float_sliding_frame.out
new file mode 100644
index 00000000000..a1d913228bf
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/window_functions/test_window_float_sliding_frame.out
@@ -0,0 +1,66 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !avg_double_preceding_1 --
+1 1.8014398509481984e+16 1.8014398509481984e+16
+2 0.5 9007199254740992
+3 0.5 0.5
+4 0.5 0.5
+
+-- !sum_double_preceding_1 --
+1 1.8014398509481984e+16 1.8014398509481984e+16
+2 0.5 1.8014398509481984e+16
+3 0.5 1
+4 0.5 1
+
+-- !avg_sum_float_preceding_1 --
+1 1.80143985E16 1.8014398509481984e+16 1.8014398509481984e+16
+2 0.5 9007199254740992 1.8014398509481984e+16
+3 0.5 0.5 1
+4 0.5 0.5 1
+
+-- !avg_sum_double_sign_flip --
+5 1.8014398509481984e+16 1.8014398509481984e+16 1.8014398509481984e+16
+6 0.5 9007199254740992 1.8014398509481984e+16
+7 -1.8014398509481984e+16 -9007199254740992 -1.8014398509481984e+16
+8 0.5 -9007199254740992 -1.8014398509481984e+16
+9 0.5 0.5 1
+
+-- !avg_sum_double_wider_frames --
+10 1.8014398509481984e+16 1.8014398509481984e+16 1.8014398509481984e+16 9007199254740992 1.8014398509481984e+16 6004799503160661
+11 0.5 9007199254740992 1.8014398509481984e+16 6004799503160661 1.8014398509481984e+16 0.5
+12 0.5 6004799503160661 1.8014398509481984e+16 0.5 1.5 0.5
+13 0.5 0.5 1.5 0.5 1.5 0.5
+14 0.5 0.5 1.5 0.5 1 0.5
+
+-- !avg_sum_nullable_partitioned --
+1 1 1.8014398509481984e+16 1.8014398509481984e+16 1.8014398509481984e+16
+10 3 \N \N \N
+11 3 0.5 0.5 0.5
+12 3 0.5 0.5 1
+13 3 0.5 0.5 1
+14 3 0.5 0.5 1
+2 1 0.5 9007199254740992 1.8014398509481984e+16
+3 1 0.5 0.5 1
+4 1 \N 0.5 0.5
+5 2 1.8014398509481984e+16 1.8014398509481984e+16 1.8014398509481984e+16
+6 2 0.5 9007199254740992 1.8014398509481984e+16
+7 2 -1.8014398509481984e+16 -9007199254740992 -1.8014398509481984e+16
+8 2 0.5 -9007199254740992 -1.8014398509481984e+16
+9 2 0.5 0.5 1
+
+-- !avg_sum_double_unbounded --
+1 1.8014398509481984e+16 1.8014398509481984e+16 1.8014398509481984e+16
+2 0.5 9007199254740992 1.8014398509481984e+16
+3 0.5 6004799503160661 1.8014398509481984e+16
+4 0.5 4503599627370496 1.8014398509481984e+16
+
+-- !avg_sum_exact_types --
+1 1.8014398509481984e+16 3.602879701896397e+16 36028797018963968 18014398509481984.0000 18014398509481984.000
+2 0.5 1.8014398509481984e+16 36028797018963969 9007199254740992.2500 18014398509481984.500
+3 0.5 1 2 0.5000 1.000
+4 0.5 1 2 0.5000 1.000
+
+-- !avg_double_cte --
+1 1.8014398509481984e+16 1.8014398509481984e+16
+2 1 9007199254740992
+3 1 1
+
diff --git a/regression-test/suites/query_p0/sql_functions/window_functions/test_window_float_sliding_frame.groovy b/regression-test/suites/query_p0/sql_functions/window_functions/test_window_float_sliding_frame.groovy
new file mode 100644
index 00000000000..15a630e0b1d
--- /dev/null
+++ b/regression-test/suites/query_p0/sql_functions/window_functions/test_window_float_sliding_frame.groovy
@@ -0,0 +1,150 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Sliding ROWS frames over floating-point columns must not carry the rounding
+// loss of a value that already left the frame into the current result.
+//
+// Expected values must not depend on how the compiler groups the additions
+// (the accumulators allow floating-point reassociation). The data therefore
+// uses 2^54 as the large value and 0.5 as the small value: half an ulp of 2^54
+// is 2, so as long as the small values inside a frame sum to less than 2 in
+// magnitude, any grouping rounds the frame sum back to 2^54, while removing
+// 2^54 incrementally would still leave 0 instead of the exact small sum.
+// Frames that mix the large positive and negative value hold two operands
+// only, because a two-operand IEEE addition has a single correctly rounded
+// result.
+suite("test_window_float_sliding_frame") {
+ sql "DROP TABLE IF EXISTS test_window_float_sliding_frame"
+ sql """
+ CREATE TABLE test_window_float_sliding_frame (
+ id INT,
+ grp INT,
+ v_double DOUBLE,
+ v_float FLOAT,
+ v_nullable DOUBLE NULL
+ ) ENGINE = OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+
+ // grp 1: a large value (2^54) leaves the frame, then only small values remain.
+ // grp 2: sign flip of a large value around small values.
+ // grp 3: consecutive small values after a large one, with NULL in the nullable column.
+ sql """
+ INSERT INTO test_window_float_sliding_frame VALUES
+ (1, 1, 18014398509481984, 18014398509481984, 18014398509481984),
+ (2, 1, 0.5, 0.5, 0.5),
+ (3, 1, 0.5, 0.5, 0.5),
+ (4, 1, 0.5, 0.5, NULL),
+ (5, 2, 18014398509481984, 18014398509481984, 18014398509481984),
+ (6, 2, 0.5, 0.5, 0.5),
+ (7, 2, -18014398509481984, -18014398509481984, -18014398509481984),
+ (8, 2, 0.5, 0.5, 0.5),
+ (9, 2, 0.5, 0.5, 0.5),
+ (10, 3, 18014398509481984, 18014398509481984, NULL),
+ (11, 3, 0.5, 0.5, 0.5),
+ (12, 3, 0.5, 0.5, 0.5),
+ (13, 3, 0.5, 0.5, 0.5),
+ (14, 3, 0.5, 0.5, 0.5)
+ """
+
+ // The reported case: after row 1 leaves the frame, avg over [0.5, 0.5] must be 0.5.
+ order_qt_avg_double_preceding_1 """
+ SELECT id, v_double,
+ avg(v_double) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS got
+ FROM test_window_float_sliding_frame
+ WHERE grp = 1
+ """
+
+ order_qt_sum_double_preceding_1 """
+ SELECT id, v_double,
+ sum(v_double) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS got
+ FROM test_window_float_sliding_frame
+ WHERE grp = 1
+ """
+
+ order_qt_avg_sum_float_preceding_1 """
+ SELECT id, v_float,
+ avg(v_float) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS got_avg,
+ sum(v_float) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS got_sum
+ FROM test_window_float_sliding_frame
+ WHERE grp = 1
+ """
+
+ // Sign flip: a large positive value leaves, then a large negative value
+ // enters and leaves. Two-operand frames keep the expected sums exact.
+ order_qt_avg_sum_double_sign_flip """
+ SELECT id, v_double,
+ avg(v_double) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS got_avg,
+ sum(v_double) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS got_sum
+ FROM test_window_float_sliding_frame
+ WHERE grp = 2
+ """
+
+ // Consecutive small values after the large one, wider frames and following bounds.
+ order_qt_avg_sum_double_wider_frames """
+ SELECT id, v_double,
+ avg(v_double) OVER (ORDER BY id ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS avg_p2,
+ sum(v_double) OVER (ORDER BY id ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS sum_p2,
+ avg(v_double) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS avg_p1f1,
+ sum(v_double) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS sum_p1f1,
+ avg(v_double) OVER (ORDER BY id ROWS BETWEEN CURRENT ROW AND 2 FOLLOWING) AS avg_f2
+ FROM test_window_float_sliding_frame
+ WHERE grp = 3
+ """
+
+ // Nullable input: NULL rows are skipped while the frame keeps sliding.
+ order_qt_avg_sum_nullable_partitioned """
+ SELECT id, grp, v_nullable,
+ avg(v_nullable) OVER (PARTITION BY grp ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS got_avg,
+ sum(v_nullable) OVER (PARTITION BY grp ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS got_sum
+ FROM test_window_float_sliding_frame
+ """
+
+ // Frames that only grow keep the incremental path and stay unchanged.
+ order_qt_avg_sum_double_unbounded """
+ SELECT id, v_double,
+ avg(v_double) OVER (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS got_avg,
+ sum(v_double) OVER (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS got_sum
+ FROM test_window_float_sliding_frame
+ WHERE grp = 1
+ """
+
+ // Exact accumulators (integer / decimal) still use the incremental path and stay exact.
+ order_qt_avg_sum_exact_types """
+ SELECT id, v_double,
+ avg(CAST(v_double * 2 AS BIGINT)) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS avg_bigint,
+ sum(CAST(v_double * 2 AS BIGINT)) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS sum_bigint,
+ avg(CAST(v_double AS DECIMAL(27, 3))) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS avg_decimal,
+ sum(CAST(v_double AS DECIMAL(27, 3))) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS sum_decimal
+ FROM test_window_float_sliding_frame
+ WHERE grp = 1
+ """
+
+ // The reported shape without a table.
+ order_qt_avg_double_cte """
+ WITH t AS (
+ SELECT 1 AS id, CAST(18014398509481984 AS DOUBLE) AS v
+ UNION ALL SELECT 2, CAST(1 AS DOUBLE)
+ UNION ALL SELECT 3, CAST(1 AS DOUBLE)
+ )
+ SELECT id, v,
+ avg(v) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS got
+ FROM t
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 24/28: [fix](fe) Fix HTTP system forwarding on follower FE (#68827)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3f8decb3429d73bb971910425c2bffcda9c24a6d
Author: Calvin Kirs <gu...@selectdb.com>
AuthorDate: Sat Oct 10 09:32:32 2026 +0800
[fix](fe) Fix HTTP system forwarding on follower FE (#68827)
https://github.com/apache/doris/pull/68823
---
.../java/org/apache/doris/qe/FEOpExecutor.java | 9 +++--
.../doris/qe/FEOpExecutorMysqlProtocolTest.java | 24 ++++++++++++
.../test_http_rest_v1_system_follower.groovy | 45 ++++++++++++++++++++++
3 files changed, 74 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java
index fe0de92857d..21420b31c22 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java
@@ -23,6 +23,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ClientPool;
import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
+import org.apache.doris.mysql.DummyMysqlChannel;
import org.apache.doris.mysql.MysqlCommand;
import org.apache.doris.mysql.MysqlCursorFetchCompatibility;
import org.apache.doris.mysql.MysqlProto;
@@ -222,10 +223,10 @@ public class FEOpExecutor {
// Propagate the client's CLIENT_DEPRECATE_EOF capability so the master FE
// generates packets matching the original client's protocol expectations.
- // Only a MySQL connection negotiates this capability and owns a MysqlChannel;
- // an Arrow Flight SQL session has none, and leaving the field unset keeps the
- // master on its default packet layout.
- if (ctx.getConnectType() == ConnectType.MYSQL) {
+ // HTTP contexts use a dummy MySQL channel without a negotiated capability.
+ // Leave their protocol fields unset so the master uses its default packet layout.
+ if (ctx.getConnectType() == ConnectType.MYSQL
+ && !(ctx.getMysqlChannel() instanceof DummyMysqlChannel && ctx.getCapability() == null)) {
params.setClientDeprecatedEOF(ctx.getMysqlChannel().clientDeprecatedEOF());
params.setMysqlCapability(ctx.getCapability().getFlags());
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/FEOpExecutorMysqlProtocolTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/FEOpExecutorMysqlProtocolTest.java
index a3e235f56f5..de0c3e2ee8e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/FEOpExecutorMysqlProtocolTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/FEOpExecutorMysqlProtocolTest.java
@@ -60,6 +60,7 @@ public class FEOpExecutorMysqlProtocolTest {
TMasterOpRequest request = new TestFEOpExecutor(context).build();
Assertions.assertTrue(request.isClientDeprecatedEOF());
+ Assertions.assertEquals(context.getCapability().getFlags(), request.getMysqlCapability());
Assertions.assertTrue(request.isCursorFetchRequested());
Assertions.assertEquals("8.2.0", request.getConnectAttributes().get("_client_version"));
@@ -107,6 +108,29 @@ public class FEOpExecutorMysqlProtocolTest {
}
}
+ @Test
+ public void testHttpForwardRequestUsesDefaultMysqlCapability() throws Exception {
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getSelfNode()).thenReturn(new SystemInfoService.HostInfo("127.0.0.1", 9010));
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ ConnectContext context = new ConnectContext();
+ context.setCurrentUserIdentity(UserIdentity.createAnalyzedUserIdentWithIp("alice", "%"));
+ context.setRemoteIP("127.0.0.1");
+ Assertions.assertNull(context.getCapability());
+
+ TMasterOpRequest request = new TestFEOpExecutor(context).build();
+ Assertions.assertFalse(request.isSetClientDeprecatedEOF());
+ Assertions.assertFalse(request.isSetMysqlCapability());
+
+ ConnectContext forwardedContext = createContext();
+ ConnectProcessor.restoreForwardedMysqlContext(forwardedContext, request);
+ int expectedFlags = MysqlCapability.DEFAULT_CAPABILITY.getFlags()
+ & ~MysqlCapability.Flag.CLIENT_DEPRECATE_EOF.getFlagBit();
+ Assertions.assertEquals(expectedFlags, forwardedContext.getCapability().getFlags());
+ }
+ }
+
@Test
public void testForwardedCapabilityAndMissingCursorFlag() throws Exception {
int legacyFlags = MysqlCapability.DEFAULT_CAPABILITY.getFlags()
diff --git a/regression-test/suites/auth_p0/test_http_rest_v1_system_follower.groovy b/regression-test/suites/auth_p0/test_http_rest_v1_system_follower.groovy
new file mode 100644
index 00000000000..b928969f2f9
--- /dev/null
+++ b/regression-test/suites/auth_p0/test_http_rest_v1_system_follower.groovy
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.suite.ClusterOptions
+
+suite("test_http_rest_v1_system_follower", "docker") {
+ def options = new ClusterOptions()
+ options.setFeNum(2)
+ options.setBeNum(1)
+ docker(options) {
+ def follower = cluster.getAllFrontends(true).find { !it.isMaster }
+ assertNotNull(follower)
+
+ // Wait for the follower to replay the journal before authenticating over HTTP.
+ String followerJdbcUrl = "jdbc:mysql://${follower.host}:${follower.queryPort}/"
+ connect(context.config.jdbcUser, context.config.jdbcPassword, followerJdbcUrl) {
+ sql "SYNC"
+ }
+
+ httpTest {
+ basicAuthorization "${context.config.jdbcUser}", "${context.config.jdbcPassword}"
+ endpoint "${follower.host}:${follower.httpPort}"
+ uri "/rest/v1/system?path=/"
+ op "get"
+ check { respCode, body ->
+ assertEquals(200, respCode)
+ assertEquals(0, parseJson(body).code)
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/28: [fix](ci) Backport MinIO image migration to branch-4.1 (#68777)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0910ff2ebcc81cdc6d530959ed02fb9aae8c26b5
Author: Dongyang Li <li...@selectdb.com>
AuthorDate: Thu Oct 8 16:34:37 2026 +0800
[fix](ci) Backport MinIO image migration to branch-4.1 (#68777)
### What problem does this PR solve?
Related PR: #67908, #67897
Problem Summary:
Backport the final MinIO image migration from master to branch-4.1. The
upstream `minio/minio` and `minio/mc` repositories reject pulls,
preventing third-party services from starting and causing External
regression to be skipped.
Update all 14 applicable references in 8 files to
`doristhirdpartydocker/minio` and `doristhirdpartydocker/mc`. Preserve
existing release tags, startup arguments, environment variables, and
branch-specific test logic. Align the two previously untagged
Iceberg/Paimon sample images with the release tags selected in #67908.
The master-only `docker/runtime/all-in-one/4.1/compose/cloud.yml` does
not exist on branch-4.1 and is not introduced by this backport.
---
docker/thirdparties/docker-compose/hudi/hudi.yaml.tpl | 4 ++--
docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl | 4 ++--
.../docker-compose/minio/minio-RELEASE.2024-11-07.yaml.tpl | 2 +-
docker/thirdparties/docker-compose/polaris/docker-compose.yaml.tpl | 4 ++--
.../cache/warm_up/test_file_cache_warmup_read_metrics_docker.groovy | 2 +-
samples/datalake/deltalake_and_kudu/trinoconnector-compose.yml | 4 ++--
samples/datalake/hudi/hudi-compose.yml | 4 ++--
samples/datalake/iceberg_and_paimon/docker-compose.yml | 4 ++--
8 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/docker/thirdparties/docker-compose/hudi/hudi.yaml.tpl b/docker/thirdparties/docker-compose/hudi/hudi.yaml.tpl
index 64e588f1a6a..94cd3adb3e0 100644
--- a/docker/thirdparties/docker-compose/hudi/hudi.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hudi/hudi.yaml.tpl
@@ -25,7 +25,7 @@ networks:
services:
${CONTAINER_UID}hudi-minio:
- image: minio/minio:RELEASE.2025-01-20T14-49-07Z
+ image: doristhirdpartydocker/minio:RELEASE.2025-01-20T14-49-07Z
container_name: ${CONTAINER_UID}hudi-minio
command: server /data --console-address ":${MINIO_CONSOLE_PORT}"
environment:
@@ -38,7 +38,7 @@ services:
- ${HUDI_NETWORK}
${CONTAINER_UID}hudi-minio-mc:
- image: minio/mc:RELEASE.2025-01-17T23-25-50Z
+ image: doristhirdpartydocker/mc:RELEASE.2025-01-17T23-25-50Z
container_name: ${CONTAINER_UID}hudi-minio-mc
entrypoint: |
/bin/bash -c "
diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
index 098a1cf4eed..0ea1088377e 100644
--- a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
+++ b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
@@ -178,7 +178,7 @@ services:
retries: 120
minio:
- image: minio/minio:RELEASE.2025-01-20T14-49-07Z
+ image: doristhirdpartydocker/minio:RELEASE.2025-01-20T14-49-07Z
container_name: doris--iceberg-minio
ports:
- ${MINIO_API_PORT}:9000
@@ -204,7 +204,7 @@ services:
depends_on:
minio:
condition: service_healthy
- image: minio/mc:RELEASE.2025-01-17T23-25-50Z
+ image: doristhirdpartydocker/mc:RELEASE.2025-01-17T23-25-50Z
container_name: doris--iceberg-mc
environment:
- AWS_ACCESS_KEY_ID=admin
diff --git a/docker/thirdparties/docker-compose/minio/minio-RELEASE.2024-11-07.yaml.tpl b/docker/thirdparties/docker-compose/minio/minio-RELEASE.2024-11-07.yaml.tpl
index c637b45c14c..81ddd8ba027 100644
--- a/docker/thirdparties/docker-compose/minio/minio-RELEASE.2024-11-07.yaml.tpl
+++ b/docker/thirdparties/docker-compose/minio/minio-RELEASE.2024-11-07.yaml.tpl
@@ -19,7 +19,7 @@ version: '3'
services:
doris--minio:
- image: minio/minio:RELEASE.2024-11-07T00-52-20Z
+ image: doristhirdpartydocker/minio:RELEASE.2024-11-07T00-52-20Z
restart: always
ports:
- ${DOCKER_MINIO_EXTERNAL_PORT}:9000
diff --git a/docker/thirdparties/docker-compose/polaris/docker-compose.yaml.tpl b/docker/thirdparties/docker-compose/polaris/docker-compose.yaml.tpl
index 8a41447b9f7..98124890872 100644
--- a/docker/thirdparties/docker-compose/polaris/docker-compose.yaml.tpl
+++ b/docker/thirdparties/docker-compose/polaris/docker-compose.yaml.tpl
@@ -20,7 +20,7 @@ version: "3.8"
services:
# MinIO: S3 compatible object storage for local dev
minio:
- image: minio/minio:RELEASE.2025-01-20T14-49-07Z
+ image: doristhirdpartydocker/minio:RELEASE.2025-01-20T14-49-07Z
container_name: ${CONTAINER_UID}polaris-minio
ports:
- "${MINIO_API_PORT}:9000"
@@ -42,7 +42,7 @@ services:
# MinIO client to bootstrap bucket and path
minio-client:
- image: minio/mc:RELEASE.2025-01-17T23-25-50Z
+ image: doristhirdpartydocker/mc:RELEASE.2025-01-17T23-25-50Z
container_name: ${CONTAINER_UID}polaris-mc
depends_on:
minio:
diff --git a/regression-test/suites/cloud_p0/cache/warm_up/test_file_cache_warmup_read_metrics_docker.groovy b/regression-test/suites/cloud_p0/cache/warm_up/test_file_cache_warmup_read_metrics_docker.groovy
index 8ef6eed35d7..70e90e51a65 100644
--- a/regression-test/suites/cloud_p0/cache/warm_up/test_file_cache_warmup_read_metrics_docker.groovy
+++ b/regression-test/suites/cloud_p0/cache/warm_up/test_file_cache_warmup_read_metrics_docker.groovy
@@ -99,7 +99,7 @@ suite('test_file_cache_warmup_read_metrics_docker', 'docker') {
-e MINIO_ROOT_USER=minioadmin \
-e MINIO_ROOT_PASSWORD=minioadmin \
-e MINIO_DOMAIN=host.docker.internal \
- minio/minio:RELEASE.2024-11-07T00-52-20Z \
+ doristhirdpartydocker/minio:RELEASE.2024-11-07T00-52-20Z \
server /data --console-address ':9001'
"""
waitForCondition({
diff --git a/samples/datalake/deltalake_and_kudu/trinoconnector-compose.yml b/samples/datalake/deltalake_and_kudu/trinoconnector-compose.yml
index e88348b3e6f..9a602293fee 100644
--- a/samples/datalake/deltalake_and_kudu/trinoconnector-compose.yml
+++ b/samples/datalake/deltalake_and_kudu/trinoconnector-compose.yml
@@ -79,7 +79,7 @@ services:
minio:
hostname: minio
- image: 'minio/minio:RELEASE.2022-05-26T05-48-41Z'
+ image: 'doristhirdpartydocker/minio:RELEASE.2022-05-26T05-48-41Z'
container_name: minio-storage
ports:
- '${MINIO_SERVICE_PORT}:9000'
@@ -90,7 +90,7 @@ services:
# This job will create the "datalake" bucket on Minio
mc-job:
- image: 'minio/mc:RELEASE.2022-05-09T04-08-26Z'
+ image: 'doristhirdpartydocker/mc:RELEASE.2022-05-09T04-08-26Z'
volumes:
- './data:/data'
entrypoint: |
diff --git a/samples/datalake/hudi/hudi-compose.yml b/samples/datalake/hudi/hudi-compose.yml
index 59d30451537..423fc701542 100644
--- a/samples/datalake/hudi/hudi-compose.yml
+++ b/samples/datalake/hudi/hudi-compose.yml
@@ -78,7 +78,7 @@ services:
minio:
hostname: minio
- image: 'minio/minio:RELEASE.2022-05-26T05-48-41Z'
+ image: 'doristhirdpartydocker/minio:RELEASE.2022-05-26T05-48-41Z'
container_name: minio-hudi-storage
ports:
- '${MINIO_SERVICE_PORT}:9000'
@@ -89,7 +89,7 @@ services:
# This job will create the "datalake" bucket on Minio
mc-job:
- image: 'minio/mc:RELEASE.2022-05-09T04-08-26Z'
+ image: 'doristhirdpartydocker/mc:RELEASE.2022-05-09T04-08-26Z'
volumes:
- './data:/data'
entrypoint: |
diff --git a/samples/datalake/iceberg_and_paimon/docker-compose.yml b/samples/datalake/iceberg_and_paimon/docker-compose.yml
index 739b2bfbed7..3cebe1391b4 100644
--- a/samples/datalake/iceberg_and_paimon/docker-compose.yml
+++ b/samples/datalake/iceberg_and_paimon/docker-compose.yml
@@ -59,7 +59,7 @@ services:
- './scripts:/opt/scripts'
minio:
- image: minio/minio
+ image: doristhirdpartydocker/minio:RELEASE.2025-09-07T16-13-09Z
container_name: doris-iceberg-paimon-minio
ports:
- ${MINIO_API_PORT}:9000
@@ -77,7 +77,7 @@ services:
mc:
depends_on:
- minio
- image: minio/mc
+ image: doristhirdpartydocker/mc:RELEASE.2025-08-13T08-35-41Z
container_name: doris-iceberg-paimon-mc
environment:
- AWS_ACCESS_KEY_ID=admin
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 21/28: branch-4.1: [fix](regression-test) Make test_analyze independent of the SQL cache #68829 (#68843)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3aa1a3c02854233eb40f7103060483ac5fc9c8cc
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Sat Oct 10 09:27:37 2026 +0800
branch-4.1: [fix](regression-test) Make test_analyze independent of the SQL cache #68829 (#68843)
Cherry-picked from #68829
Co-authored-by: yujun <yu...@selectdb.com>
---
regression-test/suites/statistics/analyze_stats.groovy | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy
index c0eaff54e2b..8f8a05e65e1 100644
--- a/regression-test/suites/statistics/analyze_stats.groovy
+++ b/regression-test/suites/statistics/analyze_stats.groovy
@@ -120,6 +120,12 @@ suite("test_analyze") {
SET forbid_unknown_col_stats=false;
"""
+ // DROP STATS does not invalidate the SQL cache and the stats version is not part of the cache
+ // key, so a cached SELECT is replayed without planning and forbid_unknown_col_stats never fires.
+ sql """
+ SET enable_sql_cache=false;
+ """
+
sql """
SELECT * FROM ${tbl}
"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 11/28: [feature](lance) Search historical versions, tags and branches with vector_search and full_text_search (#68707)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c552df550f44c6e4ecb6200895a9e209ab9b8d6d
Author: zy-kkk <zh...@gmail.com>
AuthorDate: Fri Oct 9 14:36:26 2026 +0800
[feature](lance) Search historical versions, tags and branches with vector_search and full_text_search (#68707)
### What problem does this PR solve?
Issue Number: close #68625
Related PR: #68453 (time travel, tags, branches and managed versioning
for Lance table scans)
Problem Summary:
`vector_search()` and `full_text_search()` always searched the latest
version of `main`, and their `table` argument rejected `FOR VERSION AS
OF`, `@tag` and `@branch`. A search could not be reproduced against a
released tag, compared across versions, or run on an experimental
branch.
Both TVFs now take four optional properties, with the semantics of the
table syntax from #68453:
| Property | Value | Table syntax | Combines with |
|-|-|-|-|
| `version` | positive integer | `FOR VERSION AS OF N` | `branch` |
| `timestamp` | `yyyy-MM-dd HH:mm:ss[.SSS]`, session time zone | `FOR
TIME AS OF` | `branch` |
| `tag` | tag name | `@tag(name)` | none |
| `branch` | branch name, `main` is the main chain | `@branch(name)` |
`version` or `timestamp` |
```sql
SELECT id, _distance FROM vector_search(
"table" = "lance_catalog.db.documents", "column" = "embedding",
"query_vector" = "[0.1, 0.2, 0.3]", "version" = "3")
ORDER BY _distance;
SELECT id, _score FROM full_text_search(
"table" = "lance_catalog.db.documents", "column" = "content", "query" = "database",
"branch" = "dev", "timestamp" = "2026-09-20 12:00:00")
ORDER BY _score DESC;
```
How it works:
- **FE.** The TVF parses the properties into a `LanceRefSelector` and
loads its metadata through the same
`LanceCatalogClient.readTableSnapshot` as table scans, so
storage-versioned and namespace-managed tables follow the #68453 rules
unchanged. Schema, search column and field id, vector type and
dimension, fragments and index segments all come from the selected
snapshot, and planning never reopens the dataset. The splits carry the
branch directory URI and the version, so no thrift change is needed.
- **Each execution resolves again.** A bound TVF is never cached,
prepared statements re-analyze on every `EXECUTE`, and TVF statements
skip the SQL cache, so the latest version, a future time, a branch head
and a moved tag are resolved per execution and stay fixed within its
plan. `CREATE TABLE AS SELECT` plans its query twice (schema, then
rows), so there a moving selector can resolve twice, as it already can
for a table scan with an explicit selector; pinning `version` gives an
exact copy. This is documented.
- **No fallback.** A snapshot that cannot be selected, or whose index
files are missing, is an error; nothing falls back to the latest
version.
- **EXPLAIN** of a search node now shows `lanceCatalogType`,
`lanceManagedVersioning` and `lanceBranch` next to `lanceVersion`, as
table scans do. Planner errors that name "dataset version N" also name
the branch, since branch versions overlap main's.
Decisions worth calling out:
- `version` takes only a number. `FOR VERSION AS OF 'x'` names a tag
because SQL has one clause for both; the TVF has a `tag` property, and
Lance allows tag names made of digits, which `FOR VERSION AS OF` cannot
reach.
- An empty selector value is an error rather than "not set", so an empty
client-side variable cannot silently search the latest version of
`main`.
- The selector properties must be constants. Accepting `?` would leave
PREPARE without a snapshot to take the result schema from.
`full_text_search` now rejects `?` in any property: until now a
placeholder reached it as the literal text "?", so `'query'=?` searched
for "?".
- Each search relation resolves its own snapshot, as each explicit
selector already does for table scans. `vector_search(t) JOIN t` can
read two versions if a commit lands in between; pinning both with
`version` / `FOR VERSION AS OF` avoids that. This is documented.
- A managed table's branch is opened by a URI Doris joins
(`<table>/tree/<branch>`), which Lance does not validate on that path,
so Doris now rejects names that could leave the table's `tree/`
directory or change that URI: empty segments, `..`, `\`, and ASCII
characters other than letters, digits, `.`, `-`, `_` and `/`, so
`../other` and a percent-encoded dot segment are rejected before the
branch is opened. Outside ASCII only whitespace and control characters
are rejected, so no name Lance accepts is rejected, whichever Unicode
version the JDK and Lance's Rust use. Other tables check a branch out
through Lance, which validates the name itself.
BE changes:
- A search split or a two-phase row fetch without a positive version is
rejected instead of opening the latest version.
- Errors from opening the dataset, reading batches, preparing the FTS
query context and the two-phase row fetch name the snapshot: "... at
Lance dataset version N at <uri>", with the query string and any
password removed from the URI. Missing index files surface at execution
because Lance loads indexes lazily.
- Fragment ids from the FE must exist in the opened snapshot. lance-c
silently skips unknown fragment ids, which would drop rows without an
error.
- The profile records `LanceDatasetVersion` and `LanceDatasetUri`, the
snapshot an execution actually read. EXPLAIN cannot show this for moving
selectors because it resolves again.
- **Behavior change for every vector search:** a split without index
segments now always runs flat (`use_index=false`). The FE plans such
splits as flat search, and EXPLAIN and the profile count them as flat,
but Lance used to search them with the first index on the column in
manifest order, which need not be the index the FE selected (for example
while an index is rebuilt under a new name). Indexes the FE cannot plan
with (no fragment coverage metadata, no recorded metric, or a legacy
index whose details Lance cannot infer from its files, which the FE
skips; `lanceVectorIndexStatus=UNKNOWN_COVERAGE` / `METRIC_MISMATCH` /
`NO_MATCH`) are therefore no longer used implicitly, and such searches
become exact flat searches. Results can change only where the unplanned
index was approximate; latency can grow on such datasets. Legacy indexes
that lack details but whose details Lance infers, such as those written
by Lance 0.10 to 0.24, are planned and searched as before
(`test_lance_vector_search_legacy_index`).
- A vector split that carries index segments while the request says
`use_index=false` is rejected. The FE plans segments only when
`use_index` allows them, so it never sends such a split; the BE now
takes `use_index` from the segments and would otherwise have to override
either the plan or the request.
Known limitations (documented):
- A branch of a branch, or a branch of a shallow clone, can fail to find
the index files it inherited, because Lance writers before v13 record
them under the intermediate branch (lance-format/lance#9176). The search
fails with an error.
- After a branch is deleted and recreated under the same name, or a
dataset is replaced at the same URI, version numbers repeat. Since
#68615 the BE's Lance keys the index list and the row-id index by the
manifest e_tag (lance-format/lance#8904, #8930), but the BE's
process-wide Lance session still keys two entries by version alone,
until eviction or restart. On datasets with stable row ids, the live-row
mask that vector and full-text search prefilter with can come from the
old data, so rows deleted in the new data can reappear or live rows can
be missing (lance-format/lance#9079, open). For a manifest without an
e_tag (V1 naming, or a store that returns no ETag), the index list can
come from the old data, on the BE and in the FE's Lance session until
the catalog is refreshed. Neither is introduced by this PR.
- `SHOW INDEX` and `lance_index_entries()` keep showing the latest
version of `main`.
### Release note
`vector_search()` and `full_text_search()` accept `version`,
`timestamp`, `tag` and `branch` to search a historical version, the
version at a time, a tag or a branch of a Lance table, including REST
Namespace managed tables. Vector search no longer uses an index the
planner did not select, and `full_text_search()` rejects
prepared-statement placeholders.
### Check List (For Author)
- Test: Regression test / Unit Test
- Regression: `external_table_p0/lance/test_lance_search_snapshot`
(filesystem catalog) and `test_lance_rest_search_snapshot` (REST,
storage-versioned and managed) on a new fixture `search_snapshot.lance`.
It has seven versions (appends, a vector index, an FTS index, an append
no index covers, a delete, a vector index rebuild), a tag, and a branch
`dev` whose versions and fragment ids overlap main's with different
rows, plus a tag at a non-latest branch version.
`search_snapshot_pruned.lance` lost a version to cleanup and the index
files of a tagged version. `search_snapshot_evolved.lance` adds a column
and renames the vector column after its index is built, then appends
data the index does not cover. The suites check rows, distances,
two-phase reads (on a branch, a tag, the columns of a version before and
after a rename, and a managed table through REST), EXPLAIN, full-text
search across a delete, `use_index=false` with a selector, every
selector error, and a managed branch named with a Unicode 16 letter
(`unicode_branch.lance`, built by pylance 12.0.0) through a table scan,
a search and a two-phase read. `test_lance_rest_catalog.out` and
`test_lance_rest_time_travel.out` list the new REST tables.
`test_lance_vector_search_legacy_index` searches a vector index written
by pylance 0.18.2 without index details and checks that it is planned
(`USED`, one segment) and returns the flat search's rows. All 30 suites
in `external_table_p0/lance` pass locally.
- FE unit tests: `LanceSearchSelectorTest` (property rules, time zones,
selector forwarding), `LancePreparedSearchTest` (both TVFs pass their
selector to the table; each analysis resolves it exactly once, and a
moved tag is resolved again on the next `EXECUTE`; search statements
bypass the SQL cache; placeholders rejected), `LanceScanNodeTest`
(EXPLAIN, errors naming the branch), `LanceCatalogClientTest` (branch
names, including letters newer than JDK 17's Unicode tables), and, with
real datasets and locally only like the other JNI tests,
`LanceSearchSnapshotTest` (historical index coverage, branches, tags,
schema evolution, cleanup, a snapshot planned before later commits) and
`LanceManagedVersioningTest` (managed versions, tags, branches,
unfinalized heads through the search entry points).
- BE unit tests in `lance_reader_test.cpp`: missing fragment ids on
normal, vector and FTS splits, unpinned versions, error and profile
contents, a split without index segments searching flat although an
index covers it, and a split with index segments under `use_index=false`
being rejected. The indexed variants of the existing multi-vector tests
now pass the index segments the FE would plan.
- Behavior changed: Yes. New TVF properties; vector splits without index
segments always search flat; `full_text_search` rejects `?`; managed
branch names are validated; the error for a time in removed history now
reads "Lance cannot select the version at or before '...'" instead of
"Lance cannot resolve FOR TIME AS OF '...'", because the TVF reaches it
too; BE errors name the dataset version and URI.
- Does this need documentation: Yes (apache/doris-website#4185)
---
be/src/format_v2/table/lance_reader.cpp | 141 +++++-
be/src/format_v2/table/lance_reader.h | 8 +
be/test/format_v2/table/lance_reader_test.cpp | 368 ++++++++++++++-
.../docker-compose/iceberg/iceberg.yaml.tpl | 17 +-
.../scripts/lance_build_legacy_vector_index.py | 100 ++++
.../scripts/lance_build_preinstalled_catalog.py | 41 +-
.../iceberg/scripts/lance_build_search_snapshot.py | 329 ++++++++++++++
.../iceberg/scripts/lance_build_unicode_branch.py | 135 ++++++
.../iceberg/scripts/lance_fixture_requirements.txt | 6 +
.../bc324f21-a15a-45bf-8d9e-09538f895f55/index.idx | Bin 0 -> 9341 bytes
.../0-c5e3be90-3de0-44fb-95b1-bd46928ae0f2.txn | Bin 0 -> 184 bytes
.../1-9c8d1a85-5125-46ad-aeb8-8370bba30617.txn | Bin 0 -> 99 bytes
.../legacy_vector_index.lance/_versions/1.manifest | Bin 0 -> 253 bytes
.../legacy_vector_index.lance/_versions/2.manifest | Bin 0 -> 315 bytes
.../02cdc0ff-ceee-44ef-bccd-6cb6f4b50779.lance | Bin 0 -> 10641 bytes
.../_deletions/0-5-1267712482458982649.arrow | Bin 0 -> 698 bytes
.../_deletions/1-5-6394601565774866848.arrow | Bin 0 -> 698 bytes
.../auxiliary.idx | Bin 0 -> 1201 bytes
.../85880db4-716d-49b3-8eb2-fdadcc900e25/index.idx | Bin 0 -> 332 bytes
.../auxiliary.idx | Bin 0 -> 1265 bytes
.../984593f1-89f1-4849-83f9-d3e45462c244/index.idx | Bin 0 -> 332 bytes
.../metadata.lance | Bin 0 -> 654 bytes
.../part_0_docs.lance | Bin 0 -> 626 bytes
.../part_0_invert.lance | Bin 0 -> 3479 bytes
.../part_0_tokens.lance | Bin 0 -> 731 bytes
.../search_snapshot.lance/_refs/branches/dev.json | 15 +
.../search_snapshot.lance/_refs/tags/dev_rel.json | 8 +
.../search_snapshot.lance/_refs/tags/rel.json | 8 +
.../0-154ed129-09cd-48de-89ac-618fc1060501.txn | Bin 0 -> 237 bytes
.../1-d6645060-096c-4335-8bd9-c595a08fdd34.txn | Bin 0 -> 124 bytes
.../2-1518405c-5551-4e8f-b615-a0ca0d754255.txn | Bin 0 -> 372 bytes
.../3-c7c138f8-03a8-4564-9fc5-d59c25c2223c.txn | Bin 0 -> 278 bytes
.../4-1cbab51a-2c2d-41c4-8025-30daa3cc11a4.txn | Bin 0 -> 124 bytes
.../5-630bd020-d3af-4be8-b5e8-33388c068adc.txn | Bin 0 -> 259 bytes
.../6-9a8f0e65-3b5d-49ac-b232-5990ddd60891.txn | Bin 0 -> 702 bytes
.../_versions/18446744073709551608.manifest | Bin 0 -> 1789 bytes
.../_versions/18446744073709551609.manifest | Bin 0 -> 1343 bytes
.../_versions/18446744073709551610.manifest | Bin 0 -> 1173 bytes
.../_versions/18446744073709551611.manifest | Bin 0 -> 1244 bytes
.../_versions/18446744073709551612.manifest | Bin 0 -> 1104 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 521 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 551 bytes
.../_versions/latest_version_hint.json | 1 +
...11110000001111167198a47f9880a55a8085ebb87.lance | Bin 0 -> 1211 bytes
...010111000101110284ca44e90bfc0c56be666a32f.lance | Bin 0 -> 1211 bytes
...111011110010100fc3e5d485e9c40f5a217eaa59e.lance | Bin 0 -> 1211 bytes
.../metadata.lance | Bin 0 -> 654 bytes
.../part_0_docs.lance | Bin 0 -> 690 bytes
.../part_0_invert.lance | Bin 0 -> 2987 bytes
.../part_0_tokens.lance | Bin 0 -> 731 bytes
.../4-a99a47e7-aae1-46cf-87f7-4cc649db3aa1.txn | 1 +
.../4-d995fee3-982e-4e43-8e30-94537a9b0de2.txn | Bin 0 -> 124 bytes
.../5-6028c4ff-0a8e-4938-9d58-8a13524f6812.txn | Bin 0 -> 516 bytes
.../dev/_versions/18446744073709551609.manifest | Bin 0 -> 1632 bytes
.../dev/_versions/18446744073709551610.manifest | Bin 0 -> 1240 bytes
.../dev/_versions/18446744073709551611.manifest | Bin 0 -> 1128 bytes
...01010111100101014250f4aa0a70d3d926f124125.lance | Bin 0 -> 1211 bytes
.../auxiliary.idx | Bin 0 -> 1008 bytes
.../72e585d1-b843-4463-aba0-ffabc76017c1/index.idx | Bin 0 -> 332 bytes
.../0-fecb1851-f6b0-496c-bfb8-f4dbe7f649f1.txn | Bin 0 -> 237 bytes
.../1-5a98fef6-06a5-457c-b1b1-a9c830eafd2b.txn | Bin 0 -> 370 bytes
.../2-9bdddb95-2dc3-4e77-95b9-f67ed6e6387e.txn | Bin 0 -> 346 bytes
.../3-954a5d41-5fc9-4e39-96d9-864e1ac233bc.txn | 5 +
.../4-a41517b8-29a3-410f-b3ae-768c33b38cf5.txn | Bin 0 -> 126 bytes
.../_versions/18446744073709551610.manifest | Bin 0 -> 970 bytes
.../_versions/18446744073709551611.manifest | Bin 0 -> 956 bytes
.../_versions/18446744073709551612.manifest | Bin 0 -> 1099 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 1016 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 551 bytes
.../_versions/latest_version_hint.json | 1 +
...001001000111000634254483b82696db0b43cee8b.lance | Bin 0 -> 1576 bytes
...0111000011010107529f744f0b515d3cc0aaf4111.lance | Bin 0 -> 416 bytes
...0000010110100009494424b38a01956f50baa341d.lance | Bin 0 -> 1211 bytes
.../auxiliary.idx | Bin 0 -> 1008 bytes
.../961caf93-b3e9-45b9-b1a7-8944a37213ea/index.idx | Bin 0 -> 332 bytes
.../_refs/tags/kept.json | 8 +
.../1-5c4816b8-2e0a-46bb-aa6c-0879630399b4.txn | Bin 0 -> 370 bytes
.../2-b57c17ad-85a6-415b-a250-4b2efe8d3f8c.txn | Bin 0 -> 696 bytes
.../_versions/18446744073709551612.manifest | Bin 0 -> 1343 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 1017 bytes
.../_versions/latest_version_hint.json | 1 +
...110111001110011284b0f491a947f1b18c7abddff.lance | Bin 0 -> 1211 bytes
.../_refs/branches/dev%E1%B2%89.json | 15 +
.../0-420a1091-b61a-4cfb-8006-0b8f1e322ef8.txn | Bin 0 -> 237 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 552 bytes
.../_versions/latest_version_hint.json | 1 +
...101111000101000a13eb949eca41d69d17f9ea5a6.lance | Bin 0 -> 1146 bytes
.../1-1be00bd2-8443-4851-9ab8-4bdcee3ffa5e.txn" | Bin 0 -> 124 bytes
.../1-c1998c6f-6213-4b7e-ae7c-cc00cde3b7de.txn" | 1 +
.../_versions/18446744073709551613.manifest" | Bin 0 -> 585 bytes
.../_versions/18446744073709551614.manifest" | Bin 0 -> 476 bytes
...1000100001111040aa57401ab473a6a921298540.lance" | Bin 0 -> 1017 bytes
.../doris/datasource/lance/LanceCatalogClient.java | 78 +++-
.../datasource/lance/LanceExternalCatalog.java | 12 +-
.../doris/datasource/lance/LanceExternalTable.java | 15 +-
.../lance/index/LanceDatasetIndexDiscovery.java | 6 +-
.../lance/metadata/LanceRefSelector.java | 11 +-
.../lance/metadata/LanceSnapshotResolver.java | 25 +
.../datasource/lance/source/LanceScanNode.java | 22 +-
.../datasource/lance/source/LanceScanPlan.java | 11 +-
.../datasource/lance/source/LanceScanPlanner.java | 14 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 10 +
.../FullTextSearchTableValuedFunction.java | 2 +-
.../LanceExternalSearchTableValuedFunction.java | 89 +++-
.../VectorSearchTableValuedFunction.java | 2 +-
.../datasource/lance/LanceCatalogClientTest.java | 26 ++
.../lance/LanceManagedVersioningTest.java | 82 +++-
.../datasource/lance/LanceSearchSnapshotTest.java | 506 +++++++++++++++++++++
.../datasource/lance/source/LanceScanNodeTest.java | 61 ++-
.../tablefunction/LancePreparedSearchTest.java | 202 +++++++-
.../tablefunction/LanceSearchSelectorTest.java | 189 ++++++++
.../VectorSearchTableValuedFunctionTest.java | 3 +-
.../lance/test_lance_rest_catalog.out | 5 +
.../lance/test_lance_rest_search_snapshot.out | 59 +++
.../lance/test_lance_rest_time_travel.out | 5 +
.../lance/test_lance_search_snapshot.out | 182 ++++++++
.../test_lance_vector_search_legacy_index.out | 11 +
.../lance/test_lance_rest_search_snapshot.groovy | 202 ++++++++
.../lance/test_lance_search_snapshot.groovy | 452 ++++++++++++++++++
.../test_lance_vector_search_legacy_index.groovy | 90 ++++
120 files changed, 3479 insertions(+), 103 deletions(-)
diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp
index b9e5054a2a6..73d68332f49 100644
--- a/be/src/format_v2/table/lance_reader.cpp
+++ b/be/src/format_v2/table/lance_reader.cpp
@@ -74,6 +74,37 @@ Status LanceTableReader::fetch_schema(const TFileRangeDesc& range,
return convert_arrow_schema_to_doris(schema, column_names, column_types);
}
+namespace {
+// A dataset URI without its query string or a password in its user information, either of which
+// may carry credentials, for errors and profiles. A user part without a password stays: Azure
+// URIs name the container there (abfss://container@account...).
+std::string display_uri(const std::string& uri) {
+ std::string display = uri.substr(0, uri.find('?'));
+ const auto authority = display.find("://");
+ if (authority != std::string::npos) {
+ const auto start = authority + 3;
+ const auto at = display.find('@', start);
+ const auto colon = display.find(':', start);
+ if (at != std::string::npos && at < display.find('/', start) && colon < at) {
+ display.erase(start, at + 1 - start);
+ }
+ }
+ return display;
+}
+
+// A Lance version of 0 opens the latest version, which would silently leave the snapshot FE
+// planned. Search splits and row-id fetches always carry FE's pinned version.
+Status require_fixed_version(const TFileRangeDesc& range, std::string_view what) {
+ const auto& params = range.table_format_params.lance_params;
+ if (!params.__isset.version || params.version <= 0) {
+ return Status::InvalidArgument(
+ "Lance {} requires a fixed positive dataset version, but got {}", what,
+ params.__isset.version ? std::to_string(params.version) : std::string("none"));
+ }
+ return Status::OK();
+}
+} // namespace
+
Status LanceTableReader::init(TableReadOptions&& options) {
RETURN_IF_ERROR(TableReader::init(std::move(options)));
DORIS_CHECK(_runtime_state != nullptr);
@@ -161,6 +192,9 @@ Status LanceTableReader::prepare_split(const SplitReadOptions& options) {
"Lance global row id requested without global row id context");
}
+ if (_search_kind != SearchKind::NORMAL) {
+ RETURN_IF_ERROR(require_fixed_version(options.current_range, "search split"));
+ }
RETURN_IF_ERROR(_ensure_dataset_open(options.current_range));
RETURN_IF_ERROR(_open_scanner(options.current_range));
return Status::OK();
@@ -209,7 +243,7 @@ Status LanceTableReader::get_block(Block* block, bool* eos) {
break;
}
if (scan_status != 0 || raw_batch == nullptr) {
- return lance_error("read next Lance batch");
+ return lance_error("read next Lance batch at " + _snapshot_description());
}
std::unique_ptr<LanceBatch, LanceBatchDeleter> batch(raw_batch);
@@ -261,6 +295,8 @@ Status LanceTableReader::read_by_row_ids(const TFileRangeDesc& range,
TUnit::UNIT, LANCE_READER_PROFILE, 1);
SCOPED_TIMER(_row_id_fetch_total_time);
+ // Row IDs mean something only within the snapshot phase one read.
+ RETURN_IF_ERROR(require_fixed_version(range, "row-id fetch"));
// Phase-two row fetch does not execute FTS, so a reader created only for take_rows must not
// collect query-specific global statistics.
RETURN_IF_ERROR(_ensure_dataset_open(range, false));
@@ -283,15 +319,15 @@ Status LanceTableReader::read_by_row_ids(const TFileRangeDesc& range,
if (stream.release != nullptr) {
stream.release(&stream);
}
- return lance_error("take Lance rows by row id");
+ return lance_error("take Lance rows by row id at " + _snapshot_description());
}
auto imported_reader = arrow::ImportRecordBatchReader(&stream);
if (!imported_reader.ok()) {
if (stream.release != nullptr) {
stream.release(&stream);
}
- return Status::InternalError("import Lance take-rows stream failed: {}",
- imported_reader.status().message());
+ return Status::InternalError("import Lance take-rows stream at {} failed: {}",
+ _snapshot_description(), imported_reader.status().message());
}
size_t fetched_rows = 0;
@@ -305,8 +341,8 @@ Status LanceTableReader::read_by_row_ids(const TFileRangeDesc& range,
read_status = batch_reader->ReadNext(&record_batch);
}
if (!read_status.ok()) {
- return Status::InternalError("read Lance take-rows batch failed: {}",
- read_status.message());
+ return Status::InternalError("read Lance take-rows batch at {} failed: {}",
+ _snapshot_description(), read_status.message());
}
if (record_batch == nullptr) {
break;
@@ -321,8 +357,9 @@ Status LanceTableReader::read_by_row_ids(const TFileRangeDesc& range,
COUNTER_UPDATE(fetch_rows, rows);
}
if (fetched_rows != row_ids.size()) {
- return Status::InternalError("Lance row-id fetch returned {} rows for {} requested row ids",
- fetched_rows, row_ids.size());
+ return Status::InternalError(
+ "Lance row-id fetch at {} returned {} rows for {} requested row ids",
+ _snapshot_description(), fetched_rows, row_ids.size());
}
return Status::OK();
}
@@ -593,15 +630,66 @@ Status LanceTableReader::_open_dataset(const DatasetKey& key) {
{
SCOPED_TIMER(_dataset_open_time);
LanceDataset* raw_dataset = nullptr;
- RETURN_IF_ERROR(LanceSessionManager::instance().open_dataset(
+ Status status = LanceSessionManager::instance().open_dataset(
key.uri.c_str(), key.storage_options.empty() ? nullptr : storage_option_ptrs.data(),
- static_cast<uint64_t>(key.version), &raw_dataset));
+ static_cast<uint64_t>(key.version), &raw_dataset);
+ if (!status.ok()) {
+ return status.prepend(fmt::format("Lance dataset version {} at {}: ", key.version,
+ display_uri(key.uri)));
+ }
dataset.reset(raw_dataset);
}
+
+ // EXPLAIN plans a moving selector separately, so only the profile shows what this execution
+ // read. The version is the one opened, which version 0 (the latest) resolves.
+ _opened_version = static_cast<int64_t>(lance_dataset_version(dataset.get()));
+ _scanner_profile->add_info_string("LanceDatasetVersion", std::to_string(_opened_version));
+ _scanner_profile->add_info_string("LanceDatasetUri", display_uri(key.uri));
_dataset = dataset.release();
return Status::OK();
}
+Status LanceTableReader::_check_fragment_ids(const std::vector<uint64_t>& fragment_ids) const {
+ if (fragment_ids.empty()) {
+ return Status::OK();
+ }
+ if (!_dataset_fragment_ids.has_value()) {
+ // A count of 0 is also lance-c's error value; a failed listing then leaves the list empty,
+ // so every fragment id FE sends is still rejected rather than skipped.
+ const uint64_t fragment_count = lance_dataset_fragment_count(_dataset);
+ std::vector<uint64_t> ids(fragment_count);
+ if (fragment_count > 0 && lance_dataset_fragment_ids(_dataset, ids.data()) != 0) {
+ return lance_error("list fragments of " + _snapshot_description());
+ }
+ std::sort(ids.begin(), ids.end());
+ _dataset_fragment_ids = std::move(ids);
+ }
+ std::vector<uint64_t> missing;
+ for (uint64_t id : fragment_ids) {
+ if (!std::binary_search(_dataset_fragment_ids->begin(), _dataset_fragment_ids->end(), id)) {
+ missing.emplace_back(id);
+ }
+ }
+ if (!missing.empty()) {
+ return Status::InternalError(
+ "Lance split references fragment ids [{}] that {} does not contain",
+ fmt::join(missing, ", "), _snapshot_description());
+ }
+ return Status::OK();
+}
+
+std::string LanceTableReader::_snapshot_description() const {
+ if (!_opened_dataset_key.has_value()) {
+ return "Lance dataset";
+ }
+ // Version 0 asks for the latest version; name the one that was opened. No lance-c call here:
+ // callers build this message before lance_error() reads the thread's last error, which a
+ // successful lance-c call clears.
+ const auto version = _opened_version > 0 ? _opened_version : _opened_dataset_key->version;
+ return fmt::format("Lance dataset version {} at {}", version,
+ display_uri(_opened_dataset_key->uri));
+}
+
Status LanceTableReader::_prepare_fts_query_context() {
DORIS_CHECK(_dataset != nullptr);
DORIS_CHECK(_fts_query_context == nullptr);
@@ -632,7 +720,7 @@ Status LanceTableReader::_prepare_fts_query_context() {
coverage_mode);
}
if (_fts_query_context == nullptr) {
- return lance_error("prepare Lance FTS query context");
+ return lance_error("prepare Lance FTS query context at " + _snapshot_description());
}
return Status::OK();
}
@@ -923,6 +1011,7 @@ Status LanceTableReader::_configure_normal_scan(LanceScanner* scanner,
DORIS_CHECK(scanner != nullptr);
std::vector<uint64_t> fragment_ids;
RETURN_IF_ERROR(parse_fragment_ids(lance_params, &fragment_ids));
+ RETURN_IF_ERROR(_check_fragment_ids(fragment_ids));
if (!fragment_ids.empty() &&
lance_scanner_set_fragment_ids(scanner, fragment_ids.data(), fragment_ids.size()) != 0) {
return lance_error("set Lance scanner fragment ids");
@@ -963,8 +1052,12 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner,
DORIS_CHECK(scanner != nullptr);
DORIS_CHECK(_scan_params != nullptr);
DORIS_CHECK(_scan_params->__isset.lance_scan_params);
+ const auto& lance_scan_params = _scan_params->lance_scan_params;
+ DORIS_CHECK(lance_scan_params.__isset.external_search_request);
+ const auto& request = lance_scan_params.external_search_request;
std::vector<uint64_t> fragment_ids;
RETURN_IF_ERROR(parse_fragment_ids(lance_params, &fragment_ids));
+ RETURN_IF_ERROR(_check_fragment_ids(fragment_ids));
if (!fragment_ids.empty() &&
lance_scanner_set_fragment_ids(scanner, fragment_ids.data(), fragment_ids.size()) != 0) {
return lance_error("set Lance vector scanner fragment ids");
@@ -972,6 +1065,15 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner,
std::vector<uint8_t> segment_uuids;
size_t segment_count = 0;
RETURN_IF_ERROR(parse_index_segment_uuids(lance_params, &segment_uuids, &segment_count));
+ // FE plans index segments only when use_index allows them. A split carrying both contradicts
+ // itself, and the BE would have to override either FE's plan or the request.
+ if (segment_count > 0 && request.__isset.vector_search_options &&
+ request.vector_search_options.__isset.use_index &&
+ !request.vector_search_options.use_index) {
+ return Status::InvalidArgument(
+ "Lance vector search split carries {} index segments although use_index is false",
+ segment_count);
+ }
if (segment_count > 0 &&
lance_scanner_set_index_segments(scanner, segment_uuids.data(), segment_count) != 0) {
return lance_error("set Lance vector scanner index segments");
@@ -981,9 +1083,6 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner,
if (lance_scanner_set_prefilter(scanner, true) != 0) {
return lance_error("enable Lance vector prefilter");
}
- const auto& lance_scan_params = _scan_params->lance_scan_params;
- DORIS_CHECK(lance_scan_params.__isset.external_search_request);
- const auto& request = lance_scan_params.external_search_request;
const auto& vector = request.search_query.vector_search;
const auto& query = vector.query_vector;
const auto dimension = static_cast<size_t>(query.dimension);
@@ -1102,10 +1201,13 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner,
lance_scanner_set_ef(scanner, static_cast<uint32_t>(options.ef)) != 0) {
return lance_error("set Lance vector ef");
}
- if (options.__isset.use_index &&
- lance_scanner_set_use_index(scanner, options.use_index) != 0) {
- return lance_error("set Lance vector use_index");
- }
+ }
+ // FE plans a split without index segments as a flat search, and EXPLAIN and the profile count
+ // it as one. Lance would otherwise search it with the first index on the column in manifest
+ // order, which need not be the index FE selected. A split with segments searches exactly
+ // those; one that also carries use_index=false was rejected above.
+ if (lance_scanner_set_use_index(scanner, segment_count > 0) != 0) {
+ return lance_error("set Lance vector use_index");
}
if (lance_scanner_set_offset(scanner, vector.offset) != 0) {
return lance_error("set Lance vector offset");
@@ -1133,6 +1235,7 @@ Status LanceTableReader::_configure_full_text_search(LanceScanner* scanner,
// prepared FTS context; the segment UUID is the execution boundary.
std::vector<uint64_t> fragment_ids;
RETURN_IF_ERROR(parse_fragment_ids(lance_params, &fragment_ids));
+ RETURN_IF_ERROR(_check_fragment_ids(fragment_ids));
std::vector<uint8_t> segment_uuids;
size_t segment_count = 0;
RETURN_IF_ERROR(parse_index_segment_uuids(lance_params, &segment_uuids, &segment_count));
@@ -1251,6 +1354,8 @@ void LanceTableReader::_close_dataset() {
_dataset = nullptr;
}
_dataset_schema.reset();
+ _dataset_fragment_ids.reset();
+ _opened_version = 0;
_record_batch_converter.reset_schema();
}
diff --git a/be/src/format_v2/table/lance_reader.h b/be/src/format_v2/table/lance_reader.h
index 88deea610b3..6fe11480bc6 100644
--- a/be/src/format_v2/table/lance_reader.h
+++ b/be/src/format_v2/table/lance_reader.h
@@ -78,6 +78,10 @@ private:
Status _validate_external_search_request() const;
Status _ensure_dataset_open(const TFileRangeDesc& range, bool prepare_fts_context = true);
Status _open_dataset(const DatasetKey& key);
+ // Rejects FE fragment ids the opened snapshot lacks; lance-c would silently skip them.
+ Status _check_fragment_ids(const std::vector<uint64_t>& fragment_ids) const;
+ // Names the opened snapshot in execution errors, since several share one URI or version.
+ std::string _snapshot_description() const;
Status _prepare_fts_query_context();
Status _open_scanner(const TFileRangeDesc& range);
void _init_scanner_profile();
@@ -103,6 +107,10 @@ private:
LanceScanner* _scanner = nullptr;
ShardedKVCache* _runtime_filter_cache = nullptr;
std::optional<DatasetKey> _opened_dataset_key;
+ // Sorted fragment ids of the opened snapshot, listed on the first split that names any.
+ mutable std::optional<std::vector<uint64_t>> _dataset_fragment_ids;
+ // The version the open dataset resolved to, which differs from the key's for version 0.
+ int64_t _opened_version = 0;
LanceRecordBatchConverter _record_batch_converter;
size_t _scanner_batch_size = 0;
RuntimeProfile::Counter* _planned_index_segment_count = nullptr;
diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp
index e952d8da483..d13665f2998 100644
--- a/be/test/format_v2/table/lance_reader_test.cpp
+++ b/be/test/format_v2/table/lance_reader_test.cpp
@@ -448,6 +448,18 @@ Status get_index_segment_uuids(const std::filesystem::path& dataset_uri, const c
return Status::OK();
}
+// A split over `fragment_ids` searching the segments of `index_name`, as FE plans an indexed split;
+// BE searches a vector index only through the segments FE names.
+TFileRangeDesc make_indexed_lance_range(const std::filesystem::path& dataset_uri,
+ const LanceFixtureInfo& fixture,
+ std::vector<int64_t> fragment_ids, const char* index_name) {
+ auto range = make_lance_range(dataset_uri, fixture.version, std::move(fragment_ids));
+ std::vector<std::string> segment_uuids;
+ EXPECT_TRUE(get_index_segment_uuids(dataset_uri, index_name, &segment_uuids).ok());
+ range.table_format_params.lance_params.__set_index_segment_uuids(std::move(segment_uuids));
+ return range;
+}
+
TFileRangeDesc make_full_text_search_range(const std::filesystem::path& dataset_uri,
const LanceFixtureInfo& fixture,
const char* index_name) {
@@ -880,7 +892,16 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorScoresFiltersOffsetsAndIndexed
}
LanceTableReader reader;
ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok());
- ASSERT_TRUE(prepare_fixture(&reader, uri, fixture, fixture.fragment_ids).ok());
+ // BE searches an index only through the segments FE planned; the index covers
+ // fragment 0 and fragment 1 is searched flat in the same split.
+ auto range = make_lance_range(uri, fixture.version, fixture.fragment_ids);
+ if (indexed) {
+ std::vector<std::string> segments;
+ ASSERT_TRUE(get_index_segment_uuids(uri, (column + "_idx").c_str(), &segments)
+ .ok());
+ range.table_format_params.lance_params.__set_index_segment_uuids(segments);
+ }
+ ASSERT_TRUE(prepare_range(&reader, range).ok());
Block block;
add_output_columns(&block, columns);
auto rows = read_vector_search_rows(&reader, &block);
@@ -1058,6 +1079,328 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorSegmentTopKMatchesIndependentG
}
}
+TEST(LanceTableReaderVectorSearchTest, SplitWithoutIndexSegmentsSearchesFlat) {
+ // FE plans a split without index segments as a flat search. Lance must not search it with
+ // an index of its own choosing, even one covering the split and with use_index=true.
+ const std::filesystem::path uri =
+ "./be/test/format_v2/table/lance/data/multivector_ivf_flat.lance";
+ LanceFixtureInfo fixture;
+ ASSERT_TRUE(get_fixture_info(uri, &fixture).ok());
+ ASSERT_EQ(3, fixture.fragment_ids.size());
+ std::vector<std::string> segments;
+ ASSERT_TRUE(get_index_segment_uuids(uri, "vectors_idx", &segments).ok());
+ ASSERT_EQ(2, segments.size());
+ const auto query = representative_multi_vector(7, 0);
+ const Columns columns {projected_column("row_id", TYPE_BIGINT, false),
+ projected_column("_distance", TYPE_FLOAT, true)};
+ TQueryGlobals globals;
+ RuntimeState state(globals);
+ std::map<std::string, std::vector<std::pair<int64_t, float>>> results;
+ for (const auto* mode : {"segment", "no_segment_default", "no_segment_use_index"}) {
+ SCOPED_TRACE(mode);
+ RuntimeProfile profile("lance_segmentless_split");
+ auto params = make_float32_vector_search_params({0, 0, 0}, 5, 0);
+ auto& request = params.lance_scan_params.external_search_request;
+ request.vector_search_options.__isset.use_index = false;
+ if (std::string(mode) == "no_segment_use_index") {
+ request.vector_search_options.__set_use_index(true);
+ }
+ request.vector_search_options.__set_nprobes(64);
+ auto& search = request.search_query.vector_search;
+ search.__set_column("vectors");
+ search.__set_metric(TVectorMetric::COSINE);
+ search.query_vector.__set_dimension(128);
+ search.query_vector.__set_num_vectors(1);
+ std::string bytes(query.size() * sizeof(float), '\0');
+ for (size_t j = 0; j < query.size(); ++j) {
+ LittleEndian::Store32(bytes.data() + j * sizeof(float),
+ std::bit_cast<uint32_t>(query[j]));
+ }
+ search.query_vector.__set_values(bytes);
+ auto range = make_lance_range(uri, fixture.version, {fixture.fragment_ids[0]});
+ if (std::string(mode) == "segment") {
+ range.table_format_params.lance_params.__set_index_segment_uuids({segments[0]});
+ }
+ LanceTableReader reader;
+ ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok());
+ ASSERT_TRUE(prepare_range(&reader, range).ok());
+ Block block;
+ add_output_columns(&block, columns);
+ results[mode] = read_vector_search_rows(&reader, &block);
+ ASSERT_EQ(5, results[mode].size());
+ EXPECT_TRUE(reader.close().ok());
+ auto* partitions = profile.get_counter("LanceIVFPartitionsSearched");
+ ASSERT_NE(nullptr, partitions);
+ auto* flat = profile.get_counter("LancePlannedFlatSearchFragmentCount");
+ ASSERT_NE(nullptr, flat);
+ if (std::string(mode) == "segment") {
+ EXPECT_GT(partitions->value(), 0);
+ EXPECT_EQ(0, flat->value());
+ } else {
+ EXPECT_EQ(0, partitions->value());
+ EXPECT_EQ(1, flat->value());
+ }
+ }
+ EXPECT_EQ(results["no_segment_default"], results["no_segment_use_index"]);
+}
+
+TEST(LanceTableReaderVectorSearchTest, RejectsIndexSegmentsWhenUseIndexIsFalse) {
+ // FE plans index segments only when use_index allows them. A split carrying segments under
+ // use_index=false is rejected instead of being searched with or without the index.
+ const std::filesystem::path uri =
+ "./be/test/format_v2/table/lance/data/multivector_ivf_flat.lance";
+ LanceFixtureInfo fixture;
+ ASSERT_TRUE(get_fixture_info(uri, &fixture).ok());
+ std::vector<std::string> segments;
+ ASSERT_TRUE(get_index_segment_uuids(uri, "vectors_idx", &segments).ok());
+ ASSERT_FALSE(segments.empty());
+ const auto query = representative_multi_vector(7, 0);
+ const Columns columns {projected_column("row_id", TYPE_BIGINT, false),
+ projected_column("_distance", TYPE_FLOAT, true)};
+ TQueryGlobals globals;
+ RuntimeState state(globals);
+ for (const bool with_segment : {true, false}) {
+ SCOPED_TRACE(with_segment ? "with segment" : "without segment");
+ RuntimeProfile profile("lance_segment_with_use_index_false");
+ auto params = make_float32_vector_search_params({0, 0, 0}, 5, 0);
+ auto& request = params.lance_scan_params.external_search_request;
+ ASSERT_TRUE(request.vector_search_options.__isset.use_index);
+ ASSERT_FALSE(request.vector_search_options.use_index);
+ auto& search = request.search_query.vector_search;
+ search.__set_column("vectors");
+ search.__set_metric(TVectorMetric::COSINE);
+ search.query_vector.__set_dimension(128);
+ search.query_vector.__set_num_vectors(1);
+ std::string bytes(query.size() * sizeof(float), '\0');
+ for (size_t j = 0; j < query.size(); ++j) {
+ LittleEndian::Store32(bytes.data() + j * sizeof(float),
+ std::bit_cast<uint32_t>(query[j]));
+ }
+ search.query_vector.__set_values(bytes);
+ auto range = make_lance_range(uri, fixture.version, {fixture.fragment_ids[0]});
+ if (with_segment) {
+ range.table_format_params.lance_params.__set_index_segment_uuids({segments[0]});
+ }
+ LanceTableReader reader;
+ ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok());
+ const auto status = prepare_range(&reader, range);
+ if (with_segment) {
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(
+ status.to_string().find("carries 1 index segments although use_index is false"),
+ std::string::npos)
+ << status.to_string();
+ } else {
+ ASSERT_TRUE(status.ok()) << status.to_string();
+ Block block;
+ add_output_columns(&block, columns);
+ EXPECT_EQ(5, read_vector_search_rows(&reader, &block).size());
+ auto* flat = profile.get_counter("LancePlannedFlatSearchFragmentCount");
+ ASSERT_NE(nullptr, flat);
+ EXPECT_EQ(1, flat->value());
+ }
+ EXPECT_TRUE(reader.close().ok());
+ }
+}
+
+TEST(LanceTableReaderSnapshotTest, RejectsFragmentIdsMissingFromTheSnapshot) {
+ // lance-c silently skips a fragment id its snapshot lacks, which would drop rows.
+ const std::filesystem::path dataset_uri =
+ "./be/test/format_v2/table/lance/data/all_types.lance";
+ LanceFixtureInfo fixture;
+ ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok());
+ TQueryGlobals query_globals;
+ RuntimeState state(query_globals);
+ const Columns search_columns {projected_column("row_id", TYPE_BIGINT, false),
+ projected_column("_distance", TYPE_FLOAT, true)};
+ const Columns scan_columns {projected_column("row_id", TYPE_BIGINT, false)};
+ for (const bool search : {true, false}) {
+ SCOPED_TRACE(search ? "vector search" : "normal scan");
+ RuntimeProfile profile("lance_missing_fragment");
+ auto scan_params = search ? make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 0)
+ : TFileScanRangeParams {};
+ LanceTableReader reader;
+ ASSERT_TRUE(init_reader(&reader, search ? search_columns : scan_columns, &state, &profile,
+ &scan_params)
+ .ok());
+ const auto status =
+ prepare_fixture(&reader, dataset_uri, fixture, {fixture.fragment_ids[0], 999});
+ ASSERT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find(
+ "references fragment ids [999] that Lance dataset version " +
+ std::to_string(fixture.version) + " at " + dataset_uri.string() +
+ " does not contain"),
+ std::string::npos)
+ << status.to_string();
+ EXPECT_TRUE(reader.close().ok());
+ }
+
+ // Full-text search installs no fragment filter, but its fragment ids still describe the
+ // planned coverage of this snapshot.
+ const std::filesystem::path fts_uri = "./be/test/format_v2/table/lance/data/fts_indexed.lance";
+ LanceFixtureInfo fts_fixture;
+ ASSERT_TRUE(get_fixture_info(fts_uri, &fts_fixture).ok());
+ RuntimeProfile fts_profile("lance_missing_fragment_fts");
+ auto fts_params = make_full_text_search_params("lance", 2, 0);
+ const Columns fts_columns {projected_column("row_id", TYPE_BIGINT, false),
+ projected_column("_score", TYPE_FLOAT, true)};
+ LanceTableReader fts_reader;
+ ASSERT_TRUE(init_reader(&fts_reader, fts_columns, &state, &fts_profile, &fts_params).ok());
+ auto fts_range = make_full_text_search_range(fts_uri, fts_fixture, "body_fts");
+ fts_range.table_format_params.lance_params.fragment_ids.push_back(999);
+ const auto fts_status = prepare_range(&fts_reader, fts_range);
+ ASSERT_FALSE(fts_status.ok());
+ EXPECT_NE(fts_status.to_string().find("references fragment ids [999]"), std::string::npos)
+ << fts_status.to_string();
+ EXPECT_TRUE(fts_reader.close().ok());
+}
+
+TEST(LanceTableReaderSnapshotTest, RejectsSearchSplitsAndRowIdFetchesWithoutFixedVersion) {
+ // Version 0 opens the latest version, which is not the snapshot FE planned.
+ const std::filesystem::path dataset_uri =
+ "./be/test/format_v2/table/lance/data/all_types.lance";
+ LanceFixtureInfo fixture;
+ ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok());
+ TQueryGlobals query_globals;
+ RuntimeState state(query_globals);
+ const Columns columns {projected_column("row_id", TYPE_BIGINT, false),
+ projected_column("_distance", TYPE_FLOAT, true)};
+ for (const int64_t version : {int64_t {0}, int64_t {-1}, int64_t {-2}}) {
+ // -2 stands for a version FE never set.
+ const bool unset = version == -2;
+ RuntimeProfile profile("lance_unpinned_search");
+ auto scan_params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 0);
+ LanceTableReader reader;
+ ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok());
+ auto range = make_lance_range(dataset_uri, version, fixture.fragment_ids);
+ if (unset) {
+ range.table_format_params.lance_params.__isset.version = false;
+ }
+ const auto status = prepare_range(&reader, range);
+ ASSERT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find(std::string("Lance search split requires a fixed "
+ "positive dataset version, but got ") +
+ (unset ? std::string("none") : std::to_string(version))),
+ std::string::npos)
+ << status.to_string();
+ EXPECT_TRUE(reader.close().ok());
+ }
+
+ RuntimeProfile fetch_profile("lance_unpinned_fetch");
+ TFileScanRangeParams scan_params;
+ const Columns payload_columns {projected_column("row_id", TYPE_BIGINT, false)};
+ LanceTableReader reader;
+ ASSERT_TRUE(init_reader(&reader, payload_columns, &state, &fetch_profile, &scan_params).ok());
+ Block block;
+ add_output_columns(&block, payload_columns);
+ const auto status = reader.read_by_row_ids(
+ make_lance_range(dataset_uri, 0, fixture.fragment_ids), {0}, &block);
+ ASSERT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find(
+ "Lance row-id fetch requires a fixed positive dataset version, but got 0"),
+ std::string::npos)
+ << status.to_string();
+ EXPECT_TRUE(reader.close().ok());
+}
+
+TEST(LanceTableReaderSnapshotTest, ProfileShowsTheSnapshotRead) {
+ const std::filesystem::path dataset_uri =
+ "./be/test/format_v2/table/lance/data/all_types.lance";
+ LanceFixtureInfo fixture;
+ ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok());
+ TQueryGlobals query_globals;
+ RuntimeState state(query_globals);
+ RuntimeProfile profile("lance_snapshot_profile");
+ auto scan_params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 0);
+ const Columns columns {projected_column("row_id", TYPE_BIGINT, false),
+ projected_column("_distance", TYPE_FLOAT, true)};
+ LanceTableReader reader;
+ ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok());
+ // A query string may carry credentials, so the profile leaves it out.
+ auto range = make_lance_range(dataset_uri, fixture.version, fixture.fragment_ids);
+ ASSERT_TRUE(prepare_range(&reader, range).ok());
+ const auto* version = profile.get_info_string("LanceDatasetVersion");
+ ASSERT_NE(nullptr, version);
+ EXPECT_EQ(std::to_string(fixture.version), *version);
+ const auto* uri = profile.get_info_string("LanceDatasetUri");
+ ASSERT_NE(nullptr, uri);
+ EXPECT_EQ(dataset_uri.string(), *uri);
+ EXPECT_TRUE(reader.close().ok());
+
+ // A scan of version 0 opens the latest version, and the profile shows which one that was.
+ RuntimeProfile latest_profile("lance_snapshot_profile_latest");
+ TFileScanRangeParams normal_params;
+ const Columns normal_columns {projected_column("row_id", TYPE_BIGINT, false)};
+ LanceTableReader latest_reader;
+ ASSERT_TRUE(init_reader(&latest_reader, normal_columns, &state, &latest_profile, &normal_params)
+ .ok());
+ ASSERT_TRUE(prepare_range(&latest_reader, make_latest_lance_range(dataset_uri)).ok());
+ const auto* latest_version = latest_profile.get_info_string("LanceDatasetVersion");
+ ASSERT_NE(nullptr, latest_version);
+ EXPECT_EQ(std::to_string(fixture.version), *latest_version);
+ EXPECT_TRUE(latest_reader.close().ok());
+}
+
+TEST(LanceTableReaderSnapshotTest, OpenErrorsNameTheVersionWithoutTheQueryString) {
+ const std::filesystem::path dataset_uri =
+ "./be/test/format_v2/table/lance/data/all_types.lance";
+ LanceFixtureInfo fixture;
+ ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok());
+ TQueryGlobals query_globals;
+ RuntimeState state(query_globals);
+ RuntimeProfile profile("lance_open_error");
+ auto scan_params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 0);
+ const Columns columns {projected_column("row_id", TYPE_BIGINT, false),
+ projected_column("_distance", TYPE_FLOAT, true)};
+ LanceTableReader reader;
+ ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok());
+ // The query string stands for credentials a URI may carry; Doris's own prefix leaves it out.
+ // What lance-c reports after the prefix is its own message.
+ auto range = make_lance_range(dataset_uri, 999, fixture.fragment_ids);
+ range.table_format_params.lance_params.__set_dataset_uri(dataset_uri.string() +
+ "?secret=hidden");
+ const auto status = prepare_range(&reader, range);
+ ASSERT_FALSE(status.ok());
+ EXPECT_NE(
+ status.to_string().find("Lance dataset version 999 at " + dataset_uri.string() + ": "),
+ std::string::npos)
+ << status.to_string();
+ EXPECT_TRUE(reader.close().ok());
+}
+
+TEST(LanceTableReaderSnapshotTest, OpenErrorsShowTheUriWithoutItsPassword) {
+ // A password in the user information may carry a credential and is removed; a user part
+ // without a password stays, since Azure URIs name the container there. Lance parses a
+ // memory:// URI as a URL and names only the object path in its own error text, so the whole
+ // error must be free of the password and the query string. (A file:// URI with user
+ // information is not a valid URL; Lance would open it as a relative local path and echo it.)
+ TQueryGlobals query_globals;
+ RuntimeState state(query_globals);
+ const Columns columns {projected_column("row_id", TYPE_BIGINT, false),
+ projected_column("_distance", TYPE_FLOAT, true)};
+ for (const auto& [uri, shown] : std::vector<std::pair<std::string, std::string>> {
+ {"memory://doris:secret@localhost/open_error.lance?token=hidden",
+ "memory://localhost/open_error.lance"},
+ {"memory://container@localhost/open_error.lance?token=hidden",
+ "memory://container@localhost/open_error.lance"}}) {
+ SCOPED_TRACE(uri);
+ RuntimeProfile profile("lance_open_error_uri");
+ auto scan_params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 0);
+ LanceTableReader reader;
+ ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok());
+ auto range = make_lance_range("open_error.lance", 999, {0});
+ range.table_format_params.lance_params.__set_dataset_uri(uri);
+ const auto status = prepare_range(&reader, range);
+ ASSERT_FALSE(status.ok());
+ const std::string message = status.to_string();
+ EXPECT_NE(message.find("Lance dataset version 999 at " + shown + ": "), std::string::npos)
+ << message;
+ EXPECT_EQ(message.find("secret"), std::string::npos) << message;
+ EXPECT_EQ(message.find("hidden"), std::string::npos) << message;
+ EXPECT_TRUE(reader.close().ok());
+ }
+}
+
TEST(LanceTableReaderVectorSearchTest, MultiVectorTopOnePreservesPrecisionAndBatchIndependence) {
const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/multivector.lance";
LanceFixtureInfo fixture;
@@ -1093,7 +1436,12 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorTopOnePreservesPrecisionAndBat
RuntimeProfile profile("multi_vector_top_one");
LanceTableReader reader;
ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok());
- ASSERT_TRUE(prepare_fixture(&reader, uri, fixture, fixture.fragment_ids).ok());
+ ASSERT_TRUE(prepare_range(&reader, indexed ? make_indexed_lance_range(
+ uri, fixture, fixture.fragment_ids,
+ "batch_vectors_idx")
+ : make_lance_range(uri, fixture.version,
+ fixture.fragment_ids))
+ .ok());
Block block;
add_output_columns(&block, columns);
auto rows = read_vector_search_rows(&reader, &block);
@@ -1102,6 +1450,7 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorTopOnePreservesPrecisionAndBat
EXPECT_NEAR(indexed ? 2.0F - std::sqrt(2.0F) : 1e-8F, rows[0].second,
indexed ? 1e-6F : 1e-13F);
EXPECT_TRUE(reader.close().ok());
+ EXPECT_EQ(indexed, profile.get_counter("LanceIVFPartitionsSearched")->value() > 0);
}
}
}
@@ -1136,7 +1485,12 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorCosineMasksUndefinedRows) {
RuntimeProfile profile("multi_vector_cosine_zero_norm");
LanceTableReader reader;
ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok());
- ASSERT_TRUE(prepare_fixture(&reader, uri, fixture, fixture.fragment_ids).ok());
+ ASSERT_TRUE(prepare_range(&reader, indexed ? make_indexed_lance_range(
+ uri, fixture, fixture.fragment_ids,
+ "vectors_idx")
+ : make_lance_range(uri, fixture.version,
+ fixture.fragment_ids))
+ .ok());
Block block;
add_output_columns(&block, columns);
auto rows = read_vector_search_rows(&reader, &block);
@@ -1147,10 +1501,14 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorCosineMasksUndefinedRows) {
: std::vector<std::pair<int64_t, float>> {{2, 1}, {3, 1}};
EXPECT_EQ(expected, rows);
EXPECT_TRUE(reader.close().ok());
+ EXPECT_EQ(indexed, profile.get_counter("LanceIVFPartitionsSearched")->value() > 0);
}
}
}
+// FE plans an omitted metric as L2, and this column's only index is cosine, so FE plans every split
+// flat. BE must then search flat too, with the same L2 metric in every split, although the request
+// allows indexes.
TEST(LanceTableReaderVectorSearchTest, MultiVectorDefaultMetricIsConsistentAcrossFragments) {
const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/multivector.lance";
LanceFixtureInfo fixture;
@@ -1183,6 +1541,7 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorDefaultMetricIsConsistentAcros
auto rows = read_vector_search_rows(&reader, &block);
all_rows.insert(all_rows.end(), rows.begin(), rows.end());
EXPECT_TRUE(reader.close().ok());
+ EXPECT_EQ(0, profile.get_counter("LanceIVFPartitionsSearched")->value());
}
std::sort(all_rows.begin(), all_rows.end());
EXPECT_EQ((std::vector<std::pair<int64_t, float>> {{1, 0}, {2, 1}, {3, 4}, {6, 1}}), all_rows);
@@ -1220,7 +1579,8 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorRejectsActualNullAndNonFiniteE
bool eos = false;
auto status = reader.get_block(&block, &eos);
EXPECT_FALSE(status.ok());
- EXPECT_NE(status.to_string().find("finite, non-null elements"), std::string::npos);
+ EXPECT_NE(status.to_string().find("finite, non-null elements"), std::string::npos)
+ << status.to_string();
EXPECT_TRUE(reader.close().ok());
}
}
diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
index 0ea1088377e..f2dddc0a319 100644
--- a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
+++ b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
@@ -118,11 +118,11 @@ services:
- ./scripts/lance_rest_server.py:/opt/lance-rest/server.py:ro
environment:
LANCE_REST_BEARER_TOKEN: doris-lance-rest-test-token
- LANCE_REST_TABLES_JSON: '{"all_types":"s3://warehouse/lance/all_types.lance","all_types_unprefixed":"s3://warehouse/lance/all_types.lance","time_travel":"s3://warehouse/lance/time_travel.lance"}'
+ LANCE_REST_TABLES_JSON: '{"all_types":"s3://warehouse/lance/all_types.lance","all_types_unprefixed":"s3://warehouse/lance/all_types.lance","time_travel":"s3://warehouse/lance/time_travel.lance","search_snapshot":"s3://warehouse/lance/search_snapshot.lance","search_snapshot_pruned":"s3://warehouse/lance/search_snapshot_pruned.lance"}'
# all_types_unprefixed serves the same dataset but vends its credentials under the
# unprefixed object-store spelling, which is what real namespace servers emit.
LANCE_REST_UNPREFIXED_TABLES_JSON: '["all_types_unprefixed","time_travel_managed_unprefixed"]'
- # All managed tables are the same three-version dataset; they differ in what the namespace
+ # The time_travel_managed tables are the same three-version dataset; they differ in what the namespace
# records. A reader that resolves versions through the namespace cannot see a version
# missing there even though its manifest is still in storage, and its latest version is
# the namespace's latest, not storage's:
@@ -138,10 +138,17 @@ services:
# although its canonical one exists
# time_travel_managed_misrecorded every version, version 3 recorded at a manifest Doris
# does not read
- # Versions and commit times (epoch millis, UTC) match the committed time_travel.lance,
- # see lance_build_time_travel.py. Doris resolves FOR TIME AS OF from the manifests' commit
+ # The search_snapshot_managed tables serve search_snapshot.lance (seven versions, see
+ # lance_build_search_snapshot.py) to the search TVFs:
+ # search_snapshot_managed every version, with commit times, and branch dev
+ # (versions 4 to 6 under tree/dev/)
+ # search_snapshot_managed_partial every version but 6
+ # unicode_branch_managed serves unicode_branch.lance, whose branch dev\u1c89 has a letter newer
+ # than the JDK's Unicode tables (see lance_build_unicode_branch.py).
+ # Versions and commit times (epoch millis, UTC) match the committed time_travel.lance and
+ # search_snapshot.lance, see lance_build_time_travel.py and lance_build_search_snapshot.py. Doris resolves FOR TIME AS OF from the manifests' commit
# times, so the reported ones are not what the time-travel results depend on.
- LANCE_REST_MANAGED_TABLES_JSON: '{"time_travel_managed":{"uri":"s3://warehouse/lance/time_travel.lance","versions":[{"version":1,"timestamp_millis":1789823167597},{"version":2,"timestamp_millis":1789823169113},{"version":3,"timestamp_millis":1789823170621}],"branches":{"dev":{"versions":[2,3]}}},"time_travel_managed_partial":{"uri":"s3://warehouse/lance/time_travel.lance","versions":[{"version":1,"timestamp_millis":1789823167597},{"version":3,"timestamp_millis":1789823170621}]},"ti [...]
+ LANCE_REST_MANAGED_TABLES_JSON: '{"time_travel_managed":{"uri":"s3://warehouse/lance/time_travel.lance","versions":[{"version":1,"timestamp_millis":1789823167597},{"version":2,"timestamp_millis":1789823169113},{"version":3,"timestamp_millis":1789823170621}],"branches":{"dev":{"versions":[2,3]}}},"time_travel_managed_partial":{"uri":"s3://warehouse/lance/time_travel.lance","versions":[{"version":1,"timestamp_millis":1789823167597},{"version":3,"timestamp_millis":1789823170621}]},"ti [...]
LANCE_S3_ACCESS_KEY: admin
LANCE_S3_SECRET_KEY: password
LANCE_S3_REGION: us-east-1
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_legacy_vector_index.py b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_legacy_vector_index.py
new file mode 100644
index 00000000000..7a91b2336de
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_legacy_vector_index.py
@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Generate the Lance legacy vector index regression fixture.
+
+legacy_vector_index.lance is a root table of the preinstalled Directory catalog next to
+time_travel.lance. Its vector index was written by pylance 0.18.2, which predates index details
+in the manifest, so no segment of vec_idx carries details:
+
+ version 1 create row_id 1..512 fragment 0
+ version 2 vector index vec_idx (IVF_PQ, l2, 2 partitions, 2 sub-vectors) over fragment 0
+
+vec = [row_id, row_id + 1, row_id + 2, row_id + 3] as float32. Lance infers the missing details
+from the index files when it loads the index, so the FE plans vec_idx like any other index and
+the BE searches it through the planned segment instead of falling back to a flat search.
+
+Build it with pylance 0.18.2, not the writer pinned in lance_fixture_requirements.txt; the
+catalog build carries it over as-is. --check runs with any reader.
+
+Usage:
+ pip install pylance==0.18.2 && python3 lance_build_legacy_vector_index.py preinstalled_data/lance
+ python3 lance_build_legacy_vector_index.py --check preinstalled_data/lance
+"""
+import argparse
+import shutil
+from pathlib import Path
+
+import lance
+import pyarrow as pa
+
+LEGACY_DIR = "legacy_vector_index.lance"
+WRITER_VERSION = "0.18.2"
+ROWS = 512
+DIMENSION = 4
+
+
+def build(output: Path) -> None:
+ if lance.__version__ != WRITER_VERSION:
+ raise SystemExit(f"build {LEGACY_DIR} with pylance {WRITER_VERSION}, not {lance.__version__}")
+ if output.exists():
+ shutil.rmtree(output)
+ ids = list(range(1, ROWS + 1))
+ table = pa.table({
+ "row_id": pa.array(ids, pa.int32()),
+ "vec": pa.array([[float(i + j) for j in range(DIMENSION)] for i in ids],
+ pa.list_(pa.float32(), DIMENSION)),
+ })
+ uri = str(output)
+ lance.write_dataset(table, uri, mode="create")
+ lance.dataset(uri).create_index("vec", "IVF_PQ", name="vec_idx", metric="l2", num_partitions=2,
+ num_sub_vectors=2, sample_rate=256)
+ assert lance.dataset(uri).version == 2, lance.dataset(uri).version
+
+
+def check(output: Path) -> None:
+ dataset = lance.dataset(str(output))
+ assert [v["version"] for v in dataset.versions()] == [1, 2], dataset.versions()
+ row_ids = sorted(dataset.to_table(columns=["row_id"])["row_id"].to_pylist())
+ assert row_ids == list(range(1, ROWS + 1)), "rows differ"
+ indexes = dataset.list_indices()
+ assert [index["name"] for index in indexes] == ["vec_idx"], indexes
+ assert sorted(indexes[0]["fragment_ids"]) == [0], indexes
+ manifests = sorted((output / "_versions").glob("*.manifest"))
+ assert manifests, f"no manifest under {output / '_versions'}"
+ for manifest in manifests:
+ content = manifest.read_bytes()
+ assert b"VectorIndexDetails" not in content, f"{manifest} carries index details"
+ assert WRITER_VERSION.encode() in content, f"{manifest} was not written by pylance {WRITER_VERSION}"
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("root", type=Path, help="directory holding the fixture")
+ parser.add_argument("--check", action="store_true", help="verify the existing fixture")
+ args = parser.parse_args()
+ if args.check:
+ check(args.root / LEGACY_DIR)
+ else:
+ build(args.root / LEGACY_DIR)
+ check(args.root / LEGACY_DIR)
+ print(f"{LEGACY_DIR} {'checked' if args.check else 'written'}")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py
index 2645e8748d8..d8652bf9ef8 100644
--- a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py
@@ -32,6 +32,20 @@ The generated catalog contains:
- time_travel.lance Three uncompacted versions for FOR VERSION / TIME AS OF; carried
over as-is because the suites hard-code its commit times
(see lance_build_time_travel.py).
+ - search_snapshot.lance, search_snapshot_pruned.lance
+ Index history, tags and a branch for the search TVFs' snapshot
+ selectors; carried over as-is for the same reason
+ (see lance_build_search_snapshot.py).
+ - search_snapshot_evolved.lance
+ A column added and renamed after the vector index; rebuilt, since
+ no suite depends on its commit times.
+ - legacy_vector_index.lance
+ A vector index written by pylance 0.18.2, without index details;
+ carried over as-is because this writer cannot rebuild it
+ (see lance_build_legacy_vector_index.py).
+ - unicode_branch.lance
+ A branch named with a letter newer than the JDK's Unicode, written by
+ pylance 12.0.0; carried over as-is (see lance_build_unicode_branch.py).
- The `doris` namespace with two full-text-search fixtures, one indexed vector table per cell of the
algorithm x element type x metric matrix (hash-prefixed directories), listed in
VECTOR_TABLES below; BREADTH_TABLE, one table carrying the remaining cells at plan
@@ -90,9 +104,20 @@ import lance_namespace
import pyarrow as pa
import pyarrow.ipc as ipc
from lance_build_array_predicates import build as build_array_predicates, check as check_array_predicates
+from lance_build_legacy_vector_index import LEGACY_DIR as LEGACY_VECTOR_INDEX_DIR
+from lance_build_legacy_vector_index import check as check_legacy_vector_index
from lance_build_multivector import build as build_multivector, check as check_multivector
from lance_build_nested_null import build as build_nested_null, check as check_nested_null
+from lance_build_search_snapshot import (
+ EVOLVED_DIR as SEARCH_SNAPSHOT_EVOLVED_DIR,
+ MAIN_DIR as SEARCH_SNAPSHOT_DIR,
+ PRUNED_DIR as SEARCH_SNAPSHOT_PRUNED_DIR,
+ build_evolved as build_search_snapshot_evolved,
+ check as check_search_snapshot,
+)
from lance_build_time_travel import check as check_time_travel
+from lance_build_unicode_branch import UNICODE_BRANCH_DIR
+from lance_build_unicode_branch import check as check_unicode_branch
from lance_namespace_urllib3_client.models import (
CreateNamespaceRequest,
CreateTableRequest,
@@ -937,10 +962,13 @@ def build_multi_frag(root: Path) -> None:
lance.dataset(location).delete(f"row_id in ({deleted})")
-def build(root: Path, all_types_source: Path, time_travel_source: Path) -> None:
+def build(root: Path, all_types_source: Path, time_travel_source: Path, carried_root: Path) -> None:
shutil.copytree(all_types_source, root / ALL_TYPES_DIR)
- # Not rebuilt: its commit times are hard-coded in the time-travel suites.
+ # Not rebuilt: their commit times are hard-coded in the time-travel and search-snapshot suites.
shutil.copytree(time_travel_source, root / TIME_TRAVEL_DIR)
+ for name in (SEARCH_SNAPSHOT_DIR, SEARCH_SNAPSHOT_PRUNED_DIR, LEGACY_VECTOR_INDEX_DIR, UNICODE_BRANCH_DIR):
+ shutil.copytree(carried_root / name, root / name)
+ build_search_snapshot_evolved(root / SEARCH_SNAPSHOT_EVOLVED_DIR)
build_multi_frag(root)
# Recreate this fixture in staging because promotion replaces the entire catalog tree.
build_nested_null(root / NESTED_NULL_DIR)
@@ -1737,6 +1765,9 @@ def check_catalog(root: Path) -> None:
check_multi_frag(root)
check_nested_null(root / NESTED_NULL_DIR)
check_time_travel(root / TIME_TRAVEL_DIR)
+ check_search_snapshot(root)
+ check_legacy_vector_index(root / LEGACY_VECTOR_INDEX_DIR)
+ check_unicode_branch(root / UNICODE_BRANCH_DIR)
check_multivector(root / "multivector.lance")
full_fts = namespace.describe_table(DescribeTableRequest(id=[NAMESPACE, FTS_TABLE]))
@@ -1808,11 +1839,15 @@ def main() -> int:
if not time_travel_source.is_dir():
print(f"missing time_travel source: {time_travel_source}", file=sys.stderr)
return 1
+ for name in (SEARCH_SNAPSHOT_DIR, SEARCH_SNAPSHOT_PRUNED_DIR, LEGACY_VECTOR_INDEX_DIR, UNICODE_BRANCH_DIR):
+ if not (output / name).is_dir():
+ print(f"missing carried-over source: {output / name}", file=sys.stderr)
+ return 1
with tempfile.TemporaryDirectory(prefix="lance_fixture_") as staging_name:
staging = Path(staging_name) / "lance"
staging.mkdir()
- build(staging, all_types_source, time_travel_source)
+ build(staging, all_types_source, time_travel_source, output)
build_array_predicates(staging / "predicate_arrays")
check_catalog(staging)
backup = output.with_name(output.name + ".old")
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_search_snapshot.py b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_search_snapshot.py
new file mode 100644
index 00000000000..24d056ec995
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_search_snapshot.py
@@ -0,0 +1,329 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Generate the Lance search-snapshot regression fixtures.
+
+search_snapshot.lance, search_snapshot_pruned.lance and search_snapshot_evolved.lance are root
+tables of the preinstalled Directory catalog next to time_travel.lance. They exercise
+vector_search() and full_text_search() with the version, timestamp, tag and branch properties,
+so they keep an uncompacted history in which indexes appear, stop covering new data, are
+rebuilt, and outlive schema changes.
+
+search_snapshot.lance, main chain:
+
+ version 1 create row_id 1..8 fragment 0
+ version 2 append row_id 9..16 fragment 1
+ version 3 vector index vec_idx (IVF_FLAT, l2) over fragments 0 and 1
+ version 4 FTS index body_idx over fragments 0 and 1
+ version 5 append row_id 17..24 fragment 2, covered by neither index
+ version 6 delete row_id 3 and 11
+ version 7 vector index vec_idx rebuilt over all fragments (a new index UUID)
+
+Tag "rel" points at main version 5. Branch "dev" forks from version 4:
+
+ dev version 5 append row_id 101..108 fragment 2 (the same id as main's fragment 2)
+ dev version 6 FTS index body_idx rebuilt over all fragments, stored under tree/dev/
+
+Tag "dev_rel" points at dev version 5, which is not the branch's latest version. So main and
+dev both have versions 5 and 6 and a fragment 2, with different rows and indexes.
+
+Every row has vec = [row_id, row_id + 1, row_id + 2, row_id + 3] as float32, so the exact
+squared L2 distance between rows r and n is 4 * (n - r)^2, and body = "doc <row_id> <word>"
+with <word> "lance" for even row_ids and "doris" for odd ones. The vector index has two IVF
+partitions; the suites search with nprobes=2, which makes IVF_FLAT exact, so indexed and flat
+searches return the same rows.
+
+search_snapshot_pruned.lance keeps what cleanup leaves behind:
+
+ version 1 create row_id 1..8, removed by cleanup
+ version 2 vector index vec_idx, pinned by tag "kept"; its index files are then deleted
+ version 3 vector index vec_idx rebuilt
+
+so version 1 is missing, and version 2 opens but its index files are gone.
+
+search_snapshot_evolved.lance changes its schema after the vector index is built:
+
+ version 1 create row_id 1..8 fragment 0
+ version 2 vector index vec_idx over fragment 0
+ version 3 add column note = "note <row_id>"
+ version 4 rename vec -> embedding (same field id, so vec_idx still serves it)
+ version 5 append row_id 9..16 fragment 1, not covered by vec_idx
+
+so a search must bind the column names, the index and the fragments of the version it reads.
+
+A Lance branch is a shallow clone that records its parent's absolute URI, so the branch was
+created against the fixture's final location s3://warehouse/lance/search_snapshot.lance (see
+lance_build_time_travel.py for the same procedure):
+
+ python3 lance_build_search_snapshot.py preinstalled_data/lance
+ # upload search_snapshot.lance to s3://warehouse/lance/search_snapshot.lance
+ python3 lance_build_search_snapshot.py --create-branch s3://warehouse/lance/search_snapshot.lance \
+ --storage-option endpoint=http://127.0.0.1:19000 --storage-option access_key_id=admin \
+ --storage-option secret_access_key=password --storage-option region=us-east-1 \
+ --storage-option allow_http=true
+ # then sync tree/, _refs/branches/ and _refs/tags/ from that location back
+
+The suites hard-code the commit times this script prints, as the time-travel suites do.
+Run it with the writer pinned in lance_fixture_requirements.txt.
+
+Usage:
+ python3 lance_build_search_snapshot.py preinstalled_data/lance
+ python3 lance_build_search_snapshot.py --fixture evolved preinstalled_data/lance
+ python3 lance_build_search_snapshot.py --check preinstalled_data/lance
+
+--fixture limits a build to some of the fixtures. Rebuilding search_snapshot.lance changes the
+commit times the suites hard-code and requires recreating its branch.
+"""
+import argparse
+import shutil
+import time
+from datetime import timedelta
+from pathlib import Path
+
+import lance
+import pyarrow as pa
+
+MAIN_DIR = "search_snapshot.lance"
+PRUNED_DIR = "search_snapshot_pruned.lance"
+EVOLVED_DIR = "search_snapshot_evolved.lance"
+BRANCH = "dev"
+BRANCH_PARENT_URI = "s3://warehouse/lance/search_snapshot.lance"
+DIMENSION = 4
+COMMIT_GAP_SECONDS = 1.5
+FTS_INDEX_PARAMS = {
+ "base_tokenizer": "simple",
+ "language": "English",
+ "max_token_length": 40,
+ "lower_case": True,
+ "stem": False,
+ "remove_stop_words": False,
+ "ascii_folding": False,
+ "with_position": True,
+}
+
+
+def rows_of(low: int, high: int) -> pa.Table:
+ ids = list(range(low, high + 1))
+ vectors = [[float(i + j) for j in range(DIMENSION)] for i in ids]
+ return pa.table({
+ "row_id": pa.array(ids, pa.int32()),
+ "vec": pa.array(vectors, pa.list_(pa.float32(), DIMENSION)),
+ "body": pa.array([f"doc {i} {'lance' if i % 2 == 0 else 'doris'}" for i in ids], pa.string()),
+ })
+
+
+def create_vector_index(dataset, replace: bool = False):
+ dataset.create_index("vec", "IVF_FLAT", name="vec_idx", metric="l2", num_partitions=2,
+ sample_rate=256, replace=replace)
+
+
+def create_fts_index(dataset, replace: bool = False):
+ dataset.create_scalar_index("body", "INVERTED", name="body_idx", replace=replace, **FTS_INDEX_PARAMS)
+
+
+def pause() -> None:
+ time.sleep(COMMIT_GAP_SECONDS)
+
+
+def build_main(output: Path) -> None:
+ if output.exists():
+ shutil.rmtree(output)
+ uri = str(output)
+ lance.write_dataset(rows_of(1, 8), uri, mode="create", data_storage_version="2.2")
+ pause()
+ lance.write_dataset(rows_of(9, 16), uri, mode="append", data_storage_version="2.2")
+ pause()
+ create_vector_index(lance.dataset(uri))
+ pause()
+ create_fts_index(lance.dataset(uri))
+ pause()
+ lance.write_dataset(rows_of(17, 24), uri, mode="append", data_storage_version="2.2")
+ pause()
+ lance.dataset(uri).delete("row_id IN (3, 11)")
+ pause()
+ create_vector_index(lance.dataset(uri), replace=True)
+ dataset = lance.dataset(uri)
+ assert dataset.version == 7, dataset.version
+ dataset.tags.create("rel", 5)
+
+
+def build_pruned(output: Path) -> None:
+ if output.exists():
+ shutil.rmtree(output)
+ uri = str(output)
+ lance.write_dataset(rows_of(1, 8), uri, mode="create", data_storage_version="2.2")
+ create_vector_index(lance.dataset(uri))
+ kept = lance.dataset(uri)
+ kept_index = {index["name"]: index["uuid"] for index in kept.list_indices()}["vec_idx"]
+ kept.tags.create("kept", 2)
+ create_vector_index(lance.dataset(uri), replace=True)
+ dataset = lance.dataset(uri)
+ # The tag keeps version 2 through cleanup, as a tagged release would be kept.
+ dataset.cleanup_old_versions(older_than=timedelta(0), delete_unverified=True,
+ error_if_tagged_old_versions=False)
+ shutil.rmtree(output / "_indices" / kept_index)
+ assert [v["version"] for v in lance.dataset(uri).versions()] == [2, 3]
+
+
+def evolved_rows(low: int, high: int, schema: pa.Schema) -> pa.Table:
+ rows = rows_of(low, high).rename_columns(["row_id", "embedding", "body"])
+ notes = pa.array([f"note {i}" for i in range(low, high + 1)], pa.string())
+ return rows.append_column("note", notes).cast(schema)
+
+
+def build_evolved(output: Path) -> None:
+ if output.exists():
+ shutil.rmtree(output)
+ uri = str(output)
+ lance.write_dataset(rows_of(1, 8), uri, mode="create", data_storage_version="2.2")
+ create_vector_index(lance.dataset(uri))
+ lance.dataset(uri).add_columns({"note": "concat('note ', CAST(row_id AS STRING))"})
+ lance.dataset(uri).alter_columns({"path": "vec", "name": "embedding"})
+ renamed = lance.dataset(uri)
+ lance.write_dataset(evolved_rows(9, 16, renamed.schema), uri, mode="append", data_storage_version="2.2")
+ assert lance.dataset(uri).version == 5, lance.dataset(uri).version
+
+
+FIXTURES = {"main": (MAIN_DIR, build_main), "pruned": (PRUNED_DIR, build_pruned),
+ "evolved": (EVOLVED_DIR, build_evolved)}
+
+
+def build(root: Path, fixtures: list) -> None:
+ for name in fixtures:
+ directory, builder = FIXTURES[name]
+ builder(root / directory)
+ print(f"{directory} written")
+ if "main" in fixtures:
+ print(f"create the branch with --create-branch {BRANCH_PARENT_URI}")
+
+
+def create_branch(uri: str, storage_options: dict) -> None:
+ """Forks dev at the dataset's final location, appends to it and rebuilds its FTS index there."""
+ dataset = lance.dataset(uri, storage_options=storage_options)
+ assert dataset.version == 7, "upload the main chain first"
+ dataset.create_branch(BRANCH, 4)
+ branch_uri = f"{uri}/tree/{BRANCH}"
+ pause()
+ lance.write_dataset(rows_of(101, 108), branch_uri, mode="append", data_storage_version="2.2",
+ storage_options=storage_options)
+ pause()
+ create_fts_index(lance.dataset(branch_uri, storage_options=storage_options), replace=True)
+ branch = lance.dataset(branch_uri, storage_options=storage_options)
+ assert branch.version == 6, branch.version
+ dataset.tags.create("dev_rel", (BRANCH, 5))
+ print(f"branch {BRANCH} and tag dev_rel created at {uri}; sync tree/, _refs/branches/ and _refs/tags/ back")
+
+
+def row_ids(dataset) -> list:
+ return sorted(dataset.to_table(columns=["row_id"])["row_id"].to_pylist())
+
+
+def check(root: Path) -> None:
+ output = root / MAIN_DIR
+ dataset = lance.dataset(str(output))
+ versions = dataset.versions()
+ assert [v["version"] for v in versions] == list(range(1, 8)), versions
+ timestamps = [v["timestamp"] for v in versions]
+ assert all((b - a).total_seconds() >= 1 for a, b in zip(timestamps, timestamps[1:])), timestamps
+ expected_rows = {
+ 1: list(range(1, 9)), 2: list(range(1, 17)), 3: list(range(1, 17)), 4: list(range(1, 17)),
+ 5: list(range(1, 25)), 6: [i for i in range(1, 25) if i not in (3, 11)],
+ 7: [i for i in range(1, 25) if i not in (3, 11)],
+ }
+ indexes = {}
+ for version, rows in expected_rows.items():
+ snapshot = dataset.checkout_version(version)
+ assert row_ids(snapshot) == rows, f"version {version} rows differ"
+ indexes[version] = {index["name"]: (index["uuid"], sorted(index["fragment_ids"]))
+ for index in snapshot.list_indices()}
+ assert indexes[2] == {}
+ assert set(indexes[3]) == {"vec_idx"} and indexes[3]["vec_idx"][1] == [0, 1]
+ assert set(indexes[4]) == {"vec_idx", "body_idx"} and indexes[4]["body_idx"][1] == [0, 1]
+ assert indexes[5] == indexes[4] and indexes[6]["vec_idx"] == indexes[4]["vec_idx"]
+ assert indexes[7]["vec_idx"][0] != indexes[6]["vec_idx"][0], "version 7 must rebuild vec_idx"
+ assert indexes[7]["vec_idx"][1] == [0, 1, 2]
+ tags = {name: (ref["branch"], ref["version"]) for name, ref in dataset.tags.list().items()}
+ assert tags == {"rel": (None, 5), "dev_rel": (BRANCH, 5)}, tags
+ assert list(dataset.branches.list()) == [BRANCH], dataset.branches.list()
+ branch_manifests = sorted((output / "tree" / BRANCH / "_versions").glob("*.manifest"))
+ assert len(branch_manifests) == 3, f"branch {BRANCH} must carry versions 4 to 6: {branch_manifests}"
+ for manifest in branch_manifests:
+ assert BRANCH_PARENT_URI.encode() in manifest.read_bytes(), (
+ f"{manifest} must reference the parent at {BRANCH_PARENT_URI}")
+ assert (output / "tree" / BRANCH / "_indices").is_dir(), "dev must hold its rebuilt FTS index"
+
+ pruned = lance.dataset(str(root / PRUNED_DIR))
+ assert [v["version"] for v in pruned.versions()] == [2, 3], pruned.versions()
+ kept = pruned.checkout_version(2)
+ kept_index = kept.list_indices()[0]["uuid"]
+ assert not (root / PRUNED_DIR / "_indices" / kept_index).exists(), "version 2 index files must be gone"
+ assert (root / PRUNED_DIR / "_indices" / pruned.list_indices()[0]["uuid"]).is_dir()
+ check_evolved(root / EVOLVED_DIR)
+ for version in versions:
+ print(f"{MAIN_DIR} version {version['version']} committed at {version['timestamp'].isoformat()}")
+
+
+def check_evolved(output: Path) -> None:
+ dataset = lance.dataset(str(output))
+ assert [v["version"] for v in dataset.versions()] == list(range(1, 6)), dataset.versions()
+ expected_columns = {
+ 1: ["row_id", "vec", "body"], 2: ["row_id", "vec", "body"], 3: ["row_id", "vec", "body", "note"],
+ 4: ["row_id", "embedding", "body", "note"], 5: ["row_id", "embedding", "body", "note"],
+ }
+ index_uuids = set()
+ for version, columns in expected_columns.items():
+ snapshot = dataset.checkout_version(version)
+ assert snapshot.schema.names == columns, f"version {version} columns {snapshot.schema.names}"
+ rows = list(range(1, 17)) if version == 5 else list(range(1, 9))
+ assert row_ids(snapshot) == rows, f"version {version} rows differ"
+ indexes = snapshot.list_indices()
+ if version == 1:
+ assert indexes == [], indexes
+ continue
+ assert [index["name"] for index in indexes] == ["vec_idx"], indexes
+ assert sorted(indexes[0]["fragment_ids"]) == [0], indexes
+ index_uuids.add(indexes[0]["uuid"])
+ assert len(index_uuids) == 1, f"the rename must keep vec_idx: {index_uuids}"
+ notes = dataset.checkout_version(3).to_table(columns=["row_id", "note"]).to_pylist()
+ assert all(row["note"] == f"note {row['row_id']}" for row in notes), notes
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("root", type=Path, nargs="?",
+ help="directory holding the fixtures (not used with --create-branch)")
+ parser.add_argument("--check", action="store_true", help="verify the existing fixtures")
+ parser.add_argument("--fixture", action="append", choices=sorted(FIXTURES), metavar="NAME",
+ help="build only this fixture (repeatable): " + ", ".join(sorted(FIXTURES)))
+ parser.add_argument("--create-branch", metavar="URI",
+ help="create the dev branch at the uploaded dataset URI instead of building")
+ parser.add_argument("--storage-option", action="append", default=[], metavar="KEY=VALUE",
+ help="Lance storage option for --create-branch (repeatable)")
+ args = parser.parse_args()
+ if not args.create_branch and args.root is None:
+ parser.error("the fixture directory is required unless --create-branch is given")
+ if args.create_branch:
+ create_branch(args.create_branch, dict(item.split("=", 1) for item in args.storage_option))
+ elif args.check:
+ check(args.root)
+ else:
+ build(args.root, args.fixture or list(FIXTURES))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_unicode_branch.py b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_unicode_branch.py
new file mode 100644
index 00000000000..1179ddbe6a9
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_unicode_branch.py
@@ -0,0 +1,135 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Generate the Lance regression fixture whose branch name is newer than the JDK's Unicode.
+
+unicode_branch.lance is a root table of the preinstalled Directory catalog next to
+time_travel.lance. Its branch is named "dev\\u1c89": U+1C89 was added in Unicode 16, so Lance's
+Rust validation accepts it while JDK 17, on Unicode 13, leaves it unassigned. The REST server
+serves the table as unicode_branch_managed, so Doris opens the branch by the URI it joins.
+
+ version 1 create row_id 1..8 fragment 0
+ dev\\u1c89 version 2 append row_id 201..204 on the branch only
+
+vec = [row_id, row_id + 1, row_id + 2, row_id + 3] as float32, body = "doc <row_id>".
+
+A Lance branch is a shallow clone that records its parent's absolute URI, so the branch is
+created against the fixture's final location (see lance_build_search_snapshot.py):
+
+ python3 lance_build_unicode_branch.py preinstalled_data/lance
+ # upload unicode_branch.lance to s3://warehouse/lance/unicode_branch.lance
+ python3 lance_build_unicode_branch.py --create-branch s3://warehouse/lance/unicode_branch.lance \\
+ --storage-option endpoint=http://127.0.0.1:19000 --storage-option access_key_id=admin \\
+ --storage-option secret_access_key=password --storage-option region=us-east-1 \\
+ --storage-option allow_http=true
+ # then sync tree/ and _refs/branches/ from that location back
+ python3 lance_build_unicode_branch.py --check preinstalled_data/lance
+
+Build it with pylance 12.0.0, the Lance the FE uses, not the writer pinned in
+lance_fixture_requirements.txt: Lance 7 stores a non-ASCII branch directory percent-encoded
+(tree/dev%E1%B2%89/), which Lance 12 does not find, while Lance 12 stores it as tree/dev\u1c89/.
+The catalog build carries the fixture over as-is. --check runs with any reader.
+"""
+import argparse
+import shutil
+from pathlib import Path
+
+import lance
+import pyarrow as pa
+
+UNICODE_BRANCH_DIR = "unicode_branch.lance"
+BRANCH = "dev"
+PARENT_URI = "s3://warehouse/lance/unicode_branch.lance"
+WRITER_VERSION = "12.0.0"
+DIMENSION = 4
+
+
+def rows_of(low: int, high: int) -> pa.Table:
+ ids = list(range(low, high + 1))
+ return pa.table({
+ "row_id": pa.array(ids, pa.int32()),
+ "vec": pa.array([[float(i + j) for j in range(DIMENSION)] for i in ids],
+ pa.list_(pa.float32(), DIMENSION)),
+ "body": pa.array([f"doc {i}" for i in ids], pa.string()),
+ })
+
+
+def require_writer() -> None:
+ if lance.__version__ != WRITER_VERSION:
+ raise SystemExit(f"build {UNICODE_BRANCH_DIR} with pylance {WRITER_VERSION}, not {lance.__version__}")
+
+
+def build(output: Path) -> None:
+ require_writer()
+ if output.exists():
+ shutil.rmtree(output)
+ lance.write_dataset(rows_of(1, 8), str(output), mode="create", data_storage_version="2.2")
+ print(f"{UNICODE_BRANCH_DIR} written; create the branch with --create-branch {PARENT_URI}")
+
+
+def create_branch(uri: str, storage_options: dict) -> None:
+ require_writer()
+ dataset = lance.dataset(uri, storage_options=storage_options)
+ assert dataset.version == 1, "upload the main chain first"
+ dataset.create_branch(BRANCH, 1)
+ lance.write_dataset(rows_of(201, 204), f"{uri}/tree/{BRANCH}", mode="append",
+ data_storage_version="2.2", storage_options=storage_options)
+ branch = lance.dataset(f"{uri}/tree/{BRANCH}", storage_options=storage_options)
+ assert branch.version == 2, branch.version
+ print(f"branch {BRANCH!r} created at {uri}; sync tree/ and _refs/branches/ back")
+
+
+def row_ids(dataset) -> list:
+ return sorted(dataset.to_table(columns=["row_id"])["row_id"].to_pylist())
+
+
+def check(output: Path) -> None:
+ dataset = lance.dataset(str(output))
+ assert [v["version"] for v in dataset.versions()] == [1], dataset.versions()
+ assert row_ids(dataset) == list(range(1, 9)), "main rows differ"
+ assert list(dataset.branches.list()) == [BRANCH], dataset.branches.list()
+ # The branch directory keeps the name as is; Lance 12 does not find a percent-encoded one.
+ manifests = sorted((output / "tree" / BRANCH / "_versions").glob("*.manifest"))
+ assert len(manifests) == 2, f"branch {BRANCH!r} must carry versions 1 and 2: {manifests}"
+ for manifest in manifests:
+ assert PARENT_URI.encode() in manifest.read_bytes(), f"{manifest} must reference {PARENT_URI}"
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("root", type=Path, nargs="?",
+ help="directory holding the fixture (not used with --create-branch)")
+ parser.add_argument("--check", action="store_true", help="verify the existing fixture")
+ parser.add_argument("--create-branch", metavar="URI",
+ help="create the branch at the uploaded dataset URI instead of building")
+ parser.add_argument("--storage-option", action="append", default=[], metavar="KEY=VALUE",
+ help="Lance storage option for --create-branch (repeatable)")
+ args = parser.parse_args()
+ if not args.create_branch and args.root is None:
+ parser.error("the fixture directory is required unless --create-branch is given")
+ if args.create_branch:
+ create_branch(args.create_branch, dict(item.split("=", 1) for item in args.storage_option))
+ elif args.check:
+ check(args.root / UNICODE_BRANCH_DIR)
+ print(f"{UNICODE_BRANCH_DIR} checked")
+ else:
+ build(args.root / UNICODE_BRANCH_DIR)
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/lance_fixture_requirements.txt b/docker/thirdparties/docker-compose/iceberg/scripts/lance_fixture_requirements.txt
index 1af5d1305b2..c34326bdc43 100644
--- a/docker/thirdparties/docker-compose/iceberg/scripts/lance_fixture_requirements.txt
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/lance_fixture_requirements.txt
@@ -38,6 +38,12 @@
# it: the Spark and REST catalog suites read and register into the same __manifest this
# writer produces, and they must pass for a fixture rebuild to be acceptable.
#
+# One fixture is the exception: legacy_vector_index.lance must carry a vector index without
+# index details, which only writers before 0.20 produce, so lance_build_legacy_vector_index.py
+# builds it with pylance 0.18.2 and the catalog build carries it over as-is. Likewise
+# unicode_branch.lance is built by pylance 12.0.0, because Lance 7 stores a non-ASCII branch
+# directory percent-encoded, which Lance 12 does not find (see lance_build_unicode_branch.py).
+#
# The pylance and lance-namespace pins are enforced rather than advisory:
# check_pinned_writer() refuses to build or verify the fixture unless the installed versions
# match them exactly. That check has to exist because the generator's own self-check cannot
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_indices/bc324f21-a15a-45bf-8d9e-09538f895f55/index.idx b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_indices/bc324f21-a15a-45bf-8d9e-09538f895f55/index.idx
new file mode 100644
index 00000000000..ea4702c7d12
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_indices/bc324f21-a15a-45bf-8d9e-09538f895f55/index.idx differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_transactions/0-c5e3be90-3de0-44fb-95b1-bd46928ae0f2.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_transactions/0-c5e3be90-3de0-44fb-95b1-bd46928ae0f2.txn
new file mode 100644
index 00000000000..c0c2ee59368
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_transactions/0-c5e3be90-3de0-44fb-95b1-bd46928ae0f2.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_transactions/1-9c8d1a85-5125-46ad-aeb8-8370bba30617.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_transactions/1-9c8d1a85-5125-46ad-aeb8-8370bba30617.txn
new file mode 100644
index 00000000000..ef6ab8a790c
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_transactions/1-9c8d1a85-5125-46ad-aeb8-8370bba30617.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_versions/1.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_versions/1.manifest
new file mode 100644
index 00000000000..3944e308416
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_versions/1.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_versions/2.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_versions/2.manifest
new file mode 100644
index 00000000000..097654588b9
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/_versions/2.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/data/02cdc0ff-ceee-44ef-bccd-6cb6f4b50779.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/data/02cdc0ff-ceee-44ef-bccd-6cb6f4b50779.lance
new file mode 100644
index 00000000000..045bba3e716
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/legacy_vector_index.lance/data/02cdc0ff-ceee-44ef-bccd-6cb6f4b50779.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_deletions/0-5-1267712482458982649.arrow b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_deletions/0-5-1267712482458982649.arrow
new file mode 100644
index 00000000000..6ea56a96b0a
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_deletions/0-5-1267712482458982649.arrow differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_deletions/1-5-6394601565774866848.arrow b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_deletions/1-5-6394601565774866848.arrow
new file mode 100644
index 00000000000..6ea56a96b0a
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_deletions/1-5-6394601565774866848.arrow differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/85880db4-716d-49b3-8eb2-fdadcc900e25/auxiliary.idx b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/85880db4-716d-49b3-8eb2-fdadcc900e25/auxiliary.idx
new file mode 100644
index 00000000000..4b1580b5878
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/85880db4-716d-49b3-8eb2-fdadcc900e25/auxiliary.idx differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/85880db4-716d-49b3-8eb2-fdadcc900e25/index.idx b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/85880db4-716d-49b3-8eb2-fdadcc900e25/index.idx
new file mode 100644
index 00000000000..36be1299145
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/85880db4-716d-49b3-8eb2-fdadcc900e25/index.idx differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/984593f1-89f1-4849-83f9-d3e45462c244/auxiliary.idx b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/984593f1-89f1-4849-83f9-d3e45462c244/auxiliary.idx
new file mode 100644
index 00000000000..10b801b0027
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/984593f1-89f1-4849-83f9-d3e45462c244/auxiliary.idx differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/984593f1-89f1-4849-83f9-d3e45462c244/index.idx b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/984593f1-89f1-4849-83f9-d3e45462c244/index.idx
new file mode 100644
index 00000000000..8c40d51cd61
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/984593f1-89f1-4849-83f9-d3e45462c244/index.idx differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/metadata.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/metadata.lance
new file mode 100644
index 00000000000..e0a664930f8
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/metadata.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/part_0_docs.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/part_0_docs.lance
new file mode 100644
index 00000000000..48baff15562
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/part_0_docs.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/part_0_invert.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/part_0_invert.lance
new file mode 100644
index 00000000000..b237ff0ec65
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/part_0_invert.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/part_0_tokens.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/part_0_tokens.lance
new file mode 100644
index 00000000000..9c5d074382f
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_indices/b1fd3eca-c523-401e-88cd-d4afa9b66ecf/part_0_tokens.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_refs/branches/dev.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_refs/branches/dev.json
new file mode 100644
index 00000000000..f816ab17d80
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_refs/branches/dev.json
@@ -0,0 +1,15 @@
+{
+ "parentBranch": null,
+ "identifier": {
+ "version_mapping": [
+ [
+ 4,
+ "db8ce527ff5c415b9a220a4a54108997"
+ ]
+ ]
+ },
+ "parentVersion": 4,
+ "createAt": 1790693536,
+ "manifestSize": 1244,
+ "metadata": {}
+}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_refs/tags/dev_rel.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_refs/tags/dev_rel.json
new file mode 100644
index 00000000000..c43136173a5
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_refs/tags/dev_rel.json
@@ -0,0 +1,8 @@
+{
+ "branch": "dev",
+ "version": 5,
+ "createdAt": "2026-09-29T14:52:19.682676Z",
+ "updatedAt": "2026-09-29T14:52:19.682676Z",
+ "manifestSize": 1240,
+ "metadata": {}
+}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_refs/tags/rel.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_refs/tags/rel.json
new file mode 100644
index 00000000000..cbb50015fba
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_refs/tags/rel.json
@@ -0,0 +1,8 @@
+{
+ "branch": null,
+ "version": 5,
+ "createdAt": "2026-09-29T14:52:00.568051Z",
+ "updatedAt": "2026-09-29T14:52:00.568051Z",
+ "manifestSize": 1173,
+ "metadata": {}
+}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/0-154ed129-09cd-48de-89ac-618fc1060501.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/0-154ed129-09cd-48de-89ac-618fc1060501.txn
new file mode 100644
index 00000000000..9c20aecce08
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/0-154ed129-09cd-48de-89ac-618fc1060501.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/1-d6645060-096c-4335-8bd9-c595a08fdd34.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/1-d6645060-096c-4335-8bd9-c595a08fdd34.txn
new file mode 100644
index 00000000000..a2b7fb63746
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/1-d6645060-096c-4335-8bd9-c595a08fdd34.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/2-1518405c-5551-4e8f-b615-a0ca0d754255.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/2-1518405c-5551-4e8f-b615-a0ca0d754255.txn
new file mode 100644
index 00000000000..94c30ead7c7
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/2-1518405c-5551-4e8f-b615-a0ca0d754255.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/3-c7c138f8-03a8-4564-9fc5-d59c25c2223c.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/3-c7c138f8-03a8-4564-9fc5-d59c25c2223c.txn
new file mode 100644
index 00000000000..875089678d8
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/3-c7c138f8-03a8-4564-9fc5-d59c25c2223c.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/4-1cbab51a-2c2d-41c4-8025-30daa3cc11a4.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/4-1cbab51a-2c2d-41c4-8025-30daa3cc11a4.txn
new file mode 100644
index 00000000000..4b36ae8725a
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/4-1cbab51a-2c2d-41c4-8025-30daa3cc11a4.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/5-630bd020-d3af-4be8-b5e8-33388c068adc.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/5-630bd020-d3af-4be8-b5e8-33388c068adc.txn
new file mode 100644
index 00000000000..0f22d17e335
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/5-630bd020-d3af-4be8-b5e8-33388c068adc.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/6-9a8f0e65-3b5d-49ac-b232-5990ddd60891.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/6-9a8f0e65-3b5d-49ac-b232-5990ddd60891.txn
new file mode 100644
index 00000000000..c9b01bd551f
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_transactions/6-9a8f0e65-3b5d-49ac-b232-5990ddd60891.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551608.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551608.manifest
new file mode 100644
index 00000000000..299a56d1934
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551608.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551609.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551609.manifest
new file mode 100644
index 00000000000..b39807e70dc
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551609.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551610.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551610.manifest
new file mode 100644
index 00000000000..8d1014ec813
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551610.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551611.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551611.manifest
new file mode 100644
index 00000000000..f7c56dcfe47
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551611.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551612.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551612.manifest
new file mode 100644
index 00000000000..69ae8b9f38b
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551612.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551613.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551613.manifest
new file mode 100644
index 00000000000..755c4a235a4
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551613.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551614.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551614.manifest
new file mode 100644
index 00000000000..895267a7b12
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/18446744073709551614.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/latest_version_hint.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/latest_version_hint.json
new file mode 100644
index 00000000000..30a68a6c522
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/_versions/latest_version_hint.json
@@ -0,0 +1 @@
+{"version":7}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/data/01110111111110000001111167198a47f9880a55a8085ebb87.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/data/01110111111110000001111167198a47f9880a55a8085ebb87.lance
new file mode 100644
index 00000000000..a529fba94ea
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/data/01110111111110000001111167198a47f9880a55a8085ebb87.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/data/110001100010111000101110284ca44e90bfc0c56be666a32f.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/data/110001100010111000101110284ca44e90bfc0c56be666a32f.lance
new file mode 100644
index 00000000000..a6688314a69
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/data/110001100010111000101110284ca44e90bfc0c56be666a32f.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/data/110100000111011110010100fc3e5d485e9c40f5a217eaa59e.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/data/110100000111011110010100fc3e5d485e9c40f5a217eaa59e.lance
new file mode 100644
index 00000000000..4f18f971545
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/data/110100000111011110010100fc3e5d485e9c40f5a217eaa59e.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/metadata.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/metadata.lance
new file mode 100644
index 00000000000..c2703a12343
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/metadata.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/part_0_docs.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/part_0_docs.lance
new file mode 100644
index 00000000000..8199b4ab560
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/part_0_docs.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/part_0_invert.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/part_0_invert.lance
new file mode 100644
index 00000000000..eda931361ee
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/part_0_invert.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/part_0_tokens.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/part_0_tokens.lance
new file mode 100644
index 00000000000..3ae795448b3
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_indices/9821abbf-c51c-4c59-a184-756342a17365/part_0_tokens.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_transactions/4-a99a47e7-aae1-46cf-87f7-4cc649db3aa1.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_transactions/4-a99a47e7-aae1-46cf-87f7-4cc649db3aa1.txn
new file mode 100644
index 00000000000..0a82e1c17e9
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_transactions/4-a99a47e7-aae1-46cf-87f7-4cc649db3aa1.txn
@@ -0,0 +1 @@
+$a99a47e7-aae1-46cf-87f7-4cc649db3aa1�5"*s3://warehouse/lance/search_snapshot.lance*dev
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_transactions/4-d995fee3-982e-4e43-8e30-94537a9b0de2.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_transactions/4-d995fee3-982e-4e43-8e30-94537a9b0de2.txn
new file mode 100644
index 00000000000..24e59116138
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_transactions/4-d995fee3-982e-4e43-8e30-94537a9b0de2.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_transactions/5-6028c4ff-0a8e-4938-9d58-8a13524f6812.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_transactions/5-6028c4ff-0a8e-4938-9d58-8a13524f6812.txn
new file mode 100644
index 00000000000..bbc6a0a6fa7
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_transactions/5-6028c4ff-0a8e-4938-9d58-8a13524f6812.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_versions/18446744073709551609.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_versions/18446744073709551609.manifest
new file mode 100644
index 00000000000..3ffa94a28c8
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_versions/18446744073709551609.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_versions/18446744073709551610.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_versions/18446744073709551610.manifest
new file mode 100644
index 00000000000..66430e475d2
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_versions/18446744073709551610.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_versions/18446744073709551611.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_versions/18446744073709551611.manifest
new file mode 100644
index 00000000000..31a78e836d9
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/_versions/18446744073709551611.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/data/10010100101010111100101014250f4aa0a70d3d926f124125.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/data/10010100101010111100101014250f4aa0a70d3d926f124125.lance
new file mode 100644
index 00000000000..966c2a46db1
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot.lance/tree/dev/data/10010100101010111100101014250f4aa0a70d3d926f124125.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_indices/72e585d1-b843-4463-aba0-ffabc76017c1/auxiliary.idx b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_indices/72e585d1-b843-4463-aba0-ffabc76017c1/auxiliary.idx
new file mode 100644
index 00000000000..80f7aa133f6
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_indices/72e585d1-b843-4463-aba0-ffabc76017c1/auxiliary.idx differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_indices/72e585d1-b843-4463-aba0-ffabc76017c1/index.idx b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_indices/72e585d1-b843-4463-aba0-ffabc76017c1/index.idx
new file mode 100644
index 00000000000..d544eb75b56
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_indices/72e585d1-b843-4463-aba0-ffabc76017c1/index.idx differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/0-fecb1851-f6b0-496c-bfb8-f4dbe7f649f1.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/0-fecb1851-f6b0-496c-bfb8-f4dbe7f649f1.txn
new file mode 100644
index 00000000000..b954b2e1a06
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/0-fecb1851-f6b0-496c-bfb8-f4dbe7f649f1.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/1-5a98fef6-06a5-457c-b1b1-a9c830eafd2b.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/1-5a98fef6-06a5-457c-b1b1-a9c830eafd2b.txn
new file mode 100644
index 00000000000..e08d3999e36
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/1-5a98fef6-06a5-457c-b1b1-a9c830eafd2b.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/2-9bdddb95-2dc3-4e77-95b9-f67ed6e6387e.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/2-9bdddb95-2dc3-4e77-95b9-f67ed6e6387e.txn
new file mode 100644
index 00000000000..4292a1a52fc
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/2-9bdddb95-2dc3-4e77-95b9-f67ed6e6387e.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/3-954a5d41-5fc9-4e39-96d9-864e1ac233bc.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/3-954a5d41-5fc9-4e39-96d9-864e1ac233bc.txn
new file mode 100644
index 00000000000..1ba22c72ee6
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/3-954a5d41-5fc9-4e39-96d9-864e1ac233bc.txn
@@ -0,0 +1,5 @@
+$954a5d41-5fc9-4e39-96d9-864e1ac233bc��
+ row_id ���������*int3208
+5 embedding ���������*fixed_size_list:float:408
+ body ���������*string08
+ note ���������*string08
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/4-a41517b8-29a3-410f-b3ae-768c33b38cf5.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/4-a41517b8-29a3-410f-b3ae-768c33b38cf5.txn
new file mode 100644
index 00000000000..d8b69c8d8d5
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_transactions/4-a41517b8-29a3-410f-b3ae-768c33b38cf5.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551610.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551610.manifest
new file mode 100644
index 00000000000..1be033e54a6
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551610.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551611.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551611.manifest
new file mode 100644
index 00000000000..5210652bb11
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551611.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551612.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551612.manifest
new file mode 100644
index 00000000000..f8f942177e8
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551612.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551613.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551613.manifest
new file mode 100644
index 00000000000..72583dc5c45
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551613.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551614.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551614.manifest
new file mode 100644
index 00000000000..7058db9b2bb
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/18446744073709551614.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/latest_version_hint.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/latest_version_hint.json
new file mode 100644
index 00000000000..978d5262ed3
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/_versions/latest_version_hint.json
@@ -0,0 +1 @@
+{"version":5}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/data/011110000001001000111000634254483b82696db0b43cee8b.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/data/011110000001001000111000634254483b82696db0b43cee8b.lance
new file mode 100644
index 00000000000..c306d906f99
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/data/011110000001001000111000634254483b82696db0b43cee8b.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/data/1011111000111000011010107529f744f0b515d3cc0aaf4111.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/data/1011111000111000011010107529f744f0b515d3cc0aaf4111.lance
new file mode 100644
index 00000000000..bdcee25cf40
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/data/1011111000111000011010107529f744f0b515d3cc0aaf4111.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/data/1101110010000010110100009494424b38a01956f50baa341d.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/data/1101110010000010110100009494424b38a01956f50baa341d.lance
new file mode 100644
index 00000000000..a6688314a69
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_evolved.lance/data/1101110010000010110100009494424b38a01956f50baa341d.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_indices/961caf93-b3e9-45b9-b1a7-8944a37213ea/auxiliary.idx b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_indices/961caf93-b3e9-45b9-b1a7-8944a37213ea/auxiliary.idx
new file mode 100644
index 00000000000..14f77983cb9
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_indices/961caf93-b3e9-45b9-b1a7-8944a37213ea/auxiliary.idx differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_indices/961caf93-b3e9-45b9-b1a7-8944a37213ea/index.idx b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_indices/961caf93-b3e9-45b9-b1a7-8944a37213ea/index.idx
new file mode 100644
index 00000000000..0ad4ce6fc08
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_indices/961caf93-b3e9-45b9-b1a7-8944a37213ea/index.idx differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_refs/tags/kept.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_refs/tags/kept.json
new file mode 100644
index 00000000000..9a0a0880568
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_refs/tags/kept.json
@@ -0,0 +1,8 @@
+{
+ "branch": null,
+ "version": 2,
+ "createdAt": "2026-09-29T14:52:00.582902Z",
+ "updatedAt": "2026-09-29T14:52:00.582902Z",
+ "manifestSize": 1017,
+ "metadata": {}
+}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_transactions/1-5c4816b8-2e0a-46bb-aa6c-0879630399b4.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_transactions/1-5c4816b8-2e0a-46bb-aa6c-0879630399b4.txn
new file mode 100644
index 00000000000..c8a8678b7b7
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_transactions/1-5c4816b8-2e0a-46bb-aa6c-0879630399b4.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_transactions/2-b57c17ad-85a6-415b-a250-4b2efe8d3f8c.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_transactions/2-b57c17ad-85a6-415b-a250-4b2efe8d3f8c.txn
new file mode 100644
index 00000000000..e5a32a72835
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_transactions/2-b57c17ad-85a6-415b-a250-4b2efe8d3f8c.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_versions/18446744073709551612.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_versions/18446744073709551612.manifest
new file mode 100644
index 00000000000..dd9ff6f73e9
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_versions/18446744073709551612.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_versions/18446744073709551613.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_versions/18446744073709551613.manifest
new file mode 100644
index 00000000000..efcf2d7d8ee
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_versions/18446744073709551613.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_versions/latest_version_hint.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_versions/latest_version_hint.json
new file mode 100644
index 00000000000..f93d3984472
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/_versions/latest_version_hint.json
@@ -0,0 +1 @@
+{"version":3}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/data/101101000110111001110011284b0f491a947f1b18c7abddff.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/data/101101000110111001110011284b0f491a947f1b18c7abddff.lance
new file mode 100644
index 00000000000..a6688314a69
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/search_snapshot_pruned.lance/data/101101000110111001110011284b0f491a947f1b18c7abddff.lance differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_refs/branches/dev%E1%B2%89.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_refs/branches/dev%E1%B2%89.json
new file mode 100644
index 00000000000..f60670a9280
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_refs/branches/dev%E1%B2%89.json
@@ -0,0 +1,15 @@
+{
+ "parentBranch": null,
+ "identifier": {
+ "version_mapping": [
+ [
+ 1,
+ "baa77a56726b479eacff307d13509c71"
+ ]
+ ]
+ },
+ "parentVersion": 1,
+ "createAt": 1791178981,
+ "manifestSize": 552,
+ "metadata": {}
+}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_transactions/0-420a1091-b61a-4cfb-8006-0b8f1e322ef8.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_transactions/0-420a1091-b61a-4cfb-8006-0b8f1e322ef8.txn
new file mode 100644
index 00000000000..60e03205e9d
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_transactions/0-420a1091-b61a-4cfb-8006-0b8f1e322ef8.txn differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_versions/18446744073709551614.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_versions/18446744073709551614.manifest
new file mode 100644
index 00000000000..83622779333
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_versions/18446744073709551614.manifest differ
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_versions/latest_version_hint.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_versions/latest_version_hint.json
new file mode 100644
index 00000000000..491d734467a
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/_versions/latest_version_hint.json
@@ -0,0 +1 @@
+{"version":1}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/data/001101010101111000101000a13eb949eca41d69d17f9ea5a6.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/data/001101010101111000101000a13eb949eca41d69d17f9ea5a6.lance
new file mode 100644
index 00000000000..d1446b437c5
Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/data/001101010101111000101000a13eb949eca41d69d17f9ea5a6.lance differ
diff --git "a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_transactions/1-1be00bd2-8443-4851-9ab8-4bdcee3ffa5e.txn" "b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_transactions/1-1be00bd2-8443-4851-9ab8-4bdcee3ffa5e.txn"
new file mode 100644
index 00000000000..3548df51401
Binary files /dev/null and "b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_transactions/1-1be00bd2-8443-4851-9ab8-4bdcee3ffa5e.txn" differ
diff --git "a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_transactions/1-c1998c6f-6213-4b7e-ae7c-cc00cde3b7de.txn" "b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_transactions/1-c1998c6f-6213-4b7e-ae7c-cc00cde3b7de.txn"
new file mode 100644
index 00000000000..fb3b7a57266
--- /dev/null
+++ "b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_transactions/1-c1998c6f-6213-4b7e-ae7c-cc00cde3b7de.txn"
@@ -0,0 +1 @@
+$c1998c6f-6213-4b7e-ae7c-cc00cde3b7de�7")s3://warehouse/lance/unicode_branch.lance*dev
\ No newline at end of file
diff --git "a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_versions/18446744073709551613.manifest" "b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_versions/18446744073709551613.manifest"
new file mode 100644
index 00000000000..19fd4f62f3f
Binary files /dev/null and "b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_versions/18446744073709551613.manifest" differ
diff --git "a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_versions/18446744073709551614.manifest" "b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_versions/18446744073709551614.manifest"
new file mode 100644
index 00000000000..42918df699b
Binary files /dev/null and "b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/_versions/18446744073709551614.manifest" differ
diff --git "a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/data/01110100011000100001111040aa57401ab473a6a921298540.lance" "b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/data/01110100011000100001111040aa57401ab473a6a921298540.lance"
new file mode 100644
index 00000000000..266136cc003
Binary files /dev/null and "b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev\341\262\211/data/01110100011000100001111040aa57401ab473a6a921298540.lance" differ
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java
index 8b745a868a9..4cd3ed64329 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java
@@ -20,7 +20,6 @@ package org.apache.doris.datasource.lance;
import org.apache.doris.analysis.TableSnapshot;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
-import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.lance.index.LanceIndexInspection;
import org.apache.doris.datasource.lance.index.LanceIndexInspectionExecutor;
import org.apache.doris.datasource.lance.index.LancePhysicalIndexEntry;
@@ -73,8 +72,6 @@ import java.util.function.BiFunction;
final class LanceCatalogClient implements AutoCloseable {
private static final Logger LOG = LogManager.getLogger(LanceCatalogClient.class);
- /** Lance's name for the main chain; {@code @branch(main)} and tags on it mean the main chain. */
- static final String MAIN_BRANCH = "main";
private static final long METADATA_CACHE_SIZE_BYTES = 64L * 1024 * 1024;
private static final long INDEX_CACHE_SIZE_BYTES = 128L * 1024 * 1024;
@@ -235,7 +232,11 @@ final class LanceCatalogClient implements AutoCloseable {
}
public LanceTableMetadata loadTableMetadataForSearch(String dbName, String tableName) {
- LanceTableMetadata metadata = loadQueryMetadata(dbName, tableName, Optional.empty(),
+ return loadTableMetadataForSearch(dbName, tableName, LanceRefSelector.latest());
+ }
+
+ public LanceTableMetadata loadTableMetadataForSearch(String dbName, String tableName, LanceRefSelector selector) {
+ LanceTableMetadata metadata = loadQueryMetadata(dbName, tableName, selector,
LanceMetadataLoader.MetadataScope.WITH_INDEXES);
if (!metadata.getIndexMetadataState().canPlanIndexSegments()) {
throw new IllegalArgumentException("Lance SDK cannot provide field IDs required for search planning");
@@ -244,7 +245,11 @@ final class LanceCatalogClient implements AutoCloseable {
}
public LanceTableMetadata loadBasicTableMetadata(String dbName, String tableName) {
- return loadQueryMetadata(dbName, tableName, Optional.empty(), LanceMetadataLoader.MetadataScope.BASIC);
+ return loadBasicTableMetadata(dbName, tableName, LanceRefSelector.latest());
+ }
+
+ public LanceTableMetadata loadBasicTableMetadata(String dbName, String tableName, LanceRefSelector selector) {
+ return loadQueryMetadata(dbName, tableName, selector, LanceMetadataLoader.MetadataScope.BASIC);
}
public Schema loadTableSchema(String dbName, String tableName) {
@@ -261,11 +266,6 @@ final class LanceCatalogClient implements AutoCloseable {
return loadQueryMetadata(dbName, tableName, selector, LanceMetadataLoader.MetadataScope.WITH_INDEXES);
}
- private LanceTableMetadata loadQueryMetadata(String dbName, String tableName,
- Optional<TableSnapshot> tableSnapshot, LanceMetadataLoader.MetadataScope mode) {
- return loadQueryMetadata(dbName, tableName, LanceRefSelector.snapshot(tableSnapshot), mode);
- }
-
private LanceTableMetadata loadQueryMetadata(String dbName, String tableName,
LanceRefSelector selector, LanceMetadataLoader.MetadataScope mode) {
return readTableSnapshot(dbName, tableName, selector,
@@ -473,6 +473,7 @@ final class LanceCatalogClient implements AutoCloseable {
private <T> T readManagedBranch(BufferAllocator allocator, ReadState state, SnapshotReader<T> reader,
LanceMetadataMetrics metrics) throws Exception {
String branch = state.branch.get();
+ checkBranchName(branch);
LanceTableAccess branchAccess = state.access.onBranch(branch,
branchUri(state.access.getDatasetUri(), branch));
Optional<TableSnapshot> snapshot = state.selector.getSnapshot();
@@ -494,6 +495,44 @@ final class LanceCatalogClient implements AutoCloseable {
}
}
+ /**
+ * Rejects a branch name that could leave the table's {@code tree/} directory or change the URI
+ * {@link #branchUri} joins for a managed table, which Lance does not validate on that path;
+ * the object store resolves dot segments, including percent-encoded ones. Other tables check a
+ * branch out through Lance, which validates the name itself.
+ *
+ * <p>This never rejects a name Lance's {@code check_valid_branch} accepts, whichever Unicode
+ * version either side uses: ASCII allows exactly Lance's letters, digits, '.', '-', '_' and
+ * '/', and outside ASCII only whitespace and control characters, which are never
+ * alphanumeric, are rejected. Lance cannot create a branch with a name it would reject, and the
+ * namespace decides whether such a branch exists.
+ */
+ static void checkBranchName(String branch) {
+ String reason = null;
+ if (branch.isEmpty()) {
+ reason = "it is empty";
+ } else if (branch.startsWith("/") || branch.endsWith("/") || branch.contains("//")) {
+ reason = "it starts or ends with '/' or contains an empty segment";
+ } else if (branch.contains("..") || branch.endsWith(".lock")) {
+ reason = "it contains '..' or ends with '.lock'";
+ } else if (!branch.codePoints().allMatch(LanceCatalogClient::isBranchNameCharacter)) {
+ reason = "only letters, digits, '.', '-', '_' and '/' between segments are allowed";
+ }
+ if (reason != null) {
+ throw new LanceUserFacingException("Invalid Lance branch name '" + branch + "': " + reason);
+ }
+ }
+
+ private static boolean isBranchNameCharacter(int codePoint) {
+ if (codePoint < 0x80) {
+ return (codePoint >= 'a' && codePoint <= 'z') || (codePoint >= 'A' && codePoint <= 'Z')
+ || (codePoint >= '0' && codePoint <= '9') || codePoint == '.' || codePoint == '-'
+ || codePoint == '_' || codePoint == '/';
+ }
+ return !Character.isWhitespace(codePoint) && !Character.isSpaceChar(codePoint)
+ && !Character.isISOControl(codePoint);
+ }
+
/**
* The URI of a branch's directory, joined as Lance's {@code BranchLocation} joins it:
* {@code tree/<branch>} under the table root, before the URI's query string.
@@ -746,26 +785,19 @@ final class LanceCatalogClient implements AutoCloseable {
private static LanceUserFacingException historyRemoved(long version, boolean staged, String requestedText,
ReadState state) {
- return new LanceUserFacingException("Lance cannot resolve FOR TIME AS OF '" + requestedText + "' on "
+ // Worded for FOR TIME AS OF and the search functions' timestamp alike.
+ return new LanceUserFacingException("Lance cannot select the version at or before '" + requestedText + "' on "
+ state.displayName() + ": version " + version + ", which may hold the state at that time, "
+ (staged ? "cannot be read: " + stagedOnly() : "no longer exists"));
}
- /**
- * Parses a {@code FOR TIME AS OF} value in the session time zone. Second and millisecond
- * precision are accepted. Commit times are compared in full, so a commit later within the
- * requested millisecond is not selected.
- */
private static long parseTimeTravelTimestamp(String value) {
- long timestamp = TimeUtils.timeStringToLong(value, TimeUtils.getTimeZone());
- if (timestamp < 0) {
- timestamp = TimeUtils.msTimeStringToLong(value, TimeUtils.getTimeZone());
- }
- if (timestamp < 0) {
+ OptionalLong timestamp = LanceSnapshotResolver.parseTimestamp(value);
+ if (!timestamp.isPresent()) {
throw new IllegalArgumentException("Cannot parse Lance FOR TIME AS OF value '" + value
- + "', expected 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd HH:mm:ss.SSS'");
+ + "', expected " + LanceSnapshotResolver.TIMESTAMP_FORMATS);
}
- return timestamp;
+ return timestamp.getAsLong();
}
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java
index 187d2b19121..dc62cb7ff63 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java
@@ -224,11 +224,19 @@ public class LanceExternalCatalog extends ExternalCatalog {
}
public LanceTableMetadata loadTableMetadataForSearch(String dbName, String tableName) {
- return withClient(current -> current.loadTableMetadataForSearch(dbName, tableName));
+ return loadTableMetadataForSearch(dbName, tableName, LanceRefSelector.latest());
+ }
+
+ public LanceTableMetadata loadTableMetadataForSearch(String dbName, String tableName, LanceRefSelector selector) {
+ return withClient(current -> current.loadTableMetadataForSearch(dbName, tableName, selector));
}
public LanceTableMetadata loadBasicTableMetadata(String dbName, String tableName) {
- return withClient(current -> current.loadBasicTableMetadata(dbName, tableName));
+ return loadBasicTableMetadata(dbName, tableName, LanceRefSelector.latest());
+ }
+
+ public LanceTableMetadata loadBasicTableMetadata(String dbName, String tableName, LanceRefSelector selector) {
+ return withClient(current -> current.loadBasicTableMetadata(dbName, tableName, selector));
}
public Schema loadTableSchema(String dbName, String tableName) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java
index 916976cdfaf..9069a353942 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java
@@ -64,13 +64,15 @@ public class LanceExternalTable extends ExternalTable implements MvccTable {
return ((LanceExternalCatalog) catalog).loadTableMetadata(db.getRemoteName(), remoteName);
}
- public LanceTableMetadata loadMetadataForSearch() {
+ /** Search metadata, index details included, of the snapshot {@code selector} selects. */
+ public LanceTableMetadata loadMetadataForSearch(LanceRefSelector selector) {
return ((LanceExternalCatalog) catalog).loadTableMetadataForSearch(
- db.getRemoteName(), remoteName);
+ db.getRemoteName(), remoteName, selector);
}
- public LanceTableMetadata loadBasicMetadata() {
- return ((LanceExternalCatalog) catalog).loadBasicTableMetadata(db.getRemoteName(), remoteName);
+ /** Search metadata without index details, of the snapshot {@code selector} selects. */
+ public LanceTableMetadata loadBasicMetadata(LanceRefSelector selector) {
+ return ((LanceExternalCatalog) catalog).loadBasicTableMetadata(db.getRemoteName(), remoteName, selector);
}
public List<LanceShowIndexInfo> loadIndexesForShow() throws AnalysisException {
@@ -112,10 +114,7 @@ public class LanceExternalTable extends ExternalTable implements MvccTable {
+ tableSnapshot.get().getValue() + "' names a tag, which cannot be combined with @branch;"
+ " use @tag(...) or a numeric version");
}
- // Lance calls the main chain "main"; it lives at the table root, not under tree/.
- if (!LanceCatalogClient.MAIN_BRANCH.equals(branch)) {
- selector = LanceRefSelector.branch(branch, tableSnapshot);
- }
+ selector = LanceRefSelector.branch(branch, tableSnapshot);
} else if (params.isTag()) {
if (tableSnapshot.isPresent()) {
throw new IllegalArgumentException("Lance table " + getName()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/index/LanceDatasetIndexDiscovery.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/index/LanceDatasetIndexDiscovery.java
index 21819caa90e..8747aa18ef1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/index/LanceDatasetIndexDiscovery.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/index/LanceDatasetIndexDiscovery.java
@@ -47,7 +47,9 @@ public final class LanceDatasetIndexDiscovery {
* Describes only user-created indexes. The Lance JNI bulk describe path also tries to
* materialize details for internal indexes, whose details are not supported by the SDK.
* Legacy indexes that cannot be described because all their segments lack details are
- * omitted. This disables their FE segment planning, not the scanner's automatic index use.
+ * omitted, so the FE plans no segments for them. Vector and full-text searches use only the
+ * segments the FE plans and therefore never use such an index; an ordinary scan may still use
+ * a legacy scalar index through Lance's own index selection.
*/
public static List<IndexDescription> describeUserIndexes(Dataset dataset) {
List<String> listedNames = dataset.listIndexes();
@@ -98,7 +100,7 @@ public final class LanceDatasetIndexDiscovery {
if (indexesWithoutDetails.contains(name)) {
LOG.warn("Skipping FE metadata and segment planning for legacy Lance index '{}' "
+ "at dataset version {}: all physical segments lack index details; "
- + "scanner index selection remains unchanged", name, dataset.version());
+ + "vector and full-text searches will not use it", name, dataset.version());
continue;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceRefSelector.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceRefSelector.java
index 2ef80565be7..86a72e81b5c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceRefSelector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceRefSelector.java
@@ -28,6 +28,9 @@ import java.util.Optional;
* with a snapshot inside that branch). A tag and a snapshot exclude each other.
*/
public final class LanceRefSelector {
+ /** Lance's name for the main chain. Lance rejects it as a branch name, so it never names a real branch. */
+ public static final String MAIN_BRANCH = "main";
+
private static final LanceRefSelector LATEST = new LanceRefSelector(Optional.empty(), Optional.empty(),
Optional.empty());
@@ -56,8 +59,14 @@ public final class LanceRefSelector {
return new LanceRefSelector(Optional.empty(), Optional.of(tag), Optional.empty());
}
- /** A branch, at its latest version or at the snapshot selected inside it. */
+ /**
+ * A branch, at its latest version or at the snapshot selected inside it. {@value #MAIN_BRANCH}
+ * is the main chain, which lives at the table root rather than under {@code tree/}.
+ */
public static LanceRefSelector branch(String branch, Optional<TableSnapshot> snapshot) {
+ if (MAIN_BRANCH.equals(branch)) {
+ return snapshot(snapshot);
+ }
return new LanceRefSelector(snapshot, Optional.empty(), Optional.of(branch));
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceSnapshotResolver.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceSnapshotResolver.java
index 7483e4892ad..b90b98ee4cd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceSnapshotResolver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceSnapshotResolver.java
@@ -17,6 +17,8 @@
package org.apache.doris.datasource.lance.metadata;
+import org.apache.doris.common.util.TimeUtils;
+
import org.lance.Version;
import java.time.Instant;
@@ -26,6 +28,7 @@ import java.util.Comparator;
import java.util.List;
import java.util.NavigableMap;
import java.util.NavigableSet;
+import java.util.OptionalLong;
import java.util.TreeMap;
import java.util.regex.Pattern;
@@ -54,6 +57,28 @@ public final class LanceSnapshotResolver {
private static final Pattern VERSION_NUMBER = Pattern.compile("[+-]?[0-9]+");
+ /** The time formats {@link #parseTimestamp} accepts, for error messages. */
+ public static final String TIMESTAMP_FORMATS = "'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd HH:mm:ss.SSS'";
+
+ /**
+ * Parses a time-travel timestamp in the session time zone, with second or millisecond
+ * precision, into epoch milliseconds; empty if the value is in neither format. Commit times
+ * are compared in full, so a commit later within the requested millisecond is not selected.
+ */
+ public static OptionalLong parseTimestamp(String value) {
+ long timestamp;
+ try {
+ timestamp = TimeUtils.timeStringToLong(value, TimeUtils.getTimeZone());
+ if (timestamp < 0) {
+ timestamp = TimeUtils.msTimeStringToLong(value, TimeUtils.getTimeZone());
+ }
+ } catch (ArithmeticException e) {
+ // A year too large for epoch milliseconds.
+ return OptionalLong.empty();
+ }
+ return timestamp < 0 ? OptionalLong.empty() : OptionalLong.of(timestamp);
+ }
+
/**
* Whether a {@code FOR VERSION AS OF} value is a version number rather than a tag name. A
* signed number counts as one, so that {@code '-1'} is reported as an invalid version.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java
index 3f9505b87b9..c572234f027 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java
@@ -386,8 +386,7 @@ public class LanceScanNode extends FileQueryScanNode {
.append(fullText.getPhraseSlop()).append("\n");
}
}
- result.append(prefix).append("lanceVersion=")
- .append(plannedMetadata.getVersion()).append("\n");
+ appendSnapshotExplain(result, prefix);
result.append(prefix).append("lanceSearchFragments=")
.append(scanPlan.fragmentCount).append("\n");
result.append(prefix).append("lanceSearchUnindexedFragments=")
@@ -397,13 +396,7 @@ public class LanceScanNode extends FileQueryScanNode {
result.append(prefix).append("lanceSearchIndexFragments=")
.append(scanPlan.indexedFragmentCount).append("\n");
} else {
- result.append(prefix).append("lanceCatalogType=")
- .append(((LanceExternalCatalog) lanceTable.getCatalog()).getLanceCatalogType()).append("\n");
- result.append(prefix).append("lanceVersion=").append(scanPlan.version).append("\n");
- result.append(prefix).append("lanceManagedVersioning=")
- .append(plannedMetadata.isManagedVersioning()).append("\n");
- plannedMetadata.getBranch().ifPresent(branch ->
- result.append(prefix).append("lanceBranch=").append(branch).append("\n"));
+ appendSnapshotExplain(result, prefix);
result.append(prefix).append("lanceFragments=").append(scanPlan.fragmentCount).append("\n");
if (scanPlan.fragmentsPerSplit > 0) {
result.append(prefix).append("lanceFragmentGrouping=DEBUG\n");
@@ -433,6 +426,17 @@ public class LanceScanNode extends FileQueryScanNode {
return result.toString();
}
+ /** The snapshot this node plans from; table scans and searches show it the same way. */
+ private void appendSnapshotExplain(StringBuilder result, String prefix) {
+ result.append(prefix).append("lanceCatalogType=")
+ .append(((LanceExternalCatalog) lanceTable.getCatalog()).getLanceCatalogType()).append("\n");
+ result.append(prefix).append("lanceVersion=").append(plannedMetadata.getVersion()).append("\n");
+ result.append(prefix).append("lanceManagedVersioning=")
+ .append(plannedMetadata.isManagedVersioning()).append("\n");
+ plannedMetadata.getBranch().ifPresent(branch ->
+ result.append(prefix).append("lanceBranch=").append(branch).append("\n"));
+ }
+
TExternalSearchRequest createSplitSearchRequest() {
TExternalSearchRequest splitRequest = externalSearchRequest.deepCopy();
// Every split must retain enough rows for the later global OFFSET/LIMIT. Applying the
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlan.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlan.java
index 94e604a6f16..d2259c697ed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlan.java
@@ -52,7 +52,10 @@ final class LanceScanPlan {
}
}
- /** Explains FE segment selection, not whether the BE may automatically use a Lance index. */
+ /**
+ * Explains FE segment selection. The BE searches an index only through the segments the FE
+ * plans, so a vector search under any status other than USED is a flat search.
+ */
enum VectorIndexStatus {
/** No vector planning result exists yet (also used by ordinary and FTS scans). */
NOT_PLANNED,
@@ -72,7 +75,6 @@ final class LanceScanPlan {
final VectorIndexStatus vectorIndexStatus;
private final List<Split> splits;
- final long version;
final int fragmentCount;
final int fragmentsPerSplit;
final int indexSegmentCount;
@@ -80,12 +82,11 @@ final class LanceScanPlan {
final int unindexedFragmentCount;
final String scalarIndexName;
- LanceScanPlan(List<Split> splits, long version, int fragments, int fragmentsPerSplit,
+ LanceScanPlan(List<Split> splits, int fragments, int fragmentsPerSplit,
int indexSegments, int indexFragments, int unindexedFragments, String scalarIndexName,
VectorIndexStatus vectorIndexStatus) {
this.vectorIndexStatus = vectorIndexStatus;
this.splits = Collections.unmodifiableList(new ArrayList<>(splits));
- this.version = version;
this.fragmentCount = fragments;
this.fragmentsPerSplit = fragmentsPerSplit;
this.indexSegmentCount = indexSegments;
@@ -95,7 +96,7 @@ final class LanceScanPlan {
}
static LanceScanPlan empty() {
- return new LanceScanPlan(Collections.emptyList(), -1, 0, 0, 0, 0, 0, null, VectorIndexStatus.NOT_PLANNED);
+ return new LanceScanPlan(Collections.emptyList(), 0, 0, 0, 0, 0, null, VectorIndexStatus.NOT_PLANNED);
}
/** Backend assignment shuffles its input list; keep that mutation outside the stored plan. */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java
index 23fc3eeb5dd..2d97bc802bf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java
@@ -79,7 +79,7 @@ final class LanceScanPlanner {
LanceScanPlanner planner = new LanceScanPlanner(metadata, kind, request,
fieldId, pushedConjuncts, fragmentsPerSplit, countParallelism);
List<Split> splits = planner.buildSplits();
- LanceScanPlan plan = new LanceScanPlan(splits, metadata.getVersion(), planner.plannedFragments,
+ LanceScanPlan plan = new LanceScanPlan(splits, planner.plannedFragments,
planner.plannedFragmentsPerSplit, planner.plannedIndexSegments, planner.plannedIndexFragments,
planner.plannedUnindexedFragments,
planner.scalarIndexPlan == null ? null : planner.scalarIndexPlan.indexName,
@@ -166,13 +166,19 @@ final class LanceScanPlanner {
return splits;
}
+ /** "dataset version N", naming the branch too, since branches number versions like main. */
+ private static String datasetVersion(LanceTableMetadata metadata) {
+ return "dataset version " + metadata.getVersion()
+ + metadata.getBranch().map(branch -> " of branch '" + branch + "'").orElse("");
+ }
+
private Map<Long, LanceFragmentInfo> getVisibleFragments(LanceTableMetadata metadata)
throws UserException {
Map<Long, LanceFragmentInfo> visible = new LinkedHashMap<>();
for (LanceFragmentInfo fragment : metadata.getFragments()) {
if (visible.put(fragment.getId(), fragment) != null) {
throw new UserException("Duplicate Lance fragment id " + fragment.getId()
- + " at dataset version " + metadata.getVersion());
+ + " at " + datasetVersion(metadata));
}
}
return visible;
@@ -251,7 +257,7 @@ final class LanceScanPlanner {
metadata.getIndexes(), searchFieldId, fullText.getColumn());
if (matchingSegments.isEmpty()) {
throw new UserException("No committed Lance FTS index exists for column '"
- + fullText.getColumn() + "' at dataset version " + metadata.getVersion());
+ + fullText.getColumn() + "' at " + datasetVersion(metadata));
}
// A matching index with no visible coverage is a valid empty INDEX_ONLY result.
// Unknown coverage still throws in planIndexSegments; STRICT checks uncovered fragments below.
@@ -265,7 +271,7 @@ final class LanceScanPlanner {
if (fullText.getCoverageMode() == TFtsCoverageMode.STRICT
&& plannedUnindexedFragments != 0) {
throw new UserException("Lance FTS coverage_mode=STRICT requires every fragment at "
- + "dataset version " + metadata.getVersion() + " to be indexed; column '"
+ + datasetVersion(metadata) + " to be indexed; column '"
+ fullText.getColumn() + "' has " + plannedUnindexedFragments
+ " unindexed fragments. Rebuild the index or use coverage_mode=index_only");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 492b299c7dc..7a83e22961d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -2899,6 +2899,16 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
}
}
} else {
+ if ("full_text_search".equalsIgnoreCase(functionName) && ctx.properties != null) {
+ // A placeholder would otherwise reach the function as the literal text "?".
+ for (PropertyItemContext argument : ctx.properties.properties) {
+ if (argument.key.constant() instanceof DorisParser.PlaceholderContext
+ || argument.value.constant() instanceof DorisParser.PlaceholderContext) {
+ throw new AnalysisException("full_text_search properties must be constant"
+ + " in a prepared statement");
+ }
+ }
+ }
map = visitPropertyItemList(ctx.properties);
}
LogicalPlan relation = new UnboundTVFRelation(StatementScopeIdGenerator.newRelationId(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FullTextSearchTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FullTextSearchTableValuedFunction.java
index 8c7789e4535..24d27cd06e2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FullTextSearchTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FullTextSearchTableValuedFunction.java
@@ -48,7 +48,7 @@ public class FullTextSearchTableValuedFunction extends LanceExternalSearchTableV
private static final String SLOP = "slop";
private static final Set<String> PROPERTIES = ImmutableSet.of(
TABLE, COLUMN, QUERY, TOP_K, OFFSET, FILTER, COVERAGE_MODE,
- QUERY_TYPE, OPERATOR, MAX_FUZZY_DISTANCE, SLOP);
+ QUERY_TYPE, OPERATOR, MAX_FUZZY_DISTANCE, SLOP, VERSION, TIMESTAMP, TAG, BRANCH);
public FullTextSearchTableValuedFunction(Map<String, String> properties)
throws AnalysisException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LanceExternalSearchTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LanceExternalSearchTableValuedFunction.java
index 12d078e1c9a..6b1e910def7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LanceExternalSearchTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LanceExternalSearchTableValuedFunction.java
@@ -18,6 +18,7 @@
package org.apache.doris.tablefunction;
import org.apache.doris.analysis.TableName;
+import org.apache.doris.analysis.TableSnapshot;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
@@ -29,7 +30,9 @@ import org.apache.doris.common.ErrorReport;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.lance.LanceExternalCatalog;
import org.apache.doris.datasource.lance.LanceExternalTable;
+import org.apache.doris.datasource.lance.metadata.LanceRefSelector;
import org.apache.doris.datasource.lance.metadata.LanceSchemaHelper;
+import org.apache.doris.datasource.lance.metadata.LanceSnapshotResolver;
import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
import org.apache.doris.datasource.lance.source.LanceScanNode;
import org.apache.doris.mysql.privilege.PrivPredicate;
@@ -54,6 +57,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
import java.util.OptionalInt;
import java.util.Set;
import java.util.TreeMap;
@@ -67,6 +71,10 @@ abstract class LanceExternalSearchTableValuedFunction extends TableValuedFunctio
protected static final String TOP_K = "top_k";
protected static final String OFFSET = "offset";
protected static final String FILTER = "filter";
+ protected static final String VERSION = "version";
+ protected static final String TIMESTAMP = "timestamp";
+ protected static final String TAG = "tag";
+ protected static final String BRANCH = "branch";
private static final String FULLY_QUALIFIED_TABLE_NAME_ERROR =
"'table' must be a fully qualified catalog.database.table name";
@@ -74,6 +82,7 @@ abstract class LanceExternalSearchTableValuedFunction extends TableValuedFunctio
/** A trailing @tag/@branch/... or FOR VERSION/TIME AS OF on the 'table' argument. */
private static final Pattern SELECTOR_SUFFIX = Pattern.compile(
"(?i)(@\\s*(tag|branch|incr|options)\\b|\\sfor\\s+(version|time)\\b)");
+ private static final Pattern POSITIVE_INTEGER = Pattern.compile("[0-9]+");
private final String displayName;
private final TableName sourceTableName;
@@ -165,11 +174,12 @@ abstract class LanceExternalSearchTableValuedFunction extends TableValuedFunctio
String displayName, String searchDescription, boolean loadIndexMetadata)
throws AnalysisException {
TableName sourceTableName = parseTableName(required(params, TABLE, functionName));
+ LanceRefSelector selector = parseSelector(params);
LanceExternalTable sourceTable = findLanceExternalTable(sourceTableName);
LanceTableMetadata metadata;
try {
metadata = loadIndexMetadata
- ? sourceTable.loadMetadataForSearch() : sourceTable.loadBasicMetadata();
+ ? sourceTable.loadMetadataForSearch(selector) : sourceTable.loadBasicMetadata(selector);
} catch (RuntimeException e) {
throw new AnalysisException("Failed to load Lance metadata for " + searchDescription
+ " on " + sourceTableName + ": " + e.getMessage(), e);
@@ -262,14 +272,83 @@ abstract class LanceExternalSearchTableValuedFunction extends TableValuedFunctio
}
/**
- * The search TVFs always read the latest version of the main chain. A name that only fails to
- * parse because it carries a selector gets a message saying so; a backquoted name containing
- * '@' or ' for ' parses and is never affected.
+ * The snapshot the 'version', 'timestamp', 'tag' and 'branch' properties select, under the
+ * rules of FOR VERSION AS OF, FOR TIME AS OF, @tag and @branch on a Lance table: a tag
+ * determines both its branch and its version, so it stands alone, and a version or a time
+ * selects within a branch, the main chain by default. Unlike FOR VERSION AS OF, 'version' does
+ * not take a tag name, which 'tag' names without ambiguity. No selector property selects the
+ * latest version of the main chain. Errors name the properties, and a selector the catalog
+ * cannot resolve later fails rather than falling back to the latest version.
+ */
+ static LanceRefSelector parseSelector(Map<String, String> params) throws AnalysisException {
+ Optional<String> version = selectorProperty(params, VERSION);
+ Optional<String> timestamp = selectorProperty(params, TIMESTAMP);
+ Optional<String> tag = selectorProperty(params, TAG);
+ Optional<String> branch = selectorProperty(params, BRANCH);
+ if (version.isPresent() && timestamp.isPresent()) {
+ throw new AnalysisException("'" + VERSION + "' and '" + TIMESTAMP + "' are mutually exclusive");
+ }
+ if (tag.isPresent()) {
+ for (String other : new String[] {VERSION, TIMESTAMP, BRANCH}) {
+ if (params.containsKey(other)) {
+ throw new AnalysisException("'" + TAG + "' and '" + other + "' are mutually exclusive;"
+ + " a tag already determines its branch and version");
+ }
+ }
+ return LanceRefSelector.tag(tag.get());
+ }
+ Optional<TableSnapshot> snapshot = Optional.empty();
+ if (version.isPresent()) {
+ snapshot = Optional.of(TableSnapshot.versionOf(String.valueOf(parseVersion(version.get()))));
+ } else if (timestamp.isPresent()) {
+ if (!LanceSnapshotResolver.parseTimestamp(timestamp.get()).isPresent()) {
+ throw new AnalysisException("'" + TIMESTAMP + "' must be " + LanceSnapshotResolver.TIMESTAMP_FORMATS
+ + " in the session time zone, but was '" + timestamp.get() + "'");
+ }
+ snapshot = Optional.of(TableSnapshot.timeOf(timestamp.get()));
+ }
+ return branch.isPresent()
+ ? LanceRefSelector.branch(branch.get(), snapshot) : LanceRefSelector.snapshot(snapshot);
+ }
+
+ /** A selector property's trimmed value. An empty one is an error, not an absent selector. */
+ private static Optional<String> selectorProperty(Map<String, String> params, String key)
+ throws AnalysisException {
+ String value = params.get(key);
+ if (value == null) {
+ return Optional.empty();
+ }
+ if (value.trim().isEmpty()) {
+ throw new AnalysisException("'" + key + "' must not be empty");
+ }
+ return Optional.of(value.trim());
+ }
+
+ private static long parseVersion(String value) throws AnalysisException {
+ String error = "'" + VERSION + "' must be a positive integer, but was '" + value + "'";
+ if (!POSITIVE_INTEGER.matcher(value).matches()) {
+ throw new AnalysisException(error);
+ }
+ long version;
+ try {
+ version = Long.parseLong(value);
+ } catch (NumberFormatException e) {
+ throw new AnalysisException("'" + VERSION + "' " + value + " is out of range");
+ }
+ if (version <= 0) {
+ throw new AnalysisException(error);
+ }
+ return version;
+ }
+
+ /**
+ * A name that only fails to parse because it carries a selector gets a message pointing to the
+ * selector properties; a backquoted name containing '@' or ' for ' parses and is never affected.
*/
private static String tableNameError(String value) {
return SELECTOR_SUFFIX.matcher(value).find()
? "'table' of a Lance search function cannot select a version, tag or branch;"
- + " it always searches the latest version"
+ + " use the 'version', 'timestamp', 'tag' or 'branch' property"
: FULLY_QUALIFIED_TABLE_NAME_ERROR;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunction.java
index 93f7b2f4d56..2e9e2374b7f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunction.java
@@ -67,7 +67,7 @@ public class VectorSearchTableValuedFunction extends LanceExternalSearchTableVal
private static final String USE_INDEX = "use_index";
private static final Set<String> PROPERTIES = ImmutableSet.of(
TABLE, COLUMN, QUERY_VECTOR, TOP_K, OFFSET, METRIC, FILTER,
- NPROBES, REFINE_FACTOR, EF, USE_INDEX, QUERY_PARALLELISM);
+ NPROBES, REFINE_FACTOR, EF, USE_INDEX, QUERY_PARALLELISM, VERSION, TIMESTAMP, TAG, BRANCH);
public VectorSearchTableValuedFunction(Map<String, String> properties)
throws AnalysisException {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogClientTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogClientTest.java
index 73f31711abd..3b260a86ed4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogClientTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogClientTest.java
@@ -194,6 +194,32 @@ public class LanceCatalogClientTest {
}
}
+ /**
+ * A managed table's branch is opened by a URI Doris joins, so Doris rejects names that could
+ * leave the table's tree/ directory, since the object store would resolve dot segments, even
+ * percent-encoded ones, but never a name Lance accepts.
+ */
+ @Test
+ public void testManagedBranchNamesFollowLanceRules() {
+ // Every name Lance accepts passes, including letters newer than the JDK's Unicode tables:
+ // "dev" ends with U+1C89, added in Unicode 16, which Rust's char::is_alphanumeric
+ // accepts and JDK 17 leaves unassigned. "शाखा" has a combining vowel sign, then a
+ // superscript digit and a Roman numeral.
+ for (String valid : new String[] {"dev", "team/dev", "v1.0", "a_b-c", "123", "dev2026", "分支",
+ "dev", "शाखा", "v²", "Ⅻ"}) {
+ LanceCatalogClient.checkBranchName(valid);
+ }
+ // URI and path syntax is rejected, as is whitespace, including an ideographic space.
+ for (String invalid : new String[] {"", "../other.lance", "a/../../b", "/dev", "dev/", "a//b",
+ "x.lock", "%2e%2e/x", "dev?x=1", "dev#x", "a b", "a\\b", "s3://bucket/t", "a+b", "a\tb",
+ "a b"}) {
+ RuntimeException exception = Assertions.assertThrows(RuntimeException.class,
+ () -> LanceCatalogClient.checkBranchName(invalid), invalid);
+ Assertions.assertTrue(exception.getMessage().startsWith("Invalid Lance branch name '" + invalid + "'"),
+ exception.getMessage());
+ }
+ }
+
private static LanceCatalogClient client(LanceNamespace namespace, BufferAllocator allocator, Session session) {
return new LanceCatalogClient(namespace, allocator, session, "filesystem", "default",
Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), Collections.emptyList());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManagedVersioningTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManagedVersioningTest.java
index 6d5da9f669c..d1ec70c0643 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManagedVersioningTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceManagedVersioningTest.java
@@ -391,6 +391,75 @@ public class LanceManagedVersioningTest {
}
}
+ /** The search entry points take the same selectors, under the same namespace rules. */
+ @Test
+ public void testSearchMetadataOfManagedTableFollowsTheSelector() throws Exception {
+ LanceExternalCatalog catalog = newCatalog(329, "lance_managed_search");
+ try {
+ for (boolean withIndexes : new boolean[] {true, false}) {
+ java.util.function.Function<LanceRefSelector, LanceTableMetadata> load = selector -> withIndexes
+ ? catalog.loadTableMetadataForSearch("default", FULL_TABLE, selector)
+ : catalog.loadBasicTableMetadata("default", FULL_TABLE, selector);
+ LanceTableMetadata latest = load.apply(LanceRefSelector.latest());
+ Assertions.assertTrue(latest.isManagedVersioning());
+ Assertions.assertEquals(3, latest.getVersion());
+ Assertions.assertEquals(2, load.apply(LanceRefSelector.snapshot(
+ Optional.of(TableSnapshot.versionOf("2")))).getVersion());
+ Assertions.assertEquals(2, load.apply(LanceRefSelector.tag("v2")).getVersion());
+ String betweenSecondAndThird = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
+ .withZone(TimeUtils.getTimeZone().toZoneId())
+ .format(datasetVersions.get(1).getDataTime().toInstant().plusMillis(1));
+ Assertions.assertEquals(2, load.apply(LanceRefSelector.snapshot(
+ Optional.of(TableSnapshot.timeOf(betweenSecondAndThird)))).getVersion());
+
+ LanceTableMetadata branch = load.apply(LanceRefSelector.branch(BRANCH, Optional.empty()));
+ Assertions.assertEquals(Optional.of(BRANCH), branch.getBranch());
+ Assertions.assertEquals(branchVersions, branch.getVersion());
+ Assertions.assertTrue(branch.isManagedVersioning());
+ LanceTableMetadata branchTag = load.apply(LanceRefSelector.tag(BRANCH_TAG));
+ Assertions.assertEquals(Optional.of(BRANCH), branchTag.getBranch());
+ Assertions.assertEquals(3, branchTag.getVersion());
+ // main is the table root, not a branch named main.
+ LanceTableMetadata main = load.apply(LanceRefSelector.branch(LanceRefSelector.MAIN_BRANCH,
+ Optional.of(TableSnapshot.versionOf("2"))));
+ Assertions.assertFalse(main.getBranch().isPresent());
+ Assertions.assertEquals(2, main.getVersion());
+ }
+ // A version the namespace does not record fails; it never falls back to the latest one.
+ RuntimeException unrecorded = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadataForSearch("default", PARTIAL_TABLE,
+ LanceRefSelector.snapshot(Optional.of(TableSnapshot.versionOf("2")))));
+ Assertions.assertEquals(
+ "Lance version 2 of default." + PARTIAL_TABLE + " was not found in the namespace",
+ unrecorded.getMessage());
+ // A head whose commit was never finalized cannot be searched, on main or on a branch,
+ // and nothing falls back to an older version.
+ RuntimeException pending = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadataForSearch("default", PENDING_HEAD_TABLE, LanceRefSelector.latest()));
+ Assertions.assertEquals("Lance version 4 of default." + PENDING_HEAD_TABLE + STAGED_ONLY,
+ pending.getMessage());
+ RuntimeException pendingBranch = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadBasicTableMetadata("default", PENDING_HEAD_TABLE,
+ LanceRefSelector.branch(BRANCH, Optional.empty())));
+ Assertions.assertEquals("Lance version 4 of default." + PENDING_HEAD_TABLE + "@" + BRANCH + STAGED_ONLY,
+ pendingBranch.getMessage());
+ Assertions.assertEquals(3, catalog.loadTableMetadataForSearch("default", PENDING_HEAD_TABLE,
+ LanceRefSelector.snapshot(Optional.of(TableSnapshot.versionOf("3")))).getVersion());
+ // A managed branch is opened by a URI Doris joins, so a name that would climb out of the
+ // table's tree/ directory is rejected before the branch's versions are requested.
+ requestPaths.clear();
+ RuntimeException climbing = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadataForSearch("default", FULL_TABLE,
+ LanceRefSelector.branch("../../" + PLAIN_TABLE, Optional.empty())));
+ Assertions.assertTrue(climbing.getMessage().startsWith("Invalid Lance branch name '../../"),
+ climbing.getMessage());
+ Assertions.assertTrue(requestPaths.stream().noneMatch(path -> path.contains("/version/")),
+ requestPaths.toString());
+ } finally {
+ catalog.onClose();
+ }
+ }
+
@Test
public void testVersionMissingFromNamespaceIsReported() {
LanceExternalCatalog catalog = newCatalog(301, "lance_managed_partial");
@@ -421,7 +490,8 @@ public class LanceManagedVersioningTest {
RuntimeException dropped = Assertions.assertThrows(RuntimeException.class,
() -> catalog.loadTableMetadata("default", PARTIAL_TABLE,
Optional.of(new TableSnapshot(betweenSecondAndThird, TableSnapshot.VersionType.TIME))));
- Assertions.assertEquals("Lance cannot resolve FOR TIME AS OF '" + betweenSecondAndThird + "' on default."
+ Assertions.assertEquals("Lance cannot select the version at or before '"
+ + betweenSecondAndThird + "' on default."
+ PARTIAL_TABLE + ": version 2, which may hold the state at that time, no longer exists",
dropped.getMessage());
Assertions.assertTrue(requestPaths.stream().anyMatch(
@@ -537,7 +607,7 @@ public class LanceManagedVersioningTest {
RuntimeException byTime = Assertions.assertThrows(RuntimeException.class,
() -> catalog.loadTableMetadata("default", PENDING_HEAD_TABLE,
Optional.of(new TableSnapshot("2100-01-01 00:00:00", TableSnapshot.VersionType.TIME))));
- Assertions.assertEquals("Lance cannot resolve FOR TIME AS OF '2100-01-01 00:00:00' on default."
+ Assertions.assertEquals("Lance cannot select the version at or before '2100-01-01 00:00:00' on default."
+ PENDING_HEAD_TABLE + ": version 4, which may hold the state at that time," + STAGED_ONLY,
byTime.getMessage());
Assertions.assertEquals(3, catalog.loadTableMetadata("default", PENDING_HEAD_TABLE,
@@ -564,7 +634,7 @@ public class LanceManagedVersioningTest {
() -> catalog.loadTableMetadata("default", PENDING_HEAD_TABLE,
LanceRefSelector.branch(BRANCH, Optional.of(new TableSnapshot("2100-01-01 00:00:00",
TableSnapshot.VersionType.TIME)))));
- Assertions.assertEquals("Lance cannot resolve FOR TIME AS OF '2100-01-01 00:00:00' on default."
+ Assertions.assertEquals("Lance cannot select the version at or before '2100-01-01 00:00:00' on default."
+ PENDING_HEAD_TABLE + "@" + BRANCH + ": version 4, which may hold the state at that time,"
+ STAGED_ONLY, devByTime.getMessage());
} finally {
@@ -952,7 +1022,8 @@ public class LanceManagedVersioningTest {
RuntimeException removed = Assertions.assertThrows(RuntimeException.class,
() -> catalog.loadTableMetadata("default", STAGED_UNTIMED_TABLE,
Optional.of(new TableSnapshot(betweenFirstAndSecond, TableSnapshot.VersionType.TIME))));
- Assertions.assertEquals("Lance cannot resolve FOR TIME AS OF '" + betweenFirstAndSecond + "' on default."
+ Assertions.assertEquals("Lance cannot select the version at or before '"
+ + betweenFirstAndSecond + "' on default."
+ STAGED_UNTIMED_TABLE + ": version 1, which may hold the state at that time," + STAGED_ONLY,
removed.getMessage());
RuntimeException byVersion = Assertions.assertThrows(RuntimeException.class,
@@ -995,7 +1066,8 @@ public class LanceManagedVersioningTest {
RuntimeException gap = Assertions.assertThrows(RuntimeException.class,
() -> catalog.loadTableMetadata("default", table,
Optional.of(new TableSnapshot(betweenFirstAndThird, TableSnapshot.VersionType.TIME))));
- Assertions.assertEquals("Lance cannot resolve FOR TIME AS OF '" + betweenFirstAndThird + "' on default."
+ Assertions.assertEquals("Lance cannot select the version at or before '"
+ + betweenFirstAndThird + "' on default."
+ table + ": version 2, which may hold the state at that time, no longer exists",
gap.getMessage());
Assertions.assertEquals(3, catalog.loadTableMetadata("default", table,
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSearchSnapshotTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSearchSnapshotTest.java
new file mode 100644
index 00000000000..1b043db7383
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceSearchSnapshotTest.java
@@ -0,0 +1,506 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance;
+
+import org.apache.doris.analysis.TableSnapshot;
+import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.analysis.TupleId;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.lance.metadata.LanceFragmentInfo;
+import org.apache.doris.datasource.lance.metadata.LanceRefSelector;
+import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
+import org.apache.doris.datasource.lance.source.LanceScanNode;
+import org.apache.doris.datasource.lance.source.LanceSplit;
+import org.apache.doris.planner.PlanNodeId;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.spi.Split;
+import org.apache.doris.thrift.TExplainLevel;
+import org.apache.doris.thrift.TExternalSearchQuery;
+import org.apache.doris.thrift.TExternalSearchRequest;
+import org.apache.doris.thrift.TFtsCoverageMode;
+import org.apache.doris.thrift.TFtsMatchOperator;
+import org.apache.doris.thrift.TFtsQueryType;
+import org.apache.doris.thrift.TFullTextSearchParams;
+import org.apache.doris.thrift.TVectorSearchParams;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.FixedSizeListVector;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.lance.Dataset;
+import org.lance.Fragment;
+import org.lance.FragmentMetadata;
+import org.lance.FragmentOperation;
+import org.lance.ReadOptions;
+import org.lance.Ref;
+import org.lance.WriteParams;
+import org.lance.cleanup.CleanupPolicy;
+import org.lance.index.DistanceType;
+import org.lance.index.IndexParams;
+import org.lance.index.IndexType;
+import org.lance.index.scalar.ScalarIndexParams;
+import org.lance.index.vector.VectorIndexParams;
+import org.lance.schema.ColumnAlteration;
+import org.mockito.Mockito;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+/**
+ * Search metadata and planning on historical versions, tags and branches of a real dataset.
+ *
+ * <p>Main: v1 is empty; v2 adds fragment 0 (ids 0..9); v3 fragment 1 (ids 10..19); v4 a vector
+ * index over both; v5 an FTS index over both; v6 fragment 2 (ids 20..29), which neither index
+ * covers. Tag {@code rel} points at main v6. Branch {@code dev} forks from v5 and appends ids
+ * 100..109 as its own v6. A branch continues its parent's fragment numbering, so dev v6 and main
+ * v6 both hold fragments 0, 1 and 2, with a different fragment 2. Tag {@code dev_rel} points at
+ * dev v5, the fork point, which is not the branch's latest version.
+ */
+@Disabled("Re-enable after fixing Arrow C Data JNI compatibility: CI libstdc++ lacks CXXABI_1.3.9")
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class LanceSearchSnapshotTest {
+ private static final String BRANCH = "dev";
+ private static final Schema SCHEMA = new Schema(Arrays.asList(
+ Field.nullable("id", new ArrowType.Int(64, true)),
+ new Field("vec", FieldType.nullable(new ArrowType.FixedSizeList(2)), Collections.singletonList(
+ Field.nullable("item", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)))),
+ Field.nullable("body", ArrowType.Utf8.INSTANCE)));
+
+ private Path warehouse;
+ private String uri;
+ private LanceExternalCatalog catalog;
+
+ @BeforeAll
+ public void setUp() throws Exception {
+ LanceJniTestSupport.assumeJniBindingsLoadable();
+ warehouse = Files.createTempDirectory("lance_search_snapshot");
+ uri = warehouse.resolve("docs.lance").toString();
+ try (BufferAllocator allocator = new RootAllocator()) {
+ WriteParams params = new WriteParams.Builder().withDataStorageVersion("2.0").build();
+ try (Dataset created = Dataset.create(allocator, uri, SCHEMA, params)) {
+ Assertions.assertEquals(1, created.version());
+ }
+ Assertions.assertEquals(2, append(uri, allocator, 1, 0, 10));
+ Assertions.assertEquals(3, append(uri, allocator, 2, 10, 10));
+ try (Dataset dataset = Dataset.open(uri, allocator)) {
+ dataset.createIndex(Collections.singletonList("vec"), IndexType.IVF_FLAT, Optional.of("vec_idx"),
+ IndexParams.builder().setVectorIndexParams(VectorIndexParams.ivfFlat(2, DistanceType.L2))
+ .build(), false);
+ Assertions.assertEquals(4, dataset.version());
+ dataset.createIndex(Collections.singletonList("body"), IndexType.INVERTED, Optional.of("body_idx"),
+ IndexParams.builder().setScalarIndexParams(ScalarIndexParams.create("inverted",
+ "{\"base_tokenizer\":\"simple\",\"language\":\"English\",\"lower_case\":true,"
+ + "\"stem\":false,\"remove_stop_words\":false}")).build(), false);
+ Assertions.assertEquals(5, dataset.version());
+ }
+ Assertions.assertEquals(6, append(uri, allocator, 5, 20, 10));
+ try (Dataset dataset = Dataset.open(uri, allocator)) {
+ dataset.tags().create("rel", 6);
+ try (Dataset branch = dataset.createBranch(BRANCH, Ref.ofMain(5))) {
+ Assertions.assertEquals(5, branch.version());
+ }
+ Assertions.assertEquals(6, append(uri + "/tree/" + BRANCH, allocator, 5, 100, 10));
+ dataset.tags().create("dev_rel", 5, BRANCH);
+ }
+ }
+ Map<String, String> properties = new HashMap<>();
+ properties.put("type", "lance");
+ properties.put(LanceExternalCatalog.WAREHOUSE, warehouse.toString());
+ catalog = new LanceExternalCatalog(910, "lance_search_snapshot", null, properties, "");
+ }
+
+ @AfterAll
+ public void tearDown() throws Exception {
+ if (catalog != null) {
+ catalog.onClose();
+ }
+ if (warehouse != null) {
+ try (java.util.stream.Stream<Path> paths = Files.walk(warehouse)) {
+ paths.sorted(java.util.Comparator.reverseOrder()).forEach(path -> path.toFile().delete());
+ }
+ }
+ }
+
+ /** Appends one fragment with ids {@code first..first+count-1} and returns the committed version. */
+ private static long append(String datasetUri, BufferAllocator allocator, long readVersion, long first, int count)
+ throws Exception {
+ WriteParams params = new WriteParams.Builder().withDataStorageVersion("2.0").build();
+ try (VectorSchemaRoot root = VectorSchemaRoot.create(SCHEMA, allocator)) {
+ BigIntVector id = (BigIntVector) root.getVector("id");
+ FixedSizeListVector vec = (FixedSizeListVector) root.getVector("vec");
+ VarCharVector body = (VarCharVector) root.getVector("body");
+ id.allocateNew(count);
+ vec.allocateNew();
+ Float4Vector items = (Float4Vector) vec.getDataVector();
+ items.allocateNew(count * 2);
+ body.allocateNew(count);
+ for (int i = 0; i < count; i++) {
+ long value = first + i;
+ id.set(i, value);
+ vec.setNotNull(i);
+ items.set(i * 2, value);
+ items.set(i * 2 + 1, -value);
+ body.setSafe(i, ("doc " + value + (value % 2 == 0 ? " lance" : " doris"))
+ .getBytes(StandardCharsets.UTF_8));
+ }
+ id.setValueCount(count);
+ items.setValueCount(count * 2);
+ vec.setValueCount(count);
+ body.setValueCount(count);
+ root.setRowCount(count);
+ List<FragmentMetadata> fragments = Fragment.create(datasetUri, allocator, root, params);
+ try (Dataset committed = new FragmentOperation.Append(fragments)
+ .commit(allocator, datasetUri, Optional.of(readVersion), Collections.emptyMap())) {
+ return committed.version();
+ }
+ }
+ }
+
+ private LanceTableMetadata search(LanceRefSelector selector) {
+ return catalog.loadTableMetadataForSearch("default", "docs", selector);
+ }
+
+ private static LanceRefSelector version(long version) {
+ return LanceRefSelector.snapshot(Optional.of(TableSnapshot.versionOf(String.valueOf(version))));
+ }
+
+ private static Set<Long> fragments(LanceTableMetadata metadata) {
+ return metadata.getFragments().stream().map(LanceFragmentInfo::getId)
+ .collect(Collectors.toCollection(TreeSet::new));
+ }
+
+ private static LanceScanNode node(LanceTableMetadata metadata, TExternalSearchRequest request, int fieldId) {
+ TupleDescriptor desc = new TupleDescriptor(new TupleId(0));
+ desc.setTable(Mockito.mock(TableIf.class));
+ LanceExternalCatalog lanceCatalog = Mockito.mock(LanceExternalCatalog.class);
+ Mockito.when(lanceCatalog.getLanceCatalogType()).thenReturn("filesystem");
+ LanceExternalTable table = Mockito.mock(LanceExternalTable.class);
+ Mockito.when(table.getCatalog()).thenReturn(lanceCatalog);
+ return LanceScanNode.forExternalSearch(new PlanNodeId(0), desc, table, metadata, fieldId, request,
+ new SessionVariable());
+ }
+
+ private static List<LanceSplit> vectorSplits(LanceTableMetadata metadata) throws UserException {
+ return vectorSplits(metadata, "vec");
+ }
+
+ private static List<LanceSplit> vectorSplits(LanceTableMetadata metadata, String column) throws UserException {
+ TVectorSearchParams vector = new TVectorSearchParams().setColumn(column).setTopK(5).setOffset(0);
+ LanceScanNode node = node(metadata, new TExternalSearchRequest()
+ .setSearchQuery(TExternalSearchQuery.vector_search(vector)),
+ metadata.getLanceFieldId(column).getAsInt());
+ List<LanceSplit> splits = new ArrayList<>();
+ for (Split split : node.getSplits(1)) {
+ splits.add((LanceSplit) split);
+ }
+ return splits;
+ }
+
+ private static List<LanceSplit> fullTextSplits(LanceTableMetadata metadata, TFtsCoverageMode mode)
+ throws UserException {
+ TFullTextSearchParams fullText = new TFullTextSearchParams().setColumn("body").setQuery("lance")
+ .setTopK(5).setOffset(0).setCoverageMode(mode).setQueryType(TFtsQueryType.MATCH)
+ .setMatchOperator(TFtsMatchOperator.OR).setMaxFuzzyDistance(0);
+ LanceScanNode node = node(metadata, new TExternalSearchRequest()
+ .setSearchQuery(TExternalSearchQuery.full_text_search(fullText)),
+ metadata.getLanceFieldId("body").getAsInt());
+ List<LanceSplit> splits = new ArrayList<>();
+ for (Split split : node.getSplits(1)) {
+ splits.add((LanceSplit) split);
+ }
+ return splits;
+ }
+
+ private static long indexedFragments(List<LanceSplit> splits) {
+ return splits.stream().filter(split -> split.getIndexSegmentUuid().isPresent())
+ .mapToLong(split -> split.getFragmentIds().size()).sum();
+ }
+
+ private static long flatFragments(List<LanceSplit> splits) {
+ return splits.stream().filter(split -> !split.getIndexSegmentUuid().isPresent())
+ .mapToLong(split -> split.getFragmentIds().size()).sum();
+ }
+
+ @Test
+ public void testNoSelectorSearchesLatestMain() throws Exception {
+ LanceTableMetadata latest = search(LanceRefSelector.latest());
+ Assertions.assertEquals(6, latest.getVersion());
+ Assertions.assertFalse(latest.getBranch().isPresent());
+ Assertions.assertEquals(new TreeSet<>(Arrays.asList(0L, 1L, 2L)), fragments(latest));
+ }
+
+ @Test
+ public void testVersionBeforeAnyIndexFallsBackToFlatVectorSearchAndRejectsFullText() throws Exception {
+ LanceTableMetadata v3 = search(version(3));
+ Assertions.assertEquals(3, v3.getVersion());
+ Assertions.assertEquals(new TreeSet<>(Arrays.asList(0L, 1L)), fragments(v3));
+ List<LanceSplit> splits = vectorSplits(v3);
+ Assertions.assertEquals(0, indexedFragments(splits));
+ Assertions.assertEquals(2, flatFragments(splits));
+ splits.forEach(split -> Assertions.assertEquals(3, split.getVersion()));
+ UserException exception = Assertions.assertThrows(UserException.class,
+ () -> fullTextSplits(v3, TFtsCoverageMode.INDEX_ONLY));
+ Assertions.assertTrue(exception.getMessage().contains(
+ "No committed Lance FTS index exists for column 'body' at dataset version 3"),
+ exception.getMessage());
+ }
+
+ @Test
+ public void testVersionWithVectorIndexButNoFullTextIndex() throws Exception {
+ LanceTableMetadata v4 = search(version(4));
+ List<LanceSplit> splits = vectorSplits(v4);
+ Assertions.assertEquals(2, indexedFragments(splits));
+ Assertions.assertEquals(0, flatFragments(splits));
+ Assertions.assertThrows(UserException.class, () -> fullTextSplits(v4, TFtsCoverageMode.INDEX_ONLY));
+ Assertions.assertEquals(2, indexedFragments(fullTextSplits(search(version(5)), TFtsCoverageMode.STRICT)));
+ }
+
+ @Test
+ public void testAppendAfterIndexingCoversPartially() throws Exception {
+ LanceTableMetadata v6 = search(version(6));
+ List<LanceSplit> vector = vectorSplits(v6);
+ Assertions.assertEquals(2, indexedFragments(vector));
+ Assertions.assertEquals(1, flatFragments(vector));
+ UserException strict = Assertions.assertThrows(UserException.class,
+ () -> fullTextSplits(v6, TFtsCoverageMode.STRICT));
+ Assertions.assertTrue(strict.getMessage().contains("every fragment at dataset version 6 to be indexed"),
+ strict.getMessage());
+ List<LanceSplit> indexOnly = fullTextSplits(v6, TFtsCoverageMode.INDEX_ONLY);
+ Assertions.assertEquals(2, indexedFragments(indexOnly));
+ Assertions.assertEquals(0, flatFragments(indexOnly));
+ }
+
+ @Test
+ public void testTagSelectsItsVersion() throws Exception {
+ LanceTableMetadata tagged = search(LanceRefSelector.tag("rel"));
+ Assertions.assertEquals(6, tagged.getVersion());
+ Assertions.assertFalse(tagged.getBranch().isPresent());
+ Assertions.assertEquals(new TreeSet<>(Arrays.asList(0L, 1L, 2L)), fragments(tagged));
+ }
+
+ @Test
+ public void testBranchVersionsOverlapMainWithDifferentData() throws Exception {
+ LanceTableMetadata dev = search(LanceRefSelector.branch(BRANCH, Optional.empty()));
+ Assertions.assertEquals(6, dev.getVersion());
+ Assertions.assertEquals(Optional.of(BRANCH), dev.getBranch());
+ Assertions.assertTrue(dev.getDatasetUri().endsWith("/tree/" + BRANCH), dev.getDatasetUri());
+ // Same version number and fragment ids as main v6, different rows.
+ Assertions.assertEquals(fragments(search(version(6))), fragments(dev));
+ Assertions.assertEquals(10, count(dev.getDatasetUri(), 6, "id >= 100"));
+ Assertions.assertEquals(0, count(uri, 6, "id >= 100"));
+ // Inherited indexes cover the fork point's fragments; the branch's own append is flat.
+ List<LanceSplit> vector = vectorSplits(dev);
+ Assertions.assertEquals(2, indexedFragments(vector));
+ Assertions.assertEquals(1, flatFragments(vector));
+ vector.forEach(split -> {
+ Assertions.assertEquals(6, split.getVersion());
+ Assertions.assertEquals(dev.getDatasetUri(), split.getDatasetUri());
+ });
+ UserException strict = Assertions.assertThrows(UserException.class,
+ () -> fullTextSplits(dev, TFtsCoverageMode.STRICT));
+ Assertions.assertTrue(strict.getMessage().contains("dataset version 6 of branch 'dev'"), strict.getMessage());
+
+ LanceTableMetadata fork = search(LanceRefSelector.branch(BRANCH, Optional.of(TableSnapshot.versionOf("5"))));
+ Assertions.assertEquals(5, fork.getVersion());
+ Assertions.assertEquals(Optional.of(BRANCH), fork.getBranch());
+ Assertions.assertEquals(new TreeSet<>(Arrays.asList(0L, 1L)), fragments(fork));
+ Assertions.assertEquals(2, indexedFragments(fullTextSplits(fork, TFtsCoverageMode.STRICT)));
+ }
+
+ @Test
+ public void testTagOnBranchSelectsTheBranch() throws Exception {
+ // A tag at a version of the branch other than its latest: Lance's with_tag on the branch
+ // URI fails for this (lance#9227), and Doris resolves it through the table root instead.
+ LanceTableMetadata tagged = search(LanceRefSelector.tag("dev_rel"));
+ Assertions.assertEquals(5, tagged.getVersion());
+ Assertions.assertEquals(Optional.of(BRANCH), tagged.getBranch());
+ Assertions.assertTrue(tagged.getDatasetUri().endsWith("/tree/" + BRANCH), tagged.getDatasetUri());
+ Assertions.assertEquals(new TreeSet<>(Arrays.asList(0L, 1L)), fragments(tagged));
+ }
+
+ @Test
+ public void testMainBranchIsTheTableRoot() throws Exception {
+ LanceTableMetadata main = search(LanceRefSelector.branch(LanceRefSelector.MAIN_BRANCH,
+ Optional.of(TableSnapshot.versionOf("4"))));
+ Assertions.assertEquals(4, main.getVersion());
+ Assertions.assertFalse(main.getBranch().isPresent());
+ Assertions.assertFalse(main.getDatasetUri().contains("/tree/"), main.getDatasetUri());
+ }
+
+ @Test
+ public void testTimestamps() throws Exception {
+ Assertions.assertEquals(6, search(LanceRefSelector.snapshot(
+ Optional.of(TableSnapshot.timeOf("2999-01-01 00:00:00")))).getVersion());
+ Assertions.assertEquals(6, search(LanceRefSelector.branch(BRANCH,
+ Optional.of(TableSnapshot.timeOf("2999-01-01 00:00:00")))).getVersion());
+ RuntimeException early = Assertions.assertThrows(RuntimeException.class, () -> search(
+ LanceRefSelector.snapshot(Optional.of(TableSnapshot.timeOf("2000-01-01 00:00:00")))));
+ Assertions.assertTrue(early.getMessage().contains("has no version at or before '2000-01-01 00:00:00'"),
+ early.getMessage());
+ }
+
+ @Test
+ public void testMissingSnapshotsFailWithoutFallingBackToLatest() {
+ RuntimeException version = Assertions.assertThrows(RuntimeException.class, () -> search(version(99)));
+ Assertions.assertTrue(version.getMessage().contains("Lance version 99 of default.docs was not found"),
+ version.getMessage());
+ RuntimeException tag = Assertions.assertThrows(RuntimeException.class,
+ () -> search(LanceRefSelector.tag("nope")));
+ Assertions.assertTrue(tag.getMessage().contains("Lance tag 'nope'"), tag.getMessage());
+ RuntimeException branch = Assertions.assertThrows(RuntimeException.class,
+ () -> search(LanceRefSelector.branch("nope", Optional.empty())));
+ Assertions.assertTrue(branch.getMessage().contains("Lance branch 'nope'"), branch.getMessage());
+ RuntimeException branchVersion = Assertions.assertThrows(RuntimeException.class,
+ () -> search(LanceRefSelector.branch(BRANCH, Optional.of(TableSnapshot.versionOf("99")))));
+ Assertions.assertTrue(branchVersion.getMessage().contains("Lance version 99"), branchVersion.getMessage());
+ }
+
+ @Test
+ public void testPlannedSnapshotSurvivesLaterCommits() throws Exception {
+ String pinnedUri = warehouse.resolve("pinned.lance").toString();
+ try (BufferAllocator allocator = new RootAllocator()) {
+ try (Dataset created = Dataset.create(allocator, pinnedUri, SCHEMA,
+ new WriteParams.Builder().withDataStorageVersion("2.0").build())) {
+ Assertions.assertEquals(1, created.version());
+ }
+ append(pinnedUri, allocator, 1, 0, 10);
+ try (Dataset dataset = Dataset.open(pinnedUri, allocator)) {
+ dataset.tags().create("moving", 2);
+ }
+ LanceTableMetadata planned = catalog.loadTableMetadataForSearch("default", "pinned",
+ LanceRefSelector.tag("moving"));
+ Assertions.assertEquals(2, planned.getVersion());
+ // A commit and a tag move after planning do not reach the planned splits.
+ append(pinnedUri, allocator, 2, 10, 10);
+ try (Dataset dataset = Dataset.open(pinnedUri, allocator)) {
+ dataset.tags().update("moving", 3);
+ }
+ List<LanceSplit> splits = vectorSplits(planned);
+ splits.forEach(split -> Assertions.assertEquals(2, split.getVersion()));
+ Assertions.assertEquals(1, flatFragments(splits));
+ // The next resolution sees the moved tag.
+ Assertions.assertEquals(3, catalog.loadTableMetadataForSearch("default", "pinned",
+ LanceRefSelector.tag("moving")).getVersion());
+ }
+ }
+
+ /**
+ * Each version keeps its own schema: a column added later is absent from older versions, and
+ * a renamed column keeps its field id, so the index built before the rename still serves it.
+ */
+ @Test
+ public void testSchemaFollowsTheSelectedVersion() throws Exception {
+ String schemaUri = warehouse.resolve("evolving.lance").toString();
+ try (BufferAllocator allocator = new RootAllocator()) {
+ try (Dataset created = Dataset.create(allocator, schemaUri, SCHEMA,
+ new WriteParams.Builder().withDataStorageVersion("2.0").build())) {
+ Assertions.assertEquals(1, created.version());
+ }
+ Assertions.assertEquals(2, append(schemaUri, allocator, 1, 0, 10));
+ try (Dataset dataset = Dataset.open(schemaUri, allocator)) {
+ dataset.createIndex(Collections.singletonList("vec"), IndexType.IVF_FLAT, Optional.of("vec_idx"),
+ IndexParams.builder().setVectorIndexParams(VectorIndexParams.ivfFlat(2, DistanceType.L2))
+ .build(), false);
+ dataset.addColumns(Collections.singletonList(Field.nullable("note", ArrowType.Utf8.INSTANCE)));
+ dataset.alterColumns(Collections.singletonList(
+ new ColumnAlteration.Builder("vec").rename("embedding").build()));
+ Assertions.assertEquals(5, dataset.version());
+ }
+ }
+ LanceTableMetadata v3 = catalog.loadTableMetadataForSearch("default", "evolving", version(3));
+ LanceTableMetadata v5 = catalog.loadTableMetadataForSearch("default", "evolving", version(5));
+ Set<String> v3Columns = v3.getSchema().getFields().stream().map(Field::getName).collect(Collectors.toSet());
+ Set<String> v5Columns = v5.getSchema().getFields().stream().map(Field::getName).collect(Collectors.toSet());
+ Assertions.assertEquals(new TreeSet<>(Arrays.asList("id", "vec", "body")), new TreeSet<>(v3Columns));
+ Assertions.assertEquals(new TreeSet<>(Arrays.asList("id", "embedding", "body", "note")),
+ new TreeSet<>(v5Columns));
+ Assertions.assertFalse(v3.getLanceFieldId("note").isPresent());
+ Assertions.assertEquals(v3.getLanceFieldId("vec").getAsInt(), v5.getLanceFieldId("embedding").getAsInt());
+ Assertions.assertEquals(1, indexedFragments(vectorSplits(v3, "vec")));
+ Assertions.assertEquals(1, indexedFragments(vectorSplits(v5, "embedding")));
+ }
+
+ @Test
+ public void testVersionRemovedByCleanupIsNotFound() throws Exception {
+ String cleanedUri = warehouse.resolve("cleaned.lance").toString();
+ try (BufferAllocator allocator = new RootAllocator()) {
+ try (Dataset created = Dataset.create(allocator, cleanedUri, SCHEMA,
+ new WriteParams.Builder().withDataStorageVersion("2.0").build())) {
+ Assertions.assertEquals(1, created.version());
+ }
+ append(cleanedUri, allocator, 1, 0, 10);
+ append(cleanedUri, allocator, 2, 10, 10);
+ try (Dataset dataset = Dataset.open(cleanedUri, allocator)) {
+ dataset.cleanupWithPolicy(CleanupPolicy.builder().withBeforeVersion(3)
+ .withDeleteUnverified(true).build());
+ }
+ }
+ RuntimeException removed = Assertions.assertThrows(RuntimeException.class,
+ () -> catalog.loadTableMetadataForSearch("default", "cleaned", version(2)));
+ Assertions.assertTrue(removed.getMessage().contains("Lance version 2 of default.cleaned was not found"),
+ removed.getMessage());
+ Assertions.assertEquals(3, catalog.loadTableMetadataForSearch("default", "cleaned", version(3)).getVersion());
+ }
+
+ @Test
+ public void testSearchExplainNamesBranch() throws Exception {
+ LanceTableMetadata dev = search(LanceRefSelector.branch(BRANCH, Optional.empty()));
+ TVectorSearchParams vector = new TVectorSearchParams().setColumn("vec").setTopK(5).setOffset(0);
+ LanceScanNode node = node(dev, new TExternalSearchRequest()
+ .setSearchQuery(TExternalSearchQuery.vector_search(vector)), dev.getLanceFieldId("vec").getAsInt());
+ node.getSplits(1);
+ String explain = node.getNodeExplainString("", TExplainLevel.NORMAL);
+ Assertions.assertTrue(explain.contains("lanceVersion=6\n"), explain);
+ Assertions.assertTrue(explain.contains("lanceBranch=dev\n"), explain);
+ Assertions.assertTrue(explain.contains("lanceManagedVersioning=false\n"), explain);
+ }
+
+ private static long count(String datasetUri, long version, String filter) throws Exception {
+ try (BufferAllocator allocator = new RootAllocator();
+ Dataset dataset = Dataset.open(allocator, datasetUri,
+ new ReadOptions.Builder().setVersion(version).build())) {
+ return dataset.countRows(filter);
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java
index ac1f0f7218f..bd555d1dab3 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java
@@ -30,6 +30,8 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.profile.RuntimeProfile;
import org.apache.doris.common.profile.SummaryProfile;
import org.apache.doris.datasource.FederationBackendPolicy;
+import org.apache.doris.datasource.lance.LanceExternalCatalog;
+import org.apache.doris.datasource.lance.LanceExternalTable;
import org.apache.doris.datasource.lance.index.LanceIndexSegmentInfo;
import org.apache.doris.datasource.lance.metadata.LanceFragmentInfo;
import org.apache.doris.datasource.lance.metadata.LanceTableAccess;
@@ -1011,6 +1013,59 @@ public class LanceScanNodeTest {
"Lance fragment id must be non-negative");
}
+ @Test
+ public void testExternalSearchExplainShowsSelectedSnapshot() throws Exception {
+ LanceTableAccess main = LanceTableAccess.managedByNamespace("s3://bucket/table.lance",
+ Collections.emptyMap(), Arrays.asList("db", "table"));
+ LanceTableAccess branch = main.onBranch("dev", "s3://bucket/table.lance/tree/dev");
+ for (LanceTableAccess access : Arrays.asList(main, branch)) {
+ LanceTableMetadata metadata = LanceTableMetadata.createSnapshotWithIndexes(
+ access, 7, vectorSchema(), Collections.singletonList(new LanceFragmentInfo(1, 8, 8)),
+ Collections.singletonMap("vector", 9), Collections.emptyList());
+ LanceScanNode node = newSearchNode(metadata, vectorSearchRequest(5, 0));
+ node.getSplits(1);
+ String explain = node.getNodeExplainString("", TExplainLevel.NORMAL);
+ Assert.assertTrue(explain, explain.contains("lanceCatalogType=rest\n"));
+ Assert.assertTrue(explain, explain.contains("lanceVersion=7\n"));
+ Assert.assertTrue(explain, explain.contains("lanceManagedVersioning=true\n"));
+ Assert.assertEquals(explain, access == branch, explain.contains("lanceBranch=dev\n"));
+ }
+ }
+
+ @Test
+ public void testExternalSearchErrorsNameTheBranch() {
+ LanceTableAccess branch = new LanceTableAccess("s3://bucket/table.lance", Collections.emptyMap())
+ .onBranch("dev", "s3://bucket/table.lance/tree/dev");
+ LanceTableMetadata metadata = LanceTableMetadata.createSnapshotWithIndexes(
+ branch, 7, fullTextSchema(), Collections.singletonList(new LanceFragmentInfo(1, 8, 8)),
+ Collections.singletonMap("body", 9), Collections.emptyList());
+ // Without an FTS index, and with duplicate fragment ids, the errors name the branch too.
+ for (TFtsCoverageMode mode : TFtsCoverageMode.values()) {
+ LanceScanNode node = newSearchNode(metadata, fullTextSearchRequest(5, 0, mode));
+ UserException exception = Assert.assertThrows(UserException.class, () -> node.getSplits(1));
+ Assert.assertTrue(exception.getMessage(), exception.getMessage().contains(
+ "No committed Lance FTS index exists for column 'body' at dataset version 7 of branch 'dev'"));
+ }
+ LanceTableMetadata partial = LanceTableMetadata.createSnapshotWithIndexes(
+ branch, 7, fullTextSchema(),
+ Arrays.asList(new LanceFragmentInfo(1, 8, 8), new LanceFragmentInfo(2, 8, 8)),
+ Collections.singletonMap("body", 9),
+ Collections.singletonList(new LanceIndexSegmentInfo(UUID.randomUUID(), "body_fts",
+ Collections.singletonList(9), Collections.singletonList(1L), IndexType.INVERTED, null)));
+ LanceScanNode strictNode = newSearchNode(partial, fullTextSearchRequest(5, 0, TFtsCoverageMode.STRICT));
+ UserException strict = Assert.assertThrows(UserException.class, () -> strictNode.getSplits(1));
+ Assert.assertTrue(strict.getMessage(), strict.getMessage().contains(
+ "requires every fragment at dataset version 7 of branch 'dev' to be indexed"));
+ LanceTableMetadata duplicate = LanceTableMetadata.createSnapshotWithIndexes(
+ branch, 7, vectorSchema(),
+ Arrays.asList(new LanceFragmentInfo(1, 8, 8), new LanceFragmentInfo(1, 8, 8)),
+ Collections.singletonMap("vector", 9), Collections.emptyList());
+ LanceScanNode node = newSearchNode(duplicate, vectorSearchRequest(5, 0));
+ UserException exception = Assert.assertThrows(UserException.class, () -> node.getSplits(1));
+ Assert.assertTrue(exception.getMessage(),
+ exception.getMessage().contains("Duplicate Lance fragment id 1 at dataset version 7 of branch 'dev'"));
+ }
+
private static LanceScanNode newNode() {
return newNode(new SessionVariable());
}
@@ -1035,8 +1090,12 @@ public class LanceScanNodeTest {
sessionVariable.lanceFragmentsPerSplit = 8;
TupleDescriptor desc = new TupleDescriptor(new TupleId(0));
desc.setTable(Mockito.mock(TableIf.class));
+ LanceExternalCatalog catalog = Mockito.mock(LanceExternalCatalog.class);
+ Mockito.when(catalog.getLanceCatalogType()).thenReturn("rest");
+ LanceExternalTable table = Mockito.mock(LanceExternalTable.class);
+ Mockito.when(table.getCatalog()).thenReturn(catalog);
return LanceScanNode.forExternalSearch(
- new PlanNodeId(0), desc, null,
+ new PlanNodeId(0), desc, table,
metadata, searchFieldId, request, sessionVariable);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LancePreparedSearchTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LancePreparedSearchTest.java
index 20e72540a7f..074de75998f 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LancePreparedSearchTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LancePreparedSearchTest.java
@@ -23,10 +23,12 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.common.Pair;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.datasource.lance.LanceExternalTable;
+import org.apache.doris.datasource.lance.metadata.LanceRefSelector;
import org.apache.doris.datasource.lance.metadata.LanceTableAccess;
import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
import org.apache.doris.mysql.MysqlCommand;
import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.SqlCacheContext;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.parser.NereidsParser;
@@ -224,6 +226,88 @@ public class LancePreparedSearchTest {
}
}
+ /**
+ * A constant selector is re-resolved by every EXECUTE: a tag moved between two executions
+ * selects its new version, and the selector itself is the same each time.
+ */
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testExecuteResolvesConstantSelectorAgain(boolean useIndex) {
+ ConnectContext previous = ConnectContext.get();
+ ConnectContext context = MemoTestUtils.createConnectContext();
+ AtomicLong version = new AtomicLong(42);
+ LanceExternalTable table = mockTable(version);
+ try (MockedStatic<LanceExternalSearchTableValuedFunction> lookup = Mockito.mockStatic(
+ LanceExternalSearchTableValuedFunction.class, Mockito.CALLS_REAL_METHODS)) {
+ lookup.when(() -> LanceExternalSearchTableValuedFunction.findLanceExternalTable(
+ Mockito.any(TableName.class))).thenReturn(table);
+ Pair<LogicalPlan, StatementContext> parsed = new NereidsParser().parseMultiple(
+ "select id, _distance from vector_search('table'='catalog.db.items', "
+ + "'column'='embedding', 'use_index'='" + useIndex + "', 'tag'='release', "
+ + "'query_vector'=?)").get(0);
+ StatementContext prepare = MemoTestUtils.createStatementContext(context, "");
+ prepare.setPrepareStage(true);
+ analyze(parsed.first, prepare);
+ // Each analysis resolves the selector exactly once; a second resolution within one
+ // execution could select a different version.
+ verifyResolvedOnceWithTag(table, "release");
+ Placeholder vector = parsed.second.getPlaceholders().get(0);
+ for (int i = 0; i < 2; i++) {
+ version.set(42 + i);
+ StatementContext execute = MemoTestUtils.createStatementContext(context, "");
+ execute.getIdToPlaceholderRealExpr().put(vector.getPlaceholderId(), new StringLiteral("[1,2]"));
+ Assertions.assertEquals(42 + i, analyze(parsed.first, execute).getMetadata().getVersion());
+ verifyResolvedOnceWithTag(table, "release");
+ }
+ } finally {
+ ConnectContext.remove();
+ if (previous != null) {
+ previous.setThreadLocalInfo();
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"version", "timestamp", "tag", "branch"})
+ public void testSelectorMustBeConstantInPreparedStatement(String selector) {
+ ConnectContext previous = ConnectContext.get();
+ ConnectContext context = MemoTestUtils.createConnectContext();
+ context.setStatementContext(MemoTestUtils.createStatementContext(context, ""));
+ try {
+ Exception exception = Assertions.assertThrows(Exception.class,
+ () -> new NereidsParser().parseMultiple(
+ "select * from vector_search('table'='catalog.db.items', 'column'='embedding', "
+ + "'query_vector'='[1,2]', '" + selector + "'=?)"));
+ Assertions.assertTrue(exception.getMessage().contains(
+ "vector_search property '" + selector + "' must be constant in a prepared statement"),
+ exception.getMessage());
+ // Before this was rejected, full_text_search received the placeholder as the text "?".
+ for (String property : new String[] {"'" + selector + "'=?", "'top_k'=?", "?='x'"}) {
+ exception = Assertions.assertThrows(Exception.class,
+ () -> new NereidsParser().parseMultiple(
+ "select * from full_text_search('table'='catalog.db.items', 'column'='body', "
+ + "'query'='lance', " + property + ")"));
+ Assertions.assertTrue(exception.getMessage().contains(
+ "full_text_search properties must be constant in a prepared statement"),
+ exception.getMessage());
+ }
+ } finally {
+ ConnectContext.remove();
+ if (previous != null) {
+ previous.setThreadLocalInfo();
+ }
+ }
+ }
+
+ private static void verifyResolvedOnceWithTag(LanceExternalTable table, String tag) {
+ org.mockito.ArgumentCaptor<LanceRefSelector> selector =
+ org.mockito.ArgumentCaptor.forClass(LanceRefSelector.class);
+ // loadBasicMetadata delegates to loadMetadataForSearch in mockTable, so both modes count here.
+ Mockito.verify(table, Mockito.times(1)).loadMetadataForSearch(selector.capture());
+ Assertions.assertEquals(tag, selector.getValue().getTag().orElse(null));
+ Mockito.clearInvocations(table);
+ }
+
private LanceExternalTable mockTable(AtomicLong version) {
return mockTable(version, false);
}
@@ -240,14 +324,122 @@ public class LancePreparedSearchTest {
vector.getChildren())));
}
Schema schema = new Schema(Arrays.asList(Field.nullable("id", new ArrowType.Int(64, true)), vector));
- Mockito.when(table.loadMetadataForSearch()).thenAnswer(invocation -> LanceTableMetadata.createSnapshotWithIndexes(
- new LanceTableAccess("s3://bucket/items.lance", Collections.emptyMap()),
- version.get(), schema, Collections.emptyList(),
- ImmutableMap.of("id", 0, "embedding", 1), Collections.emptyList()));
- Mockito.when(table.loadBasicMetadata()).thenAnswer(invocation -> table.loadMetadataForSearch());
+ Mockito.when(table.loadMetadataForSearch(Mockito.any(LanceRefSelector.class))).thenAnswer(
+ invocation -> LanceTableMetadata.createSnapshotWithIndexes(
+ new LanceTableAccess("s3://bucket/items.lance", Collections.emptyMap()),
+ version.get(), schema, Collections.emptyList(),
+ ImmutableMap.of("id", 0, "embedding", 1), Collections.emptyList()));
+ Mockito.when(table.loadBasicMetadata(Mockito.any(LanceRefSelector.class))).thenAnswer(
+ invocation -> table.loadMetadataForSearch(invocation.getArgument(0)));
+ return table;
+ }
+
+ /** Both search functions accept the selector properties and hand the table what they select. */
+ @org.junit.jupiter.api.Test
+ public void testBothSearchFunctionsPassTheirSelectorToTheTable() {
+ ConnectContext previous = ConnectContext.get();
+ ConnectContext context = MemoTestUtils.createConnectContext();
+ LanceExternalTable table = mockSearchTable();
+ String vector = "vector_search('table'='catalog.db.items', 'column'='embedding', 'query_vector'='[1,2]', ";
+ String text = "full_text_search('table'='catalog.db.items', 'column'='body', 'query'='lance', ";
+ try (MockedStatic<LanceExternalSearchTableValuedFunction> lookup = Mockito.mockStatic(
+ LanceExternalSearchTableValuedFunction.class, Mockito.CALLS_REAL_METHODS)) {
+ lookup.when(() -> LanceExternalSearchTableValuedFunction.findLanceExternalTable(
+ Mockito.any(TableName.class))).thenReturn(table);
+ for (String useIndex : new String[] {"true", "false"}) {
+ String prefix = vector + "'use_index'='" + useIndex + "', ";
+ LanceRefSelector selector = selectorOf(table, prefix + "'VERSION'='3', 'Branch'='dev')", context);
+ Assertions.assertEquals("dev", selector.getBranch().orElse(null));
+ Assertions.assertEquals("3", selector.getSnapshot().get().getValue());
+ selector = selectorOf(table, prefix + "'tag'='rel')", context);
+ Assertions.assertEquals("rel", selector.getTag().orElse(null));
+ }
+ LanceRefSelector selector = selectorOf(table,
+ text + "'timestamp'='2026-09-20 12:00:00', 'branch'='main')", context);
+ Assertions.assertFalse(selector.getBranch().isPresent());
+ Assertions.assertEquals(org.apache.doris.analysis.TableSnapshot.VersionType.TIME,
+ selector.getSnapshot().get().getType());
+ selector = selectorOf(table, text + "'version'='2', 'branch'='dev')", context);
+ Assertions.assertEquals("dev", selector.getBranch().orElse(null));
+ Assertions.assertEquals("2", selector.getSnapshot().get().getValue());
+ Assertions.assertEquals("rel", selectorOf(table, text + "'tag'='rel')", context).getTag().orElse(null));
+ Assertions.assertSame(LanceRefSelector.latest(), selectorOf(table, text + "'top_k'='3')", context));
+ } finally {
+ ConnectContext.remove();
+ if (previous != null) {
+ previous.setThreadLocalInfo();
+ }
+ }
+ }
+
+ /**
+ * A search resolves its snapshot when it is bound, so a statement containing one must never be
+ * answered from the SQL cache: a cached result would skip resolving a moving selector again.
+ */
+ @org.junit.jupiter.api.Test
+ public void testSearchStatementsBypassSqlCache() {
+ ConnectContext previous = ConnectContext.get();
+ ConnectContext context = MemoTestUtils.createConnectContext();
+ context.getSessionVariable().setEnableSqlCache(true);
+ LanceExternalTable table = mockSearchTable();
+ try (MockedStatic<LanceExternalSearchTableValuedFunction> lookup = Mockito.mockStatic(
+ LanceExternalSearchTableValuedFunction.class, Mockito.CALLS_REAL_METHODS)) {
+ lookup.when(() -> LanceExternalSearchTableValuedFunction.findLanceExternalTable(
+ Mockito.any(TableName.class))).thenReturn(table);
+ for (String tvf : new String[] {
+ "vector_search('table'='catalog.db.items', 'column'='embedding', 'query_vector'='[1,2]', "
+ + "'tag'='rel')",
+ "full_text_search('table'='catalog.db.items', 'column'='body', 'query'='lance', "
+ + "'tag'='rel')"}) {
+ String sql = "select * from " + tvf;
+ StatementContext statement = MemoTestUtils.createStatementContext(context, sql);
+ SqlCacheContext sqlCache = statement.getSqlCacheContext().orElseThrow(
+ () -> new IllegalStateException("SQL cache is not enabled for " + sql));
+ Assertions.assertFalse(sqlCache.isCannotProcessExpression(), sql);
+ LogicalPlan plan = new NereidsParser().parseMultiple(sql).get(0).first;
+ CascadesContext.initContext(statement, plan, PhysicalProperties.ANY).newAnalyzer().analyze();
+ Assertions.assertTrue(sqlCache.isCannotProcessExpression(), sql);
+ }
+ } finally {
+ ConnectContext.remove();
+ if (previous != null) {
+ previous.setThreadLocalInfo();
+ }
+ }
+ }
+
+ /** A table with a vector and a text column whose metadata is version 3 for every selector. */
+ private static LanceExternalTable mockSearchTable() {
+ Field item = Field.nullable("item", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE));
+ Field embedding = new Field("embedding", org.apache.arrow.vector.types.pojo.FieldType.nullable(
+ new ArrowType.FixedSizeList(2)), Collections.singletonList(item));
+ Schema schema = new Schema(Arrays.asList(Field.nullable("id", new ArrowType.Int(64, true)),
+ embedding, Field.nullable("body", ArrowType.Utf8.INSTANCE)));
+ LanceExternalTable table = Mockito.mock(LanceExternalTable.class);
+ Mockito.when(table.loadMetadataForSearch(Mockito.any(LanceRefSelector.class))).thenAnswer(
+ invocation -> LanceTableMetadata.createSnapshotWithIndexes(
+ new LanceTableAccess("s3://bucket/items.lance", Collections.emptyMap()), 3, schema,
+ Collections.emptyList(), ImmutableMap.of("id", 0, "embedding", 1, "body", 2),
+ Collections.emptyList()));
+ Mockito.when(table.loadBasicMetadata(Mockito.any(LanceRefSelector.class))).thenAnswer(
+ invocation -> table.loadMetadataForSearch(invocation.getArgument(0)));
return table;
}
+ private LanceRefSelector selectorOf(LanceExternalTable table, String tvf, ConnectContext context) {
+ Mockito.clearInvocations(table);
+ LogicalPlan plan = new NereidsParser().parseMultiple("select * from " + tvf).get(0).first;
+ CascadesContext cascades = CascadesContext.initContext(
+ MemoTestUtils.createStatementContext(context, ""), plan, PhysicalProperties.ANY);
+ cascades.newAnalyzer().analyze();
+ org.mockito.ArgumentCaptor<LanceRefSelector> selectors =
+ org.mockito.ArgumentCaptor.forClass(LanceRefSelector.class);
+ Mockito.verify(table, Mockito.atLeast(0)).loadMetadataForSearch(selectors.capture());
+ Mockito.verify(table, Mockito.atLeast(0)).loadBasicMetadata(selectors.capture());
+ Assertions.assertFalse(selectors.getAllValues().isEmpty(), tvf);
+ return selectors.getAllValues().get(0);
+ }
+
private VectorSearchTableValuedFunction analyze(LogicalPlan plan, StatementContext statement) {
CascadesContext cascades = CascadesContext.initContext(statement, plan, PhysicalProperties.ANY);
cascades.newAnalyzer().analyze();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LanceSearchSelectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LanceSearchSelectorTest.java
new file mode 100644
index 00000000000..6446785150a
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LanceSearchSelectorTest.java
@@ -0,0 +1,189 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.tablefunction;
+
+import org.apache.doris.analysis.TableSnapshot;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.datasource.lance.metadata.LanceRefSelector;
+import org.apache.doris.datasource.lance.metadata.LanceSnapshotResolver;
+import org.apache.doris.nereids.util.MemoTestUtils;
+import org.apache.doris.qe.ConnectContext;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+public class LanceSearchSelectorTest {
+
+ private static LanceRefSelector parse(String... keyValues) throws AnalysisException {
+ Map<String, String> params = new HashMap<>();
+ for (int i = 0; i < keyValues.length; i += 2) {
+ params.put(keyValues[i], keyValues[i + 1]);
+ }
+ return LanceExternalSearchTableValuedFunction.parseSelector(params);
+ }
+
+ private static String error(String... keyValues) {
+ return Assertions.assertThrows(AnalysisException.class, () -> parse(keyValues)).getDetailMessage();
+ }
+
+ private static void assertSnapshot(LanceRefSelector selector, TableSnapshot.VersionType type, String value) {
+ Assertions.assertTrue(selector.getSnapshot().isPresent());
+ Assertions.assertEquals(type, selector.getSnapshot().get().getType());
+ Assertions.assertEquals(value, selector.getSnapshot().get().getValue());
+ }
+
+ @Test
+ public void testNoSelectorIsLatestMain() throws Exception {
+ Assertions.assertSame(LanceRefSelector.latest(),
+ LanceExternalSearchTableValuedFunction.parseSelector(Collections.emptyMap()));
+ Assertions.assertSame(LanceRefSelector.latest(), parse("column", "embedding"));
+ }
+
+ @Test
+ public void testVersionIsAPositiveIntegerOnly() throws Exception {
+ LanceRefSelector selector = parse("version", " 007 ");
+ assertSnapshot(selector, TableSnapshot.VersionType.VERSION, "7");
+ Assertions.assertFalse(selector.getBranch().isPresent());
+ Assertions.assertFalse(selector.getTag().isPresent());
+ // A tag name is not a version here, although FOR VERSION AS OF takes one.
+ for (String invalid : new String[] {"0", "-1", "+3", "v3", "1.5", "release"}) {
+ Assertions.assertEquals("'version' must be a positive integer, but was '" + invalid + "'",
+ error("version", invalid));
+ }
+ Assertions.assertTrue(error("version", "9223372036854775808").contains("out of range"));
+ }
+
+ @Test
+ public void testTimestampFormats() throws Exception {
+ assertSnapshot(parse("timestamp", "2026-09-20 12:00:00"), TableSnapshot.VersionType.TIME,
+ "2026-09-20 12:00:00");
+ assertSnapshot(parse("timestamp", " 2026-09-20 12:00:00.123 "), TableSnapshot.VersionType.TIME,
+ "2026-09-20 12:00:00.123");
+ for (String invalid : new String[] {"1758369600000", "2026-09-20", "2026-09-20T12:00:00",
+ "2026-09-20 12:00:00.1234", "yesterday", "+300000000-01-01 00:00:00"}) {
+ Assertions.assertTrue(error("timestamp", invalid).startsWith("'timestamp' must be "
+ + LanceSnapshotResolver.TIMESTAMP_FORMATS + " in the session time zone"), invalid);
+ }
+ }
+
+ @Test
+ public void testTimestampUsesSessionTimeZone() {
+ ConnectContext previous = ConnectContext.get();
+ ConnectContext context = MemoTestUtils.createConnectContext();
+ try {
+ context.getSessionVariable().setTimeZone("UTC");
+ long utc = LanceSnapshotResolver.parseTimestamp("2026-09-20 12:00:00").getAsLong();
+ Assertions.assertEquals(1789905600000L, utc);
+ Assertions.assertEquals(utc + 123,
+ LanceSnapshotResolver.parseTimestamp("2026-09-20 12:00:00.123").getAsLong());
+ context.getSessionVariable().setTimeZone("Asia/Shanghai");
+ Assertions.assertEquals(utc - 8 * 3600 * 1000L,
+ LanceSnapshotResolver.parseTimestamp("2026-09-20 12:00:00").getAsLong());
+ Assertions.assertFalse(LanceSnapshotResolver.parseTimestamp("2026-09-20").isPresent());
+ } finally {
+ ConnectContext.remove();
+ if (previous != null) {
+ previous.setThreadLocalInfo();
+ }
+ }
+ }
+
+ @Test
+ public void testTagStandsAlone() throws Exception {
+ LanceRefSelector selector = parse("tag", "release");
+ Assertions.assertEquals(Optional.of("release"), selector.getTag());
+ Assertions.assertFalse(selector.getSnapshot().isPresent());
+ Assertions.assertFalse(selector.getBranch().isPresent());
+ // A numeric tag name is a tag here; FOR VERSION AS OF would read it as a version.
+ Assertions.assertEquals(Optional.of("123"), parse("tag", "123").getTag());
+ for (String other : new String[] {"version", "timestamp", "branch"}) {
+ String value = other.equals("timestamp") ? "2026-09-20 12:00:00" : other.equals("version") ? "3" : "dev";
+ Assertions.assertEquals("'tag' and '" + other + "' are mutually exclusive;"
+ + " a tag already determines its branch and version", error("tag", "release", other, value));
+ }
+ // Even main names a chain, which the tag already determines.
+ Assertions.assertTrue(error("tag", "release", "branch", "main").contains("mutually exclusive"));
+ }
+
+ @Test
+ public void testVersionAndTimestampAreMutuallyExclusive() {
+ Assertions.assertEquals("'version' and 'timestamp' are mutually exclusive",
+ error("version", "3", "timestamp", "2026-09-20 12:00:00"));
+ Assertions.assertEquals("'version' and 'timestamp' are mutually exclusive",
+ error("version", "3", "timestamp", "2026-09-20 12:00:00", "branch", "dev"));
+ }
+
+ @Test
+ public void testBranchAloneOrWithVersionOrTimestamp() throws Exception {
+ LanceRefSelector latest = parse("branch", "dev");
+ Assertions.assertEquals(Optional.of("dev"), latest.getBranch());
+ Assertions.assertFalse(latest.getSnapshot().isPresent());
+
+ LanceRefSelector version = parse("branch", "team/dev", "version", "4");
+ Assertions.assertEquals(Optional.of("team/dev"), version.getBranch());
+ assertSnapshot(version, TableSnapshot.VersionType.VERSION, "4");
+
+ LanceRefSelector time = parse("branch", "dev", "timestamp", "2026-09-20 12:00:00");
+ Assertions.assertEquals(Optional.of("dev"), time.getBranch());
+ assertSnapshot(time, TableSnapshot.VersionType.TIME, "2026-09-20 12:00:00");
+ }
+
+ @Test
+ public void testMainBranchIsTheMainChain() throws Exception {
+ Assertions.assertSame(LanceRefSelector.latest(), parse("branch", "main"));
+ LanceRefSelector version = parse("branch", "main", "version", "4");
+ Assertions.assertFalse(version.getBranch().isPresent());
+ assertSnapshot(version, TableSnapshot.VersionType.VERSION, "4");
+ LanceRefSelector time = parse("branch", "main", "timestamp", "2026-09-20 12:00:00");
+ Assertions.assertFalse(time.getBranch().isPresent());
+ assertSnapshot(time, TableSnapshot.VersionType.TIME, "2026-09-20 12:00:00");
+ }
+
+ /** The table hands the selector, unchanged, to the catalog's search metadata loaders. */
+ @Test
+ public void testTableForwardsTheSelectorToTheCatalog() throws Exception {
+ org.apache.doris.datasource.lance.LanceExternalCatalog catalog =
+ org.mockito.Mockito.mock(org.apache.doris.datasource.lance.LanceExternalCatalog.class);
+ org.apache.doris.datasource.lance.LanceExternalDatabase db =
+ org.mockito.Mockito.mock(org.apache.doris.datasource.lance.LanceExternalDatabase.class);
+ org.mockito.Mockito.when(db.getRemoteName()).thenReturn("remote_db");
+ org.apache.doris.datasource.lance.LanceExternalTable table =
+ new org.apache.doris.datasource.lance.LanceExternalTable(1, "t", "remote_t", catalog, db);
+ LanceRefSelector selector = parse("branch", "dev", "version", "4");
+ table.loadMetadataForSearch(selector);
+ org.mockito.Mockito.verify(catalog).loadTableMetadataForSearch("remote_db", "remote_t", selector);
+ table.loadBasicMetadata(selector);
+ org.mockito.Mockito.verify(catalog).loadBasicTableMetadata("remote_db", "remote_t", selector);
+ }
+
+ @Test
+ public void testEmptySelectorIsAnError() {
+ for (String key : new String[] {"version", "timestamp", "tag", "branch"}) {
+ for (String empty : new String[] {"", " "}) {
+ Assertions.assertEquals("'" + key + "' must not be empty", error(key, empty));
+ }
+ }
+ // An empty selector fails before any combination is checked.
+ Assertions.assertEquals("'branch' must not be empty", error("tag", "release", "branch", ""));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java
index c8c3da2c6c0..a141419a1ba 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java
@@ -90,7 +90,8 @@ public class VectorSearchTableValuedFunctionTest {
AnalysisException exception = Assert.assertThrows(AnalysisException.class,
() -> VectorSearchTableValuedFunction.parseTableName(table));
Assert.assertTrue(exception.getMessage(),
- exception.getMessage().contains("cannot select a version, tag or branch"));
+ exception.getMessage().contains("cannot select a version, tag or branch;"
+ + " use the 'version', 'timestamp', 'tag' or 'branch' property"));
}
}
diff --git a/regression-test/data/external_table_p0/lance/test_lance_rest_catalog.out b/regression-test/data/external_table_p0/lance/test_lance_rest_catalog.out
index 17e65ed4f29..78f2409fba1 100644
--- a/regression-test/data/external_table_p0/lance/test_lance_rest_catalog.out
+++ b/regression-test/data/external_table_p0/lance/test_lance_rest_catalog.out
@@ -7,6 +7,10 @@ mysql
-- !rest_tables --
all_types
all_types_unprefixed
+search_snapshot
+search_snapshot_managed
+search_snapshot_managed_partial
+search_snapshot_pruned
time_travel
time_travel_managed
time_travel_managed_lagging
@@ -16,6 +20,7 @@ time_travel_managed_pending
time_travel_managed_staged
time_travel_managed_unprefixed
time_travel_managed_untimed
+unicode_branch_managed
-- !rest_scan --
12 12 1 12 78
diff --git a/regression-test/data/external_table_p0/lance/test_lance_rest_search_snapshot.out b/regression-test/data/external_table_p0/lance/test_lance_rest_search_snapshot.out
new file mode 100644
index 00000000000..57462eff9ec
--- /dev/null
+++ b/regression-test/data/external_table_p0/lance/test_lance_rest_search_snapshot.out
@@ -0,0 +1,59 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !plain_version_5 --
+17 0.0
+16 4.0
+18 4.0
+
+-- !managed_version_5 --
+17 0.0
+16 4.0
+18 4.0
+
+-- !managed_branch --
+101 4.0
+102 16.0
+103 36.0
+
+-- !managed_branch_two_phase --
+101 doc 101 doris 4.0
+102 doc 102 lance 16.0
+103 doc 103 doris 36.0
+
+-- !managed_version_two_phase --
+17 doc 17 doris 0.25
+18 doc 18 lance 2.25
+16 doc 16 lance 6.25
+
+-- !managed_branch_tag_fts --
+10
+12
+14
+16
+2
+4
+6
+8
+
+-- !managed_fts_version_4 --
+10
+12
+14
+16
+2
+4
+6
+8
+
+-- !unicode_branch_scan --
+12 204
+
+-- !unicode_branch_search --
+201 4.0
+202 16.0
+203 36.0
+
+-- !unicode_branch_two_phase --
+201 doc 201 2.25
+202 doc 202 12.25
+203 doc 203 30.25
+
diff --git a/regression-test/data/external_table_p0/lance/test_lance_rest_time_travel.out b/regression-test/data/external_table_p0/lance/test_lance_rest_time_travel.out
index ea9662a455a..9e9b6777ea4 100644
--- a/regression-test/data/external_table_p0/lance/test_lance_rest_time_travel.out
+++ b/regression-test/data/external_table_p0/lance/test_lance_rest_time_travel.out
@@ -2,6 +2,10 @@
-- !tables --
all_types
all_types_unprefixed
+search_snapshot
+search_snapshot_managed
+search_snapshot_managed_partial
+search_snapshot_pruned
time_travel
time_travel_managed
time_travel_managed_lagging
@@ -11,6 +15,7 @@ time_travel_managed_pending
time_travel_managed_staged
time_travel_managed_unprefixed
time_travel_managed_untimed
+unicode_branch_managed
-- !plain_latest --
9 9
diff --git a/regression-test/data/external_table_p0/lance/test_lance_search_snapshot.out b/regression-test/data/external_table_p0/lance/test_lance_search_snapshot.out
new file mode 100644
index 00000000000..e7344fd0702
--- /dev/null
+++ b/regression-test/data/external_table_p0/lance/test_lance_search_snapshot.out
@@ -0,0 +1,182 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !latest --
+1 4.0
+2 16.0
+4 64.0
+
+-- !version_2 --
+1 4.0
+2 16.0
+3 36.0
+
+-- !version_5 --
+17 0.0
+16 4.0
+18 4.0
+
+-- !version_5_fts --
+10
+12
+14
+16
+2
+4
+6
+8
+
+-- !version_4_fts_strict --
+10
+12
+14
+16
+2
+4
+6
+8
+
+-- !version_6 --
+4 2.25
+2 6.25
+5 12.25
+
+-- !version_5_not_deleted --
+3 0.25
+4 2.25
+2 6.25
+
+-- !version_5_fts_doris --
+1
+11
+13
+15
+3
+5
+7
+9
+
+-- !version_6_fts_doris --
+1
+13
+15
+5
+7
+9
+
+-- !version_5_no_index --
+17 0.0
+16 4.0
+18 4.0
+
+-- !tag_rel --
+17 0.0
+16 4.0
+18 4.0
+
+-- !main_branch --
+1 4.0
+2 16.0
+3 36.0
+
+-- !branch_latest --
+101 4.0
+102 16.0
+103 36.0
+
+-- !main_same_query --
+24 23104.0
+23 23716.0
+22 24336.0
+
+-- !branch_fts_strict --
+10
+102
+104
+106
+108
+12
+14
+16
+2
+4
+6
+8
+
+-- !branch_no_index --
+101 4.0
+102 16.0
+103 36.0
+
+-- !branch_fork --
+16 28224.0
+15 28900.0
+14 29584.0
+
+-- !branch_version_5 --
+101 4.0
+102 16.0
+103 36.0
+
+-- !branch_version_6_fts_strict --
+10
+102
+104
+106
+108
+12
+14
+16
+2
+4
+6
+8
+
+-- !branch_tag_fts --
+10
+12
+14
+16
+2
+4
+6
+8
+
+-- !branch_two_phase --
+101 doc 101 doris 4.0
+102 doc 102 lance 16.0
+103 doc 103 doris 36.0
+
+-- !tag_two_phase --
+3 doc 3 doris 0.25
+4 doc 4 lance 2.25
+2 doc 2 lance 6.25
+
+-- !evolved_two_phase --
+2 [2, 3, 4, 5] note 2 doc 2 lance 0.25
+3 [3, 4, 5, 6] note 3 doc 3 doris 2.25
+1 [1, 2, 3, 4] note 1 doc 1 doris 6.25
+
+-- !evolved_old_two_phase --
+2 [2, 3, 4, 5] note 2 doc 2 lance 0.25
+3 [3, 4, 5, 6] note 3 doc 3 doris 2.25
+1 [1, 2, 3, 4] note 1 doc 1 doris 6.25
+
+-- !evolved_version_2 --
+2 0.25
+3 2.25
+1 6.25
+
+-- !evolved_version_4 --
+2 0.25
+3 2.25
+1 6.25
+
+-- !evolved_latest --
+12 0.25
+13 2.25
+11 6.25
+
+-- !pruned_latest --
+1 4.0
+2 16.0
+3 36.0
+
diff --git a/regression-test/data/external_table_p0/lance/test_lance_vector_search_legacy_index.out b/regression-test/data/external_table_p0/lance/test_lance_vector_search_legacy_index.out
new file mode 100644
index 00000000000..865d1ece304
--- /dev/null
+++ b/regression-test/data/external_table_p0/lance/test_lance_vector_search_legacy_index.out
@@ -0,0 +1,11 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !legacy_indexed --
+100 0.25
+101 2.25
+99 6.25
+
+-- !legacy_flat --
+100 0.25
+101 2.25
+99 6.25
+
diff --git a/regression-test/suites/external_table_p0/lance/test_lance_rest_search_snapshot.groovy b/regression-test/suites/external_table_p0/lance/test_lance_rest_search_snapshot.groovy
new file mode 100644
index 00000000000..362ed549b42
--- /dev/null
+++ b/regression-test/suites/external_table_p0/lance/test_lance_rest_search_snapshot.groovy
@@ -0,0 +1,202 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_lance_rest_search_snapshot", "p0,external") {
+ /*
+ * The search TVFs' snapshot selectors through a Lance REST Namespace, with and without
+ * managed versioning. All tables are search_snapshot.lance (seven main versions, branch dev
+ * with versions 4 to 6, tags rel and dev_rel; see test_lance_search_snapshot):
+ *
+ * search_snapshot managed_versioning=false: versions come from _versions/
+ * search_snapshot_managed managed_versioning=true, records every version and dev
+ * search_snapshot_managed_partial managed_versioning=true, records every main version but 6
+ * unicode_branch_managed managed_versioning=true, unicode_branch.lance: version 1
+ * has row_id 1..8; branch dev\u1c89 adds row_id 201..204
+ * in its version 2 (see lance_build_unicode_branch.py)
+ *
+ * For a managed table the namespace decides which versions exist, as for FOR VERSION AS OF
+ * (see test_lance_rest_time_travel); the search TVFs follow the same rules.
+ */
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable Lance REST test because the Iceberg MinIO environment is disabled.")
+ return
+ }
+
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ String lanceRestPort = context.config.otherConfigs.get("lance_rest_port")
+ String catalogName = "test_lance_rest_search_snapshot"
+ String plain = "${catalogName}.`default`.search_snapshot"
+ String managed = "${catalogName}.`default`.search_snapshot_managed"
+ String partial = "${catalogName}.`default`.search_snapshot_managed_partial"
+ String unicodeManaged = "${catalogName}.`default`.unicode_branch_managed"
+ // U+1C89 was added in Unicode 16: Lance accepts the name, and JDK 17 leaves it unassigned.
+ String unicodeBranch = "dev\u1c89"
+ def vectorSearch = { String target, String selector, String query = "[0,1,2,3]" ->
+ """vector_search(
+ "table"="${target}",
+ "column"="vec",
+ "query_vector"="${query}",
+ "top_k"="3",
+ "metric"="l2",
+ "nprobes"="2"${selector})"""
+ }
+ def fullTextSearch = { String target, String selector ->
+ """full_text_search(
+ "table"="${target}",
+ "column"="body",
+ "query"="lance",
+ "top_k"="30",
+ "coverage_mode"="index_only"${selector})"""
+ }
+ def scannerV2Rows = sql """SHOW VARIABLES LIKE 'enable_file_scanner_v2'"""
+ String originalScannerV2 = scannerV2Rows[0][1].toString()
+ String originalTimeZone = (sql """SHOW VARIABLES LIKE 'time_zone'""")[0][1].toString()
+ String originalLazy = (sql """SHOW VARIABLES LIKE 'enable_lance_lazy_materialization'""")[0][1].toString()
+
+ sql """DROP CATALOG IF EXISTS `${catalogName}`"""
+ try {
+ sql """SET enable_file_scanner_v2 = true"""
+ sql """SET time_zone = 'UTC'"""
+ // No static access key or secret key: every read below uses vended credentials.
+ sql """
+ CREATE CATALOG `${catalogName}` PROPERTIES (
+ "type" = "lance",
+ "lance.catalog.type" = "rest",
+ "lance.rest.uri" = "http://${externalEnvIp}:${lanceRestPort}",
+ "lance.rest.security.type" = "bearer",
+ "lance.rest.bearer-token" = "doris-lance-rest-test-token",
+ "lance.namespace.root_database" = "default",
+ "s3.endpoint" = "http://${externalEnvIp}:${minioPort}",
+ "s3.region" = "us-east-1",
+ "use_path_style" = "true",
+ "test_connection" = "true"
+ )
+ """
+
+ for (String target : [plain, managed]) {
+ String managedFlag = target == managed ? "true" : "false"
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(target, "")}")
+ contains "lanceCatalogType=rest"
+ contains "lanceVersion=7"
+ contains "lanceManagedVersioning=${managedFlag}"
+ }
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(target, ', "tag"="rel"')}")
+ contains "lanceVersion=5"
+ contains "lanceManagedVersioning=${managedFlag}"
+ }
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(target, ', "branch"="dev"')}")
+ contains "lanceVersion=6"
+ contains "lanceBranch=dev"
+ contains "lanceManagedVersioning=${managedFlag}"
+ }
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(target, ', "timestamp"="2026-09-29 14:51:58"')}")
+ contains "lanceVersion=5"
+ }
+ }
+ qt_plain_version_5 """
+ SELECT row_id, _distance FROM ${vectorSearch(plain, ', "version"="5"', "[17,18,19,20]")}
+ ORDER BY _distance, row_id
+ """
+ qt_managed_version_5 """
+ SELECT row_id, _distance FROM ${vectorSearch(managed, ', "version"="5"', "[17,18,19,20]")}
+ ORDER BY _distance, row_id
+ """
+ qt_managed_branch """
+ SELECT row_id, _distance FROM ${vectorSearch(managed, ', "branch"="dev"', "[100,101,102,103]")}
+ ORDER BY _distance, row_id
+ """
+ // Two-phase reads of a managed table: the second phase reopens the version and branch
+ // directory the namespace resolved, with the first phase's vended credentials.
+ sql """SET enable_lance_lazy_materialization = true"""
+ String managedTwoPhase = """
+ SELECT row_id, body, _distance
+ FROM ${vectorSearch(managed, ', "branch"="dev"', "[100,101,102,103]")}
+ ORDER BY _distance
+ LIMIT 3
+ """
+ explain {
+ sql "verbose ${managedTwoPhase}"
+ contains "VMaterializeNode"
+ }
+ qt_managed_branch_two_phase "${managedTwoPhase}"
+ qt_managed_version_two_phase """
+ SELECT row_id, body, _distance
+ FROM ${vectorSearch(managed, ', "version"="5"', "[17.25,18.25,19.25,20.25]")}
+ ORDER BY _distance
+ LIMIT 3
+ """
+ sql """SET enable_lance_lazy_materialization = ${originalLazy}"""
+ order_qt_managed_branch_tag_fts """SELECT row_id FROM ${fullTextSearch(managed, ', "tag"="dev_rel"')}"""
+ order_qt_managed_fts_version_4 """SELECT row_id FROM ${fullTextSearch(managed, ', "version"="4"')}"""
+
+ // A managed branch whose name has a letter newer than the JDK's Unicode tables is opened by
+ // the URI Doris joins, for table scans, searches and the second phase alike.
+ order_qt_unicode_branch_scan """
+ SELECT count(*), max(row_id) FROM ${unicodeManaged}@branch('name'='${unicodeBranch}')
+ """
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(unicodeManaged, ', "branch"="' + unicodeBranch + '"')}")
+ contains "lanceVersion=2"
+ contains "lanceBranch=${unicodeBranch}"
+ }
+ qt_unicode_branch_search """
+ SELECT row_id, _distance
+ FROM ${vectorSearch(unicodeManaged, ', "branch"="' + unicodeBranch + '"', "[200,201,202,203]")}
+ ORDER BY _distance, row_id
+ """
+ sql """SET enable_lance_lazy_materialization = true"""
+ String unicodeTwoPhase = """
+ SELECT row_id, body, _distance
+ FROM ${vectorSearch(unicodeManaged, ', "branch"="' + unicodeBranch + '"', "[200.25,201.25,202.25,203.25]")}
+ ORDER BY _distance
+ LIMIT 3
+ """
+ explain {
+ sql "verbose ${unicodeTwoPhase}"
+ contains "VMaterializeNode"
+ }
+ qt_unicode_branch_two_phase "${unicodeTwoPhase}"
+ sql """SET enable_lance_lazy_materialization = ${originalLazy}"""
+
+ // The namespace does not record version 6 of the partial table, so it cannot be searched
+ // even though storage holds it; the latest version is still 7.
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(partial, "")}")
+ contains "lanceVersion=7"
+ contains "lanceManagedVersioning=true"
+ }
+ test {
+ sql """SELECT row_id FROM ${vectorSearch(partial, ', "version"="6"')}"""
+ exception "Lance version 6 of default.search_snapshot_managed_partial was not found in the namespace"
+ }
+ test {
+ sql """SELECT row_id FROM ${vectorSearch(managed, ', "branch"="dev", "version"="9"')}"""
+ exception "Lance version 9 of default.search_snapshot_managed@dev was not found in the namespace"
+ }
+ } finally {
+ sql """SET enable_file_scanner_v2 = ${originalScannerV2}"""
+ sql """SET time_zone = '${originalTimeZone}'"""
+ sql """SET enable_lance_lazy_materialization = ${originalLazy}"""
+ sql """DROP CATALOG IF EXISTS `${catalogName}`"""
+ }
+}
diff --git a/regression-test/suites/external_table_p0/lance/test_lance_search_snapshot.groovy b/regression-test/suites/external_table_p0/lance/test_lance_search_snapshot.groovy
new file mode 100644
index 00000000000..b155327770d
--- /dev/null
+++ b/regression-test/suites/external_table_p0/lance/test_lance_search_snapshot.groovy
@@ -0,0 +1,452 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_lance_search_snapshot", "p0,external") {
+ /*
+ * vector_search() and full_text_search() with the version, timestamp, tag and branch
+ * properties, on a Lance filesystem catalog.
+ *
+ * search_snapshot.lance (see lance_build_search_snapshot.py), main chain:
+ * version 1 row_id 1..8 fragment 0 committed 2026-09-29 14:51:51.410834 UTC
+ * version 2 row_id 9..16 fragment 1 committed 2026-09-29 14:51:52.929307 UTC
+ * version 3 vec_idx over fragments 0, 1 committed 2026-09-29 14:51:54.457459 UTC
+ * version 4 body_idx over fragments 0, 1 committed 2026-09-29 14:51:55.985957 UTC
+ * version 5 row_id 17..24 fragment 2 committed 2026-09-29 14:51:57.507267 UTC
+ * version 6 delete row_id 3, 11 committed 2026-09-29 14:51:59.031043 UTC
+ * version 7 vec_idx rebuilt over 0, 1, 2 committed 2026-09-29 14:52:00.565680 UTC
+ * Tag rel points at version 5. Branch dev forks from version 4; its version 5 appends
+ * row_id 101..108 as fragment 2, and its version 6 rebuilds body_idx over all fragments.
+ * Tag dev_rel points at dev version 5. Dev versions 4, 5 and 6 were committed at
+ * 14:52:16.005995, 14:52:17.762386 and 14:52:19.576617 UTC, after every main version.
+ * vec is [row_id, row_id + 1, row_id + 2, row_id + 3],
+ * so the squared L2 distance between rows r and n is 4 * (n - r)^2, and nprobes=2 makes
+ * the two-partition IVF_FLAT index exact. body is "doc <row_id> lance" for even row_ids and
+ * "doc <row_id> doris" for odd ones.
+ *
+ * search_snapshot_pruned.lance lost version 1 to cleanup; version 2, pinned by tag kept,
+ * still opens but its vector index files are gone; version 3 rebuilt the index.
+ *
+ * search_snapshot_evolved.lance changes its schema after its vector index: version 1 has
+ * row_id 1..8 as fragment 0, version 2 builds vec_idx over it, version 3 adds note
+ * ("note <row_id>"), version 4 renames vec to embedding (vec_idx still serves it), and
+ * version 5 appends row_id 9..16 as fragment 1, which vec_idx does not cover.
+ */
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable Lance test because the Iceberg MinIO environment is disabled.")
+ return
+ }
+
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ String catalogName = "test_lance_search_snapshot"
+ String table = "${catalogName}.`default`.search_snapshot"
+ String pruned = "${catalogName}.`default`.search_snapshot_pruned"
+ String evolved = "${catalogName}.`default`.search_snapshot_evolved"
+ def vectorSearch = { String selector, String query = "[0,1,2,3]", String topK = "3",
+ String target = table ->
+ """vector_search(
+ "table"="${target}",
+ "column"="vec",
+ "query_vector"="${query}",
+ "top_k"="${topK}",
+ "metric"="l2",
+ "nprobes"="2"${selector})"""
+ }
+ def fullTextSearch = { String selector, String coverageMode = "index_only", String query = "lance" ->
+ """full_text_search(
+ "table"="${table}",
+ "column"="body",
+ "query"="${query}",
+ "top_k"="30",
+ "coverage_mode"="${coverageMode}"${selector})"""
+ }
+ // Queries sit a quarter step past a row, so no two rows tie.
+ def evolvedSearch = { String column, String selector, String query = "[2.25,3.25,4.25,5.25]" ->
+ """vector_search(
+ "table"="${evolved}",
+ "column"="${column}",
+ "query_vector"="${query}",
+ "top_k"="3",
+ "metric"="l2",
+ "nprobes"="2"${selector})"""
+ }
+ def scannerV2Rows = sql """SHOW VARIABLES LIKE 'enable_file_scanner_v2'"""
+ String originalScannerV2 = scannerV2Rows[0][1].toString()
+ String originalTimeZone = (sql """SHOW VARIABLES LIKE 'time_zone'""")[0][1].toString()
+ String originalLazy = (sql """SHOW VARIABLES LIKE 'enable_lance_lazy_materialization'""")[0][1].toString()
+
+ sql """DROP CATALOG IF EXISTS `${catalogName}`"""
+ try {
+ sql """SET enable_file_scanner_v2 = true"""
+ sql """SET time_zone = 'UTC'"""
+ sql """
+ CREATE CATALOG `${catalogName}` PROPERTIES (
+ "type" = "lance",
+ "lance.catalog.type" = "filesystem",
+ "warehouse" = "s3://warehouse/lance",
+ "s3.endpoint" = "http://${externalEnvIp}:${minioPort}",
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.region" = "us-east-1",
+ "use_path_style" = "true"
+ )
+ """
+
+ // Without a selector: the latest main version, where rows 3 and 11 are deleted.
+ qt_latest """SELECT row_id, _distance FROM ${vectorSearch("")} ORDER BY _distance, row_id"""
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch("")}")
+ contains "lanceCatalogType=filesystem"
+ contains "lanceVersion=7"
+ contains "lanceManagedVersioning=false"
+ notContains "lanceBranch="
+ contains "lanceSearchIndexFragments=3"
+ contains "lanceSearchUnindexedFragments=0"
+ }
+
+ // Before any index: a flat search over that version's two fragments.
+ qt_version_2 """SELECT row_id, _distance FROM ${vectorSearch(', "version"="2"')} ORDER BY _distance, row_id"""
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(', "version"="2"')}")
+ contains "lanceVersion=2"
+ contains "lanceVectorIndexStatus=NO_MATCH"
+ contains "lanceSearchIndexSegments=0"
+ contains "lanceSearchUnindexedFragments=2"
+ }
+ test {
+ sql """SELECT row_id FROM ${fullTextSearch(', "version"="3"')}"""
+ exception "No committed Lance FTS index exists for column 'body' at dataset version 3"
+ }
+
+ // Version 5 appended fragment 2 after both indexes: the vector search combines the index
+ // over fragments 0 and 1 with a flat search of fragment 2, and full-text search covers
+ // only the indexed fragments.
+ qt_version_5 """
+ SELECT row_id, _distance FROM ${vectorSearch(', "version"="5"', "[17,18,19,20]")}
+ ORDER BY _distance, row_id
+ """
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(', "version"="5"')}")
+ contains "lanceVersion=5"
+ contains "lanceSearchIndexFragments=2"
+ contains "lanceSearchUnindexedFragments=1"
+ }
+ order_qt_version_5_fts """SELECT row_id FROM ${fullTextSearch(', "version"="5"')}"""
+ test {
+ sql """SELECT row_id FROM ${fullTextSearch(', "version"="5"', "strict")}"""
+ exception "requires every fragment at dataset version 5 to be indexed"
+ }
+ order_qt_version_4_fts_strict """SELECT row_id FROM ${fullTextSearch(', "version"="4"', "strict")}"""
+
+ // Version 6 deleted rows 3 and 11 and still uses the original index; version 7 rebuilt it.
+ // The query sits a quarter step past row 3, so no two rows tie.
+ qt_version_6 """
+ SELECT row_id, _distance FROM ${vectorSearch(', "version"="6"', "[3.25,4.25,5.25,6.25]")}
+ ORDER BY _distance, row_id
+ """
+ qt_version_5_not_deleted """
+ SELECT row_id, _distance FROM ${vectorSearch(', "version"="5"', "[3.25,4.25,5.25,6.25]")}
+ ORDER BY _distance, row_id
+ """
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(', "version"="6"')}")
+ contains "lanceSearchIndexFragments=2"
+ contains "lanceSearchUnindexedFragments=1"
+ }
+ // Full-text search sees the deletion too: rows 3 and 11 are "doris" rows.
+ order_qt_version_5_fts_doris """SELECT row_id FROM ${fullTextSearch(', "version"="5"', "index_only", "doris")}"""
+ order_qt_version_6_fts_doris """SELECT row_id FROM ${fullTextSearch(', "version"="6"', "index_only", "doris")}"""
+
+ // use_index=false loads metadata without indexes, from the selected snapshot as well.
+ qt_version_5_no_index """
+ SELECT row_id, _distance FROM ${vectorSearch(', "version"="5", "use_index"="false"', "[17,18,19,20]")}
+ ORDER BY _distance, row_id
+ """
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(', "version"="5", "use_index"="false"')}")
+ contains "lanceVersion=5"
+ contains "lanceVectorIndexStatus=DISABLED"
+ contains "lanceSearchUnindexedFragments=3"
+ }
+
+ // A tag and main written as a branch select main versions.
+ qt_tag_rel """
+ SELECT row_id, _distance FROM ${vectorSearch(', "tag"="rel"', "[17,18,19,20]")}
+ ORDER BY _distance, row_id
+ """
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(', "tag"="rel"')}")
+ contains "lanceVersion=5"
+ notContains "lanceBranch="
+ }
+ qt_main_branch """
+ SELECT row_id, _distance FROM ${vectorSearch(', "branch"="main", "version"="2"')}
+ ORDER BY _distance, row_id
+ """
+
+ // Branch dev: its version 6 shares main's version numbers and fragment ids, not its rows.
+ qt_branch_latest """
+ SELECT row_id, _distance FROM ${vectorSearch(', "branch"="dev"', "[100,101,102,103]")}
+ ORDER BY _distance, row_id
+ """
+ qt_main_same_query """
+ SELECT row_id, _distance FROM ${vectorSearch(', "version"="6"', "[100,101,102,103]")}
+ ORDER BY _distance, row_id
+ """
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(', "branch"="dev"')}")
+ contains "lanceVersion=6"
+ contains "lanceBranch=dev"
+ contains "lanceSearchIndexFragments=2"
+ contains "lanceSearchUnindexedFragments=1"
+ }
+ order_qt_branch_fts_strict """SELECT row_id FROM ${fullTextSearch(', "branch"="dev"', "strict")}"""
+ qt_branch_no_index """
+ SELECT row_id, _distance
+ FROM ${vectorSearch(', "branch"="dev", "use_index"="false"', "[100,101,102,103]")}
+ ORDER BY _distance, row_id
+ """
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(', "branch"="dev", "use_index"="false"')}")
+ contains "lanceVersion=6"
+ contains "lanceBranch=dev"
+ contains "lanceVectorIndexStatus=DISABLED"
+ }
+ qt_branch_fork """
+ SELECT row_id, _distance FROM ${vectorSearch(', "branch"="dev", "version"="4"', "[100,101,102,103]")}
+ ORDER BY _distance, row_id
+ """
+ // Historical branch versions past the fork differ from main's versions of the same number:
+ // dev version 5 has row_id 101..108 where main version 5 has 17..24, and dev version 6
+ // rebuilt body_idx over fragment 2, which main version 6 does not cover.
+ qt_branch_version_5 """
+ SELECT row_id, _distance FROM ${vectorSearch(', "branch"="dev", "version"="5"', "[100,101,102,103]")}
+ ORDER BY _distance, row_id
+ """
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(', "branch"="dev", "version"="5"')}")
+ contains "lanceVersion=5"
+ contains "lanceBranch=dev"
+ contains "lanceSearchUnindexedFragments=1"
+ }
+ order_qt_branch_version_6_fts_strict """
+ SELECT row_id FROM ${fullTextSearch(', "branch"="dev", "version"="6"', "strict")}
+ """
+ test {
+ sql """SELECT row_id FROM ${fullTextSearch(', "version"="6"', "strict")}"""
+ exception "requires every fragment at dataset version 6 to be indexed"
+ }
+ // dev_rel points at dev version 5, not the branch's latest; its FTS index covers only
+ // the fork's fragments, and the error names the branch.
+ order_qt_branch_tag_fts """SELECT row_id FROM ${fullTextSearch(', "tag"="dev_rel"')}"""
+ test {
+ sql """SELECT row_id FROM ${fullTextSearch(', "tag"="dev_rel"', "strict")}"""
+ exception "dataset version 5 of branch 'dev' to be indexed"
+ }
+
+ // Timestamps in the session time zone: between versions 5 and 6 selects version 5.
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(', "timestamp"="2026-09-29 14:51:58"')}")
+ contains "lanceVersion=5"
+ }
+ sql """SET time_zone = 'Asia/Shanghai'"""
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(', "timestamp"="2026-09-29 22:51:58.000"')}")
+ contains "lanceVersion=5"
+ }
+ sql """SET time_zone = 'UTC'"""
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(', "branch"="dev", "timestamp"="2999-01-01 00:00:00"')}")
+ contains "lanceVersion=6"
+ contains "lanceBranch=dev"
+ }
+ // Between dev versions 5 and 6: ignoring the time would select dev version 6, and resolving
+ // it on main would select main version 7.
+ explain {
+ sql("SELECT row_id FROM ${vectorSearch(', "branch"="dev", "timestamp"="2026-09-29 14:52:18"')}")
+ contains "lanceVersion=5"
+ contains "lanceBranch=dev"
+ }
+ test {
+ sql """SELECT row_id FROM ${fullTextSearch(', "branch"="dev", "timestamp"="2026-09-29 14:52:18"', "strict")}"""
+ exception "dataset version 5 of branch 'dev' to be indexed"
+ }
+ test {
+ sql """SELECT row_id FROM ${vectorSearch(', "timestamp"="2026-09-29 14:51:51"')}"""
+ exception "Lance table default.search_snapshot has no version at or before '2026-09-29 14:51:51'"
+ }
+
+ // Two-phase materialization reads the branch's rows, not main's fragment 2.
+ sql """SET enable_lance_lazy_materialization = true"""
+ String twoPhase = """
+ SELECT row_id, body, _distance
+ FROM ${vectorSearch(', "branch"="dev"', "[100,101,102,103]")}
+ ORDER BY _distance
+ LIMIT 3
+ """
+ explain {
+ sql "verbose ${twoPhase}"
+ contains "VMaterializeNode"
+ }
+ qt_branch_two_phase "${twoPhase}"
+ // rel is version 5, where row 3 still exists; version 6 deleted it, so a second phase
+ // that reopened a later version would fetch fewer rows than it asked for.
+ String tagTwoPhase = """
+ SELECT row_id, body, _distance
+ FROM ${vectorSearch(', "tag"="rel"', "[3.25,4.25,5.25,6.25]")}
+ ORDER BY _distance
+ LIMIT 3
+ """
+ explain {
+ sql "verbose ${tagTwoPhase}"
+ contains "VMaterializeNode"
+ }
+ qt_tag_two_phase "${tagTwoPhase}"
+ // The second phase fetches the payload by the column names of the planned version: vec
+ // exists only up to version 3, note only from version 3 and embedding only from version 4,
+ // so fetching from any other version fails.
+ String evolvedTwoPhase = """
+ SELECT row_id, embedding, note, body, _distance
+ FROM ${evolvedSearch("embedding", ', "version"="4"')}
+ ORDER BY _distance
+ LIMIT 3
+ """
+ explain {
+ sql "verbose ${evolvedTwoPhase}"
+ contains "VMaterializeNode"
+ }
+ qt_evolved_two_phase "${evolvedTwoPhase}"
+ String evolvedOldTwoPhase = """
+ SELECT row_id, vec, note, body, _distance
+ FROM ${evolvedSearch("vec", ', "version"="3"')}
+ ORDER BY _distance
+ LIMIT 3
+ """
+ explain {
+ sql "verbose ${evolvedOldTwoPhase}"
+ contains "VMaterializeNode"
+ }
+ qt_evolved_old_two_phase "${evolvedOldTwoPhase}"
+ sql """SET enable_lance_lazy_materialization = ${originalLazy}"""
+
+ // Schema changes: each search binds the column names, the index and the fragments of the
+ // version it reads. The rename keeps the field id, so vec_idx serves embedding.
+ qt_evolved_version_2 """
+ SELECT row_id, _distance FROM ${evolvedSearch("vec", ', "version"="2"')} ORDER BY _distance, row_id
+ """
+ qt_evolved_version_4 """
+ SELECT row_id, _distance FROM ${evolvedSearch("embedding", ', "version"="4"')}
+ ORDER BY _distance, row_id
+ """
+ for (String version : ["2", "4"]) {
+ String column = version == "2" ? "vec" : "embedding"
+ explain {
+ sql("SELECT row_id FROM ${evolvedSearch(column, ', "version"="' + version + '"')}")
+ contains "lanceVersion=${version}"
+ contains "lanceVectorColumn=${column}"
+ contains "lanceVectorIndexStatus=USED"
+ contains "lanceSearchIndexFragments=1"
+ contains "lanceSearchUnindexedFragments=0"
+ }
+ }
+ qt_evolved_latest """
+ SELECT row_id, _distance FROM ${evolvedSearch("embedding", "", "[12.25,13.25,14.25,15.25]")}
+ ORDER BY _distance, row_id
+ """
+ explain {
+ sql("SELECT row_id FROM ${evolvedSearch("embedding", "")}")
+ contains "lanceVersion=5"
+ contains "lanceSearchIndexFragments=1"
+ contains "lanceSearchUnindexedFragments=1"
+ }
+ test {
+ sql """SELECT row_id FROM ${evolvedSearch("embedding", ', "version"="2"')}"""
+ exception "Lance vector column 'embedding' does not exist"
+ }
+ test {
+ sql """SELECT row_id FROM ${evolvedSearch("vec", ', "version"="4"')}"""
+ exception "Lance vector column 'vec' does not exist"
+ }
+ test {
+ sql """SELECT row_id, note FROM ${evolvedSearch("vec", ', "version"="2"')}"""
+ exception "Unknown column 'note'"
+ }
+
+ // A selector that cannot be resolved fails; nothing falls back to the latest version.
+ test {
+ sql """SELECT row_id FROM ${vectorSearch(', "version"="99"')}"""
+ exception "Lance version 99 of default.search_snapshot was not found"
+ }
+ test {
+ sql """SELECT row_id FROM ${vectorSearch(', "tag"="nope"')}"""
+ exception "Lance tag 'nope' of default.search_snapshot was not found"
+ }
+ test {
+ sql """SELECT row_id FROM ${vectorSearch(', "branch"="nope"')}"""
+ exception "Lance branch 'nope' of default.search_snapshot was not found"
+ }
+ test {
+ sql """SELECT row_id FROM ${vectorSearch(', "branch"="dev", "version"="9"')}"""
+ exception "Lance version 9 of default.search_snapshot@dev was not found"
+ }
+ test {
+ sql """SELECT row_id FROM ${vectorSearch(', "version"="2", "timestamp"="2026-09-29 14:51:58"')}"""
+ exception "'version' and 'timestamp' are mutually exclusive"
+ }
+ test {
+ sql """SELECT row_id FROM ${fullTextSearch(', "tag"="rel", "branch"="dev"')}"""
+ exception "'tag' and 'branch' are mutually exclusive"
+ }
+ test {
+ sql """SELECT row_id FROM ${vectorSearch(', "version"="rel"')}"""
+ exception "'version' must be a positive integer, but was 'rel'"
+ }
+ test {
+ sql """SELECT row_id FROM ${vectorSearch(', "branch"=""')}"""
+ exception "'branch' must not be empty"
+ }
+ test {
+ sql """SELECT row_id FROM ${vectorSearch(', "timestamp"="1790693517507"')}"""
+ exception "'timestamp' must be 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd HH:mm:ss.SSS'"
+ }
+ test {
+ sql """SELECT row_id FROM ${vectorSearch("", "[0,1,2,3]", "3", table + "@branch(dev)")}"""
+ exception "use the 'version', 'timestamp', 'tag' or 'branch' property"
+ }
+
+ // Cleanup: a removed version is missing, and a kept version whose index files are gone
+ // fails at execution instead of searching another version.
+ test {
+ sql """SELECT row_id FROM ${vectorSearch(', "version"="1"', "[0,1,2,3]", "3", pruned)}"""
+ exception "Lance version 1 of default.search_snapshot_pruned was not found"
+ }
+ test {
+ sql """SELECT row_id FROM ${vectorSearch(', "tag"="kept"', "[0,1,2,3]", "3", pruned)}"""
+ exception "at Lance dataset version 2 at s3://warehouse/lance/search_snapshot_pruned.lance failed"
+ }
+ qt_pruned_latest """
+ SELECT row_id, _distance FROM ${vectorSearch("", "[0,1,2,3]", "3", pruned)}
+ ORDER BY _distance, row_id
+ """
+ } finally {
+ sql """SET enable_file_scanner_v2 = ${originalScannerV2}"""
+ sql """SET time_zone = '${originalTimeZone}'"""
+ sql """SET enable_lance_lazy_materialization = ${originalLazy}"""
+ sql """DROP CATALOG IF EXISTS `${catalogName}`"""
+ }
+}
diff --git a/regression-test/suites/external_table_p0/lance/test_lance_vector_search_legacy_index.groovy b/regression-test/suites/external_table_p0/lance/test_lance_vector_search_legacy_index.groovy
new file mode 100644
index 00000000000..e2e5da1ae5b
--- /dev/null
+++ b/regression-test/suites/external_table_p0/lance/test_lance_vector_search_legacy_index.groovy
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_lance_vector_search_legacy_index", "p0,external") {
+ /*
+ * A vector index without index details, as Lance writers before 0.20 produced. Lance infers
+ * the missing details from the index files when it loads the index, so the FE plans the
+ * index like any other and the BE searches it through the planned segment. A split without
+ * index segments always searches flat, so a legacy index the FE skipped would lose ANN.
+ *
+ * Fixture: legacy_vector_index.lance (see lance_build_legacy_vector_index.py), written by
+ * pylance 0.18.2: row_id 1..512 in fragment 0, vec = [row_id, row_id + 1, row_id + 2,
+ * row_id + 3], and vec_idx (IVF_PQ, l2, 2 partitions, 2 sub-vectors) over fragment 0. The
+ * squared L2 distance between rows r and n is 4 * (n - r)^2; refine_factor recomputes it
+ * from the stored vectors, so the indexed search returns the flat search's rows.
+ */
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable Lance legacy vector index test because the Iceberg MinIO environment is disabled.")
+ return
+ }
+
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ String catalogName = "test_lance_vector_search_legacy_index"
+ String tableName = "${catalogName}.`default`.legacy_vector_index"
+ // The query sits a quarter step past row 100, so no two rows tie.
+ String query = "[100.25,101.25,102.25,103.25]"
+ def search = { String options ->
+ """vector_search("table"="${tableName}", "column"="vec", "query_vector"="${query}", "top_k"="3", "metric"="l2"${options})"""
+ }
+ String originalScannerV2 = (sql """SHOW VARIABLES LIKE 'enable_file_scanner_v2'""")[0][1].toString()
+
+ sql """DROP CATALOG IF EXISTS `${catalogName}`"""
+ try {
+ sql """SET enable_file_scanner_v2 = true"""
+ sql """
+ CREATE CATALOG `${catalogName}` PROPERTIES (
+ "type" = "lance",
+ "lance.catalog.type" = "filesystem",
+ "warehouse" = "s3://warehouse/lance",
+ "s3.endpoint" = "http://${externalEnvIp}:${minioPort}",
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.region" = "us-east-1",
+ "use_path_style" = "true"
+ )
+ """
+
+ explain {
+ sql("SELECT row_id FROM ${search(', "nprobes"="2", "refine_factor"="4"')}")
+ contains "lanceVectorIndexStatus=USED"
+ contains "lanceVersion=2"
+ contains "lanceSearchIndexSegments=1"
+ contains "lanceSearchIndexFragments=1"
+ contains "lanceSearchUnindexedFragments=0"
+ }
+ qt_legacy_indexed """
+ SELECT row_id, _distance FROM ${search(', "nprobes"="2", "refine_factor"="4"')}
+ ORDER BY _distance, row_id
+ """
+ qt_legacy_flat """
+ SELECT row_id, _distance FROM ${search(', "use_index"="false"')}
+ ORDER BY _distance, row_id
+ """
+ // Before the index was built, the same search plans no index.
+ explain {
+ sql("SELECT row_id FROM ${search(', "version"="1"')}")
+ contains "lanceVectorIndexStatus=NO_MATCH"
+ contains "lanceSearchUnindexedFragments=1"
+ }
+ } finally {
+ sql """SET enable_file_scanner_v2 = ${originalScannerV2}"""
+ sql """DROP CATALOG IF EXISTS `${catalogName}`"""
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 06/28: branch-4.1:[fix](rowid-fetch) fix TopN lazy materialization crash with duplicateprojected columns (#64372) (#67563)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit b621170f2822300d05f0bf03a7f80961b08b6358
Author: daidai <ch...@selectdb.com>
AuthorDate: Thu Oct 8 19:10:22 2026 +0800
branch-4.1:[fix](rowid-fetch) fix TopN lazy materialization crash with duplicateprojected columns (#64372) (#67563)
### What problem does this PR solve?
Problem Summary:
bp #64372
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
Co-authored-by: York Cao <52...@users.noreply.github.com>
Co-authored-by: caokaihua1 <ca...@jd.com>
---
be/src/exec/rowid_fetcher.cpp | 267 +++--
be/src/exec/rowid_fetcher.h | 15 +-
be/test/exec/rowid_fetcher_test.cpp | 167 ++++
.../hive/test_hive_topn_lazy_mat.out | 1047 ++++++++++++++++++++
.../tvf/test_tvf_topn_lazy_mat.out | 72 ++
.../hive/test_hive_topn_lazy_mat.groovy | 50 +-
.../tvf/test_tvf_topn_lazy_mat.groovy | 7 +
7 files changed, 1552 insertions(+), 73 deletions(-)
diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp
index 92f8de5c2e6..0219ac0b778 100644
--- a/be/src/exec/rowid_fetcher.cpp
+++ b/be/src/exec/rowid_fetcher.cpp
@@ -32,6 +32,7 @@
#include <cstdint>
#include <limits>
#include <memory>
+#include <mutex>
#include <ostream>
#include <string>
#include <unordered_map>
@@ -907,16 +908,22 @@ Status RowIdStorageReader::read_lance_rows_by_row_ids(
Status RowIdStorageReader::read_external_row_from_file_mapping(
size_t idx, const std::multimap<uint64_t, size_t>& row_ids,
- const std::shared_ptr<FileMapping>& file_mapping, const std::vector<SlotDescriptor>& slots,
- const TUniqueId& query_id, const std::shared_ptr<RuntimeState>& runtime_state,
- std::vector<Block>& scan_blocks, std::vector<std::pair<size_t, size_t>>& row_id_block_idx,
+ const std::shared_ptr<FileMapping>& file_mapping,
+ const std::vector<SlotDescriptor>& scan_slots, const TUniqueId& query_id,
+ const std::shared_ptr<RuntimeState>& runtime_state, std::vector<Block>& scan_blocks,
+ std::vector<std::pair<size_t, size_t>>& row_id_block_idx,
std::vector<RowIdStorageReader::ExternalFetchStatistics>& fetch_statistics,
const TFileScanRangeParams& rpc_scan_params,
const std::unordered_map<std::string, int>& colname_to_slot_id,
- TupleDescriptor& tuple_desc) {
+ std::counting_semaphore<>& semaphore, TupleDescriptor& tuple_desc) {
SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->rowid_storage_reader_tracker());
signal::set_signal_task_id(query_id);
+ // Release the concurrency permit on every exit path (including error returns
+ // and exceptions). Completion accounting and status publishing are owned by
+ // the caller, so the status is always published before the waiter is woken.
+ Defer defer([&] { semaphore.release(); });
+
std::vector<uint64_t> read_ids;
// Generate an ordered, deduplicated list with the help of the multimap ordering.
for (const auto& [row_id, result_block_idx] : row_ids) {
@@ -926,7 +933,7 @@ Status RowIdStorageReader::read_external_row_from_file_mapping(
row_id_block_idx[result_block_idx] = std::make_pair(idx, read_ids.size() - 1);
}
- scan_blocks[idx] = Block(slots, read_ids.size());
+ scan_blocks[idx] = Block(scan_slots, read_ids.size());
auto& external_info = file_mapping->get_external_file_info();
auto scan_range_desc = build_external_fetch_range(external_info.scan_range_desc);
@@ -936,9 +943,10 @@ Status RowIdStorageReader::read_external_row_from_file_mapping(
const auto format_type = scan_range_desc.__isset.format_type ? scan_range_desc.format_type
: rpc_scan_params.format_type;
if (format_type == TFileFormatType::FORMAT_LANCE) {
- RETURN_IF_ERROR(read_lance_rows_by_row_ids(
- scan_range_desc, read_ids, slots, runtime_state.get(), sub_runtime_profile.get(),
- rpc_scan_params, &scan_blocks[idx], &fetch_statistics[idx]));
+ RETURN_IF_ERROR(read_lance_rows_by_row_ids(scan_range_desc, read_ids, scan_slots,
+ runtime_state.get(), sub_runtime_profile.get(),
+ rpc_scan_params, &scan_blocks[idx],
+ &fetch_statistics[idx]));
} else {
// Parquet/ORC IDs remain signed physical file positions; Lance keeps native uint64 IDs.
std::list<int64_t> legacy_read_ids;
@@ -970,6 +978,23 @@ Status RowIdStorageReader::read_external_row_from_file_mapping(
}
}
+ if (scan_blocks[idx].rows() != read_ids.size()) {
+ return Status::InternalError(
+ "Row id fetch scan row count mismatch, "
+ "query_id={}, path={}, expected_rows={}, actual_rows={}",
+ print_id(query_id), scan_range_desc.path, read_ids.size(), scan_blocks[idx].rows());
+ }
+ for (size_t column_id = 0; column_id < scan_blocks[idx].columns(); ++column_id) {
+ const auto& column = scan_blocks[idx].get_by_position(column_id);
+ if (column.column->size() != read_ids.size()) {
+ return Status::InternalError(
+ "Row id fetch scan column row count mismatch, "
+ "query_id={}, path={}, column={}, expected_rows={}, actual_rows={}",
+ print_id(query_id), scan_range_desc.path, column.name, read_ids.size(),
+ column.column->size());
+ }
+ }
+
auto file_read_bytes_counter =
sub_runtime_profile->get_counter(FileScannerV2::FileReadBytesProfile);
@@ -988,6 +1013,92 @@ Status RowIdStorageReader::read_external_row_from_file_mapping(
return Status::OK();
}
+std::string RowIdStorageReader::source_column_key(const SlotDescriptor& slot, uint32_t column_idx) {
+ fmt::memory_buffer key;
+ // Length-prefix each component so distinct sequences cannot alias, e.g.
+ // paths ["a", "b"] -> "1:a1:b" while ["a:b"] -> "3:a:b".
+ auto append = [&key](std::string_view component) {
+ fmt::format_to(key, "{}:", component.size());
+ key.append(component.data(), component.data() + component.size());
+ };
+ append(slot.col_name());
+ append(std::to_string(column_idx));
+ append(std::to_string(slot.col_unique_id()));
+ append(std::to_string(slot.column_paths().size()));
+ for (const auto& path : slot.column_paths()) {
+ append(path);
+ }
+ append(std::to_string(slot.all_access_paths().size()));
+ // Encode each optional sub-path's presence bit separately from its element
+ // count so an absent path ("0") never aliases a present-but-empty path
+ // ("1" + size "0").
+ auto append_optional_path = [&append](bool is_set, const std::vector<std::string>& items) {
+ append(is_set ? "1" : "0");
+ if (is_set) {
+ append(std::to_string(items.size()));
+ for (const auto& item : items) {
+ append(item);
+ }
+ }
+ };
+ for (const auto& path : slot.all_access_paths()) {
+ append(fmt::format("{}", path.type));
+ append_optional_path(path.__isset.data_access_path, path.data_access_path.path);
+ append_optional_path(path.__isset.meta_access_path, path.meta_access_path.path);
+ }
+ return fmt::to_string(key);
+}
+
+Status RowIdStorageReader::submit_external_scan_tasks(
+ ScannerScheduler* scheduler, std::counting_semaphore<>& semaphore, size_t task_count,
+ const std::function<std::string(size_t)>& make_task_id,
+ const std::function<Status(size_t)>& run_task) {
+ // `completed_count` is a plain counter guarded by `mtx`; the same mutex guards
+ // the wait predicate below, so a worker can never notify between the waiter's
+ // predicate check and its wait.
+ AtomicStatus scan_status;
+ std::condition_variable cv;
+ std::mutex mtx;
+ size_t completed_count = 0;
+
+ // Only tasks the scheduler actually accepted are waited for. If a submission
+ // fails we stop submitting, but still wait for the already-accepted tasks so
+ // their workers cannot outlive the locals they capture by reference.
+ size_t submitted_count = 0;
+ for (size_t idx = 0; idx < task_count; ++idx) {
+ semaphore.acquire();
+ auto run_one_task = [&, idx]() -> bool {
+ Status task_status = Status::OK();
+ // Publish the status before the completion signal wakes the waiter, on every
+ // path. A scanner that throws would otherwise leave scan_status OK while this
+ // Defer still counts the task as finished, and the caller would report success
+ // over a half-filled result block.
+ Defer complete([&] {
+ scan_status.update(task_status);
+ std::lock_guard<std::mutex> lock(mtx);
+ ++completed_count;
+ cv.notify_one();
+ });
+ ASSIGN_STATUS_IF_CATCH_EXCEPTION(task_status = run_task(idx), task_status);
+ return true;
+ };
+ Status submit_st = scheduler->submit_scan_task(
+ SimplifiedScanTask(run_one_task, nullptr, nullptr), make_task_id(idx));
+ if (!submit_st.ok()) {
+ scan_status.update(submit_st);
+ semaphore.release();
+ break;
+ }
+ ++submitted_count;
+ }
+
+ {
+ std::unique_lock<std::mutex> lock(mtx);
+ cv.wait(lock, [&] { return completed_count == submitted_count; });
+ }
+ return scan_status.ok() ? Status::OK() : scan_status.status();
+}
+
Status RowIdStorageReader::read_batch_external_row(
const uint64_t workload_group_id, const PRequestBlockDesc& request_block_desc,
std::shared_ptr<IdFileMap> id_file_map, std::vector<SlotDescriptor>& slots,
@@ -998,12 +1109,21 @@ Status RowIdStorageReader::read_batch_external_row(
TupleDescriptor tuple_desc(request_block_desc.desc(), false);
std::unordered_map<std::string, int> colname_to_slot_id;
std::shared_ptr<RuntimeState> runtime_state = nullptr;
+ std::vector<SlotDescriptor> scan_slots;
+ std::vector<size_t> result_column_to_scan_column;
+ std::vector<uint32_t> scan_column_idxs;
int max_file_scanners = 0;
{
if (result_block.is_empty_column()) [[likely]] {
result_block = Block(slots, request_block_desc.row_id_size());
}
+ if (request_block_desc.column_idxs_size() != slots.size()) {
+ return Status::InternalError(
+ "Row id fetch request has mismatched slots and column indexes, "
+ "query_id={}, slots={}, column_idxs={}",
+ print_id(query_id), slots.size(), request_block_desc.column_idxs_size());
+ }
auto& external_info = first_file_mapping->get_external_file_info();
int plan_node_id = external_info.plan_node_id;
@@ -1028,16 +1148,30 @@ Status RowIdStorageReader::read_batch_external_row(
}
}
- std::vector<uint32_t> scan_column_idxs;
+ std::unordered_map<std::string, size_t> source_column_to_scan_idx;
+
+ result_column_to_scan_column.reserve(slots.size());
+ scan_slots.reserve(slots.size());
scan_column_idxs.reserve(slots.size());
- for (int slot_idx = 0; slot_idx < slots.size(); ++slot_idx) {
- auto& slot = slots[slot_idx];
+ for (auto slot_idx = 0; slot_idx < slots.size(); ++slot_idx) {
+ const auto& slot = slots[slot_idx];
+ const auto column_idx = request_block_desc.column_idxs(slot_idx);
+ const auto key = source_column_key(slot, column_idx);
+ auto [it, inserted] =
+ source_column_to_scan_idx.emplace(key, source_column_to_scan_idx.size());
+ result_column_to_scan_column.emplace_back(it->second);
+ if (inserted) {
+ scan_slots.emplace_back(slot);
+ scan_column_idxs.emplace_back(column_idx);
+ }
+ }
+
+ for (auto& slot : scan_slots) {
tuple_desc.add_slot(&slot);
- colname_to_slot_id.emplace(slot.col_name(), slot.id());
- scan_column_idxs.emplace_back(request_block_desc.column_idxs(slot_idx));
+ colname_to_slot_id[slot.col_name()] = slot.id();
}
- rpc_scan_params = build_external_scan_params(rpc_scan_params, first_scan_range_desc, slots,
- scan_column_idxs);
+ rpc_scan_params = build_external_scan_params(rpc_scan_params, first_scan_range_desc,
+ scan_slots, scan_column_idxs);
const auto& query_options = id_file_map->get_query_options();
const auto& query_globals = id_file_map->get_query_globals();
@@ -1131,75 +1265,68 @@ Status RowIdStorageReader::read_batch_external_row(
int64_t scan_running_time = 0;
RETURN_IF_ERROR(scope_timer_run(
[&]() -> Status {
- // Make sure to insert data into result_block only after all scan tasks have been executed.
- std::atomic<int> producer_count {0};
- std::condition_variable cv;
- std::mutex mtx;
- Status scan_status = Status::OK();
-
//semaphore: Limit the number of scan tasks submitted at one time
std::counting_semaphore semaphore {max_file_scanners};
- size_t idx = 0;
+ std::vector<
+ std::pair<std::multimap<uint64_t, size_t>, std::shared_ptr<FileMapping>>>
+ scan_info_list;
+ scan_info_list.reserve(scan_rows.size());
for (const auto& [_, scan_info] : scan_rows) {
- semaphore.acquire();
- RETURN_IF_ERROR(remote_scan_sched->submit_scan_task(
- SimplifiedScanTask(
- [&, idx, scan_info]() -> bool {
- Defer complete_task {[&]() {
- semaphore.release();
- if (++producer_count == scan_rows.size()) {
- std::lock_guard<std::mutex> lock(mtx);
- cv.notify_one();
- }
- }};
- const auto& [row_ids, file_mapping] = scan_info;
- auto status = read_external_row_from_file_mapping(
- idx, row_ids, file_mapping, slots, query_id,
- runtime_state, scan_blocks, row_id_block_idx,
- fetch_statistics, rpc_scan_params,
- colname_to_slot_id, tuple_desc);
- if (!status.ok()) {
- std::lock_guard<std::mutex> lock(mtx);
- if (scan_status.ok()) {
- scan_status = status;
- }
- }
- // The return value indicates whether this one-shot
- // scheduler task has completed, not whether the fetch
- // succeeded. The fetch status is propagated by scan_status.
- return true;
- },
- nullptr, nullptr),
- fmt::format("{}-read_batch_external_row-{}", print_id(query_id), idx)));
- idx++;
+ scan_info_list.emplace_back(scan_info);
}
- {
- std::unique_lock<std::mutex> lock(mtx);
- cv.wait(lock, [&] { return producer_count == scan_rows.size(); });
- }
- RETURN_IF_ERROR(scan_status);
- return Status::OK();
+ return submit_external_scan_tasks(
+ remote_scan_sched, semaphore, scan_rows.size(),
+ [&](size_t idx) {
+ return fmt::format("{}-read_batch_external_row-{}", print_id(query_id),
+ idx);
+ },
+ [&](size_t idx) -> Status {
+ const auto& [row_ids, file_mapping] = scan_info_list[idx];
+ return read_external_row_from_file_mapping(
+ idx, row_ids, file_mapping, scan_slots, query_id, runtime_state,
+ scan_blocks, row_id_block_idx, fetch_statistics,
+ rpc_scan_params, colname_to_slot_id, semaphore, tuple_desc);
+ });
},
&scan_running_time));
- // Insert the read data into result_block.
- for (size_t column_id = 0; column_id < result_block.columns(); column_id++) {
+ // Insert the read data into result_block. Use insert_indices_from() instead of
+ // scatter_scan_blocks_to_result_block()/insert_from_multi_column(), because
+ // scan_blocks may have fewer columns than result_block when duplicate physical columns
+ // are deduplicated, and insert_from_multi_column() cannot handle ColumnString
+ // cross-type (32/64) copies safely.
+ const size_t result_column_count = result_block.columns();
+ for (size_t column_id = 0; column_id < result_column_count; column_id++) {
auto dst_col_guard = result_block.mutate_column_scoped(column_id);
MutableColumnPtr& dst_col = dst_col_guard.mutable_column();
- std::vector<const IColumn*> scan_src_columns;
- scan_src_columns.reserve(row_id_block_idx.size());
- std::vector<size_t> scan_positions;
- scan_positions.reserve(row_id_block_idx.size());
+ bool dst_is_nullable = dst_col->is_nullable();
+ std::vector<ColumnPtr> nullable_src_columns(scan_blocks.size());
+ auto scan_column_id = result_column_to_scan_column[column_id];
for (const auto& [pos_block, block_idx] : row_id_block_idx) {
- DCHECK(scan_blocks.size() > pos_block);
- DCHECK(scan_blocks[pos_block].get_columns().size() > column_id);
- scan_src_columns.emplace_back(scan_blocks[pos_block].get_columns()[column_id].get());
- scan_positions.emplace_back(block_idx);
+ DCHECK_GT(scan_blocks.size(), pos_block);
+ DCHECK_GT(scan_blocks[pos_block].columns(), scan_column_id);
+ const auto& src_column_ptr =
+ scan_blocks[pos_block].get_by_position(scan_column_id).column;
+ const auto* src_col = src_column_ptr.get();
+ if (dst_is_nullable && !src_col->is_nullable()) {
+ if (!nullable_src_columns[pos_block]) {
+ nullable_src_columns[pos_block] = make_nullable(src_column_ptr);
+ }
+ src_col = nullable_src_columns[pos_block].get();
+ }
+ if (block_idx >= src_col->size()) {
+ return Status::InternalError(
+ "Row id fetch source index out of range, query_id={}, column={}, "
+ "source_block={}, source_rows={}, row_index={}",
+ print_id(query_id), result_block.get_by_position(column_id).name, pos_block,
+ src_col->size(), block_idx);
+ }
+ uint32_t scan_position = cast_set<uint32_t>(block_idx);
+ dst_col->insert_indices_from(*src_col, &scan_position, &scan_position + 1);
}
- dst_col->insert_from_multi_column(scan_src_columns, scan_positions);
}
// Statistical runtime profile information.
diff --git a/be/src/exec/rowid_fetcher.h b/be/src/exec/rowid_fetcher.h
index fed8e5ccd52..ca1f8b50690 100644
--- a/be/src/exec/rowid_fetcher.h
+++ b/be/src/exec/rowid_fetcher.h
@@ -22,6 +22,8 @@
#include <gen_cpp/DataSinks_types.h>
#include <gen_cpp/internal_service.pb.h>
+#include <condition_variable>
+#include <functional>
#include <map>
#include <memory>
#include <semaphore>
@@ -42,6 +44,7 @@ class RuntimeProfile;
class RuntimeState;
class TQueryOptions;
class TupleDescriptor;
+class ScannerScheduler;
namespace io {
enum class FileCacheMissPolicy : uint8_t;
}
@@ -172,13 +175,21 @@ private:
static Status read_external_row_from_file_mapping(
size_t idx, const std::multimap<uint64_t, size_t>& row_ids,
const std::shared_ptr<FileMapping>& file_mapping,
- const std::vector<SlotDescriptor>& slots, const TUniqueId& query_id,
+ const std::vector<SlotDescriptor>& scan_slots, const TUniqueId& query_id,
const std::shared_ptr<RuntimeState>& runtime_state, std::vector<Block>& scan_blocks,
std::vector<std::pair<size_t, size_t>>& row_id_block_idx,
std::vector<ExternalFetchStatistics>& fetch_statistics,
const TFileScanRangeParams& rpc_scan_params,
const std::unordered_map<std::string, int>& colname_to_slot_id,
- TupleDescriptor& tuple_desc);
+ std::counting_semaphore<>& semaphore, TupleDescriptor& tuple_desc);
+
+ static std::string source_column_key(const SlotDescriptor& slot, uint32_t column_idx);
+
+ static Status submit_external_scan_tasks(ScannerScheduler* scheduler,
+ std::counting_semaphore<>& semaphore,
+ size_t task_count,
+ const std::function<std::string(size_t)>& make_task_id,
+ const std::function<Status(size_t)>& run_task);
struct ExternalFetchStatistics {
int64_t init_reader_ms = 0;
diff --git a/be/test/exec/rowid_fetcher_test.cpp b/be/test/exec/rowid_fetcher_test.cpp
index 2a092a20f70..3e8fd3e9517 100644
--- a/be/test/exec/rowid_fetcher_test.cpp
+++ b/be/test/exec/rowid_fetcher_test.cpp
@@ -33,13 +33,24 @@
namespace doris {
+// source_column_key() decides which projected slots collapse onto a single scanned
+// column. Two slots that name the same physical column must produce the same key, and
+// any two slots that do not must produce different ones -- a false merge drops a column
+// the result block still expects, which is the crash this keying was introduced to fix.
class RowIdStorageReaderTest : public testing::Test {
+public:
+ static std::string key_of(const SlotDescriptor& slot, uint32_t column_idx) {
+ return RowIdStorageReader::source_column_key(slot, column_idx);
+ }
+
protected:
struct SlotSpec {
std::string col_name = "c";
int32_t slot_id = 0;
PrimitiveType type = TYPE_INT;
int32_t col_unique_id = 1;
+ std::vector<std::string> column_paths = {};
+ TColumnAccessPaths access_paths = {};
};
static SlotDescriptor make_slot(const SlotSpec& spec) {
@@ -51,8 +62,27 @@ protected:
.build();
tdesc.__set_id(spec.slot_id);
tdesc.__set_col_unique_id(spec.col_unique_id);
+ tdesc.__set_column_paths(spec.column_paths);
+ if (!spec.access_paths.empty()) {
+ tdesc.__set_all_access_paths(spec.access_paths);
+ }
return SlotDescriptor(tdesc);
}
+
+ static TColumnAccessPath data_path(const std::vector<std::string>& path) {
+ TColumnAccessPath access_path;
+ access_path.type = TAccessPathType::DATA;
+ TDataAccessPath data;
+ data.__set_path(path);
+ access_path.__set_data_access_path(data);
+ return access_path;
+ }
+
+ static TColumnAccessPath bare_path() {
+ TColumnAccessPath access_path;
+ access_path.type = TAccessPathType::DATA;
+ return access_path;
+ }
};
TEST_F(RowIdStorageReaderTest, ExternalScannerSelectionRespectsRolloutOption) {
@@ -256,4 +286,141 @@ TEST_F(RowIdStorageReaderTest, ExternalFetchPartitionSlotsPreserveHivePositionMa
}
}
+TEST_F(RowIdStorageReaderTest, SameSourceColumnSharesKey) {
+ // The bug case: one physical column projected twice must dedup onto one scan column.
+ const SlotDescriptor first = make_slot({});
+ const SlotDescriptor second = make_slot({});
+ EXPECT_EQ(key_of(first, 3), key_of(second, 3));
+}
+
+TEST_F(RowIdStorageReaderTest, ColumnIndexSeparatesKeys) {
+ const SlotDescriptor slot = make_slot({});
+ EXPECT_NE(key_of(slot, 3), key_of(slot, 4));
+}
+
+TEST_F(RowIdStorageReaderTest, ColumnNameSeparatesKeys) {
+ EXPECT_NE(key_of(make_slot({.col_name = "a"}), 0), key_of(make_slot({.col_name = "b"}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, UniqueIdSeparatesKeys) {
+ EXPECT_NE(key_of(make_slot({.col_unique_id = 1}), 0),
+ key_of(make_slot({.col_unique_id = 2}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, NameAndIndexBoundaryIsNotAmbiguous) {
+ // Without length prefixes, "a" + idx 12 and "a1" + idx 2 both flatten to "a12".
+ EXPECT_NE(key_of(make_slot({.col_name = "a"}), 12), key_of(make_slot({.col_name = "a1"}), 2));
+}
+
+TEST_F(RowIdStorageReaderTest, PathComponentBoundaryIsNotAmbiguous) {
+ // The concatenation hazard the length prefix exists for: ["a", "b"] and ["a:b"] are
+ // different nested columns but share the naive ':'-joined spelling.
+ EXPECT_NE(key_of(make_slot({.column_paths = {"a", "b"}}), 0),
+ key_of(make_slot({.column_paths = {"a:b"}}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, EmptyPathIsNotTheSameAsNoPath) {
+ EXPECT_NE(key_of(make_slot({.column_paths = {}}), 0),
+ key_of(make_slot({.column_paths = {""}}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, PathOrderMatters) {
+ EXPECT_NE(key_of(make_slot({.column_paths = {"a", "b"}}), 0),
+ key_of(make_slot({.column_paths = {"b", "a"}}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, EqualPathsShareKey) {
+ EXPECT_EQ(key_of(make_slot({.column_paths = {"a", "b"}}), 0),
+ key_of(make_slot({.column_paths = {"a", "b"}}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, AccessPathSeparatesKeys) {
+ EXPECT_NE(key_of(make_slot({.access_paths = {data_path({"a"})}}), 0),
+ key_of(make_slot({.access_paths = {data_path({"b"})}}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, AbsentAccessPathIsNotAnEmptyOne) {
+ // The presence bit: an unset data_access_path must not collide with one that is set
+ // but carries no components.
+ EXPECT_NE(key_of(make_slot({.access_paths = {bare_path()}}), 0),
+ key_of(make_slot({.access_paths = {data_path({})}}), 0));
+}
+
+TEST_F(RowIdStorageReaderTest, AccessPathCountSeparatesKeys) {
+ EXPECT_NE(key_of(make_slot({.access_paths = {data_path({"a"})}}), 0),
+ key_of(make_slot({.access_paths = {data_path({"a"}), data_path({"b"})}}), 0));
+}
+
+// Runs every submitted task on the submitting thread. The point of these cases is which
+// status reaches the caller, not the threading, and inline execution keeps them
+// deterministic.
+class InlineScanScheduler : public ScannerScheduler {
+public:
+ Status start(int, int, int, int) override { return Status::OK(); }
+ void stop() override {}
+ Status submit_scan_task(SimplifiedScanTask scan_task) override {
+ scan_task.scan_func();
+ return Status::OK();
+ }
+ Status submit_scan_task(SimplifiedScanTask scan_task, const std::string&) override {
+ scan_task.scan_func();
+ return Status::OK();
+ }
+ void reset_thread_num(int, int, int) override {}
+ int get_queue_size() override { return 0; }
+ int get_active_threads() override { return 0; }
+ std::vector<int> thread_debug_info() override { return {}; }
+ Status schedule_scan_task(std::shared_ptr<ScannerContext>, std::shared_ptr<ScanTask>,
+ std::unique_lock<std::mutex>&) override {
+ return Status::OK();
+ }
+};
+
+// submit_external_scan_tasks() signals completion from a Defer, so a worker that leaves
+// without publishing its status would still wake the waiter and the caller would report
+// success over a partially filled result block.
+class SubmitExternalScanTasksTest : public RowIdStorageReaderTest {
+protected:
+ static constexpr size_t kTaskCount = 3;
+
+ static Status run_tasks(const std::function<Status(size_t)>& run_task) {
+ InlineScanScheduler scheduler;
+ std::counting_semaphore<> semaphore {kTaskCount};
+ return RowIdStorageReader::submit_external_scan_tasks(
+ &scheduler, semaphore, kTaskCount,
+ [](size_t idx) { return fmt::format("task-{}", idx); }, run_task);
+ }
+};
+
+TEST_F(SubmitExternalScanTasksTest, AllTasksSucceedingReturnsOk) {
+ size_t ran = 0;
+ EXPECT_TRUE(run_tasks([&](size_t) -> Status {
+ ++ran;
+ return Status::OK();
+ }).ok());
+ EXPECT_EQ(ran, kTaskCount);
+}
+
+TEST_F(SubmitExternalScanTasksTest, ReturnedErrorReachesTheCaller) {
+ const Status result = run_tasks([](size_t idx) -> Status {
+ return idx == kTaskCount - 1 ? Status::InternalError("scanner returned an error")
+ : Status::OK();
+ });
+ EXPECT_FALSE(result.ok());
+ EXPECT_NE(result.to_string().find("scanner returned an error"), std::string::npos);
+}
+
+TEST_F(SubmitExternalScanTasksTest, ThrownExceptionReachesTheCaller) {
+ // The last task is the interesting one: it is the completion that releases the
+ // waiter, so a status lost here is a status the caller never sees.
+ const Status result = run_tasks([](size_t idx) -> Status {
+ if (idx == kTaskCount - 1) {
+ throw Exception(ErrorCode::INTERNAL_ERROR, "scanner threw");
+ }
+ return Status::OK();
+ });
+ EXPECT_FALSE(result.ok());
+ EXPECT_NE(result.to_string().find("scanner threw"), std::string::npos);
+}
+
} // namespace doris
diff --git a/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out b/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out
index 0d4e28fd9f0..c6533162a15 100644
--- a/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out
+++ b/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out
@@ -47,6 +47,54 @@
17.0 user17 17 2
18.0 user18 18 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+user4 user4
+user5 user5
+user6 user6
+user7 user7
+user8 user8
+user9 user9
+user10 user10
+
+-- !dup_col_thrice --
+user1 user1 user1
+user2 user2 user2
+user3 user3 user3
+user4 user4 user4
+user5 user5 user5
+user6 user6 user6
+user7 user7 user7
+user8 user8 user8
+user9 user9 user9
+user10 user10 user10
+
+-- !dup_col_mixed --
+1 user1 0.5 user1
+2 user2 1 user2
+3 user3 1.5 user3
+4 user4 2 user4
+5 user5 2.5 user5
+6 user6 3 user6
+7 user7 3.5 user7
+8 user8 4 user8
+9 user9 4.5 user9
+10 user10 5 user10
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+3 3
+3.5 3.5
+4 4
+4.5 4.5
+5 5
+
-- !test_basic --
1 user1 1.0 false 0.5 1
2 user2 2.0 true 1.0 1
@@ -293,6 +341,54 @@
17.0 user17 17 2
18.0 user18 18 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+user4 user4
+user5 user5
+user6 user6
+user7 user7
+user8 user8
+user9 user9
+user10 user10
+
+-- !dup_col_thrice --
+user1 user1 user1
+user2 user2 user2
+user3 user3 user3
+user4 user4 user4
+user5 user5 user5
+user6 user6 user6
+user7 user7 user7
+user8 user8 user8
+user9 user9 user9
+user10 user10 user10
+
+-- !dup_col_mixed --
+1 user1 0.5 user1
+2 user2 1 user2
+3 user3 1.5 user3
+4 user4 2 user4
+5 user5 2.5 user5
+6 user6 3 user6
+7 user7 3.5 user7
+8 user8 4 user8
+9 user9 4.5 user9
+10 user10 5 user10
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+3 3
+3.5 3.5
+4 4
+4.5 4.5
+5 5
+
-- !test_basic --
1 user1 1.0 false 0.5 1
2 user2 2.0 true 1.0 1
@@ -721,6 +817,11 @@
1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1
1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1
+-- !test_join_dup_cols --
+user1 user1 1 1
+user2 user2 2 2
+user3 user3 3 3
+
-- !test_join1 --
2 user2 2.0 1.0
4 user4 4.0 2.0
@@ -966,6 +1067,16 @@
1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1
1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1
+-- !test_join_dup_cols --
+user1 user1 1 1
+user2 user2 2 2
+user3 user3 3 3
+user4 user4 4 4
+user5 user5 5 5
+user6 user6 6 6
+user7 user7 7 7
+user8 user8 8 8
+
-- !test_join1 --
2 user2 2.0 1.0
4 user4 4.0 2.0
@@ -1240,6 +1351,35 @@
1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1
1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1
+-- !test_join_dup_cols --
+user1 user1 1 1
+user2 user2 2 2
+user3 user3 3 3
+user4 user4 4 4
+user5 user5 5 5
+user6 user6 6 6
+user7 user7 7 7
+user8 user8 8 8
+user9 user9 9 9
+user10 user10 10 10
+user11 user11 11 11
+user12 user12 12 12
+user13 user13 13 13
+user14 user14 14 14
+user15 user15 15 15
+user16 user16 16 16
+user17 user17 17 17
+user18 user18 18 18
+user19 user19 19 19
+user20 user20 20 20
+user21 user21 21 21
+user22 user22 22 22
+user23 user23 23 23
+user24 user24 24 24
+user25 user25 25 25
+user26 user26 26 26
+user27 user27 27 27
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -1301,6 +1441,31 @@ text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -1407,6 +1572,56 @@ text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -1597,6 +1812,123 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+{"a":9, "b":[9, 10]} {"a":9, "b":[9, 10]}
+{"a":10, "b":null} {"a":10, "b":null}
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]}
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]}
+{"a":13, "b":null} {"a":13, "b":null}
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]}
+{"a":null, "b":null} {"a":null, "b":null}
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]}
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]}
+{"a":18, "b":null} {"a":18, "b":null}
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":21, "b":null} {"a":21, "b":null}
+{"a":22, "b":[4, 5]} {"a":22, "b":[4, 5]}
+{"a":null, "b":[6, 7]} {"a":null, "b":[6, 7]}
+{"a":24, "b":null} {"a":24, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null}
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {12:["p", null], 13:["k1", "k2"], 14:null}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {15:["z", null], 16:["y1", "y2"], 17:null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {17:["t", null], 18:["u1", "u2"], 19:null}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+text_9 text_9
+text_10 text_10
+text_11 text_11
+text_12 text_12
+text_13 text_13
+text_14 text_14
+text_15 text_15
+text_16 text_16
+text_17 text_17
+text_18 text_18
+text_19 text_19
+text_20 text_20
+text_21 text_21
+text_22 text_22
+text_23 text_23
+text_24 text_24
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+9 {"a":9, "b":[9, 10]} text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} text_9
+10 {"a":10, "b":null} text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} text_10
+11 {"a":11, "b":[11, 12]} text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} text_11
+12 {"a":null, "b":[12, 13]} text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} text_12
+13 {"a":13, "b":null} text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} text_13
+14 {"a":14, "b":[14, 15]} text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} text_14
+15 {"a":null, "b":null} text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} text_15
+16 {"a":16, "b":[16, 17]} text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} text_16
+17 {"a":null, "b":[17, 18]} text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} text_17
+18 {"a":18, "b":null} text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} text_18
+19 {"a":19, "b":[1, null]} text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} text_19
+20 {"a":null, "b":[2, 3]} text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} text_20
+21 {"a":21, "b":null} text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} text_21
+22 {"a":22, "b":[4, 5]} text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} text_22
+23 {"a":null, "b":[6, 7]} text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} text_23
+24 {"a":24, "b":null} text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} text_24
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} text_17
+{"a":18, "b":null} {"a":18, "b":null} text_18
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} text_19
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -1658,6 +1990,31 @@ text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -1764,6 +2121,56 @@ text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -1954,6 +2361,123 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+{"a":9, "b":[9, 10]} {"a":9, "b":[9, 10]}
+{"a":10, "b":null} {"a":10, "b":null}
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]}
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]}
+{"a":13, "b":null} {"a":13, "b":null}
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]}
+{"a":null, "b":null} {"a":null, "b":null}
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]}
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]}
+{"a":18, "b":null} {"a":18, "b":null}
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":21, "b":null} {"a":21, "b":null}
+{"a":22, "b":[4, 5]} {"a":22, "b":[4, 5]}
+{"a":null, "b":[6, 7]} {"a":null, "b":[6, 7]}
+{"a":24, "b":null} {"a":24, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null}
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {12:["p", null], 13:["k1", "k2"], 14:null}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {15:["z", null], 16:["y1", "y2"], 17:null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {17:["t", null], 18:["u1", "u2"], 19:null}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+text_9 text_9
+text_10 text_10
+text_11 text_11
+text_12 text_12
+text_13 text_13
+text_14 text_14
+text_15 text_15
+text_16 text_16
+text_17 text_17
+text_18 text_18
+text_19 text_19
+text_20 text_20
+text_21 text_21
+text_22 text_22
+text_23 text_23
+text_24 text_24
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+9 {"a":9, "b":[9, 10]} text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} text_9
+10 {"a":10, "b":null} text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} text_10
+11 {"a":11, "b":[11, 12]} text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} text_11
+12 {"a":null, "b":[12, 13]} text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} text_12
+13 {"a":13, "b":null} text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} text_13
+14 {"a":14, "b":[14, 15]} text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} text_14
+15 {"a":null, "b":null} text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} text_15
+16 {"a":16, "b":[16, 17]} text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} text_16
+17 {"a":null, "b":[17, 18]} text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} text_17
+18 {"a":18, "b":null} text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} text_18
+19 {"a":19, "b":[1, null]} text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} text_19
+20 {"a":null, "b":[2, 3]} text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} text_20
+21 {"a":21, "b":null} text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} text_21
+22 {"a":22, "b":[4, 5]} text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} text_22
+23 {"a":null, "b":[6, 7]} text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} text_23
+24 {"a":24, "b":null} text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} text_24
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} text_17
+{"a":18, "b":null} {"a":18, "b":null} text_18
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} text_19
+
-- !1 --
1 user1 1.0 false 0.5 1
2 user2 2.0 true 1.0 1
@@ -2002,6 +2526,54 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
17.0 user17 17 2
18.0 user18 18 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+user4 user4
+user5 user5
+user6 user6
+user7 user7
+user8 user8
+user9 user9
+user10 user10
+
+-- !dup_col_thrice --
+user1 user1 user1
+user2 user2 user2
+user3 user3 user3
+user4 user4 user4
+user5 user5 user5
+user6 user6 user6
+user7 user7 user7
+user8 user8 user8
+user9 user9 user9
+user10 user10 user10
+
+-- !dup_col_mixed --
+1 user1 0.5 user1
+2 user2 1 user2
+3 user3 1.5 user3
+4 user4 2 user4
+5 user5 2.5 user5
+6 user6 3 user6
+7 user7 3.5 user7
+8 user8 4 user8
+9 user9 4.5 user9
+10 user10 5 user10
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+3 3
+3.5 3.5
+4 4
+4.5 4.5
+5 5
+
-- !test_basic --
1 user1 1.0 false 0.5 1
2 user2 2.0 true 1.0 1
@@ -2248,6 +2820,54 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
17.0 user17 17 2
18.0 user18 18 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+user4 user4
+user5 user5
+user6 user6
+user7 user7
+user8 user8
+user9 user9
+user10 user10
+
+-- !dup_col_thrice --
+user1 user1 user1
+user2 user2 user2
+user3 user3 user3
+user4 user4 user4
+user5 user5 user5
+user6 user6 user6
+user7 user7 user7
+user8 user8 user8
+user9 user9 user9
+user10 user10 user10
+
+-- !dup_col_mixed --
+1 user1 0.5 user1
+2 user2 1 user2
+3 user3 1.5 user3
+4 user4 2 user4
+5 user5 2.5 user5
+6 user6 3 user6
+7 user7 3.5 user7
+8 user8 4 user8
+9 user9 4.5 user9
+10 user10 5 user10
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+3 3
+3.5 3.5
+4 4
+4.5 4.5
+5 5
+
-- !test_basic --
1 user1 1.0 false 0.5 1
2 user2 2.0 true 1.0 1
@@ -2676,6 +3296,11 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1
1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1
+-- !test_join_dup_cols --
+user1 user1 1 1
+user2 user2 2 2
+user3 user3 3 3
+
-- !test_join1 --
2 user2 2.0 1.0
4 user4 4.0 2.0
@@ -2921,6 +3546,16 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1
1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1
+-- !test_join_dup_cols --
+user1 user1 1 1
+user2 user2 2 2
+user3 user3 3 3
+user4 user4 4 4
+user5 user5 5 5
+user6 user6 6 6
+user7 user7 7 7
+user8 user8 8 8
+
-- !test_join1 --
2 user2 2.0 1.0
4 user4 4.0 2.0
@@ -3195,6 +3830,35 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1
1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1
+-- !test_join_dup_cols --
+user1 user1 1 1
+user2 user2 2 2
+user3 user3 3 3
+user4 user4 4 4
+user5 user5 5 5
+user6 user6 6 6
+user7 user7 7 7
+user8 user8 8 8
+user9 user9 9 9
+user10 user10 10 10
+user11 user11 11 11
+user12 user12 12 12
+user13 user13 13 13
+user14 user14 14 14
+user15 user15 15 15
+user16 user16 16 16
+user17 user17 17 17
+user18 user18 18 18
+user19 user19 19 19
+user20 user20 20 20
+user21 user21 21 21
+user22 user22 22 22
+user23 user23 23 23
+user24 user24 24 24
+user25 user25 25 25
+user26 user26 26 26
+user27 user27 27 27
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -3256,6 +3920,31 @@ text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -3362,6 +4051,56 @@ text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -3552,6 +4291,123 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+{"a":9, "b":[9, 10]} {"a":9, "b":[9, 10]}
+{"a":10, "b":null} {"a":10, "b":null}
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]}
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]}
+{"a":13, "b":null} {"a":13, "b":null}
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]}
+{"a":null, "b":null} {"a":null, "b":null}
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]}
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]}
+{"a":18, "b":null} {"a":18, "b":null}
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":21, "b":null} {"a":21, "b":null}
+{"a":22, "b":[4, 5]} {"a":22, "b":[4, 5]}
+{"a":null, "b":[6, 7]} {"a":null, "b":[6, 7]}
+{"a":24, "b":null} {"a":24, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null}
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {12:["p", null], 13:["k1", "k2"], 14:null}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {15:["z", null], 16:["y1", "y2"], 17:null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {17:["t", null], 18:["u1", "u2"], 19:null}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+text_9 text_9
+text_10 text_10
+text_11 text_11
+text_12 text_12
+text_13 text_13
+text_14 text_14
+text_15 text_15
+text_16 text_16
+text_17 text_17
+text_18 text_18
+text_19 text_19
+text_20 text_20
+text_21 text_21
+text_22 text_22
+text_23 text_23
+text_24 text_24
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+9 {"a":9, "b":[9, 10]} text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} text_9
+10 {"a":10, "b":null} text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} text_10
+11 {"a":11, "b":[11, 12]} text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} text_11
+12 {"a":null, "b":[12, 13]} text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} text_12
+13 {"a":13, "b":null} text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} text_13
+14 {"a":14, "b":[14, 15]} text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} text_14
+15 {"a":null, "b":null} text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} text_15
+16 {"a":16, "b":[16, 17]} text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} text_16
+17 {"a":null, "b":[17, 18]} text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} text_17
+18 {"a":18, "b":null} text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} text_18
+19 {"a":19, "b":[1, null]} text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} text_19
+20 {"a":null, "b":[2, 3]} text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} text_20
+21 {"a":21, "b":null} text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} text_21
+22 {"a":22, "b":[4, 5]} text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} text_22
+23 {"a":null, "b":[6, 7]} text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} text_23
+24 {"a":24, "b":null} text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} text_24
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} text_17
+{"a":18, "b":null} {"a":18, "b":null} text_18
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} text_19
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -3613,6 +4469,31 @@ text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -3719,6 +4600,56 @@ text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+
-- !complex_1 --
1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
@@ -3908,3 +4839,119 @@ text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+-- !complex_dup_struct --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":3, "b":null} {"a":3, "b":null}
+{"a":4, "b":[4, 5]} {"a":4, "b":[4, 5]}
+{"a":null, "b":[5, 6]} {"a":null, "b":[5, 6]}
+{"a":6, "b":null} {"a":6, "b":null}
+{"a":7, "b":[7, 8]} {"a":7, "b":[7, 8]}
+{"a":null, "b":[8, 9]} {"a":null, "b":[8, 9]}
+{"a":9, "b":[9, 10]} {"a":9, "b":[9, 10]}
+{"a":10, "b":null} {"a":10, "b":null}
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]}
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]}
+{"a":13, "b":null} {"a":13, "b":null}
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]}
+{"a":null, "b":null} {"a":null, "b":null}
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]}
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]}
+{"a":18, "b":null} {"a":18, "b":null}
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]}
+{"a":null, "b":[2, 3]} {"a":null, "b":[2, 3]}
+{"a":21, "b":null} {"a":21, "b":null}
+{"a":22, "b":[4, 5]} {"a":22, "b":[4, 5]}
+{"a":null, "b":[6, 7]} {"a":null, "b":[6, 7]}
+{"a":24, "b":null} {"a":24, "b":null}
+
+-- !complex_dup_map --
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {2:["s1", null], 3:["v1", "v2"], 4:null}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {8:["c", null], 9:["d1", "d2"], 10:null}
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {12:["p", null], 13:["k1", "k2"], 14:null}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {15:["z", null], 16:["y1", "y2"], 17:null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {17:["t", null], 18:["u1", "u2"], 19:null}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_dup_string --
+text_1 text_1
+text_2 text_2
+text_3 text_3
+text_4 text_4
+text_5 text_5
+text_6 text_6
+text_7 text_7
+text_8 text_8
+text_9 text_9
+text_10 text_10
+text_11 text_11
+text_12 text_12
+text_13 text_13
+text_14 text_14
+text_15 text_15
+text_16 text_16
+text_17 text_17
+text_18 text_18
+text_19 text_19
+text_20 text_20
+text_21 text_21
+text_22 text_22
+text_23 text_23
+text_24 text_24
+
+-- !complex_dup_mixed --
+1 {"a":1, "b":[1, 2]} text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} text_1
+2 {"a":null, "b":[2, 3]} text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} text_2
+3 {"a":3, "b":null} text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} text_3
+4 {"a":4, "b":[4, 5]} text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} text_4
+5 {"a":null, "b":[5, 6]} text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} text_5
+6 {"a":6, "b":null} text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} text_6
+7 {"a":7, "b":[7, 8]} text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} text_7
+8 {"a":null, "b":[8, 9]} text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+9 {"a":9, "b":[9, 10]} text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} text_9
+10 {"a":10, "b":null} text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} text_10
+11 {"a":11, "b":[11, 12]} text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} text_11
+12 {"a":null, "b":[12, 13]} text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} text_12
+13 {"a":13, "b":null} text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} text_13
+14 {"a":14, "b":[14, 15]} text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} text_14
+15 {"a":null, "b":null} text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} text_15
+16 {"a":16, "b":[16, 17]} text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} text_16
+17 {"a":null, "b":[17, 18]} text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} text_17
+18 {"a":18, "b":null} text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} text_18
+19 {"a":19, "b":[1, null]} text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} text_19
+20 {"a":null, "b":[2, 3]} text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} text_20
+21 {"a":21, "b":null} text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} text_21
+22 {"a":22, "b":[4, 5]} text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} text_22
+23 {"a":null, "b":[6, 7]} text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} text_23
+24 {"a":24, "b":null} text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} text_24
+
+-- !complex_dup_pred --
+{"a":1, "b":[1, 2]} {"a":1, "b":[1, 2]} text_1
+{"a":10, "b":null} {"a":10, "b":null} text_10
+{"a":11, "b":[11, 12]} {"a":11, "b":[11, 12]} text_11
+{"a":null, "b":[12, 13]} {"a":null, "b":[12, 13]} text_12
+{"a":13, "b":null} {"a":13, "b":null} text_13
+{"a":14, "b":[14, 15]} {"a":14, "b":[14, 15]} text_14
+{"a":null, "b":null} {"a":null, "b":null} text_15
+{"a":16, "b":[16, 17]} {"a":16, "b":[16, 17]} text_16
+{"a":null, "b":[17, 18]} {"a":null, "b":[17, 18]} text_17
+{"a":18, "b":null} {"a":18, "b":null} text_18
+{"a":19, "b":[1, null]} {"a":19, "b":[1, null]} text_19
diff --git a/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out b/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out
index a104af3c757..f9647a18b3e 100644
--- a/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out
+++ b/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out
@@ -21,6 +21,24 @@
1.0 user1 1
2.0 user2 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+
+-- !dup_col_mixed --
+1 user1 0.5 user1 0.5
+2 user2 1 user2 1
+3 user3 1.5 user3 1.5
+4 user4 2 user4 2
+
-- !test_basic --
1 user1 1.0 false 0.5
2 user2 2.0 true 1.0
@@ -111,6 +129,24 @@
1.0 user1 1
2.0 user2 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+
+-- !dup_col_mixed --
+1 user1 0.5 user1 0.5
+2 user2 1 user2 1
+3 user3 1.5 user3 1.5
+4 user4 2 user4 2
+
-- !test_basic --
1 user1 1.0 false 0.5
2 user2 2.0 true 1.0
@@ -235,6 +271,24 @@
1.0 user1 1
2.0 user2 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+
+-- !dup_col_mixed --
+1 user1 0.5 user1 0.5
+2 user2 1 user2 1
+3 user3 1.5 user3 1.5
+4 user4 2 user4 2
+
-- !test_basic --
1 user1 1.0 false 0.5
2 user2 2.0 true 1.0
@@ -325,6 +379,24 @@
1.0 user1 1
2.0 user2 2
+-- !dup_col_twice --
+user1 user1
+user2 user2
+user3 user3
+
+-- !dup_col_nullable --
+0.5 0.5
+1 1
+1.5 1.5
+2 2
+2.5 2.5
+
+-- !dup_col_mixed --
+1 user1 0.5 user1 0.5
+2 user2 1 user2 1
+3 user3 1.5 user3 1.5
+4 user4 2 user4 2
+
-- !test_basic --
1 user1 1.0 false 0.5
2 user2 2.0 true 1.0
diff --git a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy
index 4eb1abf8675..2aaf7cbe8de 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy
@@ -34,6 +34,15 @@ suite("test_hive_topn_lazy_mat", "p0,external,hive,external_docker,external_dock
qt_3 """ select score, value, active,name from ${table} order by id,file_id limit 10; """
qt_4 """ select value,name,id,file_id from ${table} order by name limit 10; """
+ // Duplicate projected column - same column twice (the core bug from rowid_fetcher fix)
+ qt_dup_col_twice """ select name a, name b from ${table} order by id limit 10; """
+ // Duplicate projected column - same column three times
+ qt_dup_col_thrice """ select name a, name b, name c from ${table} order by id limit 10; """
+ // Duplicate column mixed with other columns
+ qt_dup_col_mixed """ select id, name a, score, name b from ${table} order by id limit 10; """
+ // Duplicate nullable column
+ qt_dup_col_nullable """ select score x, score y from ${table} order by id limit 10; """
+
for (int limit : limitValues) {
// Basic query
@@ -151,10 +160,18 @@ suite("test_hive_topn_lazy_mat", "p0,external,hive,external_docker,external_dock
"""
qt_test_join6 """
- select * from parquet_topn_lazy_mat_table as a join orc_topn_lazy_mat_table as b
+ select * from parquet_topn_lazy_mat_table as a join orc_topn_lazy_mat_table as b
where a.file_id = 1 and b.file_id = 1 and a.id = 1
order by a.id,b.id limit 100;
"""
+
+ // Duplicate columns from both sides of join
+ qt_test_join_dup_cols """
+ select a.name, a.name, b.id, b.id
+ from parquet_topn_lazy_mat_table as a
+ join orc_topn_lazy_mat_table as b on a.id = b.id
+ order by a.id limit ${limit};
+ """
}
for (String table : ["parquet_topn_lazy_complex_table", "parquet_topn_lazy_complex_table_multi_pages"]) {
@@ -175,6 +192,15 @@ suite("test_hive_topn_lazy_mat", "p0,external,hive,external_docker,external_dock
qt_complex_12 """ select * from ${table} where id%2 = 0 order by id limit ${limit}; """
qt_complex_13 """ select * from ${table} where id%2 = 1 order by id limit ${limit}; """
+
+ // Duplicate projected columns over nested types. The scalar cases above only
+ // cover text/double/boolean; a STRUCT or MAP column reaches the row-id fetch as a
+ // different column shape, so dedup and the result remap need coverage here too.
+ qt_complex_dup_struct """ select col2 x, col2 y from ${table} order by id limit ${limit}; """
+ qt_complex_dup_map """ select col3 m, col3 n from ${table} order by id limit ${limit}; """
+ qt_complex_dup_string """ select col1 a, col1 b from ${table} order by id limit ${limit}; """
+ qt_complex_dup_mixed """ select id, col2 a, col1 b, col2 c, col3 d, col1 e from ${table} order by id limit ${limit}; """
+ qt_complex_dup_pred """ select col2 x, col2 y, col1 z from ${table} where col1 like 'text_1%' order by id limit ${limit}; """
}
}
}
@@ -232,6 +258,28 @@ suite("test_hive_topn_lazy_mat", "p0,external,hive,external_docker,external_dock
contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]")
}
+ // The duplicate-column regressions below only prove the fix if their queries really
+ // enter phase-2 materialization. Pin the plan: two result slots must point at one
+ // physical column index, which is the input that used to drop a column.
+ explain {
+ sql "select name a, name b from orc_topn_lazy_mat_table order by id limit 10;"
+ contains("VMaterializeNode")
+ contains("projectList:[a, b]")
+ contains("column_descs_lists[[`name` text NULL, `name` text NULL]]")
+ contains("locations: [[1, 2]]")
+ contains("column_idxs_lists: [[1, 1]]")
+ contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]")
+ }
+
+ explain {
+ sql "select col2 x, col2 y from parquet_topn_lazy_complex_table order by id limit 10;"
+ contains("VMaterializeNode")
+ contains("column_descs_lists[[`col2` struct<a:int,b:array<int>> NULL, `col2` struct<a:int,b:array<int>> NULL]]")
+ contains("locations: [[1, 2]]")
+ contains("column_idxs_lists: [[2, 2]]")
+ contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__parquet_topn_lazy_complex_table]")
+ }
+
explain {
sql """ select a.name,length(a.name),a.value,b.*,a.* from parquet_topn_lazy_mat_table as a
join orc_topn_lazy_mat_table as b on a.id = b.id order by a.name limit 10 """
diff --git a/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy
index abc920df359..41f5d235c13 100644
--- a/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy
@@ -49,6 +49,13 @@ suite("test_tvf_topn_lazy_mat","external,hive,tvf,external_docker") {
qt_3 """ select score, value, active,name from ${table} order by value limit 4; """
qt_4 """ select value,name,id from ${table} order by name,score limit 2; """
+ // Duplicate projected columns: the TVF path shares read_batch_external_row() with the
+ // catalog path, so the same physical column projected twice must be fetched once and
+ // copied into both result columns.
+ qt_dup_col_twice """ select name a, name b from ${table} order by id limit 3; """
+ qt_dup_col_nullable """ select score x, score y from ${table} order by id limit 5; """
+ qt_dup_col_mixed """ select id, name a, score, name b, score c from ${table} order by id limit 4; """
+
for (int limit : limitValues) {
// Basic query
qt_test_basic """
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/28: [improvement](lance) Upgrade prewarm dependency and remove install guard (branch-4.1) (#68698)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5acff117d6e6bda4996b09b151480fbae3ffad9e
Author: Gabriel <li...@selectdb.com>
AuthorDate: Thu Oct 8 14:23:22 2026 +0800
[improvement](lance) Upgrade prewarm dependency and remove install guard (branch-4.1) (#68698)
### What problem does this PR solve?
Related issue: #68692 (M1)
Related PRs: lance-format/lance-c#94, #68687
Upgrade lance-c from `cd63420bfbe27f6f0a1edcc873b9191af7d52852` to
`98468344bc9d56aa7ed4a4192e844afa62b34ba1` to include the upstream
synchronous index-prewarm C/C++ APIs. Refresh the retained Foyer patch
without changing its added/removed implementation payload, and verify
that patch application preserves the prewarm APIs and shared-session
regression.
Remove the Lance-specific installation fingerprint mechanism introduced
by #68687: delete `lance-install.sh` and its test harness, remove the CI
harness step, and restore the existing dependency build flow. An
existing installation is no longer rejected or rebuilt because of a
Lance revision fingerprint; missing APIs are reported by the normal
compilation or linking process.
This delivers the dependency portion of M1. SQL parsing, BE RPCs, and
shared-session integration remain outside this PR; the parent issue
stays open.
### Validation
After rebasing onto the current `branch-4.1`:
- Lance archive checksum verified; the Foyer patch's added/removed
payload matches the target branch.
- `lance-prefilter-patch-test.sh`: passed fresh extraction,
cached-source reuse, re-extraction, stale-marker refresh, invalid-patch
rejection, and prewarm API/regression preservation checks.
- `download-thirdparty-fallback-test.sh` and `adbc-jni-config-test.sh`:
passed.
- Affected shell syntax, workflow YAML parsing, and `git diff --check`:
passed.
- Confirmed that the removed installation scripts and fingerprint
symbols have no remaining references, and the build/CI scripts match
their versions before the fingerprint mechanism was introduced.
No FE/BE source files change. Full Doris compilation and runtime suites
were not rerun locally and remain for PR CI.
### Release note
Prepare the Lance dependency for index prewarm and remove Lance-specific
installation fingerprint checks. No new SQL interface is exposed.
### Check List (For Author)
- Test:
- [x] Existing dependency/script regression tests
- [x] Manual validation described above
- Behavior changed:
- [x] Yes. Upgrade lance-c and restore the existing dependency build
flow without Lance revision fingerprint checks.
- Does this need documentation?
- [x] No new SQL interface; upstream API documentation is included in
the dependency.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.github/workflows/build-thirdparty.yml | 3 -
build.sh | 25 +---
thirdparty/build-thirdparty.sh | 7 -
thirdparty/lance-install.sh | 38 ------
thirdparty/patches/lance-c-foyer.patch | 22 ++--
thirdparty/test/lance-install-test.sh | 176 --------------------------
thirdparty/test/lance-prefilter-patch-test.sh | 7 +
thirdparty/vars.sh | 10 +-
8 files changed, 27 insertions(+), 261 deletions(-)
diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml
index df6339c8ab8..3d3d11ff27d 100644
--- a/.github/workflows/build-thirdparty.yml
+++ b/.github/workflows/build-thirdparty.yml
@@ -80,9 +80,6 @@ jobs:
run: |
thirdparty/test/adbc-jni-config-test.sh
- - name: Test Lance installation
- run: bash thirdparty/test/lance-install-test.sh
-
build_linux:
name: Build Third Party Libraries (Linux)
needs: changes
diff --git a/build.sh b/build.sh
index 5eac08e4078..8449eab271a 100755
--- a/build.sh
+++ b/build.sh
@@ -441,7 +441,6 @@ if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && ${BUILD_
fi
# build thirdparty libraries if necessary. check last thirdparty lib installation
-source "${DORIS_HOME}/thirdparty/lance-install.sh"
if [[ "${TARGET_SYSTEM}" == 'Darwin' ]]; then
LAST_THIRDPARTY_LIB='libbrotlienc.a'
else
@@ -457,21 +456,10 @@ if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ||
! -f "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ||
! -s "${DORIS_THIRDPARTY}/installed/lib64/libpaimon_c.a" ||
! -s "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ||
- -e "${DORIS_THIRDPARTY}/installed/lib64/.paimon-installing" ]] ||
- ! lance_c_install_is_current "${DORIS_HOME}/thirdparty" "${DORIS_THIRDPARTY}/installed"; then
- # External trees can be partially updated or pinned to another revision. Preserve
- # the existing prefix unless their build inputs can produce the requested Lance version.
- for input in build-thirdparty.sh download-thirdparty.sh vars.sh lance-install.sh patches/lance-c-foyer.patch; do
- if [[ ! -f "${DORIS_THIRDPARTY}/${input}" || ! -r "${DORIS_THIRDPARTY}/${input}" ]]; then
- echo "Third-party dependencies require a rebuild, but ${input} is missing or unreadable." >&2
- echo "Refresh the compilation image or set DORIS_THIRDPARTY to a complete third-party source tree." >&2
- exit 1
- fi
- done
- if ! expected_lance_fingerprint="$(lance_c_install_fingerprint "${DORIS_HOME}/thirdparty")" ||
- ! rebuild_lance_fingerprint="$(lance_c_install_fingerprint "${DORIS_THIRDPARTY}")" ||
- [[ "${rebuild_lance_fingerprint}" != "${expected_lance_fingerprint}" ]]; then
- echo "Lance rebuild sources do not match this checkout; installed dependencies have been preserved." >&2
+ -e "${DORIS_THIRDPARTY}/installed/lib64/.paimon-installing" ]]; then
+ # Compilation images may contain only installed artifacts; never erase them without a rebuild source.
+ if [[ ! -f "${DORIS_THIRDPARTY}/build-thirdparty.sh" ]]; then
+ echo "Third-party dependencies require a rebuild, but build-thirdparty.sh is missing." >&2
echo "Refresh the compilation image or set DORIS_THIRDPARTY to a complete third-party source tree." >&2
exit 1
fi
@@ -484,11 +472,6 @@ if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ||
else
bash "${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}" --clean
fi
- # An external build script can itself be stale. Never link its old output silently.
- if ! lance_c_install_is_current "${DORIS_HOME}/thirdparty" "${DORIS_THIRDPARTY}/installed"; then
- echo "Lance dependency revision does not match this checkout. Refresh the third-party build tree." >&2
- exit 1
- fi
fi
update_submodule() {
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 89c69090e04..a5e209a0aed 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -156,8 +156,6 @@ if [[ ! -f "${TP_DIR}/vars.sh" ]]; then
fi
. "${TP_DIR}/vars.sh"
-. "${TP_DIR}/lance-install.sh"
-LANCE_C_INSTALL_FINGERPRINT="$(lance_c_install_fingerprint "${TP_DIR}")"
cd "${TP_DIR}"
@@ -2230,14 +2228,9 @@ build_lance_c() {
env "${cargo_env[@]}" "${cargo_bin}" "${cargo_args[@]}"
mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64"
- # Invalidate before publishing either file so interrupted installs cannot reuse
- # a matching marker with a partial header/archive pair.
- rm -f "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint"
rm -rf "${TP_INSTALL_DIR}/include/lance"
cp -av include/lance "${TP_INSTALL_DIR}/include/"
install_rust_archive "${BUILD_DIR}/release/liblance_c.a"
- printf '%s\n' "${LANCE_C_INSTALL_FINGERPRINT}" > "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint.tmp"
- mv "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint.tmp" "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint"
}
# paimon-rust
diff --git a/thirdparty/lance-install.sh b/thirdparty/lance-install.sh
deleted file mode 100644
index 28594d61c79..00000000000
--- a/thirdparty/lance-install.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Keep the check independent of the installed prefix: an external compilation image
-# can carry old vars.sh alongside an ABI-compatible but behaviorally stale archive.
-lance_c_install_fingerprint() (
- local definitions="$1"
- local TP_DIR="${definitions}"
- source "${definitions}/vars.sh" || return 1
- local patch_checksum
- patch_checksum="$(cksum < "${definitions}/patches/lance-c-foyer.patch")" || return 1
- printf '%s\n' "${LANCE_C_SOURCE}" "${LANCE_C_MD5SUM}" "${patch_checksum}"
-)
-
-lance_c_install_is_current() {
- local definitions="$1" installed="$2" expected
- [[ -s "${installed}/lib64/liblance_c.a" &&
- -s "${installed}/include/lance/lance.h" &&
- -s "${installed}/include/lance/lance.hpp" &&
- -s "${installed}/lib64/.lance-c-fingerprint" ]] || return 1
- expected="$(lance_c_install_fingerprint "${definitions}")" || return 1
- [[ "$(cat "${installed}/lib64/.lance-c-fingerprint")" == "${expected}" ]]
-}
diff --git a/thirdparty/patches/lance-c-foyer.patch b/thirdparty/patches/lance-c-foyer.patch
index bb2104ab32d..a67e2a04213 100644
--- a/thirdparty/patches/lance-c-foyer.patch
+++ b/thirdparty/patches/lance-c-foyer.patch
@@ -1,5 +1,5 @@
# Foyer data-cache integration for lance-format/lance-c#73.
-# Base: cd63420bfbe27f6f0a1edcc873b9191af7d52852 (lance-format/lance-c main, including #93)
+# Base: 98468344bc9d56aa7ed4a4192e844afa62b34ba1 (lance-format/lance-c main, including #94)
# Source: https://github.com/Gabriel39/lance-c/commit/24c7ca4bcb9422c113b0d3e07e4efe1173b0bc9f
# The unchanged Foyer payload is reapplied to the newer upstream base with git apply --3way.
diff --git a/Cargo.lock b/Cargo.lock
@@ -198,7 +198,7 @@ index 18654b862440a90aa6bf0dd846e8f1ad6036fd7c..fd134a7151e1df61da201d867a642e9b
prost = "0.14"
snafu = "0.9"
diff --git a/README.md b/README.md
-index 9f0871b17b13ebfd96916947b4bee0dd06f6a468..441719c9f3904058127e55d1cd29ea1afc878935 100644
+index f13ad015826a8ac5468cfea247707d987cc0864d..56422c22969882cd43edd5e83a4a5c4d0dadd3d6 100644
--- a/README.md
+++ b/README.md
@@ -68,6 +68,7 @@ Based on the [liblance RFC](https://github.com/lance-format/lance/discussions/60
@@ -242,11 +242,11 @@ index 9f0871b17b13ebfd96916947b4bee0dd06f6a468..441719c9f3904058127e55d1cd29ea1a
+auto ds = lance::Dataset::open_with_session(session, "s3://bucket/data.lance");
+```
+
- ### Open at a specific version
+ ### Prewarm an index synchronously
- `lance_dataset_open` takes a `version` argument — `0` means the latest, any
+ Prewarm a logical index before serving queries to move cold index reads out of
diff --git a/include/lance/lance.h b/include/lance/lance.h
-index 94772e2a14bf9a8a950dfb2d39112136e27197c0..fe48eb0d9afd1c7157a3bd2230cce7e769120bac 100644
+index b91c290a40d37edbb49323d2d11efd3b4f980bb8..0997023974a5205a145a991d53da0ee1a2babc8c 100644
--- a/include/lance/lance.h
+++ b/include/lance/lance.h
@@ -214,6 +214,36 @@ typedef struct LanceSessionCacheStats {
@@ -334,7 +334,7 @@ index 94772e2a14bf9a8a950dfb2d39112136e27197c0..fe48eb0d9afd1c7157a3bd2230cce7e7
void lance_dataset_close(LanceDataset* dataset);
diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp
-index edcd39fc3caba387ab490fa84139f24e9376130a..17b71957d0f5c419fe0b3392ce11faa757ddbfe1 100644
+index 17e93486c079d3453547af160b875310328da9e1..b049cc519ee9784d4bcc1a087eac237e880b5e74 100644
--- a/include/lance/lance.hpp
+++ b/include/lance/lance.hpp
@@ -176,6 +176,13 @@ struct SqlColumn {
@@ -2243,7 +2243,7 @@ index 1971510de4a13ee6f04fc39c159b088e78e21d55..ba51c87a8cebb6897c48fb55509f71da
// SAFETY: `out_dataset` is non-NULL (checked above) and the caller
// guarantees it points to caller-owned, writable storage of size
diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index d3bea44529a842fe3ae28ee3f90cd8d45fe0b394..98e6901a2cfeb2cc98df6606e26b04788d4218b2 100644
+index b00072df3f966f58663359ccd370b399650c2023..e44871e92bc7ff9787bf5220b1ea8a4270da9710 100644
--- a/tests/c_api_test.rs
+++ b/tests/c_api_test.rs
@@ -100,10 +100,83 @@ fn create_large_dataset(num_rows: i32) -> (tempfile::TempDir, String) {
@@ -2502,7 +2502,7 @@ index d3bea44529a842fe3ae28ee3f90cd8d45fe0b394..98e6901a2cfeb2cc98df6606e26b0478
fn test_dataset_restore_to_current_latest_writes_new_manifest() {
// Restoring to the current latest still writes a new manifest. The
diff --git a/tests/cpp/test_c_api.c b/tests/cpp/test_c_api.c
-index efad5ed55987976a3f1820220e42fc58bef42638..050543aebc02454cab09f6e0bd38ec10b1d21720 100644
+index 77af1a69b872bd4152a44e642f30190720daa33f..da15749671cb9448c73d3c61546ecab2adcacb15 100644
--- a/tests/cpp/test_c_api.c
+++ b/tests/cpp/test_c_api.c
@@ -171,6 +171,37 @@ static void test_shared_session(const char *uri) {
@@ -2543,7 +2543,7 @@ index efad5ed55987976a3f1820220e42fc58bef42638..050543aebc02454cab09f6e0bd38ec10
static void test_scan(const char *uri) {
printf(" test_scan... ");
-@@ -1471,6 +1502,7 @@ int main(int argc, char **argv) {
+@@ -1476,6 +1507,7 @@ int main(int argc, char **argv) {
test_batch_nearest(uri);
test_open_and_metadata(uri);
test_shared_session(uri);
@@ -2552,7 +2552,7 @@ index efad5ed55987976a3f1820220e42fc58bef42638..050543aebc02454cab09f6e0bd38ec10
test_distance_range(uri);
test_scan_with_limit(uri);
diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp
-index e2d541e4430889aba89ab85c01cc7e3eacf97baa..ac1c10bccfcf399bbbfcfe908d3188484aff0add 100644
+index 3bc313ebecbb0de06badd9f8d75f531fc95809c9..298228f7dd68d28452e6436db5c589183361f2de 100644
--- a/tests/cpp/test_cpp_api.cpp
+++ b/tests/cpp/test_cpp_api.cpp
@@ -131,6 +131,28 @@ static void test_shared_session(const std::string& uri) {
@@ -2584,7 +2584,7 @@ index e2d541e4430889aba89ab85c01cc7e3eacf97baa..ac1c10bccfcf399bbbfcfe908d318848
static void test_dataset_schema(const std::string& uri) {
TEST(test_dataset_schema);
-@@ -1353,6 +1375,7 @@ int main(int argc, char** argv) {
+@@ -1365,6 +1387,7 @@ int main(int argc, char** argv) {
test_dataset_open(uri);
test_shared_session(uri);
diff --git a/thirdparty/test/lance-install-test.sh b/thirdparty/test/lance-install-test.sh
deleted file mode 100644
index db93c355c0f..00000000000
--- a/thirdparty/test/lance-install-test.sh
+++ /dev/null
@@ -1,176 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -euo pipefail
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-work="$(mktemp -d)"
-trap 'rm -rf "${work}"' EXIT
-mkdir -p "${work}/repo/thirdparty/patches" "${work}/external"
-cp "${ROOT}/thirdparty/vars.sh" "${work}/repo/thirdparty/"
-cp "${ROOT}/thirdparty/patches/lance-c-foyer.patch" "${work}/repo/thirdparty/patches/"
-# Extract the real build gate; all destructive operations stay inside this temporary install.
-echo 'set -eo pipefail' > "${work}/gate.sh"
-sed -n '/^# build thirdparty libraries if necessary/,/^update_submodule()/p' "${ROOT}/build.sh" \
- | sed '$d' >> "${work}/gate.sh"
-export DORIS_HOME="${work}/repo" DORIS_THIRDPARTY="${work}/external"
-export TARGET_SYSTEM=Linux CLEAN=0 PARALLEL=1
-export TEST_HELPER="${ROOT}/thirdparty/lance-install.sh"
-cp "${TEST_HELPER}" "${work}/repo/thirdparty/"
-cp -r "${DORIS_HOME}/thirdparty/." "${DORIS_THIRDPARTY}/"
-printf '#!/usr/bin/env bash\nexit 0\n' > "${DORIS_THIRDPARTY}/download-thirdparty.sh"
-cat > "${DORIS_THIRDPARTY}/build-thirdparty.sh" <<'BUILDER'
-set -euo pipefail
-TP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source "${TP_DIR}/lance-install.sh"
-fingerprint="$(lance_c_install_fingerprint "${TP_DIR}")"
-echo rebuilt >> "${DORIS_THIRDPARTY}/builds"
-installed="${DORIS_THIRDPARTY}/installed"
-mkdir -p "${installed}/lib/hadoop_hdfs/native" "${installed}/lib/hadoop_hdfs_3_4/native" "${installed}/lib64" \
- "${installed}/include/lance" "${installed}/include/paimon_rust"
-for file in lib/hadoop_hdfs/native/libhdfs.a lib/hadoop_hdfs_3_4/native/libhdfs.a \
- lib64/liblance_c.a lib64/libpaimon_c.a \
- include/lance/lance.h include/lance/lance.hpp include/paimon_rust/paimon.h; do
- echo artifact > "${installed}/${file}"
-done
-printf '%s\n' "${fingerprint}" > "${installed}/lib64/.lance-c-fingerprint"
-BUILDER
-# A complete legacy image has all old sentinels but no Lance revision marker.
-installed="${DORIS_THIRDPARTY}/installed"
-mkdir -p "${installed}/lib/hadoop_hdfs/native" "${installed}/lib/hadoop_hdfs_3_4/native" "${installed}/lib64" \
- "${installed}/include/lance" "${installed}/include/paimon_rust"
-for file in lib/hadoop_hdfs/native/libhdfs.a lib/hadoop_hdfs_3_4/native/libhdfs.a \
- lib64/liblance_c.a lib64/libpaimon_c.a \
- include/lance/lance.h include/lance/lance.hpp include/paimon_rust/paimon.h; do
- echo legacy > "${installed}/${file}"
-done
-bash "${work}/gate.sh"
-[[ -f "${DORIS_THIRDPARTY}/builds" ]] || { echo 'FAIL: reused unversioned Lance archive'; exit 1; }
-source "${TEST_HELPER}"
-lance_c_install_is_current "${DORIS_HOME}/thirdparty" "${installed}"
-check_builds() {
- [[ "$(wc -l < "${DORIS_THIRDPARTY}/builds")" -eq "$1" ]]
-}
-check_builds 1
-bash "${work}/gate.sh"
-check_builds 1
-echo 'PASS: legacy image rebuilt; matching install reused'
-# Failed preflight must preserve every dependency, not just the Lance archive.
-expect_preserved_install() {
- cp -a "${installed}" "${work}/saved-install"
- cp "${DORIS_THIRDPARTY}/builds" "${work}/saved-builds"
- if bash "${work}/gate.sh" > "${work}/preflight.log" 2>&1; then
- echo 'FAIL: accepted incomplete or mismatched rebuild sources'; exit 1
- fi
- if ! diff -r "${work}/saved-install" "${installed}"; then
- echo 'FAIL: rejected rebuild sources after changing installed dependencies'; exit 1
- fi
- cmp "${work}/saved-builds" "${DORIS_THIRDPARTY}/builds"
- rm -rf "${work}/saved-install"
-}
-echo stale > "${installed}/lib64/.lance-c-fingerprint"
-for file in lance-install.sh vars.sh patches/lance-c-foyer.patch download-thirdparty.sh build-thirdparty.sh; do
- mv "${DORIS_THIRDPARTY}/${file}" "${work}/missing-input"
- expect_preserved_install
- mv "${work}/missing-input" "${DORIS_THIRDPARTY}/${file}"
-done
-echo 'PASS: incomplete external sources rejected before changing installed dependencies'
-echo stale > "${installed}/lib64/.lance-c-fingerprint"
-bash "${work}/gate.sh"
-check_builds 2
-# Pin and patch updates both invalidate installed artifacts, even with identical ABI.
-sed 's/^LANCE_C_SOURCE=.*/LANCE_C_SOURCE="lance-c-test-revision"/' "${DORIS_HOME}/thirdparty/vars.sh" \
- > "${work}/new-vars.sh"
-mv "${work}/new-vars.sh" "${DORIS_HOME}/thirdparty/vars.sh"
-expect_preserved_install
-cp "${DORIS_HOME}/thirdparty/vars.sh" "${DORIS_THIRDPARTY}/vars.sh"
-bash "${work}/gate.sh"
-check_builds 3
-echo '# test patch update' >> "${DORIS_HOME}/thirdparty/patches/lance-c-foyer.patch"
-expect_preserved_install
-cp "${DORIS_HOME}/thirdparty/patches/lance-c-foyer.patch" "${DORIS_THIRDPARTY}/patches/"
-bash "${work}/gate.sh"
-check_builds 4
-echo 'PASS: mismatched sources rejected; synchronized pin and patch updates rebuild'
-rm "${installed}/include/lance/lance.h"
-bash "${work}/gate.sh"
-check_builds 5
-: > "${installed}/lib64/liblance_c.a"
-bash "${work}/gate.sh"
-check_builds 6
-echo 'PASS: incomplete header/archive install rebuilt'
-# A legacy external builder may exit successfully without installing the new revision.
-cp "${DORIS_THIRDPARTY}/build-thirdparty.sh" "${work}/good-builder.sh"
-printf '#!/usr/bin/env bash\nexit 0\n' > "${DORIS_THIRDPARTY}/build-thirdparty.sh"
-rm "${installed}/lib64/.lance-c-fingerprint"
-if bash "${work}/gate.sh" > "${work}/old-builder.log" 2>&1; then
- echo 'FAIL: accepted output from a stale external builder'; exit 1
-fi
-grep -q 'Lance dependency revision does not match' "${work}/old-builder.log"
-cp "${work}/good-builder.sh" "${DORIS_THIRDPARTY}/build-thirdparty.sh"
-bash "${DORIS_THIRDPARTY}/build-thirdparty.sh"
-echo 'PASS: stale external builder cannot silently satisfy the revision gate'
-# Images without rebuild sources must fail before deleting installed dependencies.
-rm "${DORIS_THIRDPARTY}/build-thirdparty.sh" "${installed}/lib64/.lance-c-fingerprint"
-if bash "${work}/gate.sh" > "${work}/missing-source.log" 2>&1; then
- echo 'FAIL: accepted stale compilation image without build sources'; exit 1
-fi
-[[ -s "${installed}/lib64/liblance_c.a" ]]
-echo 'PASS: missing rebuild source fails without deleting installed artifacts'
-# Execute the actual publication function with a tiny Cargo stand-in. A failed
-# archive copy must invalidate the old marker, and only a complete retry may stamp it.
-sed -n '/^build_lance_c()/,/^}/p' "${ROOT}/thirdparty/build-thirdparty.sh" > "${work}/publish-function.sh"
-export TP_DIR="${DORIS_HOME}/thirdparty"
-source "${TP_DIR}/vars.sh"
-export LANCE_C_SOURCE TP_SOURCE_DIR TP_INSTALL_DIR
-export LANCE_C_INSTALL_FINGERPRINT="$(lance_c_install_fingerprint "${TP_DIR}")"
-mkdir -p "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}/include/lance" "${TP_INSTALL_DIR}/bin"
-echo header > "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}/include/lance/lance.h"
-echo header > "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}/include/lance/lance.hpp"
-printf '#!/bin/sh\nexit 0\n' > "${TP_INSTALL_DIR}/bin/protoc"
-chmod +x "${TP_INSTALL_DIR}/bin/protoc"
-cat > "${work}/cargo" <<'CARGO'
-#!/usr/bin/env bash
-set -eu
-if [[ "$1" == --version ]]; then
- echo 'cargo 1.94.0'; exit 0
-fi
-mkdir -p "${CARGO_TARGET_DIR}/release"
-echo archive > "${CARGO_TARGET_DIR}/release/liblance_c.a"
-CARGO
-chmod +x "${work}/cargo"
-export LANCE_C_CARGO="${work}/cargo" RUSTUP_TOOLCHAIN=1.94.0
-export BUILD_DIR=build KERNEL=Linux LANCE_C_CARGO_OFFLINE=OFF
-cat > "${work}/publish.sh" <<'PUBLISH'
-set -eo pipefail
-check_if_source_exist() { :; }
-install_rust_archive() {
- if [[ "${FAIL_INSTALL:-0}" == 1 ]]; then return 1; fi
- cp "$1" "${TP_INSTALL_DIR}/lib64/liblance_c.a"
-}
-source "$1"
-build_lance_c
-PUBLISH
-bash "${work}/publish.sh" "${work}/publish-function.sh" > "${work}/publish.log" 2>&1
-lance_c_install_is_current "${TP_DIR}" "${TP_INSTALL_DIR}"
-if FAIL_INSTALL=1 bash "${work}/publish.sh" "${work}/publish-function.sh" >> "${work}/publish.log" 2>&1; then
- echo 'FAIL: expected archive publication failure'; exit 1
-fi
-[[ ! -e "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint" ]]
-bash "${work}/publish.sh" "${work}/publish-function.sh" >> "${work}/publish.log" 2>&1
-lance_c_install_is_current "${TP_DIR}" "${TP_INSTALL_DIR}"
-echo 'PASS: successful install stamped; failed publication invalidated; retry repaired'
diff --git a/thirdparty/test/lance-prefilter-patch-test.sh b/thirdparty/test/lance-prefilter-patch-test.sh
index e3dabff6c17..ab3587c56d1 100755
--- a/thirdparty/test/lance-prefilter-patch-test.sh
+++ b/thirdparty/test/lance-prefilter-patch-test.sh
@@ -68,6 +68,13 @@ check_sources() {
|| fail "missing retained cache statistics API"
grep -q 'source = "git+https://github.com/lance-format/lance.git' "${source}/Cargo.lock" \
|| fail "Lance must come from the upstream git dependency"
+ # Refreshing the Foyer patch must preserve the upstream prewarm binding and tests.
+ grep -q 'lance_dataset_prewarm_index' "${source}/include/lance/lance.h" \
+ || fail "missing upstream C prewarm API"
+ grep -q 'void prewarm_index(' "${source}/include/lance/lance.hpp" \
+ || fail "missing upstream C++ prewarm API"
+ grep -q 'fn test_prewarm_index_vector_segments_reuse_shared_session' "${source}/tests/c_api_test.rs" \
+ || fail "missing upstream shared-session prewarm regression"
[[ -f "${source}/patched_mark_foyer" ]] || fail "missing Foyer patch marker"
}
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 147e4e4b2c4..e77b7c450a7 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -580,11 +580,11 @@ PUGIXML_SOURCE=pugixml-1.15
PUGIXML_MD5SUM="3b894c29455eb33a40b165c6e2de5895"
# lance-c
-# Includes lance-c #93: scoped boolean scalar-index expressions and Substrait label filters.
-LANCE_C_DOWNLOAD="https://codeload.github.com/lance-format/lance-c/tar.gz/cd63420bfbe27f6f0a1edcc873b9191af7d52852"
-LANCE_C_NAME="lance-c-cd63420bfbe27f6f0a1edcc873b9191af7d52852.tar.gz"
-LANCE_C_SOURCE="lance-c-cd63420bfbe27f6f0a1edcc873b9191af7d52852"
-LANCE_C_MD5SUM="37d82907559c4fdb8ed6e5b767b0d309"
+# Includes lance-c #94: synchronous index prewarm through the upstream C/C++ APIs.
+LANCE_C_DOWNLOAD="https://codeload.github.com/lance-format/lance-c/tar.gz/98468344bc9d56aa7ed4a4192e844afa62b34ba1"
+LANCE_C_NAME="lance-c-98468344bc9d56aa7ed4a4192e844afa62b34ba1.tar.gz"
+LANCE_C_SOURCE="lance-c-98468344bc9d56aa7ed4a4192e844afa62b34ba1"
+LANCE_C_MD5SUM="b452bab5d05f5bce5f0bfebe8b1fb831"
# paimon-rust
PAIMON_RUST_DOWNLOAD="https://github.com/apache/paimon-rust/archive/refs/tags/v0.4.0-rc1.tar.gz"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 13/28: [improvement](metrics) Unify build version metric labels #68655 (#68656)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e2a64e268be8925ee37a607a2a86f1f8e2420e9f
Author: Gavin Chou <ga...@selectdb.com>
AuthorDate: Fri Oct 9 14:44:24 2026 +0800
[improvement](metrics) Unify build version metric labels #68655 (#68656)
### What problem does this PR solve?
Issue Number: None
Related PR: #68655
Problem Summary:
Backport #68655 to `branch-4.1`. FE already exposes its build identity
with `version`, `major`, `minor`, `patch`, `hotfix`, and `short_hash`
labels. BE and Cloud expose the same numeric version value without those
labels, even though the decimal concatenation is ambiguous for some
versions and does not include RC or revision information.
This change adds the same fixed label schema to the existing BE and
Cloud metrics while preserving all three metric names and their numeric
values. The version metrics still produce one series per process, so the
fixed build labels do not introduce runtime-dependent cardinality. The
regression case verifies that FE, BE, and Cloud expose identical build
metadata and values.
### Release note
Expose consistent build-version labels on FE, BE, and Cloud version
metrics.
### Check List (For Author)
- Test: Added regression coverage; ran `build-support/check-format.sh`,
`git diff --check`, and a focused C++ syntax check for the labeled bvar
construction.
- Behavior changed: Yes. Existing BE and Cloud version metric series now
include fixed build-version labels; metric names and numeric values are
unchanged.
- Does this need documentation: No.
---
be/src/common/config.cpp | 4 ++
be/src/common/config.h | 3 +
be/src/service/doris_main.cpp | 2 +
be/src/util/debug_util.cpp | 39 +++++++++++-
be/src/util/debug_util.h | 4 ++
be/test/util/debug_util_test.cpp | 63 +++++++++++++++++++
cloud/src/common/CMakeLists.txt | 1 +
cloud/src/common/version_metrics.cpp | 72 ++++++++++++++++++++++
cloud/src/common/version_metrics.h | 26 ++++++++
cloud/src/main.cpp | 16 +----
cloud/test/metric_test.cpp | 37 +++++++++++
.../java/org/apache/doris/metric/MetricsTest.java | 24 ++++++++
.../suites/metrics_p0/test_version_metrics.groovy | 51 +++++++++------
13 files changed, 308 insertions(+), 34 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 16fbfd7c2d0..ed484c622dd 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -112,6 +112,10 @@ DEFINE_Int32(brpc_num_threads, "256");
// the default value is set to -1, which means never close idle connection.
DEFINE_Int32(brpc_idle_timeout_sec, "-1");
+// Maximum number of multi-dimensional bvar samples exported by /brpc_metrics.
+// brpc 1.4 defaults this flag to 0, which disables all MultiDimension metrics.
+DEFINE_String(bvar_max_dump_multi_dimension_metric_num, "5000");
+
// Declare a selection strategy for those servers have many ips.
// Note that there should at most one ip match this list.
// this is a list in semicolon-delimited format, in CIDR notation, e.g. 10.10.10.0/24
diff --git a/be/src/common/config.h b/be/src/common/config.h
index c4ae9f406d8..d9c2ea4c750 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -157,6 +157,9 @@ DECLARE_Int32(arrow_flight_sql_proxy_port);
DECLARE_Int32(brpc_num_threads);
DECLARE_Int32(brpc_idle_timeout_sec);
+// Maximum number of multi-dimensional bvar samples exported by /brpc_metrics.
+DECLARE_String(bvar_max_dump_multi_dimension_metric_num);
+
// Declare a selection strategy for those servers have many ips.
// Note that there should at most one ip match this list.
// This is a list in semicolon-delimited format, in CIDR notation, e.g. 10.10.10.0/24
diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp
index 2f31e3055a0..8ec1da32f08 100644
--- a/be/src/service/doris_main.cpp
+++ b/be/src/service/doris_main.cpp
@@ -413,6 +413,8 @@ int main(int argc, char** argv) {
return -1;
}
+ doris::init_be_version_metrics();
+
// ATTN: Callers that want to override default gflags variables should do so before calling this method
google::ParseCommandLineFlags(&argc, &argv, true);
// ATTN: MUST init before LOG
diff --git a/be/src/util/debug_util.cpp b/be/src/util/debug_util.cpp
index b98dd76967b..7362788c169 100644
--- a/be/src/util/debug_util.cpp
+++ b/be/src/util/debug_util.cpp
@@ -18,16 +18,23 @@
#include "util/debug_util.h"
#include <bvar/bvar.h>
+#include <bvar/multi_dimension.h>
#include <gen_cpp/HeartbeatService_types.h>
#include <gen_cpp/PlanNodes_types.h>
+#include <gflags/gflags.h>
#include <stdint.h>
+#include <cstdlib>
#include <cstring>
#include <iomanip>
+#include <list>
#include <map>
#include <sstream> // IWYU pragma: keep
+#include <string>
#include <utility>
+#include "common/check.h"
+#include "common/config.h"
#include "common/version_internal.h"
#include "fmt/core.h"
#include "util/uid_util.h"
@@ -109,7 +116,9 @@ std::string hexdump(const char* buf, int len) {
return ss.str();
}
-bvar::Status<uint64_t> be_version_metrics("doris_be_version", [] {
+namespace {
+
+uint64_t get_be_version_metric_value() {
std::stringstream ss;
ss << version::doris_build_version_major() << 0 << version::doris_build_version_minor() << 0
<< version::doris_build_version_patch();
@@ -117,7 +126,33 @@ bvar::Status<uint64_t> be_version_metrics("doris_be_version", [] {
ss << 0 << version::doris_build_version_hotfix();
}
return std::strtoul(ss.str().c_str(), nullptr, 10);
-}());
+}
+
+} // namespace
+
+void init_be_version_metrics() {
+ // MultiDimension metrics are omitted from /brpc_metrics while this brpc flag is 0.
+ DORIS_CHECK(
+ !google::SetCommandLineOption("bvar_max_dump_multi_dimension_metric_number",
+ config::bvar_max_dump_multi_dimension_metric_num.c_str())
+ .empty());
+
+ static const bool initialized = [] {
+ static bvar::MultiDimension<bvar::Status<uint64_t>> metrics(
+ "doris_be_version", {"version", "major", "minor", "patch", "hotfix", "short_hash"});
+ auto* metric = metrics.get_stats(
+ std::list<std::string> {version::doris_build_version(),
+ std::to_string(version::doris_build_version_major()),
+ std::to_string(version::doris_build_version_minor()),
+ std::to_string(version::doris_build_version_patch()),
+ std::to_string(version::doris_build_version_hotfix()),
+ version::doris_build_short_hash()});
+ DORIS_CHECK(metric != nullptr);
+ metric->set_value(get_be_version_metric_value());
+ return true;
+ }();
+ static_cast<void>(initialized);
+}
std::string PrintThriftNetworkAddress(const TNetworkAddress& add) {
std::stringstream ss;
diff --git a/be/src/util/debug_util.h b/be/src/util/debug_util.h
index 19485ca07d2..5fad41ec637 100644
--- a/be/src/util/debug_util.h
+++ b/be/src/util/debug_util.h
@@ -50,6 +50,10 @@ std::string get_short_version();
// Returns "<program short name> version <GetBuildVersion(compact)>"
std::string get_version_string(bool compact);
+// Enables multi-dimensional bvar export and registers doris_be_version.
+// This must run before the brpc server starts.
+void init_be_version_metrics();
+
std::string hexdump(const char* buf, int len);
} // namespace doris
diff --git a/be/test/util/debug_util_test.cpp b/be/test/util/debug_util_test.cpp
new file mode 100644
index 00000000000..983268a4282
--- /dev/null
+++ b/be/test/util/debug_util_test.cpp
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/debug_util.h"
+
+#include <butil/iobuf.h>
+#include <butil/strings/string_piece.h>
+
+// brpc's header uses the butil types above without including their definitions.
+#include <brpc/builtin/prometheus_metrics_service.h>
+
+#include <cstdlib>
+#include <sstream>
+#include <string>
+
+#include "common/config.h"
+#include "common/version_internal.h"
+
+namespace doris {
+
+TEST(DebugUtilTest, BeVersionMetricIsExportedWithBuildLabels) {
+ config::bvar_max_dump_multi_dimension_metric_num = "5000";
+ init_be_version_metrics();
+
+ butil::IOBuf output;
+ ASSERT_EQ(0, brpc::DumpPrometheusMetricsToIOBuf(&output));
+ const std::string body = output.to_string();
+
+ std::stringstream value;
+ value << version::doris_build_version_major() << 0 << version::doris_build_version_minor() << 0
+ << version::doris_build_version_patch();
+ if (version::doris_build_version_hotfix() > 0) {
+ value << 0 << version::doris_build_version_hotfix();
+ }
+
+ std::stringstream sample;
+ sample << "doris_be_version{version=\"" << version::doris_build_version() << "\",major=\""
+ << version::doris_build_version_major() << "\",minor=\""
+ << version::doris_build_version_minor() << "\",patch=\""
+ << version::doris_build_version_patch() << "\",hotfix=\""
+ << version::doris_build_version_hotfix() << "\",short_hash=\""
+ << version::doris_build_short_hash() << "\"} "
+ << std::strtoull(value.str().c_str(), nullptr, 10);
+
+ EXPECT_NE(std::string::npos, body.find("# TYPE doris_be_version gauge")) << body;
+ EXPECT_NE(std::string::npos, body.find(sample.str())) << body;
+}
+
+} // namespace doris
diff --git a/cloud/src/common/CMakeLists.txt b/cloud/src/common/CMakeLists.txt
index a1b989cace1..9f29e1a183b 100644
--- a/cloud/src/common/CMakeLists.txt
+++ b/cloud/src/common/CMakeLists.txt
@@ -29,6 +29,7 @@ set(COMMON_FILES
encryption_util.cpp
http_helper.cpp
metric.cpp
+ version_metrics.cpp
kms.cpp
network_util.cpp
../server/oss/cloud_server_starter_factory.cpp
diff --git a/cloud/src/common/version_metrics.cpp b/cloud/src/common/version_metrics.cpp
new file mode 100644
index 00000000000..e627c853d3e
--- /dev/null
+++ b/cloud/src/common/version_metrics.cpp
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "common/version_metrics.h"
+
+#include <bvar/bvar.h>
+#include <bvar/multi_dimension.h>
+#include <gen_cpp/cloud_version.h>
+#include <gflags/gflags.h>
+
+#include <cstdint>
+#include <cstdlib>
+#include <list>
+#include <sstream>
+#include <string>
+
+#include "common/config.h"
+
+namespace doris::cloud {
+namespace {
+
+uint64_t get_doris_cloud_version_metric_value() {
+ std::stringstream ss;
+ ss << DORIS_CLOUD_BUILD_VERSION_MAJOR << 0 << DORIS_CLOUD_BUILD_VERSION_MINOR << 0
+ << DORIS_CLOUD_BUILD_VERSION_PATCH;
+ if (DORIS_CLOUD_BUILD_VERSION_HOTFIX > 0) {
+ ss << 0 << DORIS_CLOUD_BUILD_VERSION_HOTFIX;
+ }
+ return std::strtoul(ss.str().c_str(), nullptr, 10);
+}
+
+} // namespace
+
+void init_doris_cloud_version_metrics() {
+ // MultiDimension metrics are omitted from /brpc_metrics while this brpc flag is 0.
+ CHECK(!google::SetCommandLineOption("bvar_max_dump_multi_dimension_metric_number",
+ config::bvar_max_dump_multi_dimension_metric_num.c_str())
+ .empty());
+
+ static const bool initialized = [] {
+ // Keep the metric name role-neutral because one doris_cloud process can run meta-service,
+ // recycler, or both. Runtime roles should be represented by scrape target labels.
+ static bvar::MultiDimension<bvar::Status<uint64_t>> metrics(
+ "doris_cloud_version",
+ {"version", "major", "minor", "patch", "hotfix", "short_hash"});
+ auto* metric = metrics.get_stats(std::list<std::string> {
+ DORIS_CLOUD_BUILD_VERSION, std::to_string(DORIS_CLOUD_BUILD_VERSION_MAJOR),
+ std::to_string(DORIS_CLOUD_BUILD_VERSION_MINOR),
+ std::to_string(DORIS_CLOUD_BUILD_VERSION_PATCH),
+ std::to_string(DORIS_CLOUD_BUILD_VERSION_HOTFIX), DORIS_CLOUD_BUILD_SHORT_HASH});
+ CHECK(metric != nullptr);
+ metric->set_value(get_doris_cloud_version_metric_value());
+ return true;
+ }();
+ static_cast<void>(initialized);
+}
+
+} // namespace doris::cloud
diff --git a/cloud/src/common/version_metrics.h b/cloud/src/common/version_metrics.h
new file mode 100644
index 00000000000..d91b2420136
--- /dev/null
+++ b/cloud/src/common/version_metrics.h
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+namespace doris::cloud {
+
+// Enables multi-dimensional bvar export and registers doris_cloud_version.
+// This must run before the brpc server starts.
+void init_doris_cloud_version_metrics();
+
+} // namespace doris::cloud
diff --git a/cloud/src/main.cpp b/cloud/src/main.cpp
index be8b347fdc9..4e8d28dd580 100644
--- a/cloud/src/main.cpp
+++ b/cloud/src/main.cpp
@@ -30,6 +30,7 @@
#include <memory>
#include <mutex>
#include <sstream>
+#include <string>
#include <thread>
#include "common/arg_parser.h"
@@ -38,6 +39,7 @@
#include "common/encryption_util.h"
#include "common/logging.h"
#include "common/network_util.h"
+#include "common/version_metrics.h"
#include "meta-service/meta_server.h"
#include "meta-store/mem_txn_kv.h"
#include "meta-store/txn_kv.h"
@@ -151,17 +153,6 @@ static std::string build_info() {
return ss.str();
}
-// TODO(gavin): add doris cloud role to the metrics name
-bvar::Status<uint64_t> doris_cloud_version_metrics("doris_cloud_version", [] {
- std::stringstream ss;
- ss << DORIS_CLOUD_BUILD_VERSION_MAJOR << 0 << DORIS_CLOUD_BUILD_VERSION_MINOR << 0
- << DORIS_CLOUD_BUILD_VERSION_PATCH;
- if (DORIS_CLOUD_BUILD_VERSION_HOTFIX > 0) {
- ss << 0 << DORIS_CLOUD_BUILD_VERSION_HOTFIX;
- }
- return std::strtoul(ss.str().c_str(), nullptr, 10);
-}());
-
namespace brpc {
DECLARE_uint64(max_body_size);
DECLARE_int64(socket_max_unwritten_bytes);
@@ -245,8 +236,7 @@ int main(int argc, char** argv) {
std::cerr << "try to start meta_service, recycler" << std::endl;
}
- google::SetCommandLineOption("bvar_max_dump_multi_dimension_metric_number",
- config::bvar_max_dump_multi_dimension_metric_num.c_str());
+ init_doris_cloud_version_metrics();
brpc::Server server;
brpc::FLAGS_max_body_size = config::brpc_max_body_size;
diff --git a/cloud/test/metric_test.cpp b/cloud/test/metric_test.cpp
index e00f62d990e..5b5ecc8cd72 100644
--- a/cloud/test/metric_test.cpp
+++ b/cloud/test/metric_test.cpp
@@ -17,15 +17,24 @@
#include "common/metric.h"
+#include <butil/iobuf.h>
+#include <butil/strings/string_piece.h>
+
+// brpc's header uses the butil types above without including their definitions.
+#include <brpc/builtin/prometheus_metrics_service.h>
+#include <gen_cpp/cloud_version.h>
#include <gtest/gtest.h>
+#include <cstdlib>
#include <fstream>
#include <iostream>
+#include <sstream>
#include <string>
#include <thread>
#include "common/bvars.h"
#include "common/config.h"
+#include "common/version_metrics.h"
#include "meta-store/mem_txn_kv.h"
#include "meta-store/txn_kv.h"
#include "meta-store/txn_kv_error.h"
@@ -35,6 +44,34 @@ int main(int argc, char** argv) {
return RUN_ALL_TESTS();
}
+TEST(MetricTest, CloudVersionMetricIsExportedWithBuildLabels) {
+ using namespace doris::cloud;
+
+ config::bvar_max_dump_multi_dimension_metric_num = "5000";
+ init_doris_cloud_version_metrics();
+
+ butil::IOBuf output;
+ ASSERT_EQ(0, brpc::DumpPrometheusMetricsToIOBuf(&output));
+ const std::string body = output.to_string();
+
+ std::stringstream value;
+ value << DORIS_CLOUD_BUILD_VERSION_MAJOR << 0 << DORIS_CLOUD_BUILD_VERSION_MINOR << 0
+ << DORIS_CLOUD_BUILD_VERSION_PATCH;
+ if (DORIS_CLOUD_BUILD_VERSION_HOTFIX > 0) {
+ value << 0 << DORIS_CLOUD_BUILD_VERSION_HOTFIX;
+ }
+
+ std::stringstream sample;
+ sample << "doris_cloud_version{version=\"" << DORIS_CLOUD_BUILD_VERSION << "\",major=\""
+ << DORIS_CLOUD_BUILD_VERSION_MAJOR << "\",minor=\"" << DORIS_CLOUD_BUILD_VERSION_MINOR
+ << "\",patch=\"" << DORIS_CLOUD_BUILD_VERSION_PATCH << "\",hotfix=\""
+ << DORIS_CLOUD_BUILD_VERSION_HOTFIX << "\",short_hash=\"" << DORIS_CLOUD_BUILD_SHORT_HASH
+ << "\"} " << std::strtoull(value.str().c_str(), nullptr, 10);
+
+ EXPECT_NE(std::string::npos, body.find("# TYPE doris_cloud_version gauge")) << body;
+ EXPECT_NE(std::string::npos, body.find(sample.str())) << body;
+}
+
TEST(MetricTest, FdbMetricExporterTest) {
using namespace doris::cloud;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java b/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
index e4eb487c229..737647e1f84 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
@@ -23,6 +23,7 @@ import org.apache.doris.cloud.JobWarmUpStats;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
+import org.apache.doris.common.Version;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.common.util.JsonUtil;
import org.apache.doris.ha.FrontendNodeType;
@@ -63,6 +64,29 @@ public class MetricsTest {
MetricRepo.init();
}
+ @Test
+ public void testVersionMetricIsExportedWithBuildLabels() {
+ StringBuilder value = new StringBuilder()
+ .append(Version.DORIS_BUILD_VERSION_MAJOR).append(0)
+ .append(Version.DORIS_BUILD_VERSION_MINOR).append(0)
+ .append(Version.DORIS_BUILD_VERSION_PATCH);
+ if (Version.DORIS_BUILD_VERSION_HOTFIX > 0) {
+ value.append(0).append(Version.DORIS_BUILD_VERSION_HOTFIX);
+ }
+
+ String expectedSample = "doris_fe_version{version=\"" + Version.DORIS_BUILD_VERSION
+ + "\", major=\"" + Version.DORIS_BUILD_VERSION_MAJOR
+ + "\", minor=\"" + Version.DORIS_BUILD_VERSION_MINOR
+ + "\", patch=\"" + Version.DORIS_BUILD_VERSION_PATCH
+ + "\", hotfix=\"" + Version.DORIS_BUILD_VERSION_HOTFIX
+ + "\", short_hash=\"" + Version.DORIS_BUILD_SHORT_HASH
+ + "\"} " + Long.parseLong(value.toString());
+
+ String metricResult = getPrometheusMetrics();
+ Assert.assertTrue(metricResult, metricResult.contains("# TYPE doris_fe_version gauge"));
+ Assert.assertTrue(metricResult, metricResult.contains(expectedSample));
+ }
+
@Test
public void testTcpMetrics() {
List<Metric> metrics = MetricRepo.getMetricsByName("snmp");
diff --git a/regression-test/suites/metrics_p0/test_version_metrics.groovy b/regression-test/suites/metrics_p0/test_version_metrics.groovy
index e6625f698ea..39112ec6e77 100644
--- a/regression-test/suites/metrics_p0/test_version_metrics.groovy
+++ b/regression-test/suites/metrics_p0/test_version_metrics.groovy
@@ -19,6 +19,33 @@ import java.util.regex.Pattern
// under the License.
suite("test_version_metrics") {
+ def requiredVersionLabels = ["version", "major", "minor", "patch", "hotfix", "short_hash"] as Set
+ def parseVersionMetric = { body, metricName ->
+ def parsedMetric = null
+ Pattern pattern = Pattern.compile('^' + Pattern.quote(metricName) + '\\{([^}]*)}\\s+(\\d+)$')
+ for (final def line in body.readLines()) {
+ Matcher matcher = pattern.matcher(line)
+ if (!matcher.matches()) {
+ continue
+ }
+
+ def labels = [:]
+ for (String label : matcher.group(1).split(",")) {
+ String[] keyValue = label.trim().split("=", 2)
+ assertEquals(2, keyValue.length)
+ assertTrue(keyValue[1].startsWith("\"") && keyValue[1].endsWith("\""))
+ labels[keyValue[0]] = keyValue[1].substring(1, keyValue[1].length() - 1)
+ }
+ assertEquals(requiredVersionLabels, labels.keySet())
+ parsedMetric = [labels: labels, value: Long.parseLong(matcher.group(2))]
+ break
+ }
+ assertNotNull(parsedMetric)
+ assertTrue(parsedMetric.value >= 0)
+ return parsedMetric
+ }
+
+ def feVersionMetric = null
httpTest {
endpoint context.config.feHttpAddress
uri "/metrics"
@@ -27,15 +54,7 @@ suite("test_version_metrics") {
logger.debug("code:${code} body:${body}");
assertEquals(200, code)
assertTrue(body.contains("doris_fe_version"))
- for (final def line in body.split("\n")) {
- if (line.startsWith("doris_fe_version")) {
- Pattern pattern = Pattern.compile(/^doris_fe_version\{.*}\s+(\d+)$/)
- Matcher matcher = pattern.matcher(line)
- assertTrue(matcher.matches())
- assertTrue(Long.parseLong(matcher.group(1)) >= 0)
- break
- }
- }
+ feVersionMetric = parseVersionMetric(body, "doris_fe_version")
}
}
@@ -50,11 +69,8 @@ suite("test_version_metrics") {
logger.debug("code:${code} body:${body}");
assertEquals(200, code)
assertTrue(body.contains("doris_be_version"))
- for (final def line in body.split("\n")) {
- if (line.contains("doris_be_version") && !line.contains("#")) {
- assertTrue(Long.parseLong(line.split(" ")[1]) >= 0)
- }
- }
+ def beVersionMetric = parseVersionMetric(body, "doris_be_version")
+ assertEquals(feVersionMetric, beVersionMetric)
}
}
@@ -71,11 +87,8 @@ suite("test_version_metrics") {
logger.debug("code:${code} body:${body}");
assertEquals(200, code)
assertTrue(body.contains("doris_cloud_version"))
- for (final def line in body.split("\n")) {
- if (line.contains("doris_cloud_version") && !line.contains("#")) {
- assertTrue(Long.parseLong(line.split(" ")[1]) >= 0)
- }
- }
+ def cloudVersionMetric = parseVersionMetric(body, "doris_cloud_version")
+ assertEquals(feVersionMetric, cloudVersionMetric)
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 25/28: branch-4.1: [fix](load) Avoid false quorum failures with distributed commit reports #68275 (#68818)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3c4b3d93a59c3b5444b96c1f992ca3cf1e7024a6
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Sat Oct 10 09:32:58 2026 +0800
branch-4.1: [fix](load) Avoid false quorum failures with distributed commit reports #68275 (#68818)
Cherry-picked from #68275
Co-authored-by: Refrain <re...@gmail.com>
---
be/src/exec/sink/writer/vtablet_writer_v2.cpp | 36 ++++----
be/src/load/channel/load_stream.cpp | 62 +++++++++++++
be/src/load/channel/load_stream.h | 2 +
be/test/exec/sink/vtablet_writer_v2_test.cpp | 74 +++++++++++++++
.../test_insert_quorum_split_reports.out | 6 ++
.../test_insert_quorum_split_reports.groovy | 101 +++++++++++++++++++++
6 files changed, 265 insertions(+), 16 deletions(-)
diff --git a/be/src/exec/sink/writer/vtablet_writer_v2.cpp b/be/src/exec/sink/writer/vtablet_writer_v2.cpp
index cb6ec89ff48..add5daa39ca 100644
--- a/be/src/exec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/exec/sink/writer/vtablet_writer_v2.cpp
@@ -30,6 +30,7 @@
#include <ranges>
#include <string>
#include <unordered_map>
+#include <utility>
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/logging.h"
@@ -1074,15 +1075,15 @@ void VTabletWriterV2::_calc_tablets_to_commit() {
Status VTabletWriterV2::_create_commit_info(std::vector<TTabletCommitInfo>& tablet_commit_infos,
std::shared_ptr<LoadStreamMap> load_stream_map) {
- // Track per-tablet non-gap success count and failure reasons
- std::unordered_map<int64_t, int> success_tablets_replica;
- std::unordered_set<int64_t> failed_tablets;
+ // Commit results may be reported to different sources. Only reject a tablet when
+ // known failures make quorum impossible; FE checks the aggregated commit info.
+ std::unordered_map<int64_t, std::unordered_set<int64_t>> failed_tablets;
std::unordered_map<int64_t, Status> failed_reason;
load_stream_map->for_each([&](int64_t dst_id, LoadStreamStubs& streams) {
size_t num_success_tablets = 0;
size_t num_failed_tablets = 0;
for (auto [tablet_id, reason] : streams.failed_tablets()) {
- failed_tablets.insert(tablet_id);
+ failed_tablets[tablet_id].insert(dst_id);
failed_reason[tablet_id] = reason;
num_failed_tablets++;
}
@@ -1091,25 +1092,28 @@ Status VTabletWriterV2::_create_commit_info(std::vector<TTabletCommitInfo>& tabl
commit_info.tabletId = tablet_id;
commit_info.backendId = dst_id;
tablet_commit_infos.emplace_back(std::move(commit_info));
- // Only count non-gap backends toward success
- auto gap_it = _tablet_version_gap_backends.find(tablet_id);
- if (gap_it == _tablet_version_gap_backends.end() ||
- gap_it->second.find(dst_id) == gap_it->second.end()) {
- success_tablets_replica[tablet_id]++;
- }
num_success_tablets++;
}
LOG(INFO) << "streams to dst_id: " << dst_id << ", success tablets: " << num_success_tablets
<< ", failed tablets: " << num_failed_tablets;
});
- for (auto tablet_id : failed_tablets) {
- int succ_count = success_tablets_replica[tablet_id];
- int required = _load_required_replicas_num(tablet_id);
- if (succ_count < required) {
+ for (auto& [tablet_id, failed_backends] : failed_tablets) {
+ // Version-gap replicas cannot contribute to quorum, even if this write succeeds.
+ // Count a backend only once when it also reported a write failure.
+ if (auto gap_it = _tablet_version_gap_backends.find(tablet_id);
+ gap_it != _tablet_version_gap_backends.end()) {
+ failed_backends.insert(gap_it->second.begin(), gap_it->second.end());
+ }
+ auto [total_replicas_num, load_required_replicas_num] = _tablet_replica_info[tablet_id];
+ int max_failed_replicas = total_replicas_num == 0
+ ? (_num_replicas - 1) / 2
+ : total_replicas_num - load_required_replicas_num;
+ if (std::cmp_greater(failed_backends.size(), max_failed_replicas)) {
LOG(INFO) << "tablet " << tablet_id
- << " failed on majority backends (success=" << succ_count
- << ", required=" << required << "): " << failed_reason[tablet_id];
+ << " failed on majority backends (failed=" << failed_backends.size()
+ << ", max_failed=" << max_failed_replicas
+ << "): " << failed_reason[tablet_id];
return Status::InternalError("tablet {} failed on majority backends: {}", tablet_id,
failed_reason[tablet_id]);
}
diff --git a/be/src/load/channel/load_stream.cpp b/be/src/load/channel/load_stream.cpp
index 0abbd7bb13f..dc279a27613 100644
--- a/be/src/load/channel/load_stream.cpp
+++ b/be/src/load/channel/load_stream.cpp
@@ -796,12 +796,74 @@ void LoadStream::_dispatch(StreamId id, const PStreamHeader& hdr, butil::IOBuf*
} break;
case PStreamHeader::CLOSE_LOAD: {
DBUG_EXECUTE_IF("LoadStream.close_load.block", DBUG_BLOCK);
+ DBUG_EXECUTE_IF("LoadStream.close_load.force_last_source", {
+ if (_schema->table_id() == dp->param<int64_t>("table_id", -1)) {
+ MonotonicStopWatch wait_timer;
+ wait_timer.start();
+ while (true) {
+ bool ready = false;
+ bool single_source = false;
+ {
+ std::lock_guard lock_guard(_lock);
+ if (_debug_last_close_src_id < 0) {
+ int opened_streams = _close_load_cnt;
+ for (const auto& [_, count] : _open_streams) {
+ opened_streams += count;
+ }
+ // Wait for every source to open before selecting a stable
+ // min/max source ID. No CLOSE_LOAD is counted before selection.
+ if (opened_streams == _total_streams) {
+ single_source = _open_streams.size() < 2;
+ if (!single_source) {
+ const bool pick_max = dp->param<bool>("pick_max", true);
+ _debug_last_close_src_id = _open_streams.begin()->first;
+ for (const auto& [src_id, _] : _open_streams) {
+ if ((pick_max && src_id > _debug_last_close_src_id) ||
+ (!pick_max && src_id < _debug_last_close_src_id)) {
+ _debug_last_close_src_id = src_id;
+ }
+ }
+ }
+ }
+ }
+ ready = _debug_last_close_src_id >= 0 &&
+ (hdr.src_id() != _debug_last_close_src_id ||
+ _open_streams.size() == 1);
+ }
+ if (ready) {
+ break;
+ }
+ if (single_source ||
+ wait_timer.elapsed_time() / 1000000 >=
+ dp->param<int64_t>("wait_ms", 30000) ||
+ !DebugPoints::instance()->is_enable(DP_NAME)) {
+ LOG(WARNING) << "cannot force last CLOSE_LOAD source (requires multiple "
+ "sources), "
+ << *this;
+ // Closing without EOS makes the load fail instead of silently
+ // passing a test that did not establish the requested ordering.
+ brpc::StreamClose(id);
+ return;
+ }
+ bthread_usleep(1000);
+ }
+ }
+ });
std::vector<int64_t> success_tablet_ids;
FailedTablets failed_tablets;
std::vector<PTabletID> tablets_to_commit(hdr.tablets().begin(), hdr.tablets().end());
// Step 1: count this CLOSE_LOAD and, if this is the last one, commit. Under _lock.
bool all_received =
close(hdr.src_id(), tablets_to_commit, &success_tablet_ids, &failed_tablets);
+ DBUG_EXECUTE_IF("LoadStream.close_load.force_last_source", {
+ if (all_received && _schema->table_id() == dp->param<int64_t>("table_id", -1) &&
+ dp->param<bool>("require_failure", false) && failed_tablets.empty()) {
+ LOG(WARNING) << "expected a failed tablet in the final CLOSE_LOAD result, "
+ << *this;
+ brpc::StreamClose(id);
+ return;
+ }
+ });
// Step 2: send THIS stream's EOS (network IO, must be outside _lock). A stream
// must not be StreamClose'd before its own EOS is delivered, otherwise the
// sender sees on_closed without EOS and reports "Stream closed without EOS".
diff --git a/be/src/load/channel/load_stream.h b/be/src/load/channel/load_stream.h
index 4c8865e1603..3e883a39c69 100644
--- a/be/src/load/channel/load_stream.h
+++ b/be/src/load/channel/load_stream.h
@@ -186,6 +186,8 @@ private:
std::unordered_map<int64_t, IndexStreamSharedPtr> _index_streams_map;
int32_t _total_streams = 0;
int32_t _close_load_cnt = 0;
+ // Only used by close_load.force_last_source, protected by _lock.
+ int64_t _debug_last_close_src_id = -1;
std::atomic<int32_t> _close_rpc_cnt = 0;
std::vector<PTabletID> _tablets_to_commit;
bthread::Mutex _lock;
diff --git a/be/test/exec/sink/vtablet_writer_v2_test.cpp b/be/test/exec/sink/vtablet_writer_v2_test.cpp
index 6a508a9ef23..c86839d0599 100644
--- a/be/test/exec/sink/vtablet_writer_v2_test.cpp
+++ b/be/test/exec/sink/vtablet_writer_v2_test.cpp
@@ -427,6 +427,80 @@ TEST_F(TestVTabletWriterV2, fail_one) {
ASSERT_EQ(tablet_commit_infos.size(), 5);
}
+TEST_F(TestVTabletWriterV2, commit_info_with_results_split_across_sources) {
+ UniqueId load_id;
+ auto first_source = std::make_shared<LoadStreamMap>(load_id, src_id, 1, 1, nullptr);
+ auto second_source = std::make_shared<LoadStreamMap>(load_id, src_id + 1, 1, 1, nullptr);
+ add_stream(first_source, 1001, {1}, {});
+ add_stream(first_source, 1002, {1}, {});
+ add_stream(first_source, 1003, {}, {{1, Status::InternalError("write failed")}});
+ add_stream(second_source, 1001, {}, {});
+ add_stream(second_source, 1002, {}, {});
+ add_stream(second_source, 1003, {}, {{1, Status::InternalError("write failed")}});
+
+ auto first_writer = create_vtablet_writer();
+ auto second_writer = create_vtablet_writer();
+ std::vector<TTabletCommitInfo> first_commit_infos;
+ std::vector<TTabletCommitInfo> second_commit_infos;
+ ASSERT_TRUE(first_writer->_create_commit_info(first_commit_infos, first_source).ok());
+ // The second source received the failed replica's final result, but the two
+ // successful replicas reported to the first source. It must not reject the load.
+ ASSERT_TRUE(second_writer->_create_commit_info(second_commit_infos, second_source).ok());
+ ASSERT_EQ(first_commit_infos.size(), 2);
+ ASSERT_TRUE(second_commit_infos.empty());
+ for (const auto& info : first_commit_infos) {
+ EXPECT_EQ(info.tabletId, 1);
+ EXPECT_TRUE(info.backendId == 1001 || info.backendId == 1002);
+ }
+}
+
+TEST_F(TestVTabletWriterV2, write_failure_and_version_gap_exceed_failure_quorum) {
+ for (bool gap_replica_reported_success : {false, true}) {
+ SCOPED_TRACE(gap_replica_reported_success);
+ UniqueId load_id;
+ auto load_stream_map = std::make_shared<LoadStreamMap>(load_id, src_id, 1, 1, nullptr);
+ add_stream(load_stream_map, 1001, {1}, {});
+ add_stream(load_stream_map, 1002, {}, {{1, Status::InternalError("write failed")}});
+ add_stream(
+ load_stream_map, 1003,
+ gap_replica_reported_success ? std::vector<int64_t> {1} : std::vector<int64_t> {},
+ {});
+
+ auto writer = create_vtablet_writer();
+ writer->_tablet_version_gap_backends[1].insert(1003);
+ std::vector<TTabletCommitInfo> commit_infos;
+ auto st = writer->_create_commit_info(commit_infos, load_stream_map);
+ // A version-gap replica is unavailable regardless of where its result is
+ // reported. Together with the write failure, only one valid replica remains.
+ ASSERT_FALSE(st.ok());
+ EXPECT_NE(st.to_string().find("failed on majority backends"), std::string::npos);
+ EXPECT_NE(st.to_string().find("write failed"), std::string::npos);
+ EXPECT_EQ(commit_infos.size(), gap_replica_reported_success ? 2 : 1);
+ }
+}
+
+TEST_F(TestVTabletWriterV2, version_gap_and_duplicate_write_failure_count_once) {
+ UniqueId load_id;
+ auto load_stream_map = std::make_shared<LoadStreamMap>(load_id, src_id, 2, 1, nullptr);
+ add_stream(load_stream_map, 1001, {1}, {});
+ auto failed_streams = load_stream_map->get_or_create(1002);
+ failed_streams->mark_open();
+ for (const auto& stream : failed_streams->streams()) {
+ stream->add_failed_tablet(1, Status::InternalError("write failed"));
+ }
+ add_stream(load_stream_map, 1003, {}, {});
+
+ auto writer = create_vtablet_writer();
+ writer->_tablet_version_gap_backends[1].insert(1002);
+ std::vector<TTabletCommitInfo> commit_infos;
+ // Only backend 1002 is unavailable. Backend 1003 may have reported success to
+ // another source, so a missing local success must not cause a quorum failure.
+ ASSERT_TRUE(writer->_create_commit_info(commit_infos, load_stream_map).ok());
+ ASSERT_EQ(commit_infos.size(), 1);
+ EXPECT_EQ(commit_infos[0].tabletId, 1);
+ EXPECT_EQ(commit_infos[0].backendId, 1001);
+}
+
TEST_F(TestVTabletWriterV2, fail_one_duplicate) {
UniqueId load_id;
std::vector<TTabletCommitInfo> tablet_commit_infos;
diff --git a/regression-test/data/fault_injection_p0/test_insert_quorum_split_reports.out b/regression-test/data/fault_injection_p0/test_insert_quorum_split_reports.out
new file mode 100644
index 00000000000..a6a02a18cee
--- /dev/null
+++ b/regression-test/data/fault_injection_p0/test_insert_quorum_split_reports.out
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !rows --
+2048
+
+-- !mismatches --
+0
diff --git a/regression-test/suites/fault_injection_p0/test_insert_quorum_split_reports.groovy b/regression-test/suites/fault_injection_p0/test_insert_quorum_split_reports.groovy
new file mode 100644
index 00000000000..0b6e80408fa
--- /dev/null
+++ b/regression-test/suites/fault_injection_p0/test_insert_quorum_split_reports.groovy
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.util.NodeType
+
+suite("test_insert_quorum_split_reports", "nonConcurrent") {
+ if (isCloudMode()) {
+ return
+ }
+ def backendIPs = [:]
+ def backendPorts = [:]
+ getBackendIpHttpPort(backendIPs, backendPorts)
+ if (backendIPs.size() < 3) {
+ return
+ }
+
+ def closePoint = "LoadStream.close_load.force_last_source"
+ def failurePoint = "TabletStream.add_segment.unknown_segid"
+ def debugPoint = GetDebugPoint()
+ def failedBackend = null
+ def sessionVariables = ["enable_memtable_on_sink_node", "enable_local_shuffle",
+ "parallel_pipeline_task_num", "load_stream_per_node", "query_timeout"]
+ def savedVariables = sessionVariables.collectEntries { name ->
+ [(name): sql("select @@${name}")[0][0]]
+ }
+ try {
+ sql "set enable_memtable_on_sink_node = true"
+ sql "set enable_local_shuffle = false"
+ sql "set parallel_pipeline_task_num = 1"
+ sql "set load_stream_per_node = 1"
+ sql "set query_timeout = 60"
+ sql "drop table if exists insert_quorum_split_reports_source"
+ sql "drop table if exists insert_quorum_split_reports_target"
+ // Spread the scan over multiple BEs. The close injection fails the load
+ // explicitly if execution nevertheless opens streams from only one source.
+ sql """create table insert_quorum_split_reports_source (k int, v bigint)
+ duplicate key(k) distributed by hash(k) buckets ${backendIPs.size() * 4}
+ properties("replication_num" = "1")"""
+ sql """create table insert_quorum_split_reports_target (k int, v bigint)
+ duplicate key(k) distributed by hash(k) buckets 5
+ properties("replication_num" = "3")"""
+ sql """insert into insert_quorum_split_reports_source
+ select number, number * 11 + 5 from numbers("number" = "2048")"""
+
+ def replicas = sql_return_maparray "show tablets from insert_quorum_split_reports_target"
+ failedBackend = replicas.collect { it.BackendId.toString() }.unique().min { it.toLong() }
+ def tableId = getTableId("insert_quorum_split_reports_target").toString()
+ // All healthy destinations report to the largest source ID. The failed
+ // destination reports to the smallest, which receives no healthy results.
+ debugPoint.enableDebugPointForAllBEs(closePoint,
+ [table_id: tableId, pick_max: "true", wait_ms: "30000", timeout: "90"])
+ debugPoint.enableDebugPoint(backendIPs[failedBackend], backendPorts[failedBackend] as int,
+ NodeType.BE, closePoint,
+ [table_id: tableId, pick_max: "false", require_failure: "true",
+ wait_ms: "30000", timeout: "90"])
+ debugPoint.enableDebugPoint(backendIPs[failedBackend], backendPorts[failedBackend] as int,
+ NodeType.BE, failurePoint, [timeout: "90"])
+
+ // Before the fix, the source receiving the failed destination's final
+ // report rejects the INSERT with success=0, required=2.
+ sql "insert into insert_quorum_split_reports_target select * from insert_quorum_split_reports_source"
+
+ // require_failure checks the actual final response, avoiding a race with
+ // background replica repair when checking the FE's failed-version metadata.
+ qt_rows "select count(*) from insert_quorum_split_reports_target"
+ qt_mismatches """select count(*) from (
+ (select k, v from insert_quorum_split_reports_source
+ except select k, v from insert_quorum_split_reports_target)
+ union all
+ (select k, v from insert_quorum_split_reports_target
+ except select k, v from insert_quorum_split_reports_source)
+ ) differences"""
+ } finally {
+ try {
+ if (failedBackend != null) {
+ debugPoint.disableDebugPoint(backendIPs[failedBackend], backendPorts[failedBackend] as int,
+ NodeType.BE, failurePoint)
+ }
+ } finally {
+ try {
+ debugPoint.disableDebugPointForAllBEs(closePoint)
+ } finally {
+ savedVariables.each { name, value -> sql "set ${name} = '${value}'" }
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 28/28: [feature](lance) Support distance bounds in vector_search (#68815)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d3063235507ac8e7300478b701c206eda530af0e
Author: Gabriel <li...@selectdb.com>
AuthorDate: Sat Oct 10 09:39:09 2026 +0800
[feature](lance) Support distance bounds in vector_search (#68815)
### What problem does this PR solve?
Issue Number: Closes #66493
`vector_search()` cannot currently restrict results by distance. This
adds optional `distance_lower_bound` (inclusive) and
`distance_upper_bound` (exclusive) parameters for single-vector
Radius/Range Search on Lance tables.
The FE validates finite FLOAT bounds and their order, includes the range
in EXPLAIN, and carries it through the external-search request. The BE
applies the bounds through the existing pinned lance-c API after
configuring the query vector, covering indexed and Flat Search splits,
including appended data outside index coverage. The existing
external-search schema version remains unchanged at 1.
Results retain the existing table columns and FLOAT `_distance`,
prefilter semantics, offset, and Top-K cap. Bounds can be bound through
prepared-statement parameters and are revalidated on each execution.
Multi-vector distance bounds are rejected.
Bounded index search is limited to explicitly identified IVF_FLAT
segments with both bounds present. Quantized or unknown index types and
single-sided ranges fall back to exact fragment scans, reported as
`DISTANCE_RANGE_FALLBACK` in EXPLAIN. This avoids pre-refinement
quantization errors and the pinned index implementation’s finite
sentinels for absent bounds, at the cost of additional scanning.
Unbounded queries retain their index selection; indexed IVF_FLAT recall
still depends on probed partitions.
Standalone BM25 is already delivered; the distributed BM25 statistics
follow-up remains tracked separately in #67435.
### Release note
Lance `vector_search()` supports optional lower and upper distance
bounds for single-vector radius and range queries.
### Validation
- BE: rebuilt the current branch and passed all 68 `LanceTableReader*`
tests under ASAN, including exact SQ threshold crossing, the finite
FLT_MAX DOT endpoint, prefilter variants, and existing reader coverage.
- FE: 76 tests passed across `VectorSearchTableValuedFunctionTest`,
`LancePreparedSearchTest`, `LancePreparedStatementTest`, and
`LanceScanNodeTest`, including repeated bound binding, invalid
parameters, all index subtypes, and indexed/flat split selection.
- Native reproduction: the pinned indexed paths drop both the SQ
threshold-crossing row and the finite FLT_MAX DOT endpoint; exact Flat
Search returns both. Added self-contained fixtures and reader coverage
for these cases.
- Native execution: 10 cases passed against the pinned lance-c revision,
covering indexed/Flat Search, appended unindexed data,
inclusive/exclusive bounds, empty results, prefiltering, and offset.
- FE Checkstyle and clang-format 16 checks passed.
- Added SQL regression coverage for range results, projected stored
columns, filtering, offset, invalid bounds, and EXPLAIN. Its Groovy
syntax compilation passed; the SQL suite has not yet run against a live
Doris cluster and requires External Regression CI.
### Check List (For Author)
- Test
- [x] Regression test added (execution pending CI)
- [x] Unit Test
- [x] Manual native execution test
- Behavior changed:
- [x] Yes: optional distance bounds on single-vector searches; existing
unbounded queries retain their behavior.
- Does this need documentation?
- [x] Yes: bilingual 4.x Lance Catalog documentation:
https://github.com/apache/doris-website/pull/4200.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
be/src/format_v2/table/lance_reader.cpp | 33 ++++
.../auxiliary.idx | Bin 0 -> 3254 bytes
.../9634c881-df64-46a5-95dc-2468928008a5/index.idx | Bin 0 -> 329 bytes
.../0-388c379e-6e57-4a48-a567-603ce868bdea.txn | Bin 0 -> 205 bytes
.../1-831523ce-b62e-46b0-bcd2-501920af09b8.txn | Bin 0 -> 376 bytes
.../2-4546c7a5-c68b-47d3-8f54-1e3036ad6429.txn | Bin 0 -> 122 bytes
.../_versions/18446744073709551612.manifest | Bin 0 -> 824 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 997 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 486 bytes
.../_versions/latest_version_hint.json | 1 +
...100000101111111a29c2749f892029e3946d88007.lance | Bin 0 -> 661 bytes
...000101110010011f2593f448ba30e92dc54b90567.lance | Bin 0 -> 3100 bytes
.../auxiliary.idx | Bin 0 -> 3255 bytes
.../76cb567d-f24f-4dd0-85b1-b0f486361b5e/index.idx | Bin 0 -> 330 bytes
.../0-ec982602-f60a-425e-8b31-f77179548301.txn | Bin 0 -> 205 bytes
.../1-bd4b52a8-552e-4270-b52f-3f678df92fed.txn | Bin 0 -> 378 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 1001 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 487 bytes
.../_versions/latest_version_hint.json | 1 +
...000110101111101bccc4b4c19af77197f1a92b69c.lance | Bin 0 -> 3100 bytes
.../auxiliary.idx | Bin 0 -> 2805 bytes
.../16361222-c72d-4428-9826-beeb3bfd3667/index.idx | Bin 0 -> 327 bytes
.../0-2dfc92d6-8481-4406-8893-9fc051e167d2.txn | Bin 0 -> 205 bytes
.../1-f4a6b175-3626-4561-817e-c45c857597fa.txn | Bin 0 -> 407 bytes
.../_versions/18446744073709551613.manifest | Bin 0 -> 1059 bytes
.../_versions/18446744073709551614.manifest | Bin 0 -> 487 bytes
.../_versions/latest_version_hint.json | 1 +
...1010101000001014167dc46e9aa87a5e01e036b2a.lance | Bin 0 -> 4892 bytes
.../table/lance/distance_range_fixture.md | 93 ++++++++++++
be/test/format_v2/table/lance_reader_test.cpp | 167 +++++++++++++++++++++
.../datasource/lance/source/LanceScanNode.java | 7 +
.../datasource/lance/source/LanceScanPlan.java | 2 +
.../datasource/lance/source/LanceScanPlanner.java | 12 ++
.../doris/nereids/parser/LogicalPlanBuilder.java | 4 +-
.../VectorSearchTableValuedFunction.java | 44 +++++-
.../datasource/lance/source/LanceScanNodeTest.java | 40 +++++
.../tablefunction/LancePreparedSearchTest.java | 105 +++++++++++++
.../VectorSearchTableValuedFunctionTest.java | 47 ++++++
gensrc/thrift/PlanNodes.thrift | 3 +
.../test_lance_vector_search_distance_range.groovy | 82 ++++++++++
40 files changed, 638 insertions(+), 4 deletions(-)
diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp
index 73d68332f49..c445d795d94 100644
--- a/be/src/format_v2/table/lance_reader.cpp
+++ b/be/src/format_v2/table/lance_reader.cpp
@@ -448,6 +448,28 @@ Status LanceTableReader::_validate_external_search_request() const {
static_cast<int>(query_vector.element_type));
}
const bool multi_vector = query_vector.__isset.num_vectors;
+ const bool has_distance_range =
+ vector.__isset.distance_lower_bound || vector.__isset.distance_upper_bound;
+ if (has_distance_range && multi_vector) {
+ return Status::NotSupported(
+ "Distance bounds are not supported for Lance multi-vector search");
+ }
+ // Validate doubles before narrowing: non-finite or out-of-range values cannot reach the C API.
+ const auto valid_bound = [](double bound) {
+ return std::isfinite(bound) && std::abs(bound) <= std::numeric_limits<float>::max();
+ };
+ if ((vector.__isset.distance_lower_bound && !valid_bound(vector.distance_lower_bound)) ||
+ (vector.__isset.distance_upper_bound && !valid_bound(vector.distance_upper_bound))) {
+ return Status::InvalidArgument("Lance distance bounds must be finite FLOAT values");
+ }
+ if (vector.__isset.distance_lower_bound && vector.__isset.distance_upper_bound &&
+ static_cast<float>(vector.distance_lower_bound) >=
+ static_cast<float>(vector.distance_upper_bound)) {
+ return Status::InvalidArgument(
+ "distance_lower_bound must be less than distance_upper_bound at FLOAT "
+ "precision");
+ }
+
// The optional count distinguishes a query matrix, including a one-row matrix.
if (multi_vector && query_vector.num_vectors <= 0) {
return Status::InvalidArgument(
@@ -1177,6 +1199,17 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner,
}
}
+ if (vector.__isset.distance_lower_bound || vector.__isset.distance_upper_bound) {
+ const float lower = static_cast<float>(vector.distance_lower_bound);
+ const float upper = static_cast<float>(vector.distance_upper_bound);
+ // nearest() clears the range; install bounds only after setting the query vector.
+ if (lance_scanner_set_distance_range(
+ scanner, vector.__isset.distance_lower_bound ? &lower : nullptr,
+ vector.__isset.distance_upper_bound ? &upper : nullptr) != 0) {
+ return lance_error("set Lance vector distance range");
+ }
+ }
+
// Exact refinement validates stored elements and gives indexed and unindexed candidates
// the same row-level score before either path truncates its results.
if (query.__isset.num_vectors && lance_scanner_set_refine_factor(scanner, 1) != 0) {
diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_indices/9634c881-df64-46a5-95dc-2468928008a5/auxiliary.idx b/be/test/format_v2/table/lance/data/distance_range.lance/_indices/9634c881-df64-46a5-95dc-2468928008a5/auxiliary.idx
new file mode 100644
index 00000000000..f6ecf98bb6c
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_indices/9634c881-df64-46a5-95dc-2468928008a5/auxiliary.idx differ
diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_indices/9634c881-df64-46a5-95dc-2468928008a5/index.idx b/be/test/format_v2/table/lance/data/distance_range.lance/_indices/9634c881-df64-46a5-95dc-2468928008a5/index.idx
new file mode 100644
index 00000000000..d7646e4098c
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_indices/9634c881-df64-46a5-95dc-2468928008a5/index.idx differ
diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/0-388c379e-6e57-4a48-a567-603ce868bdea.txn b/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/0-388c379e-6e57-4a48-a567-603ce868bdea.txn
new file mode 100644
index 00000000000..6643a15e80a
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/0-388c379e-6e57-4a48-a567-603ce868bdea.txn differ
diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/1-831523ce-b62e-46b0-bcd2-501920af09b8.txn b/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/1-831523ce-b62e-46b0-bcd2-501920af09b8.txn
new file mode 100644
index 00000000000..949b47f39d9
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/1-831523ce-b62e-46b0-bcd2-501920af09b8.txn differ
diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/2-4546c7a5-c68b-47d3-8f54-1e3036ad6429.txn b/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/2-4546c7a5-c68b-47d3-8f54-1e3036ad6429.txn
new file mode 100644
index 00000000000..afebdade12d
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/2-4546c7a5-c68b-47d3-8f54-1e3036ad6429.txn differ
diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551612.manifest b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551612.manifest
new file mode 100644
index 00000000000..7ed8e3d926b
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551612.manifest differ
diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551613.manifest b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551613.manifest
new file mode 100644
index 00000000000..ba5cc92bc12
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551613.manifest differ
diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551614.manifest
new file mode 100644
index 00000000000..8f32f9b807f
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551614.manifest differ
diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/latest_version_hint.json
new file mode 100644
index 00000000000..f93d3984472
--- /dev/null
+++ b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/latest_version_hint.json
@@ -0,0 +1 @@
+{"version":3}
\ No newline at end of file
diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/data/011111011100000101111111a29c2749f892029e3946d88007.lance b/be/test/format_v2/table/lance/data/distance_range.lance/data/011111011100000101111111a29c2749f892029e3946d88007.lance
new file mode 100644
index 00000000000..52f7dbf29cc
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/data/011111011100000101111111a29c2749f892029e3946d88007.lance differ
diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/data/100110001000101110010011f2593f448ba30e92dc54b90567.lance b/be/test/format_v2/table/lance/data/distance_range.lance/data/100110001000101110010011f2593f448ba30e92dc54b90567.lance
new file mode 100644
index 00000000000..53e1eec28d0
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/data/100110001000101110010011f2593f448ba30e92dc54b90567.lance differ
diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/_indices/76cb567d-f24f-4dd0-85b1-b0f486361b5e/auxiliary.idx b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_indices/76cb567d-f24f-4dd0-85b1-b0f486361b5e/auxiliary.idx
new file mode 100644
index 00000000000..3ca57b8d6de
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_indices/76cb567d-f24f-4dd0-85b1-b0f486361b5e/auxiliary.idx differ
diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/_indices/76cb567d-f24f-4dd0-85b1-b0f486361b5e/index.idx b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_indices/76cb567d-f24f-4dd0-85b1-b0f486361b5e/index.idx
new file mode 100644
index 00000000000..446e4f4152b
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_indices/76cb567d-f24f-4dd0-85b1-b0f486361b5e/index.idx differ
diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/_transactions/0-ec982602-f60a-425e-8b31-f77179548301.txn b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_transactions/0-ec982602-f60a-425e-8b31-f77179548301.txn
new file mode 100644
index 00000000000..72e846c8998
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_transactions/0-ec982602-f60a-425e-8b31-f77179548301.txn differ
diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/_transactions/1-bd4b52a8-552e-4270-b52f-3f678df92fed.txn b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_transactions/1-bd4b52a8-552e-4270-b52f-3f678df92fed.txn
new file mode 100644
index 00000000000..57cf3b8d551
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_transactions/1-bd4b52a8-552e-4270-b52f-3f678df92fed.txn differ
diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/18446744073709551613.manifest b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/18446744073709551613.manifest
new file mode 100644
index 00000000000..d3526033b05
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/18446744073709551613.manifest differ
diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/18446744073709551614.manifest
new file mode 100644
index 00000000000..9aebc63bd80
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/18446744073709551614.manifest differ
diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/latest_version_hint.json
new file mode 100644
index 00000000000..218abba1699
--- /dev/null
+++ b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/latest_version_hint.json
@@ -0,0 +1 @@
+{"version":2}
\ No newline at end of file
diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/data/110000101000110101111101bccc4b4c19af77197f1a92b69c.lance b/be/test/format_v2/table/lance/data/distance_range_dot.lance/data/110000101000110101111101bccc4b4c19af77197f1a92b69c.lance
new file mode 100644
index 00000000000..b72e061da89
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_dot.lance/data/110000101000110101111101bccc4b4c19af77197f1a92b69c.lance differ
diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/_indices/16361222-c72d-4428-9826-beeb3bfd3667/auxiliary.idx b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_indices/16361222-c72d-4428-9826-beeb3bfd3667/auxiliary.idx
new file mode 100644
index 00000000000..1483b3a7c8e
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_indices/16361222-c72d-4428-9826-beeb3bfd3667/auxiliary.idx differ
diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/_indices/16361222-c72d-4428-9826-beeb3bfd3667/index.idx b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_indices/16361222-c72d-4428-9826-beeb3bfd3667/index.idx
new file mode 100644
index 00000000000..79948a808fe
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_indices/16361222-c72d-4428-9826-beeb3bfd3667/index.idx differ
diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/_transactions/0-2dfc92d6-8481-4406-8893-9fc051e167d2.txn b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_transactions/0-2dfc92d6-8481-4406-8893-9fc051e167d2.txn
new file mode 100644
index 00000000000..6cd36ac9668
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_transactions/0-2dfc92d6-8481-4406-8893-9fc051e167d2.txn differ
diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/_transactions/1-f4a6b175-3626-4561-817e-c45c857597fa.txn b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_transactions/1-f4a6b175-3626-4561-817e-c45c857597fa.txn
new file mode 100644
index 00000000000..b2616f9b6e6
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_transactions/1-f4a6b175-3626-4561-817e-c45c857597fa.txn differ
diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/18446744073709551613.manifest b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/18446744073709551613.manifest
new file mode 100644
index 00000000000..3a4c998b412
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/18446744073709551613.manifest differ
diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/18446744073709551614.manifest
new file mode 100644
index 00000000000..97194c3229a
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/18446744073709551614.manifest differ
diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/latest_version_hint.json
new file mode 100644
index 00000000000..218abba1699
--- /dev/null
+++ b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/latest_version_hint.json
@@ -0,0 +1 @@
+{"version":2}
\ No newline at end of file
diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/data/0000100001010101000001014167dc46e9aa87a5e01e036b2a.lance b/be/test/format_v2/table/lance/data/distance_range_sq.lance/data/0000100001010101000001014167dc46e9aa87a5e01e036b2a.lance
new file mode 100644
index 00000000000..ac784824743
Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_sq.lance/data/0000100001010101000001014167dc46e9aa87a5e01e036b2a.lance differ
diff --git a/be/test/format_v2/table/lance/distance_range_fixture.md b/be/test/format_v2/table/lance/distance_range_fixture.md
new file mode 100644
index 00000000000..d1f3723304b
--- /dev/null
+++ b/be/test/format_v2/table/lance/distance_range_fixture.md
@@ -0,0 +1,93 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Distance range fixture
+
+Synthetic single-vector data generated with `pylance==7.0.0`, using Lance data
+format 2.2 and an IVF_FLAT V3 index. No external service is required to read it.
+Rows 0–127 are indexed in fragment 0. Rows 128–130 were appended after indexing
+in fragment 1. For query `[127, 0, 0]`, range `[1, 4)` returns rows 126 and 128,
+which exercises both index coverage and the unindexed tail.
+
+Regenerate from the repository root:
+
+```python
+import lance
+import pyarrow as pa
+
+path = "be/test/format_v2/table/lance/data/distance_range.lance"
+schema = pa.schema([
+ pa.field("row_id", pa.int64(), nullable=False),
+ pa.field("embedding", pa.list_(pa.float32(), 3), nullable=False),
+])
+
+def batch(ids):
+ return pa.Table.from_pydict({
+ "row_id": list(ids),
+ "embedding": [[float(i), 0.0, 0.0] for i in ids],
+ }, schema=schema)
+
+dataset = lance.write_dataset(batch(range(128)), path, mode="overwrite",
+ data_storage_version="2.2")
+dataset.create_index("embedding", index_type="IVF_FLAT", num_partitions=1,
+ name="embedding_idx", index_file_version="V3")
+lance.write_dataset(batch(range(128, 131)), path, mode="append",
+ data_storage_version="2.2")
+```
+
+Delete the previous generated dataset directory before regenerating to keep the
+fixture free of obsolete files. Filenames and index UUIDs may change; tests discover
+them from the manifest.
+
+## Range fallback fixtures
+
+`distance_range_sq.lance` contains vectors `[i, 0, 0]` for integers 0 through 255,
+followed by row 256 with `[0.49, 0, 0]`, and an IVF_SQ V3 index. With query
+`[0.51, 0, 0]`, row 256 has squared L2 distance approximately `0.0004`, inside
+`[0.0001, 0.001)`. Its quantized distance is zero, so applying the lower bound
+before refinement incorrectly discards it.
+
+`distance_range_dot.lance` contains row 0 with `[1, 0, 0]` and 127 rows with
+`[0.5, 0, 0]`, and an IVF_FLAT V3 DOT index. Query `[-FLT_MAX, 0, 0]` with only
+an inclusive lower bound of `FLT_MAX` must return row 0. Replacing the absent
+upper bound with an exclusive `FLT_MAX` incorrectly discards it.
+
+These fixtures exercise fragment splits without index UUIDs, as planned by the
+FE for unsafe indexed range searches, with both `use_index=true` and `false`.
+They use the same schema and writer versions as the fixture above:
+
+```python
+from pathlib import Path
+
+root = Path("be/test/format_v2/table/lance/data")
+for name, values, index_type, metric in [
+ ("distance_range_sq", [[float(i), 0.0, 0.0] for i in range(256)]
+ + [[0.49, 0.0, 0.0]], "IVF_SQ", "l2"),
+ ("distance_range_dot", [[1.0, 0.0, 0.0]]
+ + [[0.5, 0.0, 0.0] for _ in range(127)], "IVF_FLAT", "dot"),
+]:
+ data = pa.Table.from_pydict({
+ "row_id": list(range(len(values))), "embedding": values,
+ }, schema=schema)
+ dataset = lance.write_dataset(data, str(root / (name + ".lance")),
+ data_storage_version="2.2")
+ dataset.create_index("embedding", index_type=index_type, metric=metric,
+ num_partitions=1, name="embedding_idx",
+ index_file_version="V3")
+```
diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp
index d13665f2998..aba7cfee95b 100644
--- a/be/test/format_v2/table/lance_reader_test.cpp
+++ b/be/test/format_v2/table/lance_reader_test.cpp
@@ -1600,6 +1600,173 @@ TEST(LanceTableReaderVectorSearchTest, RejectsInvalidQueryParallelismBeforeOpeni
EXPECT_NE(std::string::npos, status.to_string().find("query_parallelism"));
}
+TEST(LanceTableReaderVectorSearchTest, ValidatesDistanceRangeBeforeDatasetAccess) {
+ TQueryGlobals globals;
+ RuntimeState state(globals);
+ const Columns columns {projected_column("_distance", TYPE_FLOAT, true)};
+ for (int mode = 0; mode < 6; ++mode) {
+ SCOPED_TRACE(mode);
+ auto params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 0);
+ auto& request = params.lance_scan_params.external_search_request;
+ auto& vector = request.search_query.vector_search;
+ vector.__set_distance_lower_bound(1.0);
+ vector.__set_distance_upper_bound(2.0);
+ if (mode == 0) vector.__set_distance_upper_bound(std::numeric_limits<double>::infinity());
+ if (mode == 1) vector.__set_distance_lower_bound(std::numeric_limits<double>::quiet_NaN());
+ if (mode == 2) vector.__set_distance_upper_bound(std::numeric_limits<double>::max());
+ if (mode == 3) vector.__set_distance_upper_bound(1.0);
+ if (mode == 4) vector.__set_distance_upper_bound(1.00000001);
+ if (mode == 5) vector.query_vector.__set_num_vectors(1);
+ RuntimeProfile profile("invalid_distance_range");
+ LanceTableReader reader;
+ const auto status = init_reader(&reader, columns, &state, &profile, ¶ms);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(std::string::npos,
+ status.to_string().find(mode == 5 ? "multi-vector" : "distance"));
+ }
+}
+
+TEST(LanceTableReaderVectorSearchTest, AppliesDistanceBoundsAndOffset) {
+ const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/all_types.lance";
+ LanceFixtureInfo fixture;
+ ASSERT_TRUE(get_fixture_info(uri, &fixture).ok());
+ const Columns columns {projected_column("row_id", TYPE_BIGINT, false),
+ projected_column("_distance", TYPE_FLOAT, true)};
+ TQueryGlobals globals;
+ RuntimeState state(globals);
+ for (int mode = 0; mode < 5; ++mode) {
+ SCOPED_TRACE(mode);
+ auto params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 10, mode == 4 ? 1 : 0);
+ auto& request = params.lance_scan_params.external_search_request;
+ auto& vector = request.search_query.vector_search;
+ if (mode != 1) vector.__set_distance_lower_bound(1.0);
+ if (mode != 2) vector.__set_distance_upper_bound(mode == 3 ? 1.1 : 8.25);
+ RuntimeProfile profile("distance_range");
+ LanceTableReader reader;
+ ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok());
+ ASSERT_TRUE(prepare_fixture(&reader, uri, fixture, fixture.fragment_ids).ok());
+ Block block;
+ add_output_columns(&block, columns);
+ const auto rows = read_vector_search_rows(&reader, &block);
+ if (mode == 4) {
+ EXPECT_TRUE(rows.empty());
+ } else {
+ ASSERT_FALSE(rows.empty());
+ EXPECT_EQ(mode == 1 ? 1 : 2, rows[0].first);
+ for (const auto& [id, distance] : rows) {
+ if (mode != 1) EXPECT_GE(distance, 1.0F);
+ if (mode != 2) EXPECT_LT(distance, mode == 3 ? 1.1F : 8.25F);
+ }
+ if (mode == 0 || mode == 3) EXPECT_EQ(1U, rows.size());
+ }
+ ASSERT_TRUE(reader.close().ok());
+ }
+}
+
+TEST(LanceTableReaderVectorSearchTest, DistanceRangeFallbackPreservesExactScores) {
+ const Columns columns {projected_column("row_id", TYPE_BIGINT, false),
+ projected_column("_distance", TYPE_FLOAT, true)};
+ TQueryGlobals globals;
+ RuntimeState state(globals);
+ for (const bool dot : {false, true}) {
+ const std::filesystem::path uri =
+ dot ? "./be/test/format_v2/table/lance/data/distance_range_dot.lance"
+ : "./be/test/format_v2/table/lance/data/distance_range_sq.lance";
+ LanceFixtureInfo fixture;
+ ASSERT_TRUE(get_fixture_info(uri, &fixture).ok());
+ for (const bool use_index : {false, true}) {
+ for (const bool filtered : {false, true}) {
+ SCOPED_TRACE(std::to_string(dot) + "/" + std::to_string(use_index) + "/" +
+ std::to_string(filtered));
+ const float max = std::numeric_limits<float>::max();
+ auto params = make_float32_vector_search_params(
+ dot ? std::array<float, 3> {-max, 0.0F, 0.0F}
+ : std::array<float, 3> {0.51F, 0.0F, 0.0F},
+ 300, 0,
+ filtered ? std::optional<std::string>("row_id >= 0") : std::nullopt);
+ auto& request = params.lance_scan_params.external_search_request;
+ auto& vector = request.search_query.vector_search;
+ vector.__set_distance_lower_bound(dot ? max : 0.0001);
+ if (dot) {
+ vector.__set_metric(TVectorMetric::DOT_PRODUCT);
+ } else {
+ vector.__set_distance_upper_bound(0.001);
+ }
+ request.vector_search_options.__set_use_index(use_index);
+ request.vector_search_options.__set_refine_factor(1);
+ request.vector_search_options.__set_query_parallelism(4);
+ // Range fallback splits have no index UUIDs even when use_index was requested.
+ // SQ quantization must not drop 0.0004, nor an absent upper bound drop FLT_MAX.
+ RuntimeProfile profile("distance_range_fallback");
+ LanceTableReader reader;
+ ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok());
+ ASSERT_TRUE(prepare_fixture(&reader, uri, fixture, fixture.fragment_ids).ok());
+ Block block;
+ add_output_columns(&block, columns);
+ const auto rows = read_vector_search_rows(&reader, &block);
+ ASSERT_EQ(1U, rows.size());
+ EXPECT_EQ(dot ? 0 : 256, rows[0].first);
+ if (dot) {
+ EXPECT_FLOAT_EQ(max, rows[0].second);
+ } else {
+ EXPECT_NEAR(0.0004F, rows[0].second, 1e-8F);
+ }
+ ASSERT_TRUE(reader.close().ok());
+ }
+ }
+ }
+}
+
+TEST(LanceTableReaderVectorSearchTest, DistanceRangeCoversIndexedAndAppendedFragments) {
+ const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/distance_range.lance";
+ LanceFixtureInfo fixture;
+ ASSERT_TRUE(get_fixture_info(uri, &fixture).ok());
+ ASSERT_EQ(2U, fixture.fragment_ids.size());
+ std::vector<std::string> segments;
+ ASSERT_TRUE(get_index_segment_uuids(uri, "embedding_idx", &segments).ok());
+ ASSERT_EQ(1U, segments.size());
+ const Columns columns {projected_column("row_id", TYPE_BIGINT, false),
+ projected_column("_distance", TYPE_FLOAT, true)};
+ TQueryGlobals globals;
+ RuntimeState state(globals);
+ for (const bool use_index : {false, true}) {
+ for (const bool filtered : {false, true}) {
+ std::vector<int64_t> ids;
+ for (size_t split = 0; split < fixture.fragment_ids.size(); ++split) {
+ SCOPED_TRACE(std::to_string(use_index) + "/" + std::to_string(filtered) + "/" +
+ std::to_string(split));
+ auto params = make_float32_vector_search_params(
+ {127.0F, 0.0F, 0.0F}, 10, 0,
+ filtered ? std::optional<std::string>("row_id >= 127") : std::nullopt);
+ auto& request = params.lance_scan_params.external_search_request;
+ request.search_query.vector_search.__set_distance_lower_bound(1.0);
+ request.search_query.vector_search.__set_distance_upper_bound(4.0);
+ const bool indexed_split = use_index && split == 0;
+ request.vector_search_options.__set_use_index(indexed_split);
+ request.vector_search_options.__set_nprobes(1);
+ auto range = make_lance_range(uri, fixture.version, {fixture.fragment_ids[split]});
+ if (indexed_split) {
+ range.table_format_params.lance_params.__set_index_segment_uuids(segments);
+ }
+ RuntimeProfile profile("mixed_distance_range");
+ LanceTableReader reader;
+ ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok());
+ ASSERT_TRUE(prepare_range(&reader, range).ok());
+ Block block;
+ add_output_columns(&block, columns);
+ for (const auto& [id, distance] : read_vector_search_rows(&reader, &block)) {
+ ids.push_back(id);
+ EXPECT_FLOAT_EQ(1.0F, distance);
+ }
+ ASSERT_TRUE(reader.close().ok());
+ }
+ std::ranges::sort(ids);
+ EXPECT_EQ(filtered ? std::vector<int64_t> {128} : std::vector<int64_t>({126, 128}),
+ ids);
+ }
+ }
+}
+
TEST(LanceTableReaderVectorSearchTest, QueryParallelismPreservesResults) {
const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/all_types.lance";
LanceFixtureInfo fixture;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java
index c572234f027..4ac720cad4d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java
@@ -359,6 +359,13 @@ public class LanceScanNode extends FileQueryScanNode {
result.append(prefix).append("lanceQueryParallelism=")
.append(externalSearchRequest.getVectorSearchOptions().getQueryParallelism()).append("\n");
}
+ if (vector.isSetDistanceLowerBound() || vector.isSetDistanceUpperBound()) {
+ result.append(prefix).append("lanceDistanceRange=[")
+ .append(vector.isSetDistanceLowerBound() ? vector.getDistanceLowerBound() : "-inf")
+ .append(", ")
+ .append(vector.isSetDistanceUpperBound() ? vector.getDistanceUpperBound() : "+inf")
+ .append(")\n");
+ }
result.append(prefix).append("lanceMetric=")
.append(vector.isSetMetric()
? VectorSearchTableValuedFunction.metricName(vector.getMetric()) : "default")
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlan.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlan.java
index d2259c697ed..b4968f1d67f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlan.java
@@ -65,6 +65,8 @@ final class LanceScanPlan {
NO_MATCH,
/** The selected index has a different or unknown metric. */
METRIC_MISMATCH,
+ /** The pinned index search cannot safely evaluate this distance range. */
+ DISTANCE_RANGE_FALLBACK,
/** At least one segment has no fragment coverage metadata. */
UNKNOWN_COVERAGE,
/** All segment coverage lies outside the selected snapshot's visible fragments. */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java
index 2d97bc802bf..96925ff2399 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java
@@ -35,6 +35,8 @@ import org.apache.doris.thrift.TVectorMetric;
import org.apache.doris.thrift.TVectorSearchOptions;
import org.apache.doris.thrift.TVectorSearchParams;
+import org.lance.index.IndexType;
+
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
@@ -227,6 +229,16 @@ final class LanceScanPlanner {
vectorIndexStatus = VectorIndexStatus.METRIC_MISMATCH;
continue;
}
+ boolean hasLower = vectorSearchParam.isSetDistanceLowerBound();
+ boolean hasUpper = vectorSearchParam.isSetDistanceUpperBound();
+ // Quantized indexes apply bounds before exact refinement and can discard valid rows.
+ // The pinned parallel IVF_FLAT path also replaces absent bounds with finite sentinels.
+ // Only a two-sided IVF_FLAT range is safe; unknown index types must use exact scans.
+ if ((hasLower || hasUpper) && (!hasLower || !hasUpper || matchingSegments.stream()
+ .anyMatch(segment -> segment.getIndexType() != IndexType.IVF_FLAT))) {
+ vectorIndexStatus = VectorIndexStatus.DISTANCE_RANGE_FALLBACK;
+ continue;
+ }
Optional<LanceSplitBuilder> indexPlan = planIndexSegments(
metadata, matchingSegments, visibleFragments, IndexCoveragePolicy.VECTOR_FALLBACK);
if (!indexPlan.isPresent()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 68cd2da1898..172d3ac2422 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -2884,7 +2884,9 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
throw new AnalysisException("Duplicate vector_search property: " + key);
}
if (argument.value.constant() instanceof DorisParser.PlaceholderContext) {
- if (!ImmutableSet.of("query_vector", "top_k", "offset", "filter").contains(key)) {
+ // Bounds change the search results, but never the prepared result schema.
+ if (!ImmutableSet.of("query_vector", "top_k", "offset", "filter",
+ "distance_lower_bound", "distance_upper_bound").contains(key)) {
throw new AnalysisException("vector_search property '" + key
+ "' must be constant in a prepared statement");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunction.java
index 2e9e2374b7f..d9d121924fd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunction.java
@@ -64,10 +64,13 @@ public class VectorSearchTableValuedFunction extends LanceExternalSearchTableVal
private static final String NPROBES = "nprobes";
private static final String REFINE_FACTOR = "refine_factor";
private static final String EF = "ef";
+ private static final String DISTANCE_LOWER_BOUND = "distance_lower_bound";
+ private static final String DISTANCE_UPPER_BOUND = "distance_upper_bound";
private static final String USE_INDEX = "use_index";
private static final Set<String> PROPERTIES = ImmutableSet.of(
TABLE, COLUMN, QUERY_VECTOR, TOP_K, OFFSET, METRIC, FILTER,
- NPROBES, REFINE_FACTOR, EF, USE_INDEX, QUERY_PARALLELISM, VERSION, TIMESTAMP, TAG, BRANCH);
+ NPROBES, REFINE_FACTOR, EF, USE_INDEX, QUERY_PARALLELISM,
+ VERSION, TIMESTAMP, TAG, BRANCH, DISTANCE_LOWER_BOUND, DISTANCE_UPPER_BOUND);
public VectorSearchTableValuedFunction(Map<String, String> properties)
throws AnalysisException {
@@ -82,6 +85,8 @@ public class VectorSearchTableValuedFunction extends LanceExternalSearchTableVal
private static PreparedSearch prepare(Map<String, String> properties, boolean deferQueryVector)
throws AnalysisException {
Map<String, String> params = normalizeProperties(properties, PROPERTIES, NAME);
+ TVectorSearchParams vectorParams = parseDistanceBounds(params);
+ boolean hasDistanceRange = vectorParams.isSetDistanceLowerBound() || vectorParams.isSetDistanceUpperBound();
boolean useIndex = !params.containsKey(USE_INDEX)
|| parseBoolean(params.get(USE_INDEX), USE_INDEX);
CommonSearch common = prepareCommon(params, NAME,
@@ -89,6 +94,10 @@ public class VectorSearchTableValuedFunction extends LanceExternalSearchTableVal
Field vectorField = requireSearchColumn(
common.metadata().getSchema(), required(params, COLUMN, NAME), "vector");
+ // Reject matrices at PREPARE as well as EXECUTE; lance-c only supports single-vector ranges.
+ if (hasDistanceRange && vectorField.getType() instanceof ArrowType.List) {
+ throw new AnalysisException("Distance bounds are not supported for Lance multi-vector search");
+ }
int vectorFieldId = useIndex
? requireLanceFieldId(common.metadata(), vectorField) : -1;
// PREPARE needs the table schema, but the vector is supplied only at EXECUTE.
@@ -100,8 +109,7 @@ public class VectorSearchTableValuedFunction extends LanceExternalSearchTableVal
vectorField, required(params, QUERY_VECTOR, NAME));
}
- TVectorSearchParams vectorParams = new TVectorSearchParams()
- .setColumn(vectorField.getName())
+ vectorParams.setColumn(vectorField.getName())
.setQueryVector(queryVector)
.setTopK(common.topK())
.setOffset(common.offset());
@@ -126,6 +134,36 @@ public class VectorSearchTableValuedFunction extends LanceExternalSearchTableVal
common, vectorFieldId, searchRequest, DISTANCE_COLUMN, "vector search");
}
+ @VisibleForTesting
+ static TVectorSearchParams parseDistanceBounds(Map<String, String> params) throws AnalysisException {
+ TVectorSearchParams vector = new TVectorSearchParams();
+ if (params.containsKey(DISTANCE_LOWER_BOUND)) {
+ vector.setDistanceLowerBound(parseDistanceBound(params.get(DISTANCE_LOWER_BOUND), DISTANCE_LOWER_BOUND));
+ }
+ if (params.containsKey(DISTANCE_UPPER_BOUND)) {
+ vector.setDistanceUpperBound(parseDistanceBound(params.get(DISTANCE_UPPER_BOUND), DISTANCE_UPPER_BOUND));
+ }
+ // Compare after FLOAT rounding, matching the native API even for nearly equal inputs.
+ if (vector.isSetDistanceLowerBound() && vector.isSetDistanceUpperBound()
+ && vector.getDistanceLowerBound() >= vector.getDistanceUpperBound()) {
+ throw new AnalysisException(
+ "distance_lower_bound must be less than distance_upper_bound at FLOAT precision");
+ }
+ return vector;
+ }
+
+ private static float parseDistanceBound(String value, String property) throws AnalysisException {
+ try {
+ float bound = Float.parseFloat(value);
+ if (Float.isFinite(bound)) {
+ return bound;
+ }
+ } catch (NumberFormatException | NullPointerException ignored) {
+ // Report the SQL property rather than leaking a Java parsing exception.
+ }
+ throw new AnalysisException("'" + property + "' must be a finite FLOAT");
+ }
+
@VisibleForTesting
static TVectorSearchOptions buildVectorSearchOptions(
Map<String, String> params, boolean useIndex) throws AnalysisException {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java
index bd555d1dab3..32173603c8f 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java
@@ -605,6 +605,46 @@ public class LanceScanNodeTest {
assertSplit(splits.get(3), 4, 8, 63);
}
+ @Test
+ public void testDistanceRangesOnlyUseSafeIndexPaths() throws Exception {
+ for (IndexType type : IndexType.values()) {
+ if (type.getValue() < IndexType.VECTOR.getValue()) {
+ continue;
+ }
+ UUID segment = UUID.fromString("11111111-2222-3333-4444-555555555555");
+ LanceTableMetadata metadata = LanceTableMetadata.createSnapshotWithIndexes(
+ new LanceTableAccess("s3://bucket/table.lance", Collections.emptyMap()),
+ 42, vectorSchema(), Arrays.asList(new LanceFragmentInfo(1, 8, 8),
+ new LanceFragmentInfo(2, 7, 7)),
+ Collections.singletonMap("vector", 9),
+ Collections.singletonList(new LanceIndexSegmentInfo(segment, "vector_idx",
+ Collections.singletonList(9), Collections.singletonList(1L), type, "L2")));
+ for (int bounds = 0; bounds < 4; bounds++) {
+ TExternalSearchRequest request = vectorSearchRequest(5, 0);
+ if ((bounds & 1) != 0) {
+ request.getSearchQuery().getVectorSearch().setDistanceLowerBound(1);
+ }
+ if ((bounds & 2) != 0) {
+ request.getSearchQuery().getVectorSearch().setDistanceUpperBound(4);
+ }
+ request.setVectorSearchOptions(new TVectorSearchOptions().setUseIndex(true).setQueryParallelism(4));
+ LanceScanNode node = newSearchNode(metadata, request);
+ List<Split> splits = node.getSplits(2);
+ boolean indexed = bounds == 0 || (bounds == 3 && type == IndexType.IVF_FLAT);
+ String status = indexed ? "USED" : "DISTANCE_RANGE_FALLBACK";
+ Assert.assertTrue(type + "/" + bounds, node.getNodeExplainString("", TExplainLevel.NORMAL)
+ .contains("lanceVectorIndexStatus=" + status));
+ Assert.assertEquals(2, splits.size());
+ if (indexed) {
+ assertIndexSplit(splits.get(0), segment, Collections.singletonList(1L), 8, 100);
+ } else {
+ assertSplit(splits.get(0), 1, 8, 100);
+ }
+ assertSplit(splits.get(1), 2, 8, 88);
+ }
+ }
+ }
+
@Test
public void testExternalSearchUsesOneSplitPerIndexSegmentAndKeepsUnindexedFragments()
throws Exception {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LancePreparedSearchTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LancePreparedSearchTest.java
index 074de75998f..7c2a2361412 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LancePreparedSearchTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LancePreparedSearchTest.java
@@ -62,9 +62,114 @@ import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
public class LancePreparedSearchTest {
+ @ParameterizedTest
+ @ValueSource(strings = {"'distance_lower_bound'=?, 'distance_upper_bound'=?",
+ "'distance_lower_bound'=?", "'distance_upper_bound'=?",
+ "'distance_lower_bound'='-1', 'distance_upper_bound'=?",
+ "'distance_lower_bound'=?, 'distance_upper_bound'='4'"})
+ public void testRepeatedBindingUsesFreshDistanceBounds(String bounds) {
+ ConnectContext previous = ConnectContext.get();
+ ConnectContext context = MemoTestUtils.createConnectContext();
+ LanceExternalTable table = mockTable(new AtomicLong(42));
+ try (MockedStatic<LanceExternalSearchTableValuedFunction> lookup = Mockito.mockStatic(
+ LanceExternalSearchTableValuedFunction.class, Mockito.CALLS_REAL_METHODS)) {
+ lookup.when(() -> LanceExternalSearchTableValuedFunction.findLanceExternalTable(
+ Mockito.any(TableName.class))).thenReturn(table);
+ Pair<LogicalPlan, StatementContext> parsed = new NereidsParser().parseMultiple(
+ "select * from vector_search('table'='catalog.db.items', 'column'='embedding', "
+ + "'query_vector'='[1,2]', " + bounds + ")").get(0);
+ List<Placeholder> parameters = parsed.second.getPlaceholders();
+ StatementContext prepare = MemoTestUtils.createStatementContext(context, "");
+ prepare.setPrepareStage(true);
+ analyze(parsed.first, prepare);
+ for (int iteration = 0; iteration < 2; iteration++) {
+ StatementContext execute = MemoTestUtils.createStatementContext(context, "");
+ for (int i = 0; i < parameters.size(); i++) {
+ execute.getIdToPlaceholderRealExpr().put(parameters.get(i).getPlaceholderId(),
+ new StringLiteral(Integer.toString(iteration + i)));
+ }
+ org.apache.doris.thrift.TVectorSearchParams vector = analyze(parsed.first, execute)
+ .getSearchRequest().getSearchQuery().getVectorSearch();
+ Assertions.assertEquals(bounds.contains("distance_lower_bound"), vector.isSetDistanceLowerBound());
+ Assertions.assertEquals(bounds.contains("distance_upper_bound"), vector.isSetDistanceUpperBound());
+ if (vector.isSetDistanceLowerBound()) {
+ Assertions.assertEquals(bounds.contains("='-1'") ? -1.0 : iteration,
+ vector.getDistanceLowerBound());
+ }
+ if (vector.isSetDistanceUpperBound()) {
+ Assertions.assertEquals(bounds.contains("='4'") ? 4.0 : iteration + parameters.size() - 1,
+ vector.getDistanceUpperBound());
+ }
+ // Failed EXECUTEs must not poison the reusable unbound plan or retain old bounds.
+ for (String invalid : new String[] {"NaN", "Infinity", "1e39", "not-a-number"}) {
+ execute.getIdToPlaceholderRealExpr().put(parameters.get(0).getPlaceholderId(),
+ new StringLiteral(invalid));
+ Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
+ () -> analyze(parsed.first, execute));
+ }
+ execute.getIdToPlaceholderRealExpr().put(parameters.get(0).getPlaceholderId(), NullLiteral.INSTANCE);
+ Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
+ () -> analyze(parsed.first, execute));
+ if (parameters.size() == 2) {
+ execute.getIdToPlaceholderRealExpr().put(parameters.get(0).getPlaceholderId(),
+ new StringLiteral("4"));
+ Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
+ () -> analyze(parsed.first, execute));
+ }
+ }
+ } finally {
+ ConnectContext.remove();
+ if (previous != null) {
+ previous.setThreadLocalInfo();
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testDistanceRangeAndMultiVectorPrepare(boolean multiVector) throws Exception {
+ ConnectContext previous = ConnectContext.get();
+ MemoTestUtils.createConnectContext();
+ LanceExternalTable table = mockTable(new AtomicLong(42), multiVector);
+ try (MockedStatic<LanceExternalSearchTableValuedFunction> lookup = Mockito.mockStatic(
+ LanceExternalSearchTableValuedFunction.class, Mockito.CALLS_REAL_METHODS)) {
+ lookup.when(() -> LanceExternalSearchTableValuedFunction.findLanceExternalTable(
+ Mockito.any(TableName.class))).thenReturn(table);
+ Map<String, String> properties = ImmutableMap.<String, String>builder()
+ .put("table", "catalog.db.items").put("column", "embedding").put("use_index", "false")
+ .put("distance_lower_bound", "1").put("distance_upper_bound", "4")
+ .put("query_vector", multiVector ? "[[1,2]]" : "[1,2]").build();
+ for (boolean prepare : new boolean[] {true, false}) {
+ if (multiVector) {
+ Exception error = Assertions.assertThrows(org.apache.doris.common.AnalysisException.class,
+ () -> new VectorSearchTableValuedFunction(properties, prepare));
+ Assertions.assertTrue(error.getMessage().contains("multi-vector"));
+ } else {
+ VectorSearchTableValuedFunction function = new VectorSearchTableValuedFunction(properties, prepare);
+ Assertions.assertEquals(1, function.getSearchRequest().getSchemaVersion());
+ Assertions.assertEquals(1.0, function.getSearchRequest().getSearchQuery()
+ .getVectorSearch().getDistanceLowerBound());
+ Assertions.assertEquals(4.0, function.getSearchRequest().getSearchQuery()
+ .getVectorSearch().getDistanceUpperBound());
+ Assertions.assertEquals(!prepare, function.getSearchRequest().getSearchQuery()
+ .getVectorSearch().isSetQueryVector());
+ }
+ }
+ VectorSearchTableValuedFunction ordinary = new VectorSearchTableValuedFunction(ImmutableMap.of(
+ "table", "catalog.db.items", "column", "embedding", "use_index", "false"), true);
+ Assertions.assertEquals(1, ordinary.getSearchRequest().getSchemaVersion());
+ } finally {
+ ConnectContext.remove();
+ if (previous != null) {
+ previous.setThreadLocalInfo();
+ }
+ }
+ }
+
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testPrepareAndRepeatedBindingUseFreshVectorAndSnapshot(boolean useIndex) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java
index a141419a1ba..181abca8dc9 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/VectorSearchTableValuedFunctionTest.java
@@ -23,7 +23,9 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.datasource.lance.metadata.LanceTableAccess;
import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
import org.apache.doris.thrift.TVectorSearchOptions;
+import org.apache.doris.thrift.TVectorSearchParams;
+import com.google.common.collect.ImmutableMap;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
@@ -42,6 +44,51 @@ public class VectorSearchTableValuedFunctionTest {
return VectorSearchTableValuedFunction.buildVectorSearchOptions(params, true);
}
+ @Test
+ public void testRejectInvalidDistanceBoundsBeforeMetadataAccess() {
+ for (String value : new String[] {"NaN", "Infinity", "-Infinity", "1e100", "invalid", ""}) {
+ AnalysisException error = Assert.assertThrows(AnalysisException.class,
+ () -> new VectorSearchTableValuedFunction(
+ Collections.singletonMap("distance_upper_bound", value)));
+ Assert.assertTrue(error.getMessage(), error.getMessage().contains("finite FLOAT"));
+ }
+ }
+
+ @Test
+ public void testDistanceBoundsRoundTrip() throws Exception {
+ Assert.assertFalse(VectorSearchTableValuedFunction.parseDistanceBounds(Collections.emptyMap())
+ .isSetDistanceLowerBound());
+ for (Map<String, String> properties : Arrays.asList(
+ Collections.singletonMap("distance_lower_bound", "-0.5"),
+ Collections.singletonMap("distance_upper_bound", "0"),
+ ImmutableMap.of("distance_lower_bound", "0.1", "distance_upper_bound", "1.5"))) {
+ TVectorSearchParams params = VectorSearchTableValuedFunction.parseDistanceBounds(properties);
+ TVectorSearchParams decoded = new TVectorSearchParams();
+ new TDeserializer().deserialize(decoded, new TSerializer().serialize(params));
+ Assert.assertEquals(params, decoded);
+ Assert.assertEquals(properties.containsKey("distance_lower_bound"), decoded.isSetDistanceLowerBound());
+ Assert.assertEquals(properties.containsKey("distance_upper_bound"), decoded.isSetDistanceUpperBound());
+ if (decoded.isSetDistanceLowerBound()) {
+ Assert.assertEquals((double) Float.parseFloat(properties.get("distance_lower_bound")),
+ decoded.getDistanceLowerBound(), 0.0);
+ }
+ }
+ }
+
+ @Test
+ public void testRejectEmptyOrReversedDistanceRange() {
+ for (String upper : new String[] {"0", "-1", "0.00000000000000000000000000000000000000000000001"}) {
+ AnalysisException error = Assert.assertThrows(AnalysisException.class,
+ () -> new VectorSearchTableValuedFunction(
+ ImmutableMap.of("distance_lower_bound", "0", "distance_upper_bound", upper)));
+ Assert.assertTrue(error.getMessage(), error.getMessage().contains("must be less than"));
+ }
+ Assert.assertThrows(AnalysisException.class, () -> VectorSearchTableValuedFunction.parseDistanceBounds(
+ ImmutableMap.of("distance_lower_bound", "1", "distance_upper_bound", "1.00000001")));
+ Assert.assertThrows(AnalysisException.class, () -> VectorSearchTableValuedFunction.parseDistanceBounds(
+ Collections.singletonMap("distance_lower_bound", "NaN")));
+ }
+
@Test
public void testQueryParallelismOptions() throws Exception {
Assert.assertNull(parseOptions(Collections.emptyMap()));
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 6e4fa8fd77b..6a76a159606 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -487,6 +487,9 @@ struct TVectorSearchParams {
3: optional i64 top_k
4: optional i64 offset
5: optional TVectorMetric metric
+ // FLOAT bounds: inclusive lower and exclusive upper.
+ 6: optional double distance_lower_bound
+ 7: optional double distance_upper_bound
}
enum TFtsCoverageMode {
diff --git a/regression-test/suites/external_table_p0/lance/test_lance_vector_search_distance_range.groovy b/regression-test/suites/external_table_p0/lance/test_lance_vector_search_distance_range.groovy
new file mode 100644
index 00000000000..f1447bf8167
--- /dev/null
+++ b/regression-test/suites/external_table_p0/lance/test_lance_vector_search_distance_range.groovy
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_lance_vector_search_distance_range", "p0,external") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (!"true".equalsIgnoreCase(enabled)) {
+ logger.info("Skip Lance distance range tests: Iceberg MinIO is disabled")
+ return
+ }
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ String catalogName = "test_lance_vector_search_distance_range"
+ String tableName = "${catalogName}.doris.vs_ivf_flat_f32"
+ String query = "[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]"
+ def search = { boolean indexed, String options ->
+ """vector_search("table"="${tableName}", "column"="embedding", "query_vector"="${query}",
+ "top_k"="10", "metric"="l2", "nprobes"="4", "use_index"="${indexed}" ${options})"""
+ }
+ sql "DROP CATALOG IF EXISTS `${catalogName}`"
+ try {
+ sql """CREATE CATALOG `${catalogName}` PROPERTIES (
+ "type"="lance", "lance.catalog.type"="filesystem", "warehouse"="s3://warehouse/lance",
+ "s3.endpoint"="http://${externalEnvIp}:${minioPort}", "s3.access_key"="admin",
+ "s3.secret_key"="password", "s3.region"="us-east-1", "use_path_style"="true")"""
+ sql "SET enable_file_scanner_v2 = true"
+ // Squared L2 is 16 * (row_id - 1)^2. Full IVF_FLAT probing gives an exact oracle.
+ for (boolean indexed : [false, true]) {
+ def cases = [
+ [', "distance_upper_bound"="16"', [1L]],
+ [', "distance_lower_bound"="16", "distance_upper_bound"="144"', [2L, 3L]],
+ [', "distance_lower_bound"="17", "distance_upper_bound"="64"', []],
+ [', "distance_lower_bound"="16", "distance_upper_bound"="144", "offset"="1"', [3L]],
+ [', "distance_lower_bound"="16", "distance_upper_bound"="144", "filter"="row_id > 2"', [3L]],
+ [', "distance_upper_bound"="0"', []]
+ ]
+ for (def entry : cases) {
+ // Project stored columns too, exercising row-id materialization after range selection.
+ def rows = sql "SELECT row_id, label, _distance FROM ${search(indexed, entry[0])} ORDER BY _distance"
+ assertEquals(entry[1], rows.collect { (it[0] as Number).longValue() })
+ for (def row : rows) {
+ double expected = 16.0 * Math.pow((row[0] as Number).longValue() - 1, 2)
+ assertEquals(expected, (row[2] as Number).doubleValue())
+ }
+ }
+ def lowerOnly = sql "SELECT row_id, _distance FROM ${search(indexed, ', \"distance_lower_bound\"=\"16\"')}"
+ assertTrue(lowerOnly.size() > 0 && lowerOnly.size() <= 10)
+ assertTrue(lowerOnly.every { (it[1] as Number).doubleValue() >= 16.0 })
+ explain {
+ sql "SELECT row_id FROM ${search(indexed, ', \"distance_upper_bound\"=\"16\"')}"
+ contains "lanceDistanceRange=[-inf, 16.0)"
+ // Single-sided ranges must bypass the native finite-sentinel index filter.
+ contains "lanceVectorIndexStatus=${indexed ? 'DISTANCE_RANGE_FALLBACK' : 'DISABLED'}"
+ }
+ }
+ for (String value : ["NaN", "Infinity", "1e100", "invalid"]) {
+ test {
+ sql "SELECT * FROM ${search(false, ', \"distance_upper_bound\"=\"' + value + '\"')}"
+ exception "finite FLOAT"
+ }
+ }
+ test {
+ sql "SELECT * FROM ${search(false, ', \"distance_lower_bound\"=\"1\", \"distance_upper_bound\"=\"1.00000001\"')}"
+ exception "must be less than"
+ }
+ } finally {
+ sql "DROP CATALOG IF EXISTS `${catalogName}`"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 10/28: [feature](flight) Support native Variant V2 results for ADBC on branch-4.1 (#68667)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e48911e417ef0c6148c19006f62f8a6ab07aa427
Author: Gabriel <li...@selectdb.com>
AuthorDate: Fri Oct 9 14:30:19 2026 +0800
[feature](flight) Support native Variant V2 results for ADBC on branch-4.1 (#68667)
### What problem does this PR solve?
Flight SQL exposes VARIANT as UTF8, preventing ADBC clients from
receiving native binary Variant V2 values. On branch-4.1, return Variant
V2 through the `arrow.parquet.variant` extension by default, with
`struct<metadata: binary not null, value: binary not null>` storage. SQL
NULL is a null struct; Variant null is an encoded non-null value.
Clients without a Variant extension implementation can access the
physical struct and field metadata.
Support **Variant V2 only**. Legacy Variant returns an unsupported
error, including nested legacy fields, constants, all-null columns and
empty results. Reject legacy types during BE Flight schema conversion
and defensively in the legacy SerDe. Remove the output-format session
variable and result-sink flag entirely. Users who need text can
explicitly `CAST(... AS STRING)`.
Keep execution, GetTables, GetSchema and prepared-statement schemas
consistent, including nested Variant fields. Native Variant output does
not depend on a heartbeat capability or a cluster-wide BE support check.
Retain the Variant V2 type requirement and reject non-native execution
schemas.
Reuse the V2 binary representation and compact dictionaries per selected
row, preserving physical scalar types and decimal scales without
repeatedly copying unrelated dictionary keys. Keep the value helper
private to `data_type_variant_v2_serde.cpp`. Native encoding accepts up
to 128 nested levels. Iceberg output keeps its existing behavior.
Document the V2 requirement and client decoding limitations in the
Python Flight sample README; parsing JSON does not bypass the configured
Variant representation.
### Release note
Arrow Flight SQL / ADBC returns Variant V2 as native Arrow Variant
binary by default, without a session switch. Legacy Variant is
unsupported. Use an explicit SQL cast to STRING for text output.
### Validation
- The native-output update passed all 46 selected Arrow/Flight tests
using a focused ASAN binary; the binary conversion code is unchanged by
the capability removal.
- For the capability removal, regenerated the heartbeat Thrift bindings,
compiled the affected FE sources and BE heartbeat translation unit, and
passed all 6 focused FE tests covering native schemas, legacy/UTF8
rejection and the existing Paimon heartbeat capability.
- FE Checkstyle, clang-format 16 and diff checks passed.
- The updated Groovy regression script and Python sample syntax were
checked in the preceding update.
- A full production build and live cluster regression/ADBC execution
were not rerun for this update; CI validation is requested.
### Check List (For Author)
- Test
- [x] Regression test updated
- [x] Unit Test
- Behavior changed:
- [x] Yes. Variant V2 uses native binary by default; legacy Variant
requires an explicit text cast.
- Does this need documentation?
- [x] Yes, usage, V2-only scope and client decoding limitations are
documented in the Python Flight sample README.
---
.../data_type_serde/data_type_variant_serde.cpp | 6 +
.../data_type_serde/data_type_variant_v2_serde.cpp | 78 +++-
be/src/format/arrow/arrow_block_convertor.cpp | 22 +
be/src/format/arrow/arrow_block_convertor.h | 7 +
be/src/format/arrow/arrow_row_batch.cpp | 57 ++-
be/src/format/arrow/arrow_row_batch.h | 14 +-
.../arrow_flight/arrow_flight_batch_reader.cpp | 1 +
be/test/format/arrow/arrow_flight_variant_test.cpp | 511 +++++++++++++++++++++
be/test/format/arrow/arrow_row_batch_test.cpp | 11 +-
.../java/org/apache/doris/planner/ResultSink.java | 2 +-
.../service/arrowflight/FlightSqlQuerySchema.java | 16 +-
.../service/arrowflight/FlightSqlSchemaHelper.java | 39 +-
.../FlightSqlConnectProcessorSchemaTest.java | 8 +-
.../arrowflight/FlightSqlNativeVariantTest.java | 124 +++++
fe/pom.xml | 1 -
.../test_flight_cancel_cleanup.groovy | 4 +-
.../test_flight_native_variant.groovy | 215 +++++++++
.../flink_connector_p0/flink_connector_type.groovy | 29 +-
samples/arrow-flight-sql/python/README.md | 39 +-
samples/arrow-flight-sql/python/test_variant.py | 96 ++++
20 files changed, 1238 insertions(+), 42 deletions(-)
diff --git a/be/src/core/data_type_serde/data_type_variant_serde.cpp b/be/src/core/data_type_serde/data_type_variant_serde.cpp
index 3bd9be5797f..e0ddd20b7d6 100644
--- a/be/src/core/data_type_serde/data_type_variant_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_variant_serde.cpp
@@ -157,6 +157,12 @@ Status DataTypeVariantSerDe::write_column_to_arrow(const IColumn& column, const
int64_t start, int64_t end,
const cctz::time_zone& ctz) const {
const auto* var = check_and_get_column<ColumnVariant>(column);
+ if (array_builder->type()->id() == arrow::Type::STRUCT) {
+ // Native Flight output must not reinterpret legacy storage as Variant V2.
+ return Status::NotSupported(
+ "Native Arrow Flight output only supports Variant V2, not legacy Variant; "
+ "cast the result to STRING for text output");
+ }
if (array_builder->type()->id() == arrow::Type::LARGE_STRING) {
auto& builder = assert_cast<arrow::LargeStringBuilder&>(*array_builder);
return write_variant_column_to_arrow_impl(column, *var, null_map, builder, start, end, ctz);
diff --git a/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp b/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp
index 0e0d7a58d2f..4e9ae6f872d 100644
--- a/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp
@@ -24,6 +24,7 @@
#include <algorithm>
#include <cstring>
#include <limits>
+#include <optional>
#include <orc/Vector.hh>
#include <span>
#include <utility>
@@ -53,6 +54,48 @@
namespace doris {
namespace {
+// ColumnVariantV2 already validates its encoded dictionaries and value structure. Only walk
+// the selected row here: validating every unused dictionary key per row is quadratic for
+// shared dictionaries, including when a nested ARRAY invokes this writer on one-row slices.
+Status append_flight_variant_value(VariantRef value, VariantBatchBuilder::Row& output,
+ size_t depth = 0) {
+ const auto basic_type = value.basic_type();
+ if (depth > VARIANT_MAX_NESTING_DEPTH) {
+ return Status::NotSupported(
+ "Native Arrow Variant nesting exceeds {}; "
+ "cast the result to STRING for text output",
+ VARIANT_MAX_NESTING_DEPTH);
+ }
+ if (value.value_size() != value.value.size) {
+ throw Exception(ErrorCode::CORRUPTION,
+ "Native Arrow Variant contains trailing value bytes");
+ }
+ if (basic_type == VariantBasicType::OBJECT) {
+ auto object = output.start_object();
+ auto fields = value.object_view();
+ for (uint32_t i = 0; i < fields.size(); ++i) {
+ uint32_t field_id;
+ auto child = fields.value_at(i, &field_id);
+ object.add_key(value.metadata.key_at(field_id));
+ RETURN_IF_ERROR(append_flight_variant_value(child, output, depth + 1));
+ }
+ object.finish();
+ } else if (basic_type == VariantBasicType::ARRAY) {
+ auto array = output.start_array();
+ for (uint32_t i = 0; i < value.num_elements(); ++i) {
+ RETURN_IF_ERROR(append_flight_variant_value(value.array_at(i), output, depth + 1));
+ }
+ array.finish();
+ } else {
+ // Primitives never reference dictionary keys. Reuse physical import to retain widths,
+ // decimal scales and non-JSON types; canonical equality encoding normalizes those away.
+ static constexpr char empty_metadata[] = {0x11, 0, 0};
+ value.metadata = {empty_metadata, sizeof(empty_metadata)};
+ output.add_value(value);
+ }
+ return Status::OK();
+}
+
using MetaIdsColumn = ColumnVector<TYPE_UINT32>;
std::span<const NullMap::value_type> forced_nulls(const NullMap* null_map) {
@@ -625,13 +668,14 @@ Status write_paimon_variant(const IColumn& column, const NullMap* null_map,
return status;
}
-Status write_iceberg_variant(const IColumn& column, const NullMap* null_map,
- arrow::ArrayBuilder* array_builder, int64_t start, int64_t end) {
+Status write_parquet_variant_arrow(const IColumn& column, const NullMap* null_map,
+ arrow::ArrayBuilder* array_builder, int64_t start, int64_t end,
+ bool compact_metadata) {
if (start < 0 || end < start) {
- return Status::InvalidArgument("Invalid Iceberg Variant row range [{}, {})", start, end);
+ return Status::InvalidArgument("Invalid Variant Arrow row range [{}, {})", start, end);
}
if (array_builder->type()->id() != arrow::Type::STRUCT) {
- return Status::InvalidArgument("Iceberg Variant writer requires a struct builder, got {}",
+ return Status::InvalidArgument("Variant Arrow writer requires a struct builder, got {}",
array_builder->type()->ToString());
}
auto& builder = assert_cast<arrow::StructBuilder&>(*array_builder);
@@ -640,7 +684,7 @@ Status write_iceberg_variant(const IColumn& column, const NullMap* null_map,
type->field(1)->name() != "value" || type->field(0)->type()->id() != arrow::Type::BINARY ||
type->field(1)->type()->id() != arrow::Type::BINARY) {
return Status::InvalidArgument(
- "Iceberg Variant writer requires struct<metadata: binary, value: binary>, got {}",
+ "Variant Arrow writer requires struct<metadata: binary, value: binary>, got {}",
type->ToString());
}
auto& metadata_builder = assert_cast<arrow::BinaryBuilder&>(*builder.field_builder(0));
@@ -657,10 +701,27 @@ Status write_iceberg_variant(const IColumn& column, const NullMap* null_map,
if (!status.ok()) {
return;
}
+ std::optional<VariantBatchBuilder> compacted;
+ if (compact_metadata) {
+ const auto keys = value.metadata.dict_size();
+ // An empty dictionary or an object using every key already has row-local metadata.
+ if (keys != 0 && (value.basic_type() != VariantBasicType::OBJECT ||
+ value.num_elements() != keys)) {
+ VariantBatchBuilder encoder;
+ auto row = encoder.begin_row();
+ status = append_flight_variant_value(value, row);
+ if (!status.ok()) {
+ return;
+ }
+ row.finish();
+ compacted.emplace(encoder.finish_batch());
+ value = compacted->value_at(0);
+ }
+ }
if (value.metadata.size > std::numeric_limits<int32_t>::max() ||
value.value.size > std::numeric_limits<int32_t>::max()) {
status = Status::InvalidArgument(
- "Iceberg Variant metadata/value exceeds Arrow binary size limit");
+ "Variant Arrow metadata/value exceeds Arrow binary size limit");
return;
}
status = checkArrowStatus(builder.Append(), column, builder);
@@ -733,6 +794,9 @@ Status DataTypeVariantV2SerDe::write_column_to_arrow(const IColumn& column, cons
options.timezone = &ctz;
const size_t first = checked_row(start);
const size_t last = checked_row(end);
+ if (array_builder->type()->id() == arrow::Type::STRUCT) {
+ return write_parquet_variant_arrow(column, null_map, array_builder, start, end, true);
+ }
if (array_builder->type()->id() == arrow::Type::STRING) {
return write_arrow(column, null_map, assert_cast<arrow::StringBuilder&>(*array_builder),
first, last, options);
@@ -759,7 +823,7 @@ Status DataTypeVariantV2SerDe::write_column_to_iceberg_arrow(
const std::shared_ptr<const IDataType>&, const IColumn& column, const NullMap* null_map,
const std::shared_ptr<arrow::Field>&, arrow::ArrayBuilder* array_builder, int64_t start,
int64_t end, const cctz::time_zone&) const {
- return write_iceberg_variant(column, null_map, array_builder, start, end);
+ return write_parquet_variant_arrow(column, null_map, array_builder, start, end, false);
}
Status DataTypeVariantV2SerDe::write_column_to_orc(const std::string&, const IColumn& column,
diff --git a/be/src/format/arrow/arrow_block_convertor.cpp b/be/src/format/arrow/arrow_block_convertor.cpp
index 2148d0934af..685637fe737 100644
--- a/be/src/format/arrow/arrow_block_convertor.cpp
+++ b/be/src/format/arrow/arrow_block_convertor.cpp
@@ -473,6 +473,28 @@ Status ArrowBlockConvertor::init() {
return Status::OK();
}
+Status ArrowFlightArrowBlockConvertor::write_column(const std::shared_ptr<const IDataType>& type,
+ const DataTypeSerDe& serde,
+ const IColumn& column, const NullMap* null_map,
+ const std::shared_ptr<arrow::Field>& field,
+ arrow::ArrayBuilder* array_builder,
+ int64_t start, int64_t end,
+ const cctz::time_zone& ctz) const {
+ if (contains_extension_type(field->type())) {
+ std::shared_ptr<arrow::DataType> native_type;
+ RETURN_IF_ERROR(
+ ArrowFlightSchemaConvertor(ctz.name()).convert_to_arrow_type(type, &native_type));
+ // Check the extension identity and its complete nested shape before allowing the
+ // Variant SerDe to write binary storage. An arbitrary STRUCT is not a Variant binding.
+ // Timestamp labels may differ for equivalent fixed offsets, including inside containers.
+ if (is_declared_plain_arrow_binding(type, native_type, field->type())) {
+ return serde.write_column_to_arrow(column, null_map, array_builder, start, end, ctz);
+ }
+ }
+ return DorisArrowBlockConvertor::write_column(type, serde, column, null_map, field,
+ array_builder, start, end, ctz);
+}
+
Status ArrowFlightArrowBlockConvertor::convert_to_arrow(const Block& block, arrow::MemoryPool* pool,
std::shared_ptr<arrow::RecordBatch>* result,
size_t start_row, size_t end_row) const {
diff --git a/be/src/format/arrow/arrow_block_convertor.h b/be/src/format/arrow/arrow_block_convertor.h
index e0b58d76838..c18c28df72a 100644
--- a/be/src/format/arrow/arrow_block_convertor.h
+++ b/be/src/format/arrow/arrow_block_convertor.h
@@ -116,6 +116,13 @@ public:
Status convert_to_arrow(const Block& block, arrow::MemoryPool* pool,
std::shared_ptr<arrow::RecordBatch>* result, size_t start_row = 0,
size_t end_row = 0) const override;
+
+protected:
+ Status write_column(const std::shared_ptr<const IDataType>& type, const DataTypeSerDe& serde,
+ const IColumn& column, const NullMap* null_map,
+ const std::shared_ptr<arrow::Field>& field,
+ arrow::ArrayBuilder* array_builder, int64_t start, int64_t end,
+ const cctz::time_zone& ctz) const override;
};
class PythonArrowBlockConvertor final : public DorisArrowBlockConvertor {
diff --git a/be/src/format/arrow/arrow_row_batch.cpp b/be/src/format/arrow/arrow_row_batch.cpp
index a958843e519..9589f6e59ac 100644
--- a/be/src/format/arrow/arrow_row_batch.cpp
+++ b/be/src/format/arrow/arrow_row_batch.cpp
@@ -18,6 +18,7 @@
#include "format/arrow/arrow_row_batch.h"
#include <arrow/buffer.h>
+#include <arrow/extension/parquet_variant.h>
#include <arrow/io/memory.h>
#include <arrow/ipc/writer.h>
#include <arrow/record_batch.h>
@@ -40,14 +41,30 @@
#include "core/data_type/data_type_array.h"
#include "core/data_type/data_type_map.h"
#include "core/data_type/data_type_struct.h"
+#include "core/data_type/data_type_variant_v2.h"
#include "core/data_type/define_primitive_type.h"
#include "exprs/vexpr.h"
#include "exprs/vexpr_context.h"
#include "format/arrow/arrow_block_convertor.h"
+#include "format/arrow/arrow_utils.h"
#include "runtime/descriptors.h"
namespace doris {
+Status register_arrow_variant_extension() {
+ // Remote Flight readers must restore the extension before decoding the result schema.
+ static const auto status = [] {
+ if (arrow::GetExtensionType("arrow.parquet.variant") != nullptr) {
+ return arrow::Status::OK();
+ }
+ return arrow::RegisterExtensionType(
+ std::static_pointer_cast<arrow::ExtensionType>(arrow::extension::variant(
+ arrow::struct_({arrow::field("metadata", arrow::binary(), false),
+ arrow::field("value", arrow::binary(), false)}))));
+ }();
+ return status.ok() ? Status::OK() : Status::InternalError(status.ToString());
+}
+
Status DorisArrowSchemaConvertor::convert_to_arrow_type(
const DataTypePtr& origin_type, std::shared_ptr<arrow::DataType>* result) const {
auto type = get_serialized_type(origin_type);
@@ -157,10 +174,9 @@ Status DorisArrowSchemaConvertor::convert_to_arrow_type(
*result = std::make_shared<arrow::StructType>(fields);
break;
}
- case TYPE_VARIANT: {
+ case TYPE_VARIANT:
*result = arrow::utf8();
break;
- }
case TYPE_QUANTILE_STATE:
case TYPE_BITMAP:
case TYPE_HLL: {
@@ -229,6 +245,25 @@ std::string DorisArrowSchemaConvertor::timestamp_timezone(PrimitiveType) const {
return _timezone == "Z" ? "UTC" : _timezone;
}
+Status ArrowFlightSchemaConvertor::convert_to_arrow_type(
+ const DataTypePtr& type, std::shared_ptr<arrow::DataType>* result) const {
+ // Flight always uses native Variant, including recursively converted children.
+ if (type->get_primitive_type() == TYPE_VARIANT) {
+ // Reject by type before reading rows, including empty results and nested legacy leaves.
+ if (dynamic_cast<const DataTypeVariantV2*>(remove_nullable(type).get()) == nullptr) {
+ return Status::NotSupported(
+ "Native Arrow Flight output only supports Variant V2, not legacy Variant; "
+ "cast the result to STRING for text output");
+ }
+ RETURN_IF_ERROR(register_arrow_variant_extension());
+ *result = arrow::extension::variant(
+ arrow::struct_({arrow::field("metadata", arrow::binary(), false),
+ arrow::field("value", arrow::binary(), false)}));
+ return Status::OK();
+ }
+ return DorisArrowSchemaConvertor::convert_to_arrow_type(type, result);
+}
+
std::string ArrowFlightSchemaConvertor::timestamp_timezone(PrimitiveType type) const {
// DATETIMEV2 is wall-clock time; TIMESTAMPTZ must still describe an instant.
return type == TYPE_DATETIMEV2 ? "" : DorisArrowSchemaConvertor::timestamp_timezone(type);
@@ -323,13 +358,17 @@ Status serialize_record_batch(const arrow::RecordBatch& record_batch, std::strin
}
Status serialize_arrow_schema(std::shared_ptr<arrow::Schema>* schema, std::string* result) {
- auto make_empty_result = arrow::RecordBatch::MakeEmpty(*schema);
- if (!make_empty_result.ok()) {
- return Status::InternalError("serialize_arrow_schema failed, reason: {}",
- make_empty_result.status().ToString());
- }
- auto batch = make_empty_result.ValueOrDie();
- return serialize_record_batch(*batch, result);
+ // Schema RPC readers only consume the IPC schema. Building an empty batch would require
+ // nested extension builders, which Arrow does not provide for ARRAY/MAP/STRUCT<VARIANT>.
+ std::shared_ptr<arrow::io::BufferOutputStream> sink;
+ RETURN_DORIS_STATUS_IF_RESULT_ERROR(sink, arrow::io::BufferOutputStream::Create());
+ std::shared_ptr<arrow::ipc::RecordBatchWriter> writer;
+ RETURN_DORIS_STATUS_IF_RESULT_ERROR(writer, arrow::ipc::MakeStreamWriter(sink.get(), *schema));
+ RETURN_DORIS_STATUS_IF_ERROR(writer->Close());
+ std::shared_ptr<arrow::Buffer> buffer;
+ RETURN_DORIS_STATUS_IF_RESULT_ERROR(buffer, sink->Finish());
+ *result = buffer->ToString();
+ return Status::OK();
}
} // namespace doris
diff --git a/be/src/format/arrow/arrow_row_batch.h b/be/src/format/arrow/arrow_row_batch.h
index 60b192716fe..debe570fe10 100644
--- a/be/src/format/arrow/arrow_row_batch.h
+++ b/be/src/format/arrow/arrow_row_batch.h
@@ -63,8 +63,8 @@ public:
std::shared_ptr<arrow::Schema>* result) const;
Status get_arrow_schema_from_expr_ctxs(const VExprContextSPtrs& output_vexpr_ctxs,
std::shared_ptr<arrow::Schema>* result) const;
- Status convert_to_arrow_type(const DataTypePtr& type,
- std::shared_ptr<arrow::DataType>* result) const;
+ virtual Status convert_to_arrow_type(const DataTypePtr& type,
+ std::shared_ptr<arrow::DataType>* result) const;
protected:
virtual std::string timestamp_timezone(PrimitiveType type) const;
@@ -82,7 +82,13 @@ private:
class ArrowFlightSchemaConvertor : public DorisArrowSchemaConvertor {
public:
- using DorisArrowSchemaConvertor::DorisArrowSchemaConvertor;
+ explicit ArrowFlightSchemaConvertor(std::string timezone)
+ : DorisArrowSchemaConvertor(std::move(timezone)) {}
+ ArrowFlightSchemaConvertor(const Block& header, std::string timezone)
+ : DorisArrowSchemaConvertor(header, std::move(timezone)) {}
+
+ Status convert_to_arrow_type(const DataTypePtr& type,
+ std::shared_ptr<arrow::DataType>* result) const override;
protected:
std::string timestamp_timezone(PrimitiveType type) const override;
@@ -103,6 +109,8 @@ protected:
PrimitiveType primitive) const override;
};
+Status register_arrow_variant_extension();
+
std::shared_ptr<arrow::Field> create_arrow_field_with_metadata(
const std::string& field_name, const std::shared_ptr<arrow::DataType>& arrow_type,
bool is_nullable, PrimitiveType primitive_type);
diff --git a/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp b/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp
index 7e11fe9a423..32c6ca53d32 100644
--- a/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp
+++ b/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp
@@ -298,6 +298,7 @@ arrow::Status ArrowFlightBatchRemoteReader::_fetch_schema() {
st = Status::create(callback->response_->status());
ARROW_RETURN_NOT_OK(to_arrow_status(st));
+ ARROW_RETURN_NOT_OK(to_arrow_status(register_arrow_variant_extension()));
if (callback->response_->has_schema() && !callback->response_->schema().empty()) {
auto input =
arrow::io::BufferReader::FromString(std::string(callback->response_->schema()));
diff --git a/be/test/format/arrow/arrow_flight_variant_test.cpp b/be/test/format/arrow/arrow_flight_variant_test.cpp
new file mode 100644
index 00000000000..792a3d2b8fe
--- /dev/null
+++ b/be/test/format/arrow/arrow_flight_variant_test.cpp
@@ -0,0 +1,511 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/api.h>
+#include <arrow/extension/parquet_variant.h>
+#include <arrow/io/api.h>
+#include <arrow/ipc/api.h>
+#include <gtest/gtest.h>
+
+#include <cmath>
+#include <limits>
+
+#include "core/column/column_array.h"
+#include "core/column/column_const.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_struct.h"
+#include "core/column/column_variant.h"
+#include "core/column/variant_v2/column_variant_v2.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_factory.hpp"
+#include "core/data_type/data_type_map.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
+#include "core/data_type/data_type_variant.h"
+#include "core/data_type/data_type_variant_v2.h"
+#include "core/data_type_serde/data_type_serde.h"
+#include "exprs/function/parse/variant_string_parse.h"
+#include "format/arrow/arrow_block_convertor.h"
+#include "format/arrow/arrow_row_batch.h"
+#include "util/timezone_utils.h"
+
+namespace doris {
+namespace {
+
+std::shared_ptr<arrow::DataType> native_variant() {
+ return arrow::extension::variant(
+ arrow::struct_({arrow::field("metadata", arrow::binary(), false),
+ arrow::field("value", arrow::binary(), false)}));
+}
+
+MutableColumnPtr documents(const DataTypePtr& type) {
+ auto column = type->create_column();
+ auto serde = type->get_serde();
+ DataTypeSerDe::FormatOptions options;
+ for (std::string json : {R"({"a":[1,null,"x"]})", "null", "42", R"("text")"}) {
+ Slice slice(json.data(), json.size());
+ EXPECT_TRUE(serde->deserialize_one_cell_from_json(*column, slice, options).ok());
+ }
+ if (auto* legacy = check_and_get_column<ColumnVariant>(*column)) {
+ legacy->finalize();
+ }
+ return column;
+}
+
+VariantRef value_at(const arrow::Array& array, int row) {
+ const auto& storage = static_cast<const arrow::StructArray&>(
+ *static_cast<const arrow::ExtensionArray&>(array).storage());
+ auto metadata = static_cast<const arrow::BinaryArray&>(*storage.field(0)).GetView(row);
+ auto value = static_cast<const arrow::BinaryArray&>(*storage.field(1)).GetView(row);
+ return {{metadata.data(), metadata.size()}, {value.data(), value.size()}};
+}
+
+TEST(ArrowFlightVariantTest, NativeSchemaRejectsLegacyIncludingNestedAndEmptyResults) {
+ auto legacy = std::make_shared<DataTypeVariant>();
+ auto nullable = make_nullable(legacy);
+ DataTypes types {legacy, nullable, std::make_shared<DataTypeArray>(nullable),
+ std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(), nullable),
+ std::make_shared<DataTypeStruct>(
+ DataTypes {std::make_shared<DataTypeVariantV2>(), nullable},
+ Strings {"v2", "legacy"})};
+ for (const auto& type : types) {
+ SCOPED_TRACE(type->get_name());
+ Block block {{type->create_column(), type, "v"}};
+ std::shared_ptr<arrow::Schema> schema;
+ auto status = ArrowFlightSchemaConvertor(block, "UTC").get_arrow_schema(&schema);
+ EXPECT_TRUE(status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) << status;
+ EXPECT_NE(status.to_string().find("only supports Variant V2"), std::string::npos);
+ EXPECT_NE(status.to_string().find("cast the result to STRING"), std::string::npos);
+ EXPECT_TRUE(DorisArrowSchemaConvertor(block, "UTC").get_arrow_schema(&schema).ok());
+ }
+}
+
+TEST(ArrowFlightVariantTest, LegacyNativeOutputRejectsValuesConstantsAndNulls) {
+ auto type = std::make_shared<DataTypeVariant>();
+ auto nulls = ColumnUInt8::create();
+ nulls->get_data().resize_fill(4, 1);
+ auto constant = type->create_column();
+ DataTypeSerDe::FormatOptions options;
+ std::string json = "42";
+ Slice slice(json.data(), json.size());
+ ASSERT_TRUE(type->get_serde()->deserialize_one_cell_from_json(*constant, slice, options).ok());
+ assert_cast<ColumnVariant&>(*constant).finalize();
+ ColumnsWithTypeAndName inputs {
+ {documents(type), type, "v"},
+ {ColumnConst::create(std::move(constant), 3), type, "v"},
+ {ColumnNullable::create(documents(type), std::move(nulls)), make_nullable(type), "v"},
+ {type->create_column(), type, "v"}};
+ for (const auto& input : inputs) {
+ Block block {input};
+ ArrowFlightArrowBlockConvertor native(arrow::schema({arrow::field("v", native_variant())}),
+ cctz::utc_time_zone());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ auto status = native.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ EXPECT_TRUE(status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) << status;
+ EXPECT_NE(status.to_string().find("only supports Variant V2"), std::string::npos);
+
+ DorisArrowBlockConvertor utf8(block, "UTC", cctz::utc_time_zone());
+ ASSERT_TRUE(utf8.init().ok());
+ ASSERT_TRUE(utf8.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
+ EXPECT_EQ(batch->num_rows(), block.rows());
+ EXPECT_EQ(batch->column(0)->type_id(), arrow::Type::STRING);
+ if (input.type->is_nullable()) {
+ EXPECT_EQ(batch->column(0)->null_count(), 4);
+ } else if (block.rows() == 4) {
+ const auto& values = static_cast<const arrow::StringArray&>(*batch->column(0));
+ EXPECT_EQ(values.GetString(0), R"({"a":[1, null, "x"]})");
+ EXPECT_EQ(values.GetString(1), "{}");
+ EXPECT_EQ(values.GetString(2), "42");
+ EXPECT_EQ(values.GetString(3), R"("text")");
+ }
+ }
+}
+
+TEST(ArrowFlightVariantTest, LegacySerdeRejectsNativeStorageBeforeInspectingRows) {
+ auto type = std::make_shared<DataTypeVariant>();
+ auto column = documents(type);
+ auto storage = std::static_pointer_cast<arrow::ExtensionType>(native_variant())->storage_type();
+ auto builder = arrow::MakeBuilder(storage, arrow::default_memory_pool()).ValueOrDie();
+ NullMap nulls(4, 1);
+ for (int end : {0, 4}) {
+ auto status = type->get_serde()->write_column_to_arrow(*column, &nulls, builder.get(), 0,
+ end, cctz::utc_time_zone());
+ EXPECT_TRUE(status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) << status;
+ EXPECT_NE(status.to_string().find("only supports Variant V2"), std::string::npos);
+ EXPECT_EQ(builder->length(), 0);
+ }
+}
+
+TEST(ArrowFlightVariantTest, NativeResultPreservesValuesAndSqlNulls) {
+ TimezoneUtils::load_timezones_to_cache();
+ ASSERT_TRUE(register_arrow_variant_extension().ok());
+ auto type = std::make_shared<DataTypeVariantV2>();
+ auto nulls = ColumnUInt8::create();
+ nulls->get_data().assign({0, 0, 0, 1});
+ Block block;
+ block.insert(
+ {ColumnNullable::create(documents(type), std::move(nulls)), make_nullable(type), "v"});
+ auto schema = arrow::schema({arrow::field("v", native_variant())});
+ ArrowFlightArrowBlockConvertor converter(schema, cctz::utc_time_zone());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ EXPECT_EQ(batch->num_rows(), 4);
+ EXPECT_FALSE(batch->column(0)->IsNull(1));
+ EXPECT_TRUE(value_at(*batch->column(0), 1).is_null());
+ EXPECT_EQ(value_at(*batch->column(0), 2).get_int(), 42);
+ EXPECT_TRUE(batch->column(0)->IsNull(3));
+ EXPECT_EQ(value_at(*batch->column(0), 0).basic_type(), VariantBasicType::OBJECT);
+
+ // Extension metadata and storage must survive the same IPC boundary used by Flight.
+ auto output = arrow::io::BufferOutputStream::Create().ValueOrDie();
+ auto writer = arrow::ipc::MakeStreamWriter(output, batch->schema()).ValueOrDie();
+ ASSERT_TRUE(writer->WriteRecordBatch(*batch).ok());
+ ASSERT_TRUE(writer->Close().ok());
+ auto input = std::make_shared<arrow::io::BufferReader>(output->Finish().ValueOrDie());
+ auto reader = arrow::ipc::RecordBatchStreamReader::Open(input).ValueOrDie();
+ auto round_trip = reader->Next().ValueOrDie();
+ EXPECT_TRUE(batch->Equals(*round_trip));
+ ASSERT_TRUE(converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch, 1, 3).ok());
+ EXPECT_TRUE(value_at(*batch->column(0), 0).is_null());
+ EXPECT_EQ(value_at(*batch->column(0), 1).get_int(), 42);
+}
+
+TEST(ArrowFlightVariantTest, SchemaMappingAndConstantScalar) {
+ auto type = std::make_shared<DataTypeVariantV2>();
+ std::shared_ptr<arrow::DataType> mapped;
+ ASSERT_TRUE(DorisArrowSchemaConvertor("UTC").convert_to_arrow_type(type, &mapped).ok());
+ EXPECT_TRUE(mapped->Equals(arrow::utf8()));
+ ASSERT_TRUE(ArrowFlightSchemaConvertor("UTC").convert_to_arrow_type(type, &mapped).ok());
+ EXPECT_TRUE(mapped->Equals(native_variant()));
+ auto column = type->create_column();
+ std::string json = R"("te\"xt\n\u4e2d")";
+ Slice slice(json.data(), json.size());
+ DataTypeSerDe::FormatOptions options;
+ ASSERT_TRUE(type->get_serde()->deserialize_one_cell_from_json(*column, slice, options).ok());
+ Block block;
+ block.insert({ColumnConst::create(std::move(column), 3), type, "v"});
+ ArrowFlightArrowBlockConvertor converter(arrow::schema({arrow::field("v", mapped, false)}),
+ cctz::utc_time_zone());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ EXPECT_EQ(batch->num_rows(), 3);
+ EXPECT_EQ(value_at(*batch->column(0), 2).get_string().to_string(), "te\"xt\n中");
+}
+
+TEST(ArrowFlightVariantTest, NativeSchemaPreservesNestedLogicalMetadata) {
+ auto variant = std::make_shared<DataTypeVariantV2>();
+ const auto nullable = make_nullable(variant);
+ const auto string = std::make_shared<DataTypeString>();
+ DataTypes types {nullable, std::make_shared<DataTypeArray>(nullable),
+ std::make_shared<DataTypeMap>(string, nullable),
+ std::make_shared<DataTypeStruct>(DataTypes {nullable}, Strings {"v"})};
+ Block block;
+ for (size_t i = 0; i < types.size(); ++i) {
+ block.insert({types[i]->create_column(), types[i], std::to_string(i)});
+ }
+ {
+ std::shared_ptr<arrow::Schema> schema;
+ ASSERT_TRUE(ArrowFlightSchemaConvertor(block, "UTC").get_arrow_schema(&schema).ok());
+ const auto& map = static_cast<const arrow::MapType&>(*schema->field(2)->type());
+ for (const auto& field : {schema->field(0), schema->field(1)->type()->field(0),
+ map.item_field(), schema->field(3)->type()->field(0)}) {
+ EXPECT_TRUE(field->type()->Equals(native_variant()));
+ EXPECT_TRUE(field->nullable());
+ ASSERT_NE(nullptr, field->metadata());
+ EXPECT_EQ("VARIANT", field->metadata()->Get("doris_type").ValueOrDie());
+ }
+ EXPECT_FALSE(map.key_field()->nullable());
+ }
+ std::shared_ptr<arrow::Schema> schema;
+ ASSERT_TRUE(DorisArrowSchemaConvertor(block, "UTC").get_arrow_schema(&schema).ok());
+ EXPECT_TRUE(schema->field(0)->type()->Equals(arrow::utf8()));
+}
+
+TEST(ArrowFlightVariantTest, TypedV2AndNestedStructPreserveNonJsonNumbers) {
+ auto numbers = ColumnFloat64::create();
+ numbers->insert_value(std::numeric_limits<double>::quiet_NaN());
+ numbers->insert_value(std::numeric_limits<double>::infinity());
+ auto values = ColumnVariantV2::create_typed(make_nullable(std::move(numbers)),
+ std::make_shared<DataTypeFloat64>());
+ auto variant = std::make_shared<DataTypeVariantV2>();
+ auto type = std::make_shared<DataTypeStruct>(DataTypes {variant}, Strings {"v"});
+ Block block;
+ block.insert({ColumnStruct::create(Columns {std::move(values)}), type, "s"});
+ std::shared_ptr<arrow::DataType> mapped;
+ ASSERT_TRUE(ArrowFlightSchemaConvertor("UTC").convert_to_arrow_type(type, &mapped).ok());
+ ArrowFlightArrowBlockConvertor converter(arrow::schema({arrow::field("s", mapped, false)}),
+ cctz::utc_time_zone());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ const auto& child = *static_cast<const arrow::StructArray&>(*batch->column(0)).field(0);
+ EXPECT_TRUE(std::isnan(value_at(child, 0).get_double()));
+ EXPECT_TRUE(std::isinf(value_at(child, 1).get_double()));
+}
+
+TEST(ArrowFlightVariantTest, NestedTimezoneAliasesMatchPublishedSchema) {
+ TimezoneUtils::load_timezones_to_cache();
+ auto variant = std::make_shared<DataTypeVariantV2>();
+ auto timestamp = DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMPTZ, false, 0, 6);
+ auto type =
+ std::make_shared<DataTypeStruct>(DataTypes {variant, timestamp}, Strings {"v", "t"});
+ auto times = timestamp->create_column();
+ for (int i = 0; i < 4; ++i) {
+ times->insert_default();
+ }
+ Block block {{ColumnStruct::create(Columns {documents(variant), std::move(times)}), type, "s"}};
+ for (const std::string zone : {"+08:00", "+05:45", "-03:30"}) {
+ cctz::time_zone timezone;
+ ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone(zone, timezone));
+ std::shared_ptr<arrow::DataType> mapped;
+ ASSERT_TRUE(ArrowFlightSchemaConvertor(zone).convert_to_arrow_type(type, &mapped).ok());
+ ArrowFlightArrowBlockConvertor converter(arrow::schema({arrow::field("s", mapped, false)}),
+ timezone);
+ std::shared_ptr<arrow::RecordBatch> batch;
+ auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_TRUE(batch->schema()->field(0)->type()->Equals(mapped));
+ }
+}
+
+TEST(ArrowFlightVariantTest, NativeMetadataContainsOnlySelectedRowKeys) {
+ for (int rows : {32, 64}) {
+ SCOPED_TRACE(rows);
+ auto type = std::make_shared<DataTypeVariantV2>();
+ auto column = type->create_column();
+ JsonStringToVariantEncoder encoder;
+ for (int i = 0; i < rows; ++i) {
+ std::string key = std::string(200, 'k') + std::to_string(i);
+ std::string json = "{\"" + key + "\":" + std::to_string(i) + "}";
+ encoder.add_json({json.data(), json.size()});
+ }
+ auto encoded = encoder.finish_batch();
+ ASSERT_EQ(encoded.metadata_ref().dict_size(), rows);
+ assert_cast<ColumnVariantV2&>(*column).insert_encoded_batch(encoded);
+ auto nulls = ColumnUInt8::create();
+ nulls->get_data().resize_fill(rows, 0);
+ nulls->get_data()[rows / 2] = 1;
+ Block block {{ColumnNullable::create(std::move(column), std::move(nulls)),
+ make_nullable(type), "v"}};
+ ArrowFlightArrowBlockConvertor converter(
+ arrow::schema({arrow::field("v", native_variant())}), cctz::utc_time_zone());
+ for (int start : {0, 3}) {
+ std::shared_ptr<arrow::RecordBatch> batch;
+ auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch,
+ start, rows);
+ ASSERT_TRUE(status.ok()) << status;
+ size_t metadata_bytes = 0;
+ for (int i = start; i < rows; ++i) {
+ if (i == rows / 2) {
+ EXPECT_TRUE(batch->column(0)->IsNull(i - start));
+ continue;
+ }
+ VariantRef value = value_at(*batch->column(0), i - start);
+ // Internal dictionary sharing must not multiply every other row's keys on the wire.
+ EXPECT_EQ(value.metadata.dict_size(), 1);
+ metadata_bytes += value.metadata.size;
+ VariantRef child;
+ std::string key = std::string(200, 'k') + std::to_string(i);
+ ASSERT_TRUE(value.object_find({key.data(), key.size()}, &child));
+ EXPECT_EQ(child.get_int(), i);
+ }
+ EXPECT_LT(metadata_bytes, static_cast<size_t>(rows - start) * 220);
+ }
+ }
+}
+
+TEST(ArrowFlightVariantTest, NestedNativeCompactionPreservesPhysicalScalars) {
+ VariantBatchBuilder builder;
+ for (std::string key : {"first", "second"}) {
+ auto row = builder.begin_row();
+ auto object = row.start_object();
+ object.add_key({key.data(), key.size()});
+ row.add_int(1);
+ object.finish();
+ row.finish();
+ }
+ for (int kind = 0; kind < 5; ++kind) {
+ auto row = builder.begin_row();
+ switch (kind) {
+ case 0:
+ row.add_decimal(4200, 2, 4);
+ break;
+ case 1:
+ row.add_float(42.0F);
+ break;
+ case 2:
+ row.add_date(1);
+ break;
+ case 3:
+ row.add_binary({"\0\xff", 2});
+ break;
+ case 4:
+ row.add_string({"42", 2});
+ break;
+ }
+ row.finish();
+ }
+ {
+ auto row = builder.begin_row();
+ auto array = row.start_array();
+ row.add_decimal(4200, 2, 4);
+ row.add_float(42.0F);
+ row.add_string({"42", 2});
+ array.finish();
+ row.finish();
+ }
+ auto encoded = builder.finish_batch();
+ ASSERT_EQ(encoded.metadata_ref().dict_size(), 2);
+ auto values = ColumnVariantV2::create();
+ values->insert_encoded_batch(encoded);
+ auto offsets = ColumnArray::ColumnOffsets::create();
+ for (size_t i = 0; i < encoded.num_rows(); ++i) {
+ offsets->get_data().push_back(i + 1);
+ }
+ auto type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeVariantV2>());
+ Block block {
+ {ColumnArray::create(make_nullable(std::move(values)), std::move(offsets)), type, "a"}};
+ ArrowFlightArrowBlockConvertor converter(
+ arrow::schema({arrow::field("a", arrow::list(native_variant()), false)}),
+ cctz::utc_time_zone());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ ASSERT_TRUE(status.ok()) << status;
+ const auto& elements = *static_cast<const arrow::ListArray&>(*batch->column(0)).values();
+ for (int i = 0; i < elements.length(); ++i) {
+ auto actual = value_at(elements, i);
+ EXPECT_EQ(actual.metadata.dict_size(), i < 2 ? 1 : 0);
+ if (i >= 2) {
+ // Integral decimals and floats must retain physical type/scale rather than normalize to integers.
+ auto expected = encoded.value_at(i);
+ EXPECT_EQ(actual.basic_type(), expected.basic_type());
+ if (actual.basic_type() == VariantBasicType::PRIMITIVE) {
+ EXPECT_EQ(actual.primitive_id(), expected.primitive_id());
+ }
+ EXPECT_EQ(actual.value, expected.value);
+ }
+ }
+}
+
+TEST(ArrowFlightVariantTest, NativeCompactionKeepsTerminalEmptyContainersAtDepthLimit) {
+ for (std::string terminal : {"[]", "{}"}) {
+ std::string json = terminal;
+ for (size_t i = 0; i < VARIANT_MAX_NESTING_DEPTH; ++i) {
+ json = "[" + json + "]";
+ }
+ JsonStringToVariantEncoder encoder;
+ encoder.add_json({json.data(), json.size()});
+ encoder.add_json({R"({"unused":0})", 12});
+ auto encoded = encoder.finish_batch();
+ auto values = ColumnVariantV2::create();
+ values->insert_encoded_batch(encoded);
+ Block block {{std::move(values), std::make_shared<DataTypeVariantV2>(), "v"}};
+ ArrowFlightArrowBlockConvertor converter(
+ arrow::schema({arrow::field("v", native_variant(), false)}), cctz::utc_time_zone());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ ASSERT_TRUE(status.ok()) << status;
+ auto value = value_at(*batch->column(0), 0);
+ EXPECT_EQ(value.metadata.dict_size(), 0);
+ for (size_t i = 0; i < VARIANT_MAX_NESTING_DEPTH; ++i) {
+ value = value.array_at(0);
+ }
+ EXPECT_EQ(value.num_elements(), 0);
+ EXPECT_EQ(value.basic_type(),
+ terminal == "[]" ? VariantBasicType::ARRAY : VariantBasicType::OBJECT);
+ }
+}
+
+TEST(ArrowFlightVariantTest, SchemaRpcPreservesNestedVariantExtensions) {
+ ASSERT_TRUE(register_arrow_variant_extension().ok());
+ for (const auto& type : std::vector<std::shared_ptr<arrow::DataType>> {
+ arrow::utf8(), native_variant(), arrow::list(native_variant()),
+ arrow::map(arrow::utf8(), native_variant()),
+ arrow::struct_({arrow::field("v", native_variant())}),
+ arrow::list(arrow::struct_({arrow::field("v", native_variant())}))}) {
+ SCOPED_TRACE(type->ToString());
+ auto schema = arrow::schema({arrow::field("result", type)});
+ std::string serialized;
+ // Result schema discovery happens before batch conversion and must also support nested extensions.
+ auto status = serialize_arrow_schema(&schema, &serialized);
+ ASSERT_TRUE(status.ok()) << status;
+ auto input = arrow::io::BufferReader::FromString(serialized);
+ auto opened = arrow::ipc::RecordBatchStreamReader::Open(input.get());
+ ASSERT_TRUE(opened.ok()) << opened.status();
+ auto reader = opened.ValueOrDie();
+ EXPECT_TRUE(reader->schema()->Equals(*schema, true));
+ auto next = reader->Next();
+ ASSERT_TRUE(next.ok()) << next.status();
+ if (next.ValueOrDie() != nullptr) {
+ EXPECT_EQ(next.ValueOrDie()->num_rows(), 0);
+ }
+ }
+}
+
+TEST(ArrowFlightVariantTest, EmptyResultHasNativeSchema) {
+ auto type = std::make_shared<DataTypeVariantV2>();
+ Block block;
+ block.insert({type->create_column(), type, "v"});
+ ArrowFlightArrowBlockConvertor converter(
+ arrow::schema({arrow::field("v", native_variant(), false)}), cctz::utc_time_zone());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ EXPECT_EQ(batch->num_rows(), 0);
+ EXPECT_TRUE(batch->column(0)->type()->Equals(native_variant()));
+}
+
+TEST(ArrowFlightVariantTest, NestedArrayAndDefaultJsonMode) {
+ auto type = std::make_shared<DataTypeVariantV2>();
+ auto offsets = ColumnArray::ColumnOffsets::create();
+ offsets->get_data().assign({2, 4});
+ auto array_type = std::make_shared<DataTypeArray>(type);
+ Block block;
+ block.insert({ColumnArray::create(make_nullable(documents(type)), std::move(offsets)),
+ array_type, "a"});
+ auto schema = arrow::schema(
+ {arrow::field("a", arrow::list(arrow::field("item", native_variant(), true)), false)});
+ ArrowFlightArrowBlockConvertor converter(schema, cctz::utc_time_zone());
+ std::shared_ptr<arrow::RecordBatch> batch;
+ auto status = converter.convert_to_arrow(block, arrow::default_memory_pool(), &batch);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_TRUE(batch->ValidateFull().ok());
+ const auto& values = *static_cast<const arrow::ListArray&>(*batch->column(0)).values();
+ EXPECT_EQ(value_at(values, 2).get_int(), 42);
+ EXPECT_EQ(value_at(values, 3).get_string().to_string(), "text");
+
+ DorisArrowBlockConvertor json(block, "UTC", cctz::utc_time_zone());
+ ASSERT_TRUE(json.init().ok());
+ ASSERT_TRUE(json.convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok());
+ EXPECT_EQ(static_cast<const arrow::ListArray&>(*batch->column(0)).values()->type_id(),
+ arrow::Type::STRING);
+ // Native Variant bindings belong to Flight, not the ordinary Arrow export path.
+ EXPECT_FALSE(DorisArrowBlockConvertor(schema, cctz::utc_time_zone())
+ .convert_to_arrow(block, arrow::default_memory_pool(), &batch)
+ .ok());
+}
+
+} // namespace
+} // namespace doris
diff --git a/be/test/format/arrow/arrow_row_batch_test.cpp b/be/test/format/arrow/arrow_row_batch_test.cpp
index 8b720c675d8..a77b72f7009 100644
--- a/be/test/format/arrow/arrow_row_batch_test.cpp
+++ b/be/test/format/arrow/arrow_row_batch_test.cpp
@@ -29,6 +29,7 @@
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
#include "core/data_type/data_type_struct.h"
+#include "core/data_type/data_type_variant_v2.h"
#include "exprs/vexpr_context.h"
#include "exprs/vslot_ref.h"
#include "format/arrow/arrow_block_convertor.h"
@@ -104,7 +105,10 @@ class ArrowLogicalTypeMetadataTest
TEST_P(ArrowLogicalTypeMetadataTest, PreservesTopLevelAndNestedFields) {
const auto [primitive, name] = GetParam();
- auto type = make_nullable(DataTypeFactory::instance().create_data_type(primitive, false));
+ auto type =
+ make_nullable(primitive == TYPE_VARIANT
+ ? DataTypePtr(std::make_shared<DataTypeVariantV2>())
+ : DataTypeFactory::instance().create_data_type(primitive, false));
auto string_type = std::make_shared<DataTypeString>();
DataTypes types {type, std::make_shared<DataTypeArray>(type),
std::make_shared<DataTypeStruct>(DataTypes {type, string_type},
@@ -129,7 +133,10 @@ TEST_P(ArrowLogicalTypeMetadataTest, PreservesTopLevelAndNestedFields) {
TEST_P(ArrowLogicalTypeMetadataTest, OldFeReceivesLegacySchema) {
const auto [primitive, name] = GetParam();
- auto type = make_nullable(DataTypeFactory::instance().create_data_type(primitive, false));
+ auto type =
+ make_nullable(primitive == TYPE_VARIANT
+ ? DataTypePtr(std::make_shared<DataTypeVariantV2>())
+ : DataTypeFactory::instance().create_data_type(primitive, false));
auto string_type = std::make_shared<DataTypeString>();
DataTypes types {
type, std::make_shared<DataTypeArray>(type),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ResultSink.java
index b7d7dd4d876..30fa7705b0c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ResultSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ResultSink.java
@@ -37,7 +37,7 @@ public class ResultSink extends DataSink {
private TResultSinkType resultSinkType = TResultSinkType.MYSQL_PROTOCOL;
public ResultSink(PlanNodeId exchNodeId) {
- this.exchNodeId = exchNodeId;
+ this(exchNodeId, TResultSinkType.MYSQL_PROTOCOL);
}
public ResultSink(PlanNodeId exchNodeId, TResultSinkType resultSinkType) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java
index e0f89217690..678cab055a8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java
@@ -378,6 +378,10 @@ final class FlightSqlQuerySchema {
type = Type.STRING;
}
PrimitiveType primitive = type.getPrimitiveType();
+ if (primitive == PrimitiveType.VARIANT) {
+ return FlightSqlSchemaHelper.nativeVariantField(name, nullable,
+ Collections.singletonMap("doris_type", primitive.toString()));
+ }
int precision = type instanceof ScalarType ? ((ScalarType) type).getScalarPrecision() : 0;
int scale = type instanceof ScalarType ? ((ScalarType) type).getScalarScale() : 0;
ArrowType arrowType = FlightSqlSchemaHelper.getArrowType(primitive, precision, scale);
@@ -400,13 +404,17 @@ final class FlightSqlQuerySchema {
field("value", map.getValueType(), true, false, timezone))));
} else if (type instanceof StructType) {
for (StructField child : ((StructType) type).getFields()) {
- children.add(field(child.getName(), child.getType(), child.getContainsNull(), false, timezone));
+ children.add(field(child.getName(), child.getType(), child.getContainsNull(),
+ false, timezone));
}
}
Map<String, String> metadata = null;
- if (topLevel && (primitive == PrimitiveType.LARGEINT || primitive == PrimitiveType.IPV4
- || primitive == PrimitiveType.IPV6)) {
- metadata = Collections.singletonMap("doris_type", primitive.toString());
+ // Execution preserves logical type markers at every depth; Prepare must match them exactly.
+ if (primitive == PrimitiveType.LARGEINT || primitive == PrimitiveType.IPV4
+ || primitive == PrimitiveType.IPV6 || primitive == PrimitiveType.VARIANT
+ || primitive == PrimitiveType.JSONB) {
+ metadata = Collections.singletonMap("doris_type",
+ primitive == PrimitiveType.JSONB ? "JSON" : primitive.toString());
}
return new Field(name, new FieldType(nullable, arrowType, null, metadata), children);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java
index 6af37014132..0ce25bfb81c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java
@@ -23,6 +23,7 @@ import org.apache.doris.catalog.MapType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.Type;
+import org.apache.doris.common.Config;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.service.ExecuteEnv;
@@ -35,8 +36,10 @@ import org.apache.doris.thrift.TGetDbsParams;
import org.apache.doris.thrift.TGetDbsResult;
import org.apache.doris.thrift.TGetTablesParams;
import org.apache.doris.thrift.TListTableStatusResult;
+import org.apache.doris.thrift.TPrimitiveType;
import org.apache.doris.thrift.TTableStatus;
+import org.apache.arrow.flight.CallStatus;
import org.apache.arrow.flight.sql.FlightSqlColumnMetadata;
import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetDbSchemas;
import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetTables;
@@ -170,6 +173,15 @@ public class FlightSqlSchemaHelper {
}
static Field withDorisTypeMetadata(Field field, Type type) {
+ if (type.isVariantType()) {
+ requireVariantV2();
+ if (field.getMetadata() == null
+ || !"arrow.parquet.variant".equals(field.getMetadata().get("ARROW:extension:name"))) {
+ throw CallStatus.UNIMPLEMENTED.withDescription(
+ "Backend returned a non-native Variant schema; use Variant V2 "
+ + "or cast the result to STRING").toRuntimeException();
+ }
+ }
List<Field> children = new ArrayList<>(field.getChildren());
if (type.isArrayType()) {
children.set(0, withDorisTypeMetadata(children.get(0), ((ArrayType) type).getItemType()));
@@ -341,6 +353,10 @@ public class FlightSqlSchemaHelper {
/** One column, with its nested types described down to the leaves. */
private static Field buildField(String dbName, String tableName, TColumnDesc desc) {
+ if (desc.getColumnType() == TPrimitiveType.VARIANT) {
+ return nativeVariantField(desc.getColumnName(), desc.isIsAllowNull(),
+ createFlightSqlColumnMetadata(dbName, tableName, desc));
+ }
ArrowType arrowType = columnDescToArrowType(desc);
return new Field(desc.getColumnName(),
new FieldType(desc.isIsAllowNull(), arrowType, null,
@@ -348,6 +364,26 @@ public class FlightSqlSchemaHelper {
arrowChildren(dbName, tableName, desc, arrowType));
}
+ private static void requireVariantV2() {
+ // Schema discovery must reject legacy Variant before publishing a native binary layout.
+ if (!Config.enable_variant_v2) {
+ throw CallStatus.UNIMPLEMENTED.withDescription(
+ "Native Arrow Flight output only supports Variant V2, not legacy Variant; "
+ + "cast the result to STRING for text output").toRuntimeException();
+ }
+ }
+
+ static Field nativeVariantField(String name, boolean nullable, Map<String, String> columnMetadata) {
+ requireVariantV2();
+ Map<String, String> metadata = new HashMap<>(columnMetadata);
+ // Discovery and execution must share the extension metadata as well as its storage type.
+ metadata.put("ARROW:extension:name", "arrow.parquet.variant");
+ metadata.put("ARROW:extension:metadata", "");
+ return new Field(name, new FieldType(nullable, new ArrowType.Struct(), null, metadata),
+ Arrays.asList(Field.notNullable("metadata", new ArrowType.Binary()),
+ Field.notNullable("value", new ArrowType.Binary())));
+ }
+
/**
* The Arrow children of a complex column, built from the descriptor's own children.
*
@@ -389,7 +425,8 @@ public class FlightSqlSchemaHelper {
Field entries = new Field(MapVector.DATA_VECTOR_NAME,
new FieldType(false, new ArrowType.Struct(), null),
Arrays.asList(new Field(key.getName(),
- new FieldType(false, key.getType(), null), key.getChildren()),
+ new FieldType(false, key.getType(), null, key.getMetadata()),
+ key.getChildren()),
value));
return Collections.singletonList(entries);
case Struct:
diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessorSchemaTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessorSchemaTest.java
index b7f6725478b..144d615d5e4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessorSchemaTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessorSchemaTest.java
@@ -75,18 +75,16 @@ class FlightSqlConnectProcessorSchemaTest {
field("ip4", new ArrowType.Int(32, true), true, annotated ? "IPV4" : null),
field("ip6", new ArrowType.Utf8(), true, annotated ? "IPV6" : null),
field("json", new ArrowType.Utf8(), true, annotated ? "JSON" : null),
- field("variant", new ArrowType.Utf8(), true, annotated ? "VARIANT" : null),
field("text", new ArrowType.Utf8(), true, null)))),
- field("json", new ArrowType.Utf8(), true, annotated ? "JSON" : null),
- field("variant", new ArrowType.Utf8(), true, annotated ? "VARIANT" : null));
+ field("json", new ArrowType.Utf8(), true, annotated ? "JSON" : null));
}
private static List<Type> resultTypes() {
return Arrays.asList(new ArrayType(Type.LARGEINT),
new MapType(Type.LARGEINT, new StructType(
new StructField("ip4", Type.IPV4), new StructField("ip6", Type.IPV6),
- new StructField("json", Type.JSONB), new StructField("variant", Type.VARIANT),
- new StructField("text", Type.STRING))), Type.JSONB, Type.VARIANT);
+ new StructField("json", Type.JSONB),
+ new StructField("text", Type.STRING))), Type.JSONB);
}
private static Schema fetch(List<Type> types, Schema... schemas) throws Exception {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlNativeVariantTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlNativeVariantTest.java
new file mode 100644
index 00000000000..ebf73875a12
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlNativeVariantTest.java
@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service.arrowflight;
+
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.StructField;
+import org.apache.doris.catalog.StructType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.jmockit.Deencapsulation;
+import org.apache.doris.thrift.TColumnDesc;
+import org.apache.doris.thrift.TPrimitiveType;
+
+import org.apache.arrow.vector.ipc.ReadChannel;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.nio.channels.Channels;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+
+public class FlightSqlNativeVariantTest {
+ private boolean originalVariantV2;
+
+ @Before
+ public void enableVariantV2() {
+ originalVariantV2 = Config.enable_variant_v2;
+ Config.enable_variant_v2 = true;
+ }
+
+ @After
+ public void restoreVariantV2() {
+ Config.enable_variant_v2 = originalVariantV2;
+ }
+
+ @Test
+ public void legacyVariantIsRejectedWithoutAFormatSwitch() {
+ Config.enable_variant_v2 = false;
+ Assert.assertThrows(org.apache.arrow.flight.FlightRuntimeException.class,
+ () -> FlightSqlSchemaHelper.nativeVariantField("v", true, Collections.emptyMap()));
+ }
+
+ @Test
+ public void executionCannotPublishUtf8ForVariant() {
+ Assert.assertThrows(org.apache.arrow.flight.FlightRuntimeException.class,
+ () -> FlightSqlSchemaHelper.withDorisTypeMetadata(Field.nullable("v", new ArrowType.Utf8()),
+ Type.VARIANT));
+ }
+
+ @Test
+ public void schemaKeepsExtensionAcrossIpc() throws Exception {
+ TColumnDesc variant = new TColumnDesc("item", TPrimitiveType.VARIANT);
+ variant.setIsAllowNull(true);
+ TColumnDesc array = new TColumnDesc("a", TPrimitiveType.ARRAY);
+ array.setChildren(Collections.singletonList(variant));
+ Field field = Deencapsulation.invoke(FlightSqlSchemaHelper.class, "buildField",
+ "test_db", "test_table", array);
+ Field child = field.getChildren().get(0);
+ Assert.assertEquals(new ArrowType.Struct(), child.getType());
+ Assert.assertEquals("arrow.parquet.variant", child.getMetadata().get("ARROW:extension:name"));
+ Assert.assertEquals("metadata", child.getChildren().get(0).getName());
+ Assert.assertEquals("value", child.getChildren().get(1).getName());
+ for (Field storage : child.getChildren()) {
+ Assert.assertFalse(storage.isNullable());
+ Assert.assertEquals(new ArrowType.Binary(), storage.getType());
+ }
+ Schema schema = new Schema(Collections.singletonList(field));
+ try (ReadChannel channel = new ReadChannel(Channels.newChannel(
+ new ByteArrayInputStream(schema.serializeAsMessage())))) {
+ Assert.assertEquals(schema, MessageSerializer.deserializeSchema(channel));
+ }
+ }
+
+ @Test
+ public void querySchemaPreservesNativeVariantInNestedFields() {
+ Type nested = new StructType(new ArrayList<>(Arrays.asList(
+ new StructField("scalar", Type.VARIANT),
+ new StructField("array", new ArrayType(Type.VARIANT, true)),
+ new StructField("map", new MapType(Type.STRING, Type.VARIANT)))));
+ Field result = Deencapsulation.invoke(FlightSqlQuerySchema.class, "field",
+ "s", nested, true, true, "UTC");
+ Field scalar = result.getChildren().get(0);
+ Field item = result.getChildren().get(1).getChildren().get(0);
+ Field value = result.getChildren().get(2).getChildren().get(0).getChildren().get(1);
+ for (Field leaf : Arrays.asList(scalar, item, value)) {
+ Assert.assertEquals(new ArrowType.Struct(), leaf.getType());
+ Assert.assertEquals("arrow.parquet.variant", leaf.getMetadata().get("ARROW:extension:name"));
+ Assert.assertEquals("", leaf.getMetadata().get("ARROW:extension:metadata"));
+ Assert.assertEquals(Arrays.asList(Field.notNullable("metadata", new ArrowType.Binary()),
+ Field.notNullable("value", new ArrowType.Binary())), leaf.getChildren());
+ Assert.assertEquals("VARIANT", leaf.getMetadata().get("doris_type"));
+ }
+ // Model execution's metadata enrichment to catch Prepare/DoGet schema mismatches.
+ Field execution = FlightSqlSchemaHelper.withDorisTypeMetadata(result, nested);
+ Assert.assertTrue(FlightSqlQuerySchema.matchesExecutionSchema(
+ new Schema(Collections.singletonList(result)),
+ new Schema(Collections.singletonList(execution)), Collections.singletonList("s")));
+ }
+
+}
diff --git a/fe/pom.xml b/fe/pom.xml
index 5f1223aa793..556c47639a6 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -341,7 +341,6 @@ under the License.
<hudi-spark.version>hudi-spark3.4.x</hudi-spark.version>
<hive.version>3.1.3</hive.version>
<hive.common.version>2.3.9</hive.common.version>
- <nimbusds.version>9.35</nimbusds.version>
<mapreduce.client.version>2.10.1</mapreduce.client.version>
<calcite.version>1.33.0</calcite.version>
<avatica.version>1.22.0</avatica.version>
diff --git a/regression-test/suites/arrow_flight_sql_p0/test_flight_cancel_cleanup.groovy b/regression-test/suites/arrow_flight_sql_p0/test_flight_cancel_cleanup.groovy
index aff57aeac4d..9b3a21dd30b 100644
--- a/regression-test/suites/arrow_flight_sql_p0/test_flight_cancel_cleanup.groovy
+++ b/regression-test/suites/arrow_flight_sql_p0/test_flight_cancel_cleanup.groovy
@@ -73,8 +73,10 @@ suite("test_flight_cancel_cleanup", "arrow_flight_sql") {
assertTrue(info.endpoints.size() > 1, "Expected distributed Flight result endpoints")
}
def ids = info.endpoints.collect { endpoint ->
- Any.parseFrom(endpoint.ticket.bytes).unpack(FlightSql.TicketStatementQuery.class)
+ def queryId = Any.parseFrom(endpoint.ticket.bytes).unpack(FlightSql.TicketStatementQuery.class)
.statementHandle.toStringUtf8().split("&")[0]
+ // Flight tickets omit leading zeroes; the BE diagnostic API requires two 16-digit halves.
+ queryId.split("-").collect { it.padLeft(16, "0") }.join("-")
}.unique()
// Abort only one endpoint; the cancellation must reach every participating BE.
[info.endpoints[0]].each { endpoint ->
diff --git a/regression-test/suites/arrow_flight_sql_p0/test_flight_native_variant.groovy b/regression-test/suites/arrow_flight_sql_p0/test_flight_native_variant.groovy
new file mode 100644
index 00000000000..ad490bf332f
--- /dev/null
+++ b/regression-test/suites/arrow_flight_sql_p0/test_flight_native_variant.groovy
@@ -0,0 +1,215 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.arrow.driver.jdbc.shaded.com.google.protobuf.Any
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CallOptions
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightClient
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.Location
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.sql.FlightSqlClient
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.sql.impl.FlightSql
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.memory.RootAllocator
+
+import java.util.concurrent.TimeUnit
+
+suite("test_flight_native_variant", "arrow_flight_sql") {
+ def frontend = jdbc_sql_return_maparray("SHOW FRONTENDS").find {
+ it.IsMaster.toString().equalsIgnoreCase("true") && it.Alive.toString().equalsIgnoreCase("true")
+ }
+ assertNotNull(frontend)
+ assertTrue(frontend.ArrowFlightSqlPort.toString().toInteger() > 0)
+ def database = jdbc_sql("SELECT DATABASE()")[0][0]
+ def variantV2 = getFeConfig("enable_variant_v2").toBoolean()
+ def variantV2Function = variantV2 ? "parse_to_variant" : ""
+ def table = "${database}.flight_native_variant_input"
+ def allocator = new RootAllocator(Long.MAX_VALUE)
+ def feClient = FlightClient.builder(allocator,
+ Location.forGrpcInsecure(frontend.Host.toString(), frontend.ArrowFlightSqlPort.toString().toInteger())).build()
+ def client = new FlightSqlClient(feClient)
+ def auth
+ def read = { String query, Closure inspect, boolean parallel = false, int resultBackendCount = 1, def prepared = null ->
+ int count = 0
+ def info = prepared == null ? client.execute(query, auth) : prepared.execute(auth)
+ assertFalse(info.endpoints.isEmpty())
+ // Multiple buckets alone do not prove coverage of native output from different result BEs.
+ if (parallel) {
+ def resultAddresses = info.endpoints.collect { endpoint ->
+ def fields = Any.parseFrom(endpoint.ticket.bytes).unpack(FlightSql.TicketStatementQuery.class)
+ .statementHandle.toStringUtf8().split("&")
+ "${fields[1]}:${fields[2]}".toString()
+ }
+ assertEquals(info.endpoints.size(), resultAddresses.toSet().size(), "Duplicate result backends")
+ if (resultBackendCount > 1) {
+ assertTrue(info.endpoints.size() > 1, "Expected multiple native Variant result backends")
+ }
+ }
+ info.endpoints.each { endpoint ->
+ FlightClient.builder(allocator, endpoint.locations[0]).build().withCloseable { beClient ->
+ beClient.getStream(endpoint.ticket, auth, CallOptions.timeout(30, TimeUnit.SECONDS)).withCloseable { stream ->
+ assertEquals(info.schema, stream.schema, "Result endpoints must share the published schema")
+ while (stream.next()) {
+ inspect(stream.root)
+ count += stream.root.rowCount
+ }
+ }
+ }
+ }
+ count
+ }
+ def executeSetting = { String query -> read(query, { root -> }) }
+ def expectUnsupported = { Closure operation ->
+ Exception failure = null
+ try {
+ operation()
+ } catch (Exception e) {
+ failure = e
+ }
+ assertNotNull(failure, "Native legacy Variant output must fail")
+ assertTrue(failure.toString().contains("only supports Variant V2"), failure.toString())
+ }
+ try {
+ auth = feClient.authenticateBasicToken(context.config.otherConfigs.get("extArrowFlightSqlUser"),
+ context.config.otherConfigs.get("extArrowFlightSqlPassword")).get()
+ executeSetting("SET enable_sql_cache=false")
+ executeSetting("SET enable_nereids_distribute_planner=true")
+ executeSetting("SET parallel_pipeline_task_num=8")
+ jdbc_sql("DROP TABLE IF EXISTS ${table}")
+ jdbc_sql("""CREATE TABLE ${table} (id INT, v VARIANT)
+ DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 60
+ PROPERTIES("replication_num"="1")""")
+ jdbc_sql("""INSERT INTO ${table}
+ SELECT number + 1, ${variantV2Function}(CASE number % 4
+ WHEN 0 THEN '42' WHEN 1 THEN '"text"'
+ WHEN 2 THEN '{"a":[1,null,"x"]}' ELSE NULL END)
+ FROM numbers("number"="60")""")
+ def resultBackendCount = jdbc_sql_return_maparray("SHOW TABLETS FROM ${table}")
+ .collect { it.BackendId }.unique().size()
+ [false, true].each { parallel ->
+ executeSetting("SET enable_parallel_result_sink=${parallel}")
+ // Text output is an explicit SQL conversion, for either Variant representation.
+ assertEquals(60, read("SELECT CAST(v AS STRING) AS text_value FROM ${table}", { root ->
+ assertEquals("Utf8", root.getVector(0).field.type.toString())
+ }))
+ if (!variantV2) {
+ // Reject by type even for constants, SQL NULL, and empty result sets.
+ ["SELECT id, v FROM ${table}",
+ "SELECT CAST(42 AS VARIANT) AS v",
+ "SELECT CAST(NULL AS VARIANT) AS v",
+ "SELECT v FROM ${table} WHERE id < 0"].each { query ->
+ expectUnsupported { read(query.toString(), { root -> }) }
+ expectUnsupported { client.getExecuteSchema(query.toString(), auth) }
+ expectUnsupported {
+ def prepared = client.prepare(query.toString(), auth)
+ try {
+ prepared.resultSetSchema
+ } finally {
+ prepared.close(auth)
+ }
+ }
+ }
+ return
+ }
+ def seen = []
+ assertEquals(60, read("SELECT id, v FROM ${table}", { root ->
+ def vector = root.getVector(1)
+ def field = vector.field
+ assertEquals("arrow.parquet.variant", field.metadata.get("ARROW:extension:name"))
+ assertEquals("Struct", field.type.toString())
+ assertEquals(["metadata", "value"], field.children.collect { it.name })
+ field.children.each { child ->
+ assertFalse(child.nullable)
+ assertEquals("Binary", child.type.toString())
+ }
+ for (int i = 0; i < root.rowCount; i++) {
+ int id = root.getVector(0).get(i)
+ seen.add(id)
+ assertEquals(id % 4 == 0, vector.isNull(i))
+ if (id % 4 != 0) {
+ assertTrue(vector.getChild("metadata").get(i).length > 0)
+ assertTrue(vector.getChild("value").get(i).length > 0)
+ if (id % 4 == 1) {
+ assertEquals([12, 42], vector.getChild("value").get(i).collect { it & 0xff })
+ }
+ }
+ }
+ }, parallel, resultBackendCount))
+ assertEquals((1..60).toList(), seen.sort())
+ def scannedColumns = "v, ARRAY(v) AS a, MAP('key', v) AS m, STRUCT(v) AS s"
+ // Prepare and GetSchema must advertise the same Variant leaves as execution.
+ ["SELECT CAST(42 AS VARIANT) AS v",
+ "SELECT ${scannedColumns} FROM ${table} WHERE id = 1",
+ "SELECT ${scannedColumns} FROM ${table} WHERE id < 0"].eachWithIndex { query, index ->
+ def prepared = client.prepare(query.toString(), auth)
+ try {
+ def schema = prepared.resultSetSchema
+ assertEquals(schema, client.getExecuteSchema(query.toString(), auth).schema)
+ assertEquals(schema, prepared.fetchSchema(auth).schema)
+ def leaves = [schema.fields[0]]
+ if (index > 0) {
+ leaves.add(schema.fields[1].children[0])
+ leaves.add(schema.fields[2].children[0].children[1])
+ leaves.add(schema.fields[3].children[0])
+ }
+ leaves.each { field ->
+ assertEquals("Struct", field.type.toString())
+ assertEquals("arrow.parquet.variant", field.metadata.get("ARROW:extension:name"))
+ }
+ assertEquals(index == 2 ? 0 : 1, read(query.toString(), { root -> }, false, 1, prepared))
+ } finally {
+ prepared.close(auth)
+ }
+ }
+ }
+ if (!variantV2) {
+ return
+ }
+ // Each row owns its wire dictionary; unrelated keys must not multiply Arrow metadata.
+ def keyExpression = "CONCAT(REPEAT('k', 244), LPAD(CAST(number AS STRING), 6, '0'))"
+ def variantExpression = """parse_to_variant(CONCAT('{"', ${keyExpression}, '":', CAST(number AS STRING), '}'))"""
+ def metadataRows = []
+ assertEquals(128, read("SELECT number AS id, ${variantExpression} AS v FROM numbers(\"number\"=\"128\")", { root ->
+ def variant = root.getVector(1)
+ assertEquals("arrow.parquet.variant", variant.field.metadata.get("ARROW:extension:name"))
+ for (int i = 0; i < root.rowCount; ++i) {
+ metadataRows.add(root.getVector(0).get(i).intValue())
+ assertTrue(variant.getChild("metadata").get(i).length < 270,
+ "A row must not repeat the batch dictionary")
+ }
+ }))
+ assertEquals((0..<128).toList(), metadataRows.sort())
+ // A folded constant must use the same wire representation as a scanned Variant column.
+ assertEquals(1, read("SELECT parse_to_variant('42') AS v", { root ->
+ assertEquals("arrow.parquet.variant", root.getVector(0).field.metadata.get("ARROW:extension:name"))
+ }))
+ assertEquals(0, read("SELECT parse_to_variant(CAST(id AS STRING)) AS v FROM ${table} WHERE id < 0", { root -> }))
+ } finally {
+ try {
+ if (auth != null) {
+ feClient.closeSession(new org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CloseSessionRequest(), auth)
+ }
+ } finally {
+ try {
+ client.close()
+ } finally {
+ try {
+ allocator.close()
+ } finally {
+ jdbc_sql("DROP TABLE IF EXISTS ${table}")
+ }
+ }
+ }
+ }
+}
diff --git a/regression-test/suites/flink_connector_p0/flink_connector_type.groovy b/regression-test/suites/flink_connector_p0/flink_connector_type.groovy
index 26d4b33b1b8..6e6a8d25744 100644
--- a/regression-test/suites/flink_connector_p0/flink_connector_type.groovy
+++ b/regression-test/suites/flink_connector_p0/flink_connector_type.groovy
@@ -24,13 +24,15 @@ import org.awaitility.Awaitility
suite("flink_connector_type") {
+ def inputTable = "test_types_input"
def tableName1 = "test_types_source"
def tableName2 = "test_types_sink"
+ sql """DROP TABLE IF EXISTS ${inputTable}"""
sql """DROP TABLE IF EXISTS ${tableName1}"""
sql """DROP TABLE IF EXISTS ${tableName2}"""
sql """
- CREATE TABLE `test_types_source` (
+ CREATE TABLE `${inputTable}` (
`id` int,
`c1` boolean,
`c2` tinyint,
@@ -59,9 +61,9 @@ PROPERTIES (
);
""";
- sql """CREATE TABLE `test_types_sink` like `test_types_source` """
+ sql """CREATE TABLE `${tableName2}` like `${inputTable}` """
- sql """ INSERT INTO `test_types_source`
+ sql """ INSERT INTO `${inputTable}`
VALUES
(
1,
@@ -106,6 +108,17 @@ VALUES
'{"B":"variant_value1"}'
);"""
+ // The connector declares c18 as STRING and cannot decode native Arrow Variant.
+ // Materialize an explicit text projection while retaining Variant ingestion in the sink.
+ sql """
+ CREATE TABLE `${tableName1}`
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ AS SELECT id, c1, c2, c3, c4, c5, c6, c7, c8, c9,
+ c10, c11, c12, c13, c14, c15, c16, c17, CAST(c18 AS STRING) AS c18
+ FROM `${inputTable}`
+ """
+
def thisDb = sql """select database()""";
thisDb = thisDb[0][0];
logger.info("current database is ${thisDb}");
@@ -143,19 +156,21 @@ VALUES
run_cmd.addAll(addOpens.tokenize())
run_cmd.addAll(["-cp", jarName, "org.apache.doris.FlinkConnectorTypeCase",
"--doris-fe-address", context.config.feHttpAddress,
- "--doris-database", "regression_test_flink_connector_p0",
+ "--doris-database", thisDb,
"--doris-user", context.config.feHttpUser,
"--doris-password", context.config.feHttpPassword])
run_cmd.addAll(getDorisConnectorTlsArgs())
logger.info("run_cmd : ${run_cmd.join(' ')}")
- def run_flink_jar = run_cmd.execute().getText()
- logger.info("result: $run_flink_jar")
+ // Drain both streams and check the exit code so a failed Flink job cannot look successful.
+ def flinkProcess = new ProcessBuilder(run_cmd.collect { it.toString() }).redirectErrorStream(true).start()
+ logger.info("result: ${flinkProcess.text}")
+ assertEquals(0, flinkProcess.waitFor())
// The publish in the commit phase is asynchronous
Awaitility.await().atMost(30, SECONDS).pollInterval(1, SECONDS).await().until(
{
def resultTbl = sql """ select count(1) from test_types_sink"""
logger.info("retry test_types_sink count: $resultTbl")
- resultTbl.size() >= 1
+ resultTbl[0][0] == 2
})
logger.info("flink job execute finished.");
diff --git a/samples/arrow-flight-sql/python/README.md b/samples/arrow-flight-sql/python/README.md
index 7a7586940a1..42762c29cd7 100644
--- a/samples/arrow-flight-sql/python/README.md
+++ b/samples/arrow-flight-sql/python/README.md
@@ -60,4 +60,41 @@ The tests execute only read-only queries.
# Notes
For more details, refer to [Python Usage] in the document https://doris.apache.org/zh-CN/docs/dev/db-connect/arrow-flight-sql-connect
-
\ No newline at end of file
+
+
+# Native Variant V2 results
+
+On branch-4.1 builds with this feature and Variant V2 enabled on FE and BE, Arrow
+Flight SQL / ADBC returns Variant V2 as native binary values automatically:
+
+```sql
+SELECT parse_to_variant('{"key":42}') AS v;
+-- Request text explicitly when the client needs JSON strings.
+SELECT CAST(variant_column AS STRING) FROM example_table;
+```
+
+Variant V2 fields, including nested fields, use the `arrow.parquet.variant` extension
+with `struct<metadata: binary not null, value: binary not null>` storage. SQL NULL is
+a null struct. Variant null is a non-null struct containing the encoded null value.
+V2 values retain their physical scalar types and decimal scales. Each Arrow row carries
+only the dictionary keys it uses, rather than copying keys from unrelated rows.
+
+Legacy Variant is unsupported, including nested legacy fields, SQL NULL and empty
+query results. Use an explicit SQL cast to STRING for text output. `parse_to_variant`
+follows the configured Variant representation; it does not convert legacy storage to
+V2. Native encoding accepts up to 128 nested levels.
+
+ADBC can transport this schema and its binary values. A client without a registered
+Variant extension exposes the struct with `ARROW:extension:name` field metadata.
+Receiving native VARIANT does not automatically decode it to Python dictionaries or
+pandas objects; use a Parquet Variant decoder, or explicitly cast the result to STRING.
+
+To check ADBC query and partition reads against a running Variant V2 cluster:
+
+```bash
+pip install adbc_driver_flightsql pyarrow
+export DORIS_FLIGHT_URI='grpc://localhost:8815'
+export DORIS_USER='root'
+# Set DORIS_PASSWORD in the environment if authentication requires it.
+python test_variant.py
+```
diff --git a/samples/arrow-flight-sql/python/test_variant.py b/samples/arrow-flight-sql/python/test_variant.py
new file mode 100644
index 00000000000..35ab784f5b9
--- /dev/null
+++ b/samples/arrow-flight-sql/python/test_variant.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+"""
+
+"""Run against a Variant V2 cluster with DORIS_FLIGHT_URI, DORIS_USER and DORIS_PASSWORD."""
+
+import os
+import unittest
+
+import adbc_driver_flightsql
+import adbc_driver_manager
+import pyarrow as pa
+
+
+class NativeVariantTest(unittest.TestCase):
+ def test_query_and_partitions(self):
+ uri = os.environ["DORIS_FLIGHT_URI"]
+ options = {
+ adbc_driver_manager.DatabaseOptions.USERNAME.value: os.environ.get("DORIS_USER", "root"),
+ adbc_driver_manager.DatabaseOptions.PASSWORD.value: os.environ.get("DORIS_PASSWORD", ""),
+ }
+ query = """SELECT 1 AS id, parse_to_variant('42') AS v
+ UNION ALL SELECT 2, parse_to_variant(CAST(NULL AS STRING))"""
+ with adbc_driver_flightsql.connect(uri, db_kwargs=options) as database:
+ with adbc_driver_manager.AdbcConnection(database) as connection:
+ def execute(sql):
+ with adbc_driver_manager.AdbcStatement(connection) as statement:
+ statement.set_sql_query(sql)
+ stream, _ = statement.execute_query()
+ return pa.RecordBatchReader._import_from_c(stream.address).read_all()
+
+ execute("SET enable_sql_cache=false")
+ for parallel in (False, True):
+ execute(f"SET enable_parallel_result_sink={str(parallel).lower()}")
+ table = execute(query)
+ self.check_result(table)
+ # ExecuteSchema and Prepare must agree with the subsequently fetched batches.
+ with adbc_driver_manager.AdbcStatement(connection) as statement:
+ statement.set_sql_query(query)
+ schema_handle = statement.execute_schema()
+ schema = pa.Schema._import_from_c(schema_handle.address)
+ self.assertEqual(schema, table.schema)
+ statement.prepare()
+ stream, _ = statement.execute_query()
+ prepared = pa.RecordBatchReader._import_from_c(stream.address).read_all()
+ self.assertEqual(prepared.schema, schema)
+ self.check_result(prepared)
+
+ with adbc_driver_manager.AdbcStatement(connection) as statement:
+ statement.set_sql_query(query)
+ partitions, _, _ = statement.execute_partitions()
+ tables = []
+ for partition in partitions:
+ stream = connection.read_partition(partition)
+ tables.append(pa.RecordBatchReader._import_from_c(stream.address).read_all())
+ self.check_result(pa.concat_tables(tables))
+
+ def check_result(self, table):
+ table = table.sort_by("id")
+ self.assertEqual(table.num_rows, 2)
+ field = table.schema.field("v")
+ values = table.column("v").combine_chunks()
+ # Clients without a registered extension expose its storage plus field metadata.
+ if isinstance(values, pa.ExtensionArray):
+ self.assertEqual(values.type.extension_name, "arrow.parquet.variant")
+ values = values.storage
+ else:
+ self.assertEqual(field.metadata[b"ARROW:extension:name"], b"arrow.parquet.variant")
+ self.assertTrue(pa.types.is_struct(values.type))
+ self.assertEqual([child.name for child in values.type], ["metadata", "value"])
+ self.assertIsNone(values[1].as_py())
+ row = values[0].as_py()
+ self.assertTrue(row["metadata"])
+ # The Parquet Variant primitive INT8 tag is 3 << 2, followed by its signed byte.
+ self.assertEqual(row["value"], bytes([12, 42]))
+
+
+if __name__ == "__main__":
+ unittest.main()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 12/28: [fix](flight-sql) Support prepared query parameter binding on branch-4.1 (#68768)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit cd3af8974c7a564f489a60b82137ef3bc13f9faa
Author: Gabriel <li...@selectdb.com>
AuthorDate: Fri Oct 9 14:37:12 2026 +0800
[fix](flight-sql) Support prepared query parameter binding on branch-4.1 (#68768)
### What problem does this PR solve?
Flight SQL prepared queries containing `?` fail because Prepare rejects
placeholders and `acceptPutPreparedStatementQuery` returns
`UNIMPLEMENTED`. For example, ADBC cannot bind an integer to `SELECT
CAST(? AS BIGINT)` or execute a range predicate with two bound values.
This change implements single-row typed parameter binding on branch-4.1.
Prepare analyzes explicit casts and comparison columns to advertise
parameter types and a best-effort result schema. DoPut converts Arrow
values into detached Nereids literals, and execution binds those
literals to a fresh statement context. Result schemas are reanalyzed
after binding. Values are never interpolated into SQL text.
Arrow JDBC receives typed parameter metadata and a result schema before
binding, so it can construct parameter vectors and classify SELECT
correctly. Types that cannot be inferred remain unknown; JDBC callers
should use an explicit CAST for queries such as `SELECT ?`.
Timezone-bearing timestamp uploads preserve their UTC instants as
TIMESTAMPTZ literals; timestamps without a timezone retain wall-clock
semantics. Unsupported parameter vector types are not advertised as
bindable. Invalid SQL returns `INVALID_ARGUMENT` consistently through
Prepare and GetSchema.
Failed rebinds invalidate previous values. Handle ownership, concurrent
closure, upload versions, and per-binding/session memory limits protect
retained state. Multi-row uploads and unsupported types fail explicitly.
Queries requiring forwarding to the master FE reject bound parameters
because the forwarding protocol cannot carry typed bindings.
The PR also removes an obsolete duplicate JWT version property,
preserving the effective dependency version. The target branch already
contains the dependency declaration cleanup.
### Release note
Support single-row scalar parameter binding for Flight SQL prepared
queries, including repeated execution and NULL values.
### Check List (For Author)
- Test
- [x] Regression test
- [x] Unit Test
- [x] Manual test
- Behavior changed:
- [x] Yes. Parameterized Flight SQL queries can bind and execute
supported scalar values.
- Does this need documentation?
- [x] No. This implements the existing prepared-query protocol.
Validation:
- Full FE Checkstyle passed (`mvn -B clean checkstyle:check`).
- Focused compilation of all 11 changed Java source/test files passed
with `javac --release 8` using existing FE dependency artifacts. Java
8-compatible Guava copies preserve immutable parameter snapshots.
- 23 new tests and 33 existing Flight SQL schema tests passed, including
invalid-SQL status checks over Flight RPC and connection recovery.
- The new regression suite passed against a local FE service test
harness and a real BE, including actual JDBC `PreparedStatement`
execution, result fetching, rebinding, NULL, mixed types, range
predicates, and recovery after invalid uploads. TIMESTAMPTZ predicates
are checked under UTC, Asia/Shanghai and America/New_York, including a
daylight-saving transition.
- ADBC 1.11.0 with PyArrow 23.0.1 passed 18 concurrent connections and
540 parameterized queries, with result verification plus NULL and
connection-reuse checks. A further 72 timestamp result checks cover all
four Arrow units, three session timezones, two input timezone
annotations, negative epochs and NULL values.
The local integration harness uses one FE service and one real BE; a
full multi-FE cluster run and full FE build are left to CI.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../nereids/rules/analysis/ExpressionAnalyzer.java | 5 +-
.../java/org/apache/doris/qe/ConnectContext.java | 176 +++++-
.../java/org/apache/doris/qe/StmtExecutor.java | 6 +
.../arrowflight/DorisFlightSqlProducer.java | 132 +++-
.../arrowflight/FlightSqlConnectProcessor.java | 38 ++
.../service/arrowflight/FlightSqlParameters.java | 336 +++++++++++
.../service/arrowflight/FlightSqlQuerySchema.java | 65 +-
.../sessions/FlightSqlConnectPoolMgr.java | 3 +
.../arrowflight/DorisFlightSqlSchemaTest.java | 15 +-
.../arrowflight/FlightSqlParameterBindingTest.java | 41 ++
.../arrowflight/FlightSqlPreparedQueryTest.java | 666 +++++++++++++++++++++
.../test_prepared_query_parameters.groovy | 223 +++++++
12 files changed, 1648 insertions(+), 58 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
index d9cc562d305..a68d539d284 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
@@ -816,7 +816,10 @@ public class ExpressionAnalyzer extends SubExprAnalyzer<ExpressionRewriteContext
private void registerPlaceholderIdToSlot(ComparisonPredicate cp,
ExpressionRewriteContext context, Expression left, Expression right) {
if (ConnectContext.get() != null
- && ConnectContext.get().getCommand() == MysqlCommand.COM_STMT_EXECUTE) {
+ && (ConnectContext.get().getCommand() == MysqlCommand.COM_STMT_EXECUTE
+ // Flight Prepare needs the bound column type before comparison coercion erases that constraint.
+ || (ConnectContext.get().getConnectType() == ConnectContext.ConnectType.ARROW_FLIGHT_SQL
+ && context.cascadesContext.getStatementContext().isPrepareStage()))) {
// Used to replace expression in ShortCircuit plan
if (cp.right() instanceof Placeholder && left instanceof SlotReference) {
PlaceholderId id = ((Placeholder) cp.right()).getPlaceholderId();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
index 674c2a99548..e51a0031676 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
@@ -87,6 +87,7 @@ import org.apache.doris.transaction.TransactionEntry;
import org.apache.doris.transaction.TransactionStatus;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import lombok.Getter;
@@ -156,12 +157,19 @@ public class ConnectContext {
// for arrow flight
protected volatile String peerIdentity;
private final Map<String, PreparedQuery> preparedQuerys = new HashMap<>();
+ private static final long MAX_FLIGHT_PARAMETER_BYTES = 16L * 1024 * 1024;
+ private long flightParameterBytes;
+ private boolean flightPreparedQueriesClosed;
private static class PreparedQuery {
private final String sql;
- private final String catalog;
- private final String database;
- private final Schema schema;
+ private String catalog;
+ private String database;
+ private Schema schema;
+ private int parameterCount;
+ private List<Literal> parameters;
+ private long bindingVersion;
+ private long parameterBytes;
private PreparedQuery(String sql, String catalog, String database, Schema schema) {
this.sql = sql;
@@ -427,7 +435,7 @@ public class ConnectContext {
}
resetSessionVariable();
userVars = new HashMap<>();
- preparedQuerys.clear();
+ clearPreparedQueries();
preparedStatementContextMap.clear();
runningQuery = null;
queryId = null;
@@ -924,35 +932,157 @@ public class ConnectContext {
this.loginTime = System.currentTimeMillis();
}
- public synchronized void addPreparedQuery(String preparedStatementId, String preparedQuery) {
- addPreparedQuery(preparedStatementId, preparedQuery, null);
+ public void addPreparedQuery(String preparedStatementId, String preparedQuery) {
+ synchronized (preparedQuerys) {
+ addPreparedQuery(preparedStatementId, preparedQuery, null);
+ }
}
- public synchronized void addPreparedQuery(String preparedStatementId, String preparedQuery, Schema schema) {
- preparedQuerys.put(preparedStatementId,
- new PreparedQuery(preparedQuery, getDefaultCatalog(), getDatabase(), schema));
+ public void addPreparedQuery(String preparedStatementId, String preparedQuery, Schema schema) {
+ synchronized (preparedQuerys) {
+ if (flightPreparedQueriesClosed) {
+ throw new IllegalStateException("Flight SQL session is closed");
+ }
+ removePreparedQuery(preparedStatementId);
+ preparedQuerys.put(preparedStatementId,
+ new PreparedQuery(preparedQuery, getDefaultCatalog(), getDatabase(), schema));
+ }
}
- public synchronized Schema getPreparedQuerySchema(String preparedStatementId) {
- PreparedQuery query = preparedQuerys.get(preparedStatementId);
- return query == null ? null : query.schema;
+ public void addPreparedQuery(String id, String sql, Schema schema, int parameterCount) {
+ synchronized (preparedQuerys) {
+ addPreparedQuery(id, sql, schema);
+ preparedQuerys.get(id).parameterCount = parameterCount;
+ }
}
- public synchronized String getPreparedQuery(String preparedStatementId) {
- PreparedQuery query = preparedQuerys.get(preparedStatementId);
- if (query == null) {
- return null;
+ public int getPreparedQueryParameterCount(String id) {
+ synchronized (preparedQuerys) {
+ PreparedQuery query = preparedQuerys.get(id);
+ return query == null ? -1 : query.parameterCount;
}
- // A handle must not execute unqualified SQL in a different namespace than its advertised schema.
- if (!Objects.equals(query.catalog, getDefaultCatalog()) || !Objects.equals(query.database, getDatabase())) {
- preparedQuerys.remove(preparedStatementId);
- return null;
+ }
+
+ public List<Literal> getPreparedQueryParameters(String id) {
+ synchronized (preparedQuerys) {
+ PreparedQuery query = preparedQuerys.get(id);
+ return query == null ? null : query.parameters;
+ }
+ }
+
+ public long beginPreparedQueryBinding(String id) {
+ synchronized (preparedQuerys) {
+ PreparedQuery query = preparedQuerys.get(id);
+ if (query == null) {
+ return -1;
+ }
+ flightParameterBytes -= query.parameterBytes;
+ query.parameterBytes = 0;
+ query.parameters = null;
+ return ++query.bindingVersion;
}
- return query.sql;
}
- public synchronized void removePreparedQuery(String preparedStatementId) {
- preparedQuerys.remove(preparedStatementId);
+ public boolean isPreparedQueryBindingCurrent(String id, long version) {
+ synchronized (preparedQuerys) {
+ PreparedQuery query = preparedQuerys.get(id);
+ return query != null && query.bindingVersion == version;
+ }
+ }
+
+ public void setPreparedQueryParameters(String id, List<Literal> parameters, Schema schema) {
+ synchronized (preparedQuerys) {
+ PreparedQuery query = preparedQuerys.get(id);
+ if (query == null) {
+ throw new IllegalStateException("Prepared statement expired");
+ }
+ long bytes = parameters == null ? 0 : parameters.size() * 128L;
+ if (parameters != null) {
+ for (Literal parameter : parameters) {
+ if (parameter.getValue() instanceof String) {
+ bytes += 2L * ((String) parameter.getValue()).length();
+ }
+ }
+ }
+ // A per-upload limit alone allows many handles to retain unbounded parameter memory in one session.
+ if (flightParameterBytes - query.parameterBytes + bytes > MAX_FLIGHT_PARAMETER_BYTES) {
+ throw new IllegalArgumentException("Prepared query parameters exceed the 16 MiB session limit");
+ }
+ flightParameterBytes += bytes - query.parameterBytes;
+ query.parameterBytes = bytes;
+ // Keep a detached immutable snapshot using a Java 8-compatible copy implementation.
+ query.parameters = parameters == null ? null : ImmutableList.copyOf(parameters);
+ query.schema = schema;
+ }
+ }
+
+ public boolean setPreparedQueryParameters(String id, List<Literal> parameters, Schema schema, long version) {
+ synchronized (preparedQuerys) {
+ if (!isPreparedQueryBindingCurrent(id, version)) {
+ return false;
+ }
+ setPreparedQueryParameters(id, parameters, schema);
+ return true;
+ }
+ }
+
+ public boolean refreshPreparedQueryNamespace(String id) {
+ synchronized (preparedQuerys) {
+ PreparedQuery query = preparedQuerys.get(id);
+ if (query == null) {
+ return false;
+ }
+ if (query.parameterCount == 0) {
+ query.catalog = getDefaultCatalog();
+ query.database = getDatabase();
+ }
+ return true;
+ }
+ }
+
+ public Schema getPreparedQuerySchema(String preparedStatementId) {
+ synchronized (preparedQuerys) {
+ PreparedQuery query = preparedQuerys.get(preparedStatementId);
+ return query == null ? null : query.schema;
+ }
+ }
+
+ public String getPreparedQuery(String preparedStatementId) {
+ synchronized (preparedQuerys) {
+ PreparedQuery query = preparedQuerys.get(preparedStatementId);
+ if (query == null) {
+ return null;
+ }
+ // A handle must not execute unqualified SQL in a different namespace than its advertised schema.
+ if (!Objects.equals(query.catalog, getDefaultCatalog()) || !Objects.equals(query.database, getDatabase())) {
+ removePreparedQuery(preparedStatementId);
+ return null;
+ }
+ return query.sql;
+ }
+ }
+
+ public void closePreparedQueries() {
+ synchronized (preparedQuerys) {
+ flightPreparedQueriesClosed = true;
+ clearPreparedQueries();
+ }
+ }
+
+ public void clearPreparedQueries() {
+ synchronized (preparedQuerys) {
+ preparedQuerys.clear();
+ flightParameterBytes = 0;
+ }
+ }
+
+ public void removePreparedQuery(String preparedStatementId) {
+ synchronized (preparedQuerys) {
+ PreparedQuery query = preparedQuerys.remove(preparedStatementId);
+ if (query != null) {
+ flightParameterBytes -= query.parameterBytes;
+ }
+ }
}
public void setRunningQuery(String runningQuery) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 1919141ec00..3fd32020d92 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -1183,6 +1183,12 @@ public class StmtExecutor {
}
private void forwardToMaster() throws Exception {
+ // COM_QUERY forwarding carries SQL text but not Flight's typed placeholder bindings.
+ if (context.getConnectType() == ConnectType.ARROW_FLIGHT_SQL
+ && !statementContext.getIdToPlaceholderRealExpr().isEmpty()) {
+ throw new UserException("Flight SQL queries with bound parameters cannot be forwarded; "
+ + "connect to master FE");
+ }
masterOpExecutor = new MasterOpExecutor(originStmt, context, redirectStatus, isQuery());
if (LOG.isDebugEnabled()) {
LOG.debug("need to transfer to Master. stmt: {}", context.getStmtId());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java
index 2aae87d527b..13b3f506fb9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java
@@ -27,6 +27,7 @@ import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.Util;
import org.apache.doris.mysql.MysqlCommand;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.QueryState.MysqlStateType;
import org.apache.doris.qe.StmtExecutor;
@@ -186,10 +187,12 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
final StreamListener<Result> listener) {
executorService.submit(() -> {
try {
- String[] handleParts = request.getPreparedStatementHandle().toStringUtf8().split(":");
- String executedPeerIdentity = handleParts[0];
- String preparedStatementId = handleParts[1];
- flightSessionsManager.getConnectContext(executedPeerIdentity).removePreparedQuery(preparedStatementId);
+ String preparedStatementId = preparedStatementId(context, request.getPreparedStatementHandle());
+ ConnectContext connection = flightSessionsManager.getConnectContext(context.peerIdentity());
+ // Closing one handle serializes with its execution; closing a session can still cancel immediately.
+ synchronized (connection) {
+ connection.removePreparedQuery(preparedStatementId);
+ }
} catch (final Throwable e) {
listener.onError(e);
return;
@@ -211,6 +214,11 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
private Pair<FlightInfo, StmtExecutor> executeQueryStatementLocked(String peerIdentity,
ConnectContext connectContext, String query,
final FlightDescriptor descriptor) {
+ return executeQueryStatementLocked(peerIdentity, connectContext, query, descriptor, null);
+ }
+
+ private Pair<FlightInfo, StmtExecutor> executeQueryStatementLocked(String peerIdentity,
+ ConnectContext connectContext, String query, FlightDescriptor descriptor, List<Literal> parameters) {
try {
Preconditions.checkState(null != connectContext);
Preconditions.checkState(!query.isEmpty());
@@ -222,7 +230,11 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
connectContext.getFlightSqlChannel().reset();
connectContext.clearFlightSqlEndpointsLocations();
try (FlightSqlConnectProcessor flightSQLConnectProcessor = new FlightSqlConnectProcessor(connectContext)) {
- flightSQLConnectProcessor.handleQuery(query);
+ if (parameters == null) {
+ flightSQLConnectProcessor.handleQuery(query);
+ } else {
+ flightSQLConnectProcessor.handleQuery(query, parameters);
+ }
if (connectContext.getState().getStateType() == MysqlStateType.ERR) {
throw new RuntimeException("after executeQueryStatement handleQuery");
}
@@ -348,8 +360,10 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
ConnectContext connection = flightSessionsManager.getConnectContext(context.peerIdentity());
synchronized (connection) {
Pair<String, Schema> prepared = preparedQuery(connection, context, command);
+ String preparedId = preparedStatementId(context, command.getPreparedStatementHandle());
Pair<FlightInfo, StmtExecutor> result = executeQueryStatementLocked(
- context.peerIdentity(), connection, prepared.getLeft(), descriptor);
+ context.peerIdentity(), connection, prepared.getLeft(), descriptor,
+ connection.getPreparedQueryParameters(preparedId));
FlightInfo info = result.getLeft();
String id = command.getPreparedStatementHandle().toStringUtf8()
.substring(context.peerIdentity().length() + 1);
@@ -361,7 +375,8 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
? FlightSqlQuerySchema.matchesExecutionSchema(prepared.getRight(), info.getSchema(),
((LogicalPlanAdapter) executor.getParsedStmt()).getColLabels())
: prepared.getRight().equals(info.getSchema());
- if (!matches) {
+ // Session teardown can invalidate the handle during execution without taking the connection monitor.
+ if (!matches || !connection.refreshPreparedQueryNamespace(id)) {
connection.removePreparedQuery(id);
try {
if (executor != null) {
@@ -377,9 +392,6 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
throw CallStatus.NOT_FOUND.withDescription("Prepared statement schema changed; prepare again")
.toRuntimeException();
}
- // A successful USE/SWITCH in this statement may change its own namespace. Rebind
- // only this handle; unrelated handles must still reject an external namespace change.
- connection.addPreparedQuery(id, prepared.getLeft(), prepared.getRight());
return info;
}
}
@@ -396,25 +408,27 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
final CallContext context, final FlightDescriptor descriptor) {
ConnectContext connection = flightSessionsManager.getConnectContext(context.peerIdentity());
synchronized (connection) {
+ String id = preparedStatementId(context, command.getPreparedStatementHandle());
+ String query = connection.getPreparedQuery(id);
+ if (query != null && connection.getPreparedQueryParameterCount(id) > 0
+ && connection.getPreparedQueryParameters(id) == null) {
+ // Metadata discovery does not execute a query and must work before values are bound.
+ return new SchemaResult(prepareQuerySchema(connection, query).first);
+ }
return new SchemaResult(preparedQuery(connection, context, command).getRight());
}
}
private Pair<String, Schema> preparedQuery(ConnectContext connection, CallContext context,
CommandPreparedStatementQuery command) {
- String prefix = context.peerIdentity() + ":";
- String handle = command.getPreparedStatementHandle().toStringUtf8();
- if (!handle.startsWith(prefix)) {
- throw CallStatus.INVALID_ARGUMENT.withDescription("Invalid prepared statement handle").toRuntimeException();
- }
- String id = handle.substring(prefix.length());
+ String id = preparedStatementId(context, command.getPreparedStatementHandle());
String query = connection.getPreparedQuery(id);
if (query == null) {
throw CallStatus.NOT_FOUND
.withDescription("Prepared statement expired; prepare again in the current namespace")
.toRuntimeException();
}
- Schema schema = analyzeQuerySchema(connection, query);
+ Schema schema = analyzeQuerySchema(connection, query, connection.getPreparedQueryParameters(id));
// Execution reparses SQL using the current session. Never silently replace the schema
// advertised by Prepare when settings such as sql_mode or time_zone change its result.
if (!schema.equals(connection.getPreparedQuerySchema(id))) {
@@ -425,9 +439,22 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
return Pair.of(query, schema);
}
+ private String preparedStatementId(CallContext context, ByteString handleBytes) {
+ String prefix = context.peerIdentity() + ":";
+ String handle = handleBytes.toStringUtf8();
+ if (!handle.startsWith(prefix) || handle.length() == prefix.length()) {
+ throw CallStatus.INVALID_ARGUMENT.withDescription("Invalid prepared statement handle").toRuntimeException();
+ }
+ return handle.substring(prefix.length());
+ }
+
private Schema analyzeQuerySchema(ConnectContext context, String query) {
+ return analyzeQuerySchema(context, query, null);
+ }
+
+ private Schema analyzeQuerySchema(ConnectContext context, String query, List<Literal> parameters) {
try {
- return FlightSqlQuerySchema.analyze(context, query);
+ return FlightSqlQuerySchema.analyze(context, query, parameters);
} catch (FlightRuntimeException e) {
throw e;
} catch (Exception e) {
@@ -436,6 +463,18 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
}
}
+ private org.apache.doris.common.Pair<Schema, Schema> prepareQuerySchema(ConnectContext context, String query) {
+ try {
+ return FlightSqlQuerySchema.prepare(context, query);
+ } catch (FlightRuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ // Invalid SQL must report the same client error through Prepare and GetSchema.
+ throw CallStatus.INVALID_ARGUMENT.withDescription("Cannot determine query schema: " + e.getMessage())
+ .withCause(e).toRuntimeException();
+ }
+ }
+
@Override
public void close() throws Exception {
AutoCloseables.close(rootAllocator);
@@ -448,16 +487,18 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
private ActionCreatePreparedStatementResult buildCreatePreparedStatementResult(ByteString handle,
Schema parameterSchema, Schema metaData) {
- Preconditions.checkState(!Objects.isNull(metaData));
final ByteString bytes = Objects.isNull(parameterSchema) ? ByteString.EMPTY
: ByteString.copyFrom(serializeMetadata(parameterSchema));
return ActionCreatePreparedStatementResult.newBuilder()
- .setDatasetSchema(ByteString.copyFrom(serializeMetadata(metaData))).setParameterSchema(bytes)
+ .setDatasetSchema(metaData == null ? ByteString.EMPTY
+ : ByteString.copyFrom(serializeMetadata(metaData)))
+ .setParameterSchema(bytes)
.setPreparedStatementHandle(handle).build();
}
@Override
- public void createPreparedStatement(final ActionCreatePreparedStatementRequest request, final CallContext context,
+ public void createPreparedStatement(final ActionCreatePreparedStatementRequest request,
+ final CallContext context,
final StreamListener<Result> listener) {
executorService.submit(() -> {
ConnectContext connectContext = null;
@@ -468,12 +509,15 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
String query = request.getQuery();
// ADBC ExecuteSchema reads this dataset schema directly without calling GetSchema.
// Analyze before registering a handle so failed preparation does not retain a query.
- Schema schema = analyzeQuerySchema(connectContext, query);
+ org.apache.doris.common.Pair<Schema, Schema> prepared = prepareQuerySchema(
+ connectContext, query);
+ Schema schema = prepared.first;
preparedStatementId = UUID.randomUUID().toString();
ByteString handle = ByteString.copyFromUtf8(context.peerIdentity() + ":" + preparedStatementId);
Result result = new Result(Any.pack(buildCreatePreparedStatementResult(handle,
- new Schema(Collections.emptyList()), schema)).toByteArray());
- connectContext.addPreparedQuery(preparedStatementId, query, schema);
+ prepared.second, schema)).toByteArray());
+ connectContext.addPreparedQuery(preparedStatementId, query, schema,
+ prepared.second.getFields().size());
listener.onNext(result);
listener.onCompleted();
}
@@ -530,8 +574,44 @@ public class DorisFlightSqlProducer implements FlightSqlProducer, AutoCloseable
@Override
public Runnable acceptPutPreparedStatementQuery(CommandPreparedStatementQuery command, CallContext context,
FlightStream flightStream, StreamListener<PutResult> ackStream) {
- throw CallStatus.UNIMPLEMENTED.withDescription("acceptPutPreparedStatementQuery unimplemented")
- .toRuntimeException();
+ return () -> {
+ try {
+ ConnectContext connection = flightSessionsManager.getConnectContext(context.peerIdentity());
+ String id = preparedStatementId(context, command.getPreparedStatementHandle());
+ int count;
+ long version;
+ synchronized (connection) {
+ if (connection.getPreparedQuery(id) == null) {
+ throw CallStatus.NOT_FOUND.withDescription("Prepared statement expired").toRuntimeException();
+ }
+ count = connection.getPreparedQueryParameterCount(id);
+ // A failed rebind must not leave the previous values executable.
+ version = connection.beginPreparedQueryBinding(id);
+ if (version < 0) {
+ throw CallStatus.NOT_FOUND.withDescription("Prepared statement expired").toRuntimeException();
+ }
+ }
+ List<Literal> parameters = FlightSqlParameters.read(flightStream, count);
+ synchronized (connection) {
+ String query = connection.getPreparedQuery(id);
+ if (query == null || !connection.isPreparedQueryBindingCurrent(id, version)) {
+ throw CallStatus.CANCELLED.withDescription("Prepared statement binding was superseded")
+ .toRuntimeException();
+ }
+ Schema schema = analyzeQuerySchema(connection, query, parameters);
+ if (!connection.setPreparedQueryParameters(id, parameters, schema, version)) {
+ throw CallStatus.CANCELLED.withDescription("Prepared statement binding was superseded")
+ .toRuntimeException();
+ }
+ }
+ ackStream.onCompleted();
+ } catch (FlightRuntimeException e) {
+ ackStream.onError(e);
+ } catch (Exception e) {
+ ackStream.onError(CallStatus.INVALID_ARGUMENT.withDescription("Cannot bind query parameters: "
+ + e.getMessage()).withCause(e).toRuntimeException());
+ }
+ };
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java
index 5942424750c..87f00e763b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java
@@ -18,16 +18,20 @@
package org.apache.doris.service.arrowflight;
import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.StatementBase;
import org.apache.doris.common.ConnectionException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.Status;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.mysql.MysqlCommand;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.proto.InternalService;
import org.apache.doris.proto.Types;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ConnectProcessor;
+import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.rpc.BackendServiceProxy;
import org.apache.doris.rpc.RpcException;
@@ -61,6 +65,7 @@ import java.util.concurrent.TimeoutException;
public class FlightSqlConnectProcessor extends ConnectProcessor implements AutoCloseable {
private static final Logger LOG = LogManager.getLogger(FlightSqlConnectProcessor.class);
private Schema arrowSchema;
+ private List<Literal> parameters;
public FlightSqlConnectProcessor(ConnectContext context) {
super(context);
@@ -100,6 +105,39 @@ public class FlightSqlConnectProcessor extends ConnectProcessor implements AutoC
super.handleQuery(query);
}
+ public void handleQuery(String query, List<Literal> parameters) throws ConnectionException {
+ this.parameters = parameters;
+ try {
+ handleQuery(query);
+ } finally {
+ this.parameters = null;
+ }
+ }
+
+ @Override
+ protected List<StatementBase> parseWithFallback(String originStmt, String convertedStmt,
+ SessionVariable sessionVariable) throws ConnectionException {
+ List<StatementBase> statements = super.parseWithFallback(originStmt, convertedStmt, sessionVariable);
+ if (parameters != null && !parameters.isEmpty() && statements != null) {
+ try {
+ if (statements.size() != 1 || !(statements.get(0) instanceof LogicalPlanAdapter)) {
+ throw new IllegalArgumentException("Parameters require a single query statement");
+ }
+ // Reparse on every execution so bindings cannot retain planner state from a previous query.
+ FlightSqlParameters.bind(((LogicalPlanAdapter) statements.get(0)).getStatementContext(), parameters);
+ } catch (RuntimeException e) {
+ handleQueryException(e, originStmt, null, null);
+ for (StatementBase statement : statements) {
+ if (statement instanceof LogicalPlanAdapter) {
+ ((LogicalPlanAdapter) statement).getStatementContext().close();
+ }
+ }
+ return null;
+ }
+ }
+ return statements;
+ }
+
// TODO
// private void handleInitDb() {
// handleInitDb(fullDbName);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlParameters.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlParameters.java
new file mode 100644
index 00000000000..59d21fc71ce
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlParameters.java
@@ -0,0 +1,336 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service.arrowflight;
+
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.trees.expressions.Cast;
+import org.apache.doris.nereids.trees.expressions.Placeholder;
+import org.apache.doris.nereids.trees.expressions.SubqueryExpr;
+import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
+import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
+import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal;
+import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
+import org.apache.doris.nereids.trees.plans.PlaceholderId;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.types.BigIntType;
+import org.apache.doris.nereids.types.BooleanType;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.DateTimeV2Type;
+import org.apache.doris.nereids.types.DateV2Type;
+import org.apache.doris.nereids.types.DecimalV3Type;
+import org.apache.doris.nereids.types.DoubleType;
+import org.apache.doris.nereids.types.FloatType;
+import org.apache.doris.nereids.types.IntegerType;
+import org.apache.doris.nereids.types.NullType;
+import org.apache.doris.nereids.types.SmallIntType;
+import org.apache.doris.nereids.types.StringType;
+import org.apache.doris.nereids.types.TimeStampTzType;
+import org.apache.doris.nereids.types.TinyIntType;
+
+import org.apache.arrow.flight.CallStatus;
+import org.apache.arrow.flight.FlightRuntimeException;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.vector.DateDayVector;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.TimeStampVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.StandardCharsets;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Converts one parameter row into detached values consumed by Nereids placeholder analysis. */
+final class FlightSqlParameters {
+ private static final long MAX_PARAMETER_BYTES = 1024 * 1024;
+ static final int MAX_PARAMETERS = 1024;
+
+ private FlightSqlParameters() {
+ }
+
+ static Map<PlaceholderId, DataType> inferCastTypes(Plan plan) {
+ Map<PlaceholderId, DataType> types = new HashMap<>();
+ plan.foreach(node -> {
+ ((Plan) node).getExpressions().forEach(expression -> expression.foreach(child -> {
+ if (child instanceof Cast && ((Cast) child).child() instanceof Placeholder) {
+ // Only the innermost explicit cast constrains the value supplied for this placeholder.
+ types.put(((Placeholder) ((Cast) child).child()).getPlaceholderId(), ((Cast) child).getDataType());
+ } else if (child instanceof SubqueryExpr) {
+ types.putAll(inferCastTypes(((SubqueryExpr) child).getQueryPlan()));
+ }
+ }));
+ });
+ return types;
+ }
+
+ static boolean supportsParameterType(ArrowType type) {
+ switch (Types.getMinorTypeForArrowType(type)) {
+ case NULL:
+ case BIT:
+ case TINYINT:
+ case SMALLINT:
+ case INT:
+ case BIGINT:
+ case FLOAT4:
+ case FLOAT8:
+ case VARCHAR:
+ case DATEDAY:
+ case TIMESTAMPSEC:
+ case TIMESTAMPMILLI:
+ case TIMESTAMPMICRO:
+ case TIMESTAMPNANO:
+ case TIMESTAMPSECTZ:
+ case TIMESTAMPMILLITZ:
+ case TIMESTAMPMICROTZ:
+ case TIMESTAMPNANOTZ:
+ case DECIMAL:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ static void bind(StatementContext context, List<Literal> parameters) {
+ if (parameters == null || parameters.size() != context.getPlaceholders().size()) {
+ throw CallStatus.INVALID_ARGUMENT.withDescription("Bind all query parameters before execution")
+ .toRuntimeException();
+ }
+ for (int i = 0; i < parameters.size(); i++) {
+ context.getIdToPlaceholderRealExpr().put(
+ context.getPlaceholders().get(i).getPlaceholderId(), parameters.get(i));
+ }
+ }
+
+ static List<Literal> read(FlightStream stream, int parameterCount) {
+ List<Literal> parameters = null;
+ FlightRuntimeException failure = null;
+ while (stream.next()) {
+ // Consume the upload even after validation fails so clients can receive the final gRPC status.
+ if (failure != null) {
+ continue;
+ }
+ try {
+ VectorSchemaRoot root = stream.getRoot();
+ if (root.getFieldVectors().size() != parameterCount) {
+ throw invalid("Parameter count does not match the prepared query");
+ }
+ if (root.getRowCount() == 0) {
+ continue;
+ }
+ if (root.getRowCount() != 1 || parameters != null) {
+ throw CallStatus.UNIMPLEMENTED.withDescription("Only one parameter row per binding is supported")
+ .toRuntimeException();
+ }
+ parameters = convert(root);
+ } catch (FlightRuntimeException e) {
+ failure = e;
+ } catch (RuntimeException e) {
+ failure = invalid("Invalid parameter value: " + e.getMessage());
+ }
+ }
+ if (failure != null) {
+ throw failure;
+ }
+ if (parameters == null) {
+ if (parameterCount == 0) {
+ return Collections.emptyList();
+ }
+ throw invalid("Parameter upload contains no row");
+ }
+ return parameters;
+ }
+
+ static List<Literal> convert(VectorSchemaRoot root) {
+ if (root.getFieldVectors().size() > MAX_PARAMETERS) {
+ throw invalid("Too many query parameters (maximum 1024)");
+ }
+ long bytes = 0;
+ List<Literal> parameters = new ArrayList<>();
+ for (FieldVector vector : root.getFieldVectors()) {
+ bytes += vector.getBufferSize();
+ if (bytes > MAX_PARAMETER_BYTES) {
+ throw invalid("Query parameters exceed the 1 MiB binding limit");
+ }
+ parameters.add(literal(vector));
+ }
+ return parameters;
+ }
+
+ private static Literal literal(FieldVector vector) {
+ if (vector.getField().getDictionary() != null) {
+ throw CallStatus.UNIMPLEMENTED.withDescription("Dictionary encoded parameters are not supported")
+ .toRuntimeException();
+ }
+ DataType type;
+ switch (vector.getMinorType()) {
+ case NULL:
+ type = NullType.INSTANCE;
+ break;
+ case BIT:
+ type = BooleanType.INSTANCE;
+ break;
+ case TINYINT:
+ type = TinyIntType.INSTANCE;
+ break;
+ case SMALLINT:
+ type = SmallIntType.INSTANCE;
+ break;
+ case INT:
+ type = IntegerType.INSTANCE;
+ break;
+ case BIGINT:
+ type = BigIntType.INSTANCE;
+ break;
+ case FLOAT4:
+ type = FloatType.INSTANCE;
+ break;
+ case FLOAT8:
+ type = DoubleType.INSTANCE;
+ break;
+ case VARCHAR:
+ type = StringType.INSTANCE;
+ break;
+ case DATEDAY:
+ type = DateV2Type.INSTANCE;
+ break;
+ case TIMESTAMPSEC:
+ case TIMESTAMPMILLI:
+ case TIMESTAMPMICRO:
+ case TIMESTAMPNANO:
+ type = DateTimeV2Type.of(6);
+ break;
+ case TIMESTAMPSECTZ:
+ case TIMESTAMPMILLITZ:
+ case TIMESTAMPMICROTZ:
+ case TIMESTAMPNANOTZ:
+ // Arrow Java uses TZ vectors even for an empty annotation, which still means wall-clock time.
+ type = ((ArrowType.Timestamp) vector.getField().getType()).getTimezone().isEmpty()
+ ? DateTimeV2Type.of(6) : TimeStampTzType.of(6);
+ break;
+ case DECIMAL:
+ ArrowType.Decimal decimal = (ArrowType.Decimal) vector.getField().getType();
+ if (decimal.getScale() < 0 || decimal.getScale() > decimal.getPrecision()) {
+ throw invalid("Unsupported decimal parameter scale");
+ }
+ type = DecimalV3Type.createDecimalV3Type(decimal.getPrecision(), decimal.getScale());
+ break;
+ default:
+ throw CallStatus.UNIMPLEMENTED.withDescription(
+ "Unsupported query parameter type: " + vector.getField().getType()).toRuntimeException();
+ }
+ if (vector.isNull(0)) {
+ return new NullLiteral(type);
+ }
+ Object value = vector.getObject(0);
+ switch (vector.getMinorType()) {
+ case BIT: return BooleanLiteral.of((Boolean) value);
+ case TINYINT: return new TinyIntLiteral(((Number) value).byteValue());
+ case SMALLINT: return new SmallIntLiteral(((Number) value).shortValue());
+ case INT: return new IntegerLiteral(((Number) value).intValue());
+ case BIGINT: return new BigIntLiteral(((Number) value).longValue());
+ case FLOAT4:
+ case FLOAT8:
+ double number = ((Number) value).doubleValue();
+ if (!Double.isFinite(number)) {
+ throw invalid("Non-finite floating point parameters are not supported");
+ }
+ return type instanceof FloatType ? new FloatLiteral((float) number) : new DoubleLiteral(number);
+ case VARCHAR:
+ try {
+ // Reject malformed UTF-8 instead of silently replacing bytes in a bound predicate.
+ return new StringLiteral(StandardCharsets.UTF_8.newDecoder()
+ .decode(ByteBuffer.wrap(((VarCharVector) vector).get(0))).toString());
+ } catch (CharacterCodingException e) {
+ throw invalid("String parameter is not valid UTF-8");
+ }
+ case DECIMAL: return new DecimalV3Literal((DecimalV3Type) type, (BigDecimal) value);
+ case DATEDAY:
+ LocalDate date = LocalDate.ofEpochDay(((DateDayVector) vector).get(0));
+ checkYear(date.getYear());
+ return new DateV2Literal(date.getYear(), date.getMonthValue(), date.getDayOfMonth());
+ default:
+ long timestamp = ((TimeStampVector) vector).get(0);
+ ArrowType.Timestamp timestampType = (ArrowType.Timestamp) vector.getField().getType();
+ long units;
+ switch (timestampType.getUnit()) {
+ case SECOND:
+ units = 1;
+ break;
+ case MILLISECOND:
+ units = 1000;
+ break;
+ case MICROSECOND:
+ units = 1000000;
+ break;
+ case NANOSECOND:
+ units = 1000000000;
+ break;
+ default:
+ throw invalid("Unsupported timestamp unit");
+ }
+ long nanos = Math.floorMod(timestamp, units) * (1000000000 / units);
+ if (nanos % 1000 != 0) {
+ throw invalid("Timestamp parameter exceeds microsecond precision");
+ }
+ LocalDateTime time = LocalDateTime.ofEpochSecond(
+ Math.floorDiv(timestamp, units), (int) nanos, ZoneOffset.UTC);
+ checkYear(time.getYear());
+ if (type instanceof TimeStampTzType) {
+ // Zoned Arrow timestamps already encode a UTC instant. A DATETIMEV2 literal would
+ // reinterpret these fields in the session timezone when cast to TIMESTAMPTZ.
+ return new TimestampTzLiteral((TimeStampTzType) type, time.getYear(), time.getMonthValue(),
+ time.getDayOfMonth(), time.getHour(), time.getMinute(), time.getSecond(),
+ time.getNano() / 1000);
+ }
+ return new DateTimeV2Literal((DateTimeV2Type) type, time.getYear(), time.getMonthValue(),
+ time.getDayOfMonth(), time.getHour(), time.getMinute(), time.getSecond(),
+ time.getNano() / 1000);
+ }
+ }
+
+ private static void checkYear(int year) {
+ if (year < 0 || year > 9999) {
+ throw invalid("Date parameter is outside the supported year range 0000..9999");
+ }
+ }
+
+ private static FlightRuntimeException invalid(String message) {
+ return CallStatus.INVALID_ARGUMENT.withDescription(message).toRuntimeException();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java
index 678cab055a8..0ed2b77e92f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java
@@ -27,6 +27,7 @@ import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.StructField;
import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.Type;
+import org.apache.doris.common.Pair;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.es.EsExternalCatalog;
import org.apache.doris.datasource.lance.LanceExternalCatalog;
@@ -38,7 +39,11 @@ import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.parser.SqlDialectHelper;
import org.apache.doris.nereids.rules.rewrite.CheckPrivileges;
+import org.apache.doris.nereids.trees.expressions.Placeholder;
import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.plans.PlaceholderId;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PrepareCommandPlanner;
import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand;
@@ -68,6 +73,8 @@ import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTable
import org.apache.doris.nereids.trees.plans.commands.merge.MergeIntoCommand;
import org.apache.doris.nereids.trees.plans.commands.use.SwitchCommand;
import org.apache.doris.nereids.trees.plans.commands.use.UseCommand;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.NullType;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.QueryState;
import org.apache.doris.qe.ResultSetMetaData;
@@ -97,6 +104,19 @@ final class FlightSqlQuerySchema {
}
static Schema analyze(ConnectContext context, String query) throws Exception {
+ return analyze(context, query, null);
+ }
+
+ static Schema analyze(ConnectContext context, String query, List<Literal> parameters) throws Exception {
+ return analyze(context, query, parameters, false).first;
+ }
+
+ static Pair<Schema, Schema> prepare(ConnectContext context, String query) throws Exception {
+ return analyze(context, query, null, true);
+ }
+
+ private static Pair<Schema, Schema> analyze(ConnectContext context, String query,
+ List<Literal> parameters, boolean preparing) throws Exception {
synchronized (context) {
ConnectContext previousThreadContext = ConnectContext.get();
StatementContext previousStatement = context.getStatementContext();
@@ -149,12 +169,33 @@ final class FlightSqlQuerySchema {
StatementContext statementContext = statement.getStatementContext();
context.setStatementContext(statementContext);
statementContext.setParsedStatement(statement);
- if (!statementContext.getPlaceholders().isEmpty()) {
- throw CallStatus.UNIMPLEMENTED.withDescription(
- "Flight SQL parameter binding is not supported").toRuntimeException();
+ int parameterCount = statementContext.getPlaceholders().size();
+ if (parameterCount > FlightSqlParameters.MAX_PARAMETERS) {
+ throw CallStatus.INVALID_ARGUMENT.withDescription("Too many query parameters (maximum 1024)")
+ .toRuntimeException();
}
- List<Field> fields = new ArrayList<>();
Plan plan = statement.getLogicalPlan();
+ Map<PlaceholderId, DataType> parameterTypes = new HashMap<>();
+ if (parameterCount > 0) {
+ if (statements.size() != 1 || plan instanceof Command) {
+ throw CallStatus.INVALID_ARGUMENT.withDescription(
+ "Parameters require a single query statement").toRuntimeException();
+ }
+ if (preparing) {
+ statementContext.setPrepareStage(true);
+ parameterTypes.putAll(FlightSqlParameters.inferCastTypes(plan));
+ parameters = new ArrayList<>();
+ for (Placeholder placeholder : statementContext.getPlaceholders()) {
+ // Unknown values must not acquire the synthetic STRING type used by MySQL Prepare.
+ parameters.add(new NullLiteral(parameterTypes.getOrDefault(
+ placeholder.getPlaceholderId(), NullType.INSTANCE)));
+ }
+ }
+ }
+ if (parameters != null || parameterCount > 0) {
+ FlightSqlParameters.bind(statementContext, parameters);
+ }
+ List<Field> fields = new ArrayList<>();
if (plan instanceof Command) {
resolveNamespace(context, plan, scopedDatabases);
ResultSetMetaData metadata = commandMetadata(context, (Command) plan);
@@ -194,12 +235,26 @@ final class FlightSqlQuerySchema {
Plan analyzed = cascades.getRewritePlan();
// PrepareCommandPlanner stops before the rewrite phase that normally checks privileges.
new CheckPrivileges().rewriteRoot(analyzed, cascades.getCurrentJobContext());
+ if (preparing) {
+ statementContext.getIdToComparisonSlot().forEach((id, slot) ->
+ parameterTypes.putIfAbsent(id, slot.getDataType()));
+ }
for (Slot slot : analyzed.getOutput()) {
fields.add(field(slot.getName(), slot.getDataType().toCatalogDataType(), slot.nullable(),
true, context.getSessionVariable().getTimeZone()));
}
}
- return new Schema(fields);
+ List<Field> parameterFields = new ArrayList<>();
+ for (int i = 0; preparing && i < parameterCount; i++) {
+ DataType type = parameterTypes.getOrDefault(
+ statementContext.getPlaceholders().get(i).getPlaceholderId(), NullType.INSTANCE);
+ Field parameterField = field(String.valueOf(i), type.toCatalogDataType(), true,
+ true, context.getSessionVariable().getTimeZone());
+ // Result schemas support more Arrow types than parameter uploads do.
+ parameterFields.add(FlightSqlParameters.supportsParameterType(parameterField.getType())
+ ? parameterField : Field.nullable(String.valueOf(i), new ArrowType.Null()));
+ }
+ return Pair.of(new Schema(fields), new Schema(parameterFields));
} finally {
try {
List<AutoCloseable> resources = new ArrayList<>();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java
index c8854507e00..63d01a87a8a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java
@@ -57,6 +57,9 @@ public class FlightSqlConnectPoolMgr extends ConnectPoolMgr {
@Override
public void unregisterConnection(ConnectContext ctx) {
+ // Use the short-lived prepared-state lock so teardown never waits for a running query
+ // holding the connection monitor before it can cancel that query.
+ ctx.closePreparedQueries();
// All Flight SQL session teardown paths (idle/query timeout, bearer token expiry, and
// explicit CloseSession) reach here. Release channel-cached Arrow results before removing
// the context from the pool.
diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlSchemaTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlSchemaTest.java
index 70fb507e502..86fea280901 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlSchemaTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlSchemaTest.java
@@ -182,11 +182,20 @@ public class DorisFlightSqlSchemaTest extends TestWithFeService {
@Test
void invalidSqlIsRejectedAndConnectionRecovers() throws Exception {
- for (String sql : Arrays.asList("SELEC 1", "SELECT missing_column FROM schema_input")) {
- Assertions.assertThrows(FlightRuntimeException.class, () -> schema(sql));
+ for (String sql : Arrays.asList("SELEC 1", "SELECT missing_column FROM schema_input",
+ "SELECT CAST(? AS BIGINT) FROM missing_schema_table")) {
+ FlightRuntimeException schemaFailure = Assertions.assertThrows(
+ FlightRuntimeException.class, () -> schema(sql));
+ Assertions.assertEquals(FlightStatusCode.INVALID_ARGUMENT, schemaFailure.status().code());
Mockito.clearInvocations(connectContext);
- Assertions.assertThrows(java.util.concurrent.ExecutionException.class, () -> prepare(sql));
+ java.util.concurrent.ExecutionException prepareFailure = Assertions.assertThrows(
+ java.util.concurrent.ExecutionException.class, () -> prepare(sql));
+ Assertions.assertTrue(prepareFailure.getCause() instanceof FlightRuntimeException);
+ Assertions.assertEquals(FlightStatusCode.INVALID_ARGUMENT,
+ ((FlightRuntimeException) prepareFailure.getCause()).status().code());
Mockito.verify(connectContext, Mockito.never()).addPreparedQuery(Mockito.anyString(), Mockito.anyString(), Mockito.any());
+ Mockito.verify(connectContext, Mockito.never()).addPreparedQuery(
+ Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.anyInt());
Assertions.assertEquals("id", preparedSchema("SELECT id FROM schema_input")
.getFields().get(0).getName());
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlParameterBindingTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlParameterBindingTest.java
new file mode 100644
index 00000000000..1f6c7cc2a7f
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlParameterBindingTest.java
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service.arrowflight;
+
+import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager;
+
+import org.apache.arrow.flight.FlightProducer.CallContext;
+import org.apache.arrow.flight.FlightProducer.StreamListener;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.Location;
+import org.apache.arrow.flight.sql.impl.FlightSql.CommandPreparedStatementQuery;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class FlightSqlParameterBindingTest {
+ @Test
+ public void acceptsParameterUpload() throws Exception {
+ try (DorisFlightSqlProducer producer = new DorisFlightSqlProducer(
+ Location.forGrpcInsecure("127.0.0.1", 0), Mockito.mock(FlightSessionsManager.class))) {
+ Assert.assertNotNull(producer.acceptPutPreparedStatementQuery(
+ CommandPreparedStatementQuery.getDefaultInstance(), Mockito.mock(CallContext.class),
+ Mockito.mock(FlightStream.class), Mockito.mock(StreamListener.class)));
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlPreparedQueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlPreparedQueryTest.java
new file mode 100644
index 00000000000..aa6d37582bb
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlPreparedQueryTest.java
@@ -0,0 +1,666 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service.arrowflight;
+
+import org.apache.doris.analysis.StatementBase;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral;
+import org.apache.doris.nereids.types.DateTimeV2Type;
+import org.apache.doris.nereids.types.StringType;
+import org.apache.doris.nereids.types.TimeStampTzType;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager;
+import org.apache.doris.service.arrowflight.sessions.FlightSqlConnectContext;
+import org.apache.doris.utframe.TestWithFeService;
+
+import com.google.common.base.Strings;
+import com.google.protobuf.Any;
+import com.google.protobuf.ByteString;
+import org.apache.arrow.flight.Action;
+import org.apache.arrow.flight.AsyncPutListener;
+import org.apache.arrow.flight.CallHeaders;
+import org.apache.arrow.flight.FlightClient;
+import org.apache.arrow.flight.FlightDescriptor;
+import org.apache.arrow.flight.FlightProducer.CallContext;
+import org.apache.arrow.flight.FlightRuntimeException;
+import org.apache.arrow.flight.FlightServer;
+import org.apache.arrow.flight.FlightStatusCode;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.Location;
+import org.apache.arrow.flight.auth2.CallHeaderAuthenticator;
+import org.apache.arrow.flight.sql.FlightSqlClient;
+import org.apache.arrow.flight.sql.impl.FlightSql.ActionClosePreparedStatementRequest;
+import org.apache.arrow.flight.sql.impl.FlightSql.ActionCreatePreparedStatementRequest;
+import org.apache.arrow.flight.sql.impl.FlightSql.ActionCreatePreparedStatementResult;
+import org.apache.arrow.flight.sql.impl.FlightSql.CommandPreparedStatementQuery;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.DateDayVector;
+import org.apache.arrow.vector.DecimalVector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.TimeStampMicroVector;
+import org.apache.arrow.vector.TimeStampVector;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedConstruction;
+import org.mockito.Mockito;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.Types;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+public class FlightSqlPreparedQueryTest extends TestWithFeService {
+ private RootAllocator allocator;
+ private FlightServer server;
+ private FlightClient client;
+ private FlightSqlClient sqlClient;
+ private DorisFlightSqlProducer producer;
+ private FlightSqlConnectContext flightContext;
+
+ @Override
+ protected void runBeforeAll() throws Exception {
+ allocator = new RootAllocator();
+ flightContext = new FlightSqlConnectContext("parameter-peer");
+ flightContext.setEnv(connectContext.getEnv());
+ flightContext.setCurrentUserIdentity(connectContext.getCurrentUserIdentity());
+ flightContext.setSessionVariable(connectContext.getSessionVariable());
+ FlightSessionsManager sessions = Mockito.mock(FlightSessionsManager.class);
+ Mockito.when(sessions.getConnectContext("parameter-peer")).thenReturn(flightContext);
+ producer = new DorisFlightSqlProducer(Location.forGrpcInsecure("127.0.0.1", 0), sessions);
+ server = FlightServer.builder(allocator, Location.forGrpcInsecure("127.0.0.1", 0), producer)
+ .headerAuthenticator(headers -> new CallHeaderAuthenticator.AuthResult() {
+ @Override
+ public String getPeerIdentity() {
+ return "parameter-peer";
+ }
+
+ @Override
+ public void appendToOutgoingHeaders(CallHeaders headers) {
+ headers.insert("authorization", "Bearer parameter-peer");
+ }
+ }).build().start();
+ client = FlightClient.builder(allocator, server.getLocation()).build();
+ sqlClient = new FlightSqlClient(client);
+ }
+
+ @Override
+ protected void runAfterAll() throws Exception {
+ sqlClient.close();
+ server.close();
+ java.lang.reflect.Field executor = DorisFlightSqlProducer.class.getDeclaredField("executorService");
+ executor.setAccessible(true);
+ ((ExecutorService) executor.get(producer)).shutdownNow();
+ producer.close();
+ flightContext.getFlightSqlChannel().close();
+ allocator.close();
+ }
+
+ private CommandPreparedStatementQuery prepare(String sql) throws Exception {
+ Action action = new Action("CreatePreparedStatement", Any.pack(
+ ActionCreatePreparedStatementRequest.newBuilder().setQuery(sql).build()).toByteArray());
+ ActionCreatePreparedStatementResult prepared = Any.parseFrom(client.doAction(action).next().getBody())
+ .unpack(ActionCreatePreparedStatementResult.class);
+ Assertions.assertFalse(prepared.getDatasetSchema().isEmpty());
+ return CommandPreparedStatementQuery.newBuilder()
+ .setPreparedStatementHandle(prepared.getPreparedStatementHandle()).build();
+ }
+
+ private void bind(CommandPreparedStatementQuery command, VectorSchemaRoot root) {
+ AsyncPutListener listener = new AsyncPutListener();
+ FlightClient.ClientStreamListener writer = client.startPut(
+ FlightDescriptor.command(Any.pack(command).toByteArray()), root, listener);
+ writer.putNext();
+ writer.completed();
+ listener.getResult();
+ }
+
+ private String id(CommandPreparedStatementQuery command) {
+ return command.getPreparedStatementHandle().toStringUtf8().substring("parameter-peer:".length());
+ }
+
+ private Schema schema(CommandPreparedStatementQuery command) {
+ return client.getSchema(FlightDescriptor.command(Any.pack(command).toByteArray())).getSchema();
+ }
+
+ @Test
+ public void jdbcPrepareAdvertisesResultAndParameterTypes() throws Exception {
+ Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver");
+ try (Connection connection = DriverManager.getConnection("jdbc:arrow-flight-sql://127.0.0.1:"
+ + server.getPort() + "?useEncryption=false", "fixture", "fixture");
+ PreparedStatement statement = connection.prepareStatement(
+ "SELECT CAST(? AS BIGINT) AS n, CAST(? AS STRING) AS s, CAST(? AS DOUBLE) AS d")) {
+ Assertions.assertEquals(3, statement.getMetaData().getColumnCount());
+ for (int i = 0; i < 3; i++) {
+ int type = new int[] {Types.BIGINT, Types.VARCHAR, Types.DOUBLE}[i];
+ Assertions.assertEquals(type, statement.getMetaData().getColumnType(i + 1));
+ Assertions.assertEquals(type, statement.getParameterMetaData().getParameterType(i + 1));
+ }
+ }
+ }
+
+ @Test
+ public void jdbcPrepareInfersRangePredicateTypes() throws Exception {
+ Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver");
+ try (Connection connection = DriverManager.getConnection("jdbc:arrow-flight-sql://127.0.0.1:"
+ + server.getPort() + "?useEncryption=false", "fixture", "fixture");
+ PreparedStatement statement = connection.prepareStatement(
+ "SELECT number FROM numbers(\"number\"=\"10\") WHERE number >= ? AND ? > number")) {
+ Assertions.assertEquals(Types.BIGINT, statement.getParameterMetaData().getParameterType(1));
+ Assertions.assertEquals(Types.BIGINT, statement.getParameterMetaData().getParameterType(2));
+ Assertions.assertEquals(Types.BIGINT, statement.getMetaData().getColumnType(1));
+ }
+ }
+
+ @Test
+ public void jdbcPreparePreservesInnerCastsAndSubqueryTypes() throws Exception {
+ Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver");
+ try (Connection connection = DriverManager.getConnection("jdbc:arrow-flight-sql://127.0.0.1:"
+ + server.getPort() + "?useEncryption=false", "fixture", "fixture");
+ PreparedStatement statement = connection.prepareStatement(
+ "SELECT CAST(CAST(? AS BIGINT) AS STRING), (SELECT CAST(? AS DOUBLE))")) {
+ Assertions.assertEquals(Types.BIGINT, statement.getParameterMetaData().getParameterType(1));
+ Assertions.assertEquals(Types.DOUBLE, statement.getParameterMetaData().getParameterType(2));
+ Assertions.assertEquals(Types.VARCHAR, statement.getMetaData().getColumnType(1));
+ Assertions.assertEquals(Types.DOUBLE, statement.getMetaData().getColumnType(2));
+ }
+ }
+
+ @Test
+ public void unknownParameterTypesAreNotInvented() throws Exception {
+ try (FlightSqlClient.PreparedStatement statement = sqlClient.prepare("SELECT ? AS value")) {
+ Assertions.assertEquals(new ArrowType.Null(),
+ statement.getParameterSchema().getFields().get(0).getType());
+ Assertions.assertEquals(new ArrowType.Null(),
+ statement.getResultSetSchema().getFields().get(0).getType());
+ }
+ }
+
+ @Test
+ public void invalidSqlReportsClientErrorOverFlight() throws Exception {
+ for (String query : Arrays.asList("SELEC 1", "SELECT definitely_missing_schema_column",
+ "SELECT CAST(? AS BIGINT) FROM missing_parameter_table")) {
+ FlightRuntimeException prepareFailure = Assertions.assertThrows(FlightRuntimeException.class,
+ () -> sqlClient.prepare(query));
+ Assertions.assertEquals(FlightStatusCode.INVALID_ARGUMENT, prepareFailure.status().code());
+ FlightRuntimeException schemaFailure = Assertions.assertThrows(FlightRuntimeException.class,
+ () -> sqlClient.getExecuteSchema(query));
+ Assertions.assertEquals(FlightStatusCode.INVALID_ARGUMENT, schemaFailure.status().code());
+ try (FlightSqlClient.PreparedStatement statement = sqlClient.prepare("SELECT 1 AS id")) {
+ Assertions.assertEquals("id", statement.getResultSetSchema().getFields().get(0).getName());
+ }
+ }
+ }
+
+ @Test
+ public void unsupportedParameterVectorsDoNotRestrictResultTypes() throws Exception {
+ try (FlightSqlClient.PreparedStatement statement = sqlClient.prepare("SELECT CAST(? AS ARRAY<INT>)")) {
+ Assertions.assertEquals(new ArrowType.Null(),
+ statement.getParameterSchema().getFields().get(0).getType());
+ Assertions.assertEquals(new ArrowType.List(),
+ statement.getResultSetSchema().getFields().get(0).getType());
+ }
+ try (FlightSqlClient.PreparedStatement statement = sqlClient.prepare(
+ "SELECT CAST(CAST(? AS STRING) AS ARRAY<INT>)")) {
+ Assertions.assertEquals(new ArrowType.Utf8(),
+ statement.getParameterSchema().getFields().get(0).getType());
+ Assertions.assertEquals(new ArrowType.List(),
+ statement.getResultSetSchema().getFields().get(0).getType());
+ }
+ }
+
+ @Test
+ public void bindsAndRebindsIntegerQueryOverFlight() throws Exception {
+ CommandPreparedStatementQuery command = prepare("SELECT CAST(? AS BIGINT) AS value");
+ Assertions.assertEquals(new ArrowType.Int(64, true), schema(command).getFields().get(0).getType());
+ Assertions.assertThrows(FlightRuntimeException.class,
+ () -> client.getInfo(FlightDescriptor.command(Any.pack(command).toByteArray())));
+ try (BigIntVector value = new BigIntVector("parameter", allocator);
+ VectorSchemaRoot root = VectorSchemaRoot.of(value)) {
+ for (long expected : new long[] {42, -7, Long.MAX_VALUE}) {
+ value.setSafe(0, expected);
+ root.setRowCount(1);
+ bind(command, root);
+ Assertions.assertEquals(new ArrowType.Int(64, true), schema(command).getFields().get(0).getType());
+ Assertions.assertEquals(expected, flightContext.getPreparedQueryParameters(id(command)).get(0).getValue());
+ }
+ } finally {
+ flightContext.removePreparedQuery(id(command));
+ }
+ }
+
+ @Test
+ public void bindsMixedTypesAndPreservesQuotedText() throws Exception {
+ String query = "SELECT CAST(? AS BIGINT) AS n, CAST(? AS STRING) AS text, CAST(? AS DOUBLE) AS fraction";
+ CommandPreparedStatementQuery command = prepare(query);
+ try (BigIntVector n = new BigIntVector("n", allocator);
+ VarCharVector text = new VarCharVector("text", allocator);
+ Float8Vector fraction = new Float8Vector("fraction", allocator);
+ VectorSchemaRoot root = VectorSchemaRoot.of(n, text, fraction)) {
+ String expected = "中文 ' ? \\ ; SELECT 2";
+ n.setSafe(0, 17);
+ text.setSafe(0, expected.getBytes(StandardCharsets.UTF_8));
+ fraction.setSafe(0, 2.5);
+ root.setRowCount(1);
+ bind(command, root);
+ List<Literal> parameters = flightContext.getPreparedQueryParameters(id(command));
+ Assertions.assertEquals(17L, parameters.get(0).getValue());
+ Assertions.assertEquals(expected, parameters.get(1).getValue());
+ Assertions.assertEquals(2.5, parameters.get(2).getValue());
+ Assertions.assertEquals(3, schema(command).getFields().size());
+ // Exercise the execution parser hook without requiring a BE in the FE unit test fixture.
+ try (FlightSqlConnectProcessor processor = new FlightSqlConnectProcessor(flightContext)) {
+ java.lang.reflect.Field bindings = FlightSqlConnectProcessor.class.getDeclaredField("parameters");
+ bindings.setAccessible(true);
+ bindings.set(processor, parameters);
+ List<StatementBase> statements = processor.parseWithFallback(query, query,
+ flightContext.getSessionVariable());
+ StatementContext statement = ((LogicalPlanAdapter) statements.get(0))
+ .getStatementContext();
+ try {
+ for (int i = 0; i < parameters.size(); i++) {
+ Assertions.assertEquals(parameters.get(i), statement.getIdToPlaceholderRealExpr().get(
+ statement.getPlaceholders().get(i).getPlaceholderId()));
+ }
+ } finally {
+ statement.close();
+ }
+ }
+ } finally {
+ flightContext.removePreparedQuery(id(command));
+ }
+ }
+
+ @Test
+ public void failedRebindInvalidatesOldValuesAndConnectionRecovers() throws Exception {
+ CommandPreparedStatementQuery command = prepare("SELECT ? AS value");
+ try (IntVector value = new IntVector("value", allocator);
+ VectorSchemaRoot root = VectorSchemaRoot.of(value)) {
+ value.setSafe(0, 1);
+ root.setRowCount(1);
+ bind(command, root);
+ value.setSafe(1, 2);
+ root.setRowCount(2);
+ FlightRuntimeException failure = Assertions.assertThrows(FlightRuntimeException.class,
+ () -> bind(command, root));
+ Assertions.assertEquals(FlightStatusCode.UNIMPLEMENTED, failure.status().code());
+ Assertions.assertNull(flightContext.getPreparedQueryParameters(id(command)));
+ Assertions.assertThrows(FlightRuntimeException.class,
+ () -> client.getInfo(FlightDescriptor.command(Any.pack(command).toByteArray())));
+ root.setRowCount(1);
+ value.setSafe(0, 3);
+ bind(command, root);
+ Assertions.assertEquals(new ArrowType.Int(32, true), schema(command).getFields().get(0).getType());
+ value.setNull(0);
+ bind(command, root);
+ Assertions.assertTrue(flightContext.getPreparedQueryParameters(id(command)).get(0) instanceof NullLiteral);
+ } finally {
+ flightContext.removePreparedQuery(id(command));
+ }
+ Assertions.assertNotNull(sqlClient.getExecuteSchema("SELECT 1"));
+ }
+
+ @Test
+ public void convertsDetachedValuesAndTypedNulls() {
+ try (IntVector value = new IntVector("value", allocator);
+ VarCharVector text = new VarCharVector("text", allocator);
+ VectorSchemaRoot root = VectorSchemaRoot.of(value, text)) {
+ value.setSafe(0, 12);
+ text.setNull(0);
+ root.setRowCount(1);
+ List<Literal> literals = FlightSqlParameters.convert(root);
+ value.setSafe(0, 99);
+ Assertions.assertEquals(12, literals.get(0).getValue());
+ Assertions.assertTrue(literals.get(1) instanceof NullLiteral);
+ Assertions.assertEquals(StringType.INSTANCE, literals.get(1).getDataType());
+ }
+ }
+
+ @Test
+ public void validatesWholeUploadInsteadOfUsingFirstBatch() {
+ try (IntVector value = new IntVector("value", allocator);
+ VectorSchemaRoot root = VectorSchemaRoot.of(value)) {
+ value.setSafe(0, 1);
+ root.setRowCount(1);
+ FlightStream stream = Mockito.mock(FlightStream.class);
+ Mockito.when(stream.next()).thenReturn(true, true, false);
+ Mockito.when(stream.getRoot()).thenReturn(root);
+ Assertions.assertThrows(Exception.class, () -> FlightSqlParameters.read(stream, 1));
+ Mockito.verify(stream, Mockito.times(3)).next();
+ FlightStream empty = Mockito.mock(FlightStream.class);
+ Assertions.assertThrows(Exception.class, () -> FlightSqlParameters.read(empty, 1));
+ Assertions.assertEquals(Arrays.asList(), FlightSqlParameters.read(empty, 0));
+ }
+ }
+
+ @Test
+ public void limitsRetainedParametersAcrossHandlesAndReleasesTheirBudget() {
+ ConnectContext connection = new ConnectContext();
+ // Test sources also target Java 8, where String.repeat is unavailable.
+ List<Literal> parameters = Arrays.asList(
+ new StringLiteral(Strings.repeat("x", 512 * 1024)));
+ for (int i = 0; i < 15; i++) {
+ connection.addPreparedQuery("p" + i, "SELECT ?", null, 1);
+ connection.setPreparedQueryParameters("p" + i, parameters, null);
+ }
+ connection.addPreparedQuery("overflow", "SELECT ?", null, 1);
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> connection.setPreparedQueryParameters("overflow", parameters, null));
+ connection.removePreparedQuery("p0");
+ connection.setPreparedQueryParameters("overflow", parameters, null);
+ connection.beginPreparedQueryBinding("overflow");
+ connection.addPreparedQuery("replacement", "SELECT ?", null, 1);
+ connection.setPreparedQueryParameters("replacement", parameters, null);
+ long upload = connection.beginPreparedQueryBinding("replacement");
+ connection.clearPreparedQueries();
+ Assertions.assertFalse(connection.isPreparedQueryBindingCurrent("replacement", upload));
+ Assertions.assertNull(connection.getPreparedQuery("replacement"));
+ connection.addPreparedQuery("fresh", "SELECT ?", null, 1);
+ connection.setPreparedQueryParameters("fresh", parameters, null);
+ }
+
+ @Test
+ public void rejectsInvalidValuesAndUnsupportedTypes() {
+ try (VarCharVector text = new VarCharVector("text", allocator);
+ VectorSchemaRoot root = VectorSchemaRoot.of(text)) {
+ text.setSafe(0, new byte[] {(byte) 0xc3, (byte) 0x28});
+ root.setRowCount(1);
+ Assertions.assertThrows(FlightRuntimeException.class, () -> FlightSqlParameters.convert(root));
+ }
+ try (Float8Vector number = new Float8Vector("number", allocator);
+ VectorSchemaRoot root = VectorSchemaRoot.of(number)) {
+ number.setSafe(0, Double.POSITIVE_INFINITY);
+ root.setRowCount(1);
+ Assertions.assertThrows(FlightRuntimeException.class, () -> FlightSqlParameters.convert(root));
+ }
+ try (VarBinaryVector binary =
+ new VarBinaryVector("binary", allocator);
+ VectorSchemaRoot root = VectorSchemaRoot.of(binary)) {
+ binary.setSafe(0, new byte[] {0, 1});
+ root.setRowCount(1);
+ FlightRuntimeException error = Assertions.assertThrows(FlightRuntimeException.class,
+ () -> FlightSqlParameters.convert(root));
+ Assertions.assertEquals(FlightStatusCode.UNIMPLEMENTED, error.status().code());
+ }
+ }
+
+ @Test
+ public void rejectsForwardingInsteadOfLosingTypedBindings() throws Exception {
+ flightContext.setThreadLocalInfo();
+ LogicalPlanAdapter statement = (LogicalPlanAdapter) new NereidsParser()
+ .parseSQL("SELECT ?", flightContext.getSessionVariable()).get(0);
+ try {
+ FlightSqlParameters.bind(statement.getStatementContext(), Arrays.asList(
+ new BigIntLiteral(7)));
+ StmtExecutor executor = new StmtExecutor(flightContext, statement);
+ Method forward = StmtExecutor.class
+ .getDeclaredMethod("forwardToMaster");
+ forward.setAccessible(true);
+ InvocationTargetException failure = Assertions.assertThrows(
+ InvocationTargetException.class, () -> forward.invoke(executor));
+ Assertions.assertTrue(failure.getCause().getMessage().contains("connect to master FE"));
+ } finally {
+ statement.getStatementContext().close();
+ ConnectContext.remove();
+ }
+ }
+
+ @Test
+ public void preservesDecimalAndTemporalPrecision() {
+ try (DecimalVector decimal =
+ new DecimalVector("decimal", allocator, 20, 6);
+ DateDayVector date = new DateDayVector("date", allocator);
+ TimeStampMicroVector timestamp =
+ new TimeStampMicroVector("timestamp", allocator);
+ VectorSchemaRoot root = VectorSchemaRoot.of(decimal, date, timestamp)) {
+ BigDecimal expected = new BigDecimal("12345678901234.567890");
+ decimal.setSafe(0, expected);
+ date.setSafe(0, -1);
+ timestamp.setSafe(0, -1);
+ root.setRowCount(1);
+ List<Literal> values = FlightSqlParameters.convert(root);
+ Assertions.assertEquals(expected, values.get(0).getValue());
+ Assertions.assertEquals("1969-12-31", values.get(1).getStringValue());
+ Assertions.assertEquals("1969-12-31 23:59:59.999999", values.get(2).getStringValue());
+ date.setSafe(0, Integer.MAX_VALUE);
+ Assertions.assertThrows(FlightRuntimeException.class, () -> FlightSqlParameters.convert(root));
+ }
+ }
+
+ @Test
+ public void preservesTimestampSemanticsAndNulls() {
+ for (String zone : Arrays.asList(null, "", "UTC", "Asia/Shanghai", "America/New_York")) {
+ for (org.apache.arrow.vector.types.TimeUnit unit : org.apache.arrow.vector.types.TimeUnit.values()) {
+ ArrowType.Timestamp type = new ArrowType.Timestamp(unit, zone);
+ long nanosPerUnit = java.util.concurrent.TimeUnit.valueOf(unit.name() + "S").toNanos(1);
+ try (VectorSchemaRoot root = VectorSchemaRoot.create(
+ new Schema(Arrays.asList(Field.nullable("value", type))), allocator)) {
+ root.allocateNew();
+ TimeStampVector vector = (TimeStampVector) root.getVector(0);
+ for (long value : new long[] {-1000, 0, 1000}) {
+ vector.setSafe(0, value);
+ root.setRowCount(1);
+ Literal literal = FlightSqlParameters.convert(root).get(0);
+ Assertions.assertEquals(zone != null && !zone.isEmpty(), literal instanceof TimestampTzLiteral);
+ Assertions.assertEquals(java.time.LocalDateTime.of(1970, 1, 1, 0, 0)
+ .plusNanos(value * nanosPerUnit), ((DateTimeLiteral) literal).toJavaDateType());
+ }
+ vector.setNull(0);
+ Literal literal = FlightSqlParameters.convert(root).get(0);
+ Assertions.assertTrue(literal instanceof NullLiteral);
+ Assertions.assertEquals(zone != null && !zone.isEmpty()
+ ? TimeStampTzType.of(6) : DateTimeV2Type.of(6), literal.getDataType());
+ }
+ }
+ }
+ }
+
+ @Test
+ public void rejectsTimezoneTimestampPrecisionLossAndOverflow() {
+ for (org.apache.arrow.vector.types.TimeUnit unit : Arrays.asList(
+ org.apache.arrow.vector.types.TimeUnit.NANOSECOND, org.apache.arrow.vector.types.TimeUnit.SECOND)) {
+ try (VectorSchemaRoot root = VectorSchemaRoot.create(new Schema(Arrays.asList(
+ Field.nullable("value", new ArrowType.Timestamp(unit, "UTC")))), allocator)) {
+ root.allocateNew();
+ TimeStampVector vector = (TimeStampVector) root.getVector(0);
+ long[] values = unit == org.apache.arrow.vector.types.TimeUnit.NANOSECOND
+ ? new long[] {-1, 1, 1001} : new long[] {Long.MIN_VALUE, Long.MAX_VALUE};
+ for (long value : values) {
+ vector.setSafe(0, value);
+ root.setRowCount(1);
+ FlightStream stream = Mockito.mock(FlightStream.class);
+ Mockito.when(stream.next()).thenReturn(true, false);
+ Mockito.when(stream.getRoot()).thenReturn(root);
+ FlightRuntimeException failure = Assertions.assertThrows(FlightRuntimeException.class,
+ () -> FlightSqlParameters.read(stream, 1));
+ Assertions.assertEquals(FlightStatusCode.INVALID_ARGUMENT, failure.status().code());
+ }
+ }
+ }
+ }
+
+ @Test
+ public void bindsTimezoneTimestampInNonUtcSession() throws Exception {
+ String previousZone = flightContext.getSessionVariable().getTimeZone();
+ try {
+ for (String zone : Arrays.asList("Asia/Shanghai", "America/New_York")) {
+ flightContext.getSessionVariable().setTimeZone(zone);
+ try (FlightSqlClient.PreparedStatement statement = sqlClient.prepare("SELECT CAST(? AS TIMESTAMPTZ(6))")) {
+ Assertions.assertEquals(new ArrowType.Timestamp(
+ org.apache.arrow.vector.types.TimeUnit.MICROSECOND, zone),
+ statement.getParameterSchema().getFields().get(0).getType());
+ }
+ CommandPreparedStatementQuery command = prepare("SELECT CAST(? AS TIMESTAMPTZ(6))");
+ try (VectorSchemaRoot root = VectorSchemaRoot.create(new Schema(Arrays.asList(Field.nullable("value",
+ new ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, "UTC")))), allocator)) {
+ root.allocateNew();
+ TimeStampVector vector = (TimeStampVector) root.getVector(0);
+ for (long value : new long[] {0, -1, 1234567}) {
+ vector.setSafe(0, value);
+ root.setRowCount(1);
+ bind(command, root);
+ TimestampTzLiteral literal = (TimestampTzLiteral) flightContext
+ .getPreparedQueryParameters(id(command)).get(0);
+ Assertions.assertEquals(java.time.LocalDateTime.of(1970, 1, 1, 0, 0)
+ .plusNanos(value * 1000), literal.toJavaDateType());
+ Assertions.assertEquals(new ArrowType.Timestamp(
+ org.apache.arrow.vector.types.TimeUnit.MICROSECOND, zone),
+ schema(command).getFields().get(0).getType());
+ }
+ vector.setNull(0);
+ bind(command, root);
+ Assertions.assertEquals(TimeStampTzType.of(6), flightContext
+ .getPreparedQueryParameters(id(command)).get(0).getDataType());
+ } finally {
+ client.doAction(new Action("ClosePreparedStatement", Any.pack(
+ ActionClosePreparedStatementRequest.newBuilder()
+ .setPreparedStatementHandle(command.getPreparedStatementHandle()).build())
+ .toByteArray())).forEachRemaining(result -> { });
+ }
+ }
+ } finally {
+ flightContext.getSessionVariable().setTimeZone(previousZone);
+ }
+ }
+
+ @Test
+ public void rejectsForeignAndClosedHandles() throws Exception {
+ CommandPreparedStatementQuery command = prepare("SELECT ?");
+ CommandPreparedStatementQuery foreign = CommandPreparedStatementQuery.newBuilder()
+ .setPreparedStatementHandle(ByteString.copyFromUtf8("another-peer:handle")).build();
+ FlightRuntimeException failure = Assertions.assertThrows(FlightRuntimeException.class, () -> schema(foreign));
+ Assertions.assertEquals(FlightStatusCode.INVALID_ARGUMENT, failure.status().code());
+ ActionClosePreparedStatementRequest close =
+ ActionClosePreparedStatementRequest.newBuilder()
+ .setPreparedStatementHandle(command.getPreparedStatementHandle()).build();
+ client.doAction(new Action("ClosePreparedStatement", Any.pack(close).toByteArray())).forEachRemaining(r -> { });
+ FlightRuntimeException closed = Assertions.assertThrows(FlightRuntimeException.class, () -> schema(command));
+ Assertions.assertEquals(FlightStatusCode.NOT_FOUND, closed.status().code());
+ }
+
+ @Test
+ public void bindsRangePredicatesAndRejectsWrongParameterCount() throws Exception {
+ CommandPreparedStatementQuery command = prepare("SELECT number FROM numbers(\"number\"=\"10\") "
+ + "WHERE number >= ? AND number < ? ORDER BY number");
+ try (BigIntVector lower = new BigIntVector("lower", allocator);
+ BigIntVector upper = new BigIntVector("upper", allocator);
+ VectorSchemaRoot root = VectorSchemaRoot.of(lower, upper)) {
+ lower.setSafe(0, 2);
+ upper.setSafe(0, 5);
+ root.setRowCount(1);
+ bind(command, root);
+ Assertions.assertEquals(new ArrowType.Int(64, true), schema(command).getFields().get(0).getType());
+ }
+ try (IntVector only = new IntVector("only", allocator);
+ VectorSchemaRoot wrong = VectorSchemaRoot.of(only)) {
+ only.setSafe(0, 1);
+ wrong.setRowCount(1);
+ FlightRuntimeException error = Assertions.assertThrows(FlightRuntimeException.class,
+ () -> bind(command, wrong));
+ Assertions.assertEquals(FlightStatusCode.INVALID_ARGUMENT, error.status().code());
+ Assertions.assertNull(flightContext.getPreparedQueryParameters(id(command)));
+ } finally {
+ flightContext.removePreparedQuery(id(command));
+ }
+ }
+
+ @Test
+ public void sessionCloseDoesNotWaitForExecutionMonitor() throws Exception {
+ ConnectContext connection = new ConnectContext();
+ connection.addPreparedQuery("p", "SELECT ?", null, 1);
+ long upload = connection.beginPreparedQueryBinding("p");
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ synchronized (connection) {
+ executor.submit(connection::closePreparedQueries).get(5, TimeUnit.SECONDS);
+ }
+ Assertions.assertFalse(connection.isPreparedQueryBindingCurrent("p", upload));
+ Assertions.assertThrows(IllegalStateException.class,
+ () -> connection.addPreparedQuery("new", "SELECT ?", null, 1));
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void closedHandleCannotCommitAnUploadOrLeaveUnreachableResults() throws Exception {
+ ConnectContext connection = new ConnectContext();
+ connection.addPreparedQuery("p", "SELECT ?", null, 1);
+ long version = connection.beginPreparedQueryBinding("p");
+ connection.closePreparedQueries();
+ Assertions.assertFalse(connection.setPreparedQueryParameters("p", Arrays.asList(new BigIntLiteral(1)),
+ null, version));
+ Assertions.assertEquals(-1, connection.getPreparedQueryParameterCount("p"));
+ Assertions.assertNull(connection.getPreparedQueryParameters("p"));
+ Assertions.assertEquals(-1, connection.beginPreparedQueryBinding("p"));
+ String query = "SET query_timeout = 300";
+ flightContext.getState().reset();
+ flightContext.addPreparedQuery("close-race", query, FlightSqlQuerySchema.analyze(flightContext, query));
+ CallContext call = Mockito.mock(CallContext.class);
+ Mockito.when(call.peerIdentity()).thenReturn("parameter-peer");
+ CommandPreparedStatementQuery command = CommandPreparedStatementQuery.newBuilder()
+ .setPreparedStatementHandle(ByteString.copyFromUtf8("parameter-peer:close-race")).build();
+ StmtExecutor deferred = Mockito.mock(StmtExecutor.class);
+ boolean previousLocal = flightContext.isReturnResultFromLocal();
+ try (MockedConstruction<FlightSqlConnectProcessor> ignored = Mockito.mockConstruction(
+ FlightSqlConnectProcessor.class, (processor, context) -> {
+ Mockito.doAnswer(invocation -> {
+ flightContext.clearPreparedQueries();
+ flightContext.setReturnResultFromLocal(true);
+ flightContext.addFlightSqlDeferredExecutor(deferred);
+ return null;
+ }).when(processor).handleQuery(Mockito.anyString());
+ })) {
+ FlightRuntimeException error = Assertions.assertThrows(FlightRuntimeException.class,
+ () -> producer.getFlightInfoPreparedStatement(
+ command, call, FlightDescriptor.command(Any.pack(command).toByteArray())));
+ Assertions.assertEquals(FlightStatusCode.NOT_FOUND, error.status().code());
+ Assertions.assertEquals(0, flightContext.getFlightSqlChannel().resultNum());
+ Mockito.verify(deferred).finalizeArrowFlightQuery();
+ } finally {
+ flightContext.setReturnResultFromLocal(previousLocal);
+ flightContext.closeFlightSqlDeferredExecutors();
+ flightContext.getFlightSqlChannel().reset();
+ }
+ }
+
+}
diff --git a/regression-test/suites/arrow_flight_sql_p0/test_prepared_query_parameters.groovy b/regression-test/suites/arrow_flight_sql_p0/test_prepared_query_parameters.groovy
new file mode 100644
index 00000000000..e41a3bbbbaa
--- /dev/null
+++ b/regression-test/suites/arrow_flight_sql_p0/test_prepared_query_parameters.groovy
@@ -0,0 +1,223 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CallOption
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CallOptions
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.CloseSessionRequest
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightClient
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightRuntimeException
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightStatusCode
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.Location
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.sql.FlightSqlClient
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.memory.RootAllocator
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.vector.BigIntVector
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.vector.Float8Vector
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.vector.TimeStampMicroTZVector
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.vector.VarCharVector
+import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.vector.VectorSchemaRoot
+
+import java.sql.DriverManager
+import java.sql.Types
+
+suite("test_prepared_query_parameters", "arrow_flight_sql") {
+ def config = context.config.otherConfigs
+ def location = Location.forGrpcInsecure(config.get("extArrowFlightSqlHost"),
+ Integer.parseInt(config.get("extArrowFlightSqlPort")))
+ Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver")
+ def jdbcUrl = "jdbc:arrow-flight-sql://${config.get('extArrowFlightSqlHost')}:" +
+ "${config.get('extArrowFlightSqlPort')}?useEncryption=false"
+ DriverManager.getConnection(jdbcUrl, config.get("extArrowFlightSqlUser"),
+ config.get("extArrowFlightSqlPassword")).withCloseable { connection ->
+ // Exercise JDBC's schema-driven binder and query classifier, not only direct vector uploads.
+ connection.prepareStatement("SELECT CAST(? AS BIGINT) AS value").withCloseable { statement ->
+ assertEquals(1, statement.getMetaData().getColumnCount())
+ assertEquals(Types.BIGINT, statement.getParameterMetaData().getParameterType(1))
+ [42L, -7L, Long.MAX_VALUE].each { value ->
+ statement.setLong(1, value)
+ assertTrue(statement.execute())
+ statement.getResultSet().withCloseable { result ->
+ assertTrue(result.next())
+ assertEquals(value, result.getLong(1))
+ assertTrue(!result.next())
+ }
+ }
+ statement.setNull(1, Types.BIGINT)
+ statement.executeQuery().withCloseable { result ->
+ assertTrue(result.next())
+ assertEquals(null, result.getObject(1))
+ assertTrue(!result.next())
+ }
+ }
+ connection.prepareStatement("SELECT CAST(? AS BIGINT), CAST(? AS STRING), CAST(? AS DOUBLE)")
+ .withCloseable { statement ->
+ statement.setLong(1, 17L)
+ statement.setString(2, "quoted ' ? 中文")
+ statement.setDouble(3, 2.5d)
+ statement.executeQuery().withCloseable { result ->
+ assertTrue(result.next())
+ assertEquals(17L, result.getLong(1))
+ assertEquals("quoted ' ? 中文", result.getString(2))
+ assertEquals(2.5d, result.getDouble(3))
+ assertTrue(!result.next())
+ }
+ }
+ connection.prepareStatement('SELECT number FROM numbers("number"="10") '
+ + 'WHERE number >= ? AND ? > number ORDER BY number').withCloseable { statement ->
+ [[2L, 5L], [6L, 9L]].each { bounds ->
+ statement.setLong(1, bounds[0])
+ statement.setLong(2, bounds[1])
+ statement.executeQuery().withCloseable { result ->
+ def rows = []
+ while (result.next()) {
+ rows.add(result.getLong(1))
+ }
+ assertEquals((bounds[0]..<bounds[1]).toList(), rows)
+ }
+ }
+ }
+ }
+ new RootAllocator().withCloseable { allocator ->
+ FlightClient.builder(allocator, location).build().withCloseable { flight ->
+ def token = flight.authenticateBasicToken(config.get("extArrowFlightSqlUser"),
+ config.get("extArrowFlightSqlPassword")).get()
+ CallOption[] options = [token, CallOptions.timeout(30, java.util.concurrent.TimeUnit.SECONDS)]
+ def client = new FlightSqlClient(flight)
+ def fetch = { info ->
+ def rows = []
+ info.getEndpoints().each { endpoint ->
+ def resultLocation = endpoint.getLocations().isEmpty() ? location : endpoint.getLocations()[0]
+ FlightClient.builder(allocator, resultLocation).build().withCloseable { resultClient ->
+ resultClient.getStream(endpoint.getTicket(), options).withCloseable { stream ->
+ while (stream.next()) {
+ def root = stream.getRoot()
+ for (int row = 0; row < root.getRowCount(); row++) {
+ rows.add(root.getFieldVectors().collect { vector ->
+ def value = vector.getObject(row)
+ value == null ? null : value.toString()
+ })
+ }
+ }
+ }
+ }
+ }
+ rows
+ }
+ try {
+ def prepared = client.prepare("SELECT CAST(? AS BIGINT) AS value", options)
+ try {
+ new BigIntVector("value", allocator).withCloseable { value ->
+ VectorSchemaRoot.of(value).withCloseable { root ->
+ prepared.setParameters(root)
+ [42L, -7L, Long.MAX_VALUE].each { expected ->
+ value.setSafe(0, expected)
+ root.setRowCount(1)
+ assertEquals([[expected.toString()]], fetch(prepared.execute(options)))
+ }
+ value.setNull(0)
+ assertEquals([[null]], fetch(prepared.execute(options)))
+ value.setSafe(0, 1L)
+ value.setSafe(1, 2L)
+ root.setRowCount(2)
+ try {
+ prepared.execute(options)
+ assertTrue(false, "Multiple parameter rows must not silently execute only the first")
+ } catch (FlightRuntimeException e) {
+ assertEquals(FlightStatusCode.UNIMPLEMENTED, e.status().code())
+ }
+ root.setRowCount(1)
+ value.setSafe(0, 3L)
+ assertEquals([["3"]], fetch(prepared.execute(options)))
+ }
+ }
+ } finally {
+ // Close must use the same authenticated session that owns the prepared handle.
+ prepared.close(options)
+ }
+ prepared = client.prepare("SELECT CAST(? AS BIGINT), CAST(? AS STRING), CAST(? AS DOUBLE)", options)
+ try {
+ VectorSchemaRoot.of(new BigIntVector("n", allocator), new VarCharVector("s", allocator),
+ new Float8Vector("f", allocator)).withCloseable { root ->
+ String text = "quoted ' text ? with Unicode 中文"
+ root.getVector(0).setSafe(0, 17L)
+ root.getVector(1).setSafe(0, text.getBytes("UTF-8"))
+ root.getVector(2).setSafe(0, 2.5d)
+ root.setRowCount(1)
+ prepared.setParameters(root)
+ assertEquals([["17", text, "2.5"]], fetch(prepared.execute(options)))
+ }
+ } finally {
+ prepared.close(options)
+ }
+ prepared = client.prepare('SELECT number FROM numbers("number"="10") '
+ + 'WHERE number >= ? AND number < ? ORDER BY number', options)
+ try {
+ VectorSchemaRoot.of(new BigIntVector("lower", allocator),
+ new BigIntVector("upper", allocator)).withCloseable { root ->
+ root.setRowCount(1)
+ prepared.setParameters(root)
+ [[2L, 5L], [6L, 9L]].each { bounds ->
+ root.getVector(0).setSafe(0, bounds[0])
+ root.getVector(1).setSafe(0, bounds[1])
+ root.setRowCount(1)
+ assertEquals((bounds[0]..<bounds[1]).collect { [it.toString()] },
+ fetch(prepared.execute(options)))
+ }
+ }
+ } finally {
+ prepared.close(options)
+ }
+ def originalZone = fetch(client.execute("SHOW VARIABLES LIKE 'time_zone'", options))[0][1]
+ try {
+ ["UTC", "Asia/Shanghai", "America/New_York"].each { sessionZone ->
+ fetch(client.execute("SET time_zone='${sessionZone}'".toString(), options))
+ prepared = client.prepare('SELECT number FROM numbers("number"="3") '
+ + 'WHERE CAST(? AS TIMESTAMPTZ(6)) = CAST(? AS TIMESTAMPTZ(6)) ORDER BY number', options)
+ try {
+ assertEquals(sessionZone,
+ prepared.getParameterSchema().getFields()[0].getType().getTimezone())
+ ["UTC", "America/New_York"].each { inputZone ->
+ VectorSchemaRoot.of(new TimeStampMicroTZVector("instant", allocator, inputZone),
+ new VarCharVector("expected", allocator)).withCloseable { root ->
+ prepared.setParameters(root)
+ // The timezone annotation must not shift the epoch or change predicate matches.
+ [[0L, "1970-01-01 00:00:00+00:00"],
+ [-1L, "1969-12-31 23:59:59.999999+00:00"],
+ [1234567L, "1970-01-01 00:00:01.234567+00:00"],
+ [1710054000000000L, "2024-03-10 07:00:00+00:00"]].each { value ->
+ root.getVector(0).setSafe(0, value[0])
+ root.getVector(1).setSafe(0, value[1].getBytes("UTF-8"))
+ root.setRowCount(1)
+ assertEquals([["0"], ["1"], ["2"]], fetch(prepared.execute(options)))
+ }
+ root.getVector(0).setNull(0)
+ assertEquals([], fetch(prepared.execute(options)))
+ }
+ }
+ } finally {
+ prepared.close(options)
+ }
+ }
+ } finally {
+ fetch(client.execute("SET time_zone='${originalZone}'".toString(), options))
+ }
+ assertEquals([["1"]], fetch(client.execute("SELECT 1", options)))
+ } finally {
+ flight.closeSession(new CloseSessionRequest(), options)
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 20/28: [fix](be) Fix null-safe string joins in 4.1 (backport #65975) (#68809)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f5dfeb66974bc4940d9b5ffbf3235b3a7ca9afc8
Author: HappenLee <ha...@selectdb.com>
AuthorDate: Sat Oct 10 09:26:41 2026 +0800
[fix](be) Fix null-safe string joins in 4.1 (backport #65975) (#68809)
### What problem does this PR solve?
Issue Number: None
Related PR: #65975
Problem Summary:
Backport #65975 to `branch-4.1`.
A single-column string hash join using `<=>` can fail to match two NULL
keys when expression evaluation leaves different bytes in their nested
string columns. For example, joining `CONCAT(COALESCE(p.ch, ''),
CAST(p.v % 5 AS STRING)) <=> c.s` should match a NULL expression result
with a NULL `c.s`, but previously could miss that row depending on block
composition.
Normalize NULL keys to an empty `StringRef` in the serialization shared
by build and probe. Real empty strings remain in ordinary hash buckets,
separate from the dedicated NULL bucket. Also avoid passing default
empty refs to `memcmp`, and reject unsupported `<=>` ON predicates in
ASOF JOIN.
Resolve the parser conflict by retaining the 4.1 UNNEST handling and
changing only the ASOF equality check. Resolve the BE test conflict by
adding the original NULL normalization test without importing unrelated
master-only iterator tests. Adapt the existing ASOF regression to its
`nereids_p0` location on 4.1.
### Release note
Fix incorrect results of single-column null-safe string joins when NULL
keys come from expressions. ASOF JOIN rejects unsupported null-safe
equality ON predicates.
### Check List (For Author)
- Test:
- BE formatting: repository clang-format/check-format scripts passed
with clang-format 16.0.6.
- Regression: preserve the upstream expression-NULL, mixed-block,
NULL-versus-empty-string, and ASOF rejection suites. The generated
expected output is copied unchanged from the original commit; local
cluster execution has not been run.
- FE style: `mvn checkstyle:check -pl fe-core` passed (0 violations).
- FE Unit Test:
`NereidsParserTest#testParseAsofJoinRejectNullSafeEquality+testParseJoin`
passed locally (2 tests, 0 failures/errors).
- BE Unit Test: the original normalization test is preserved. Local ASAN
UT execution is blocked before compilation because the prebuilt
dependency bundle lacks `libpaimon_c.a`; no BE UT execution success is
claimed. The missing simdutf library was built with the repository
script before retrying.
- CI: `run buildall` requested for this backport.
- Behavior changed: Yes, correct NULL string key matching and reject
unsupported ASOF ON predicates.
- Does this need documentation: No.
---
be/src/core/string_ref.h | 4 +-
be/src/exec/common/hash_table/hash_map_context.h | 19 ++-
be/test/exec/hash_map/hash_table_method_test.cpp | 27 ++++
.../doris/nereids/parser/LogicalPlanBuilder.java | 3 +-
.../doris/nereids/parser/NereidsParserTest.java | 13 ++
.../join/test_null_safe_eq_join_string_key.out | 66 +++++++++
.../nereids_p0/join/asof/test_asof_join.groovy | 24 ++++
.../join/test_null_safe_eq_join_string_key.groovy | 154 +++++++++++++++++++++
8 files changed, 302 insertions(+), 8 deletions(-)
diff --git a/be/src/core/string_ref.h b/be/src/core/string_ref.h
index ca67ba91c06..c2424f98fe5 100644
--- a/be/src/core/string_ref.h
+++ b/be/src/core/string_ref.h
@@ -268,7 +268,9 @@ struct StringRef {
// ==
bool eq(const StringRef& other) const {
- return (size == other.size) && (memcmp(data, other.data, size) == 0);
+ // memcmp requires valid pointers even when size is 0, so short-circuit empty strings
+ // to avoid passing nullptr data of default-constructed StringRef to memcmp.
+ return (size == other.size) && (size == 0 || memcmp(data, other.data, size) == 0);
}
bool operator==(const StringRef& other) const { return eq(other); }
diff --git a/be/src/exec/common/hash_table/hash_map_context.h b/be/src/exec/common/hash_table/hash_map_context.h
index 19aff643c40..afb7ae3fc9d 100644
--- a/be/src/exec/common/hash_table/hash_map_context.h
+++ b/be/src/exec/common/hash_table/hash_map_context.h
@@ -371,19 +371,28 @@ struct MethodStringNoCache : public MethodBase<TData> {
}
void init_serialized_keys_impl(const ColumnRawPtrs& key_columns, uint32_t num_rows,
- DorisVector<StringRef>& stored_keys) {
+ DorisVector<StringRef>& stored_keys, const uint8_t* null_map) {
const IColumn& column = *key_columns[0];
const auto& nested_column =
column.is_nullable()
? assert_cast<const ColumnNullable&>(column).get_nested_column()
: column;
- auto serialized_str = [](const auto& column_string, DorisVector<StringRef>& stored_keys) {
+ // For join, rows with null keys are routed to a dedicated null bucket and matched by
+ // raw key comparison, so their keys must be normalized to a canonical empty StringRef
+ // to make null keys equal (e.g. single-column null-safe equal join). The nested
+ // column of a null row may hold residual bytes left by expression evaluation.
+ // Real empty strings are not affected: they are not null, so they are still hashed
+ // into normal buckets and never meet the null keys in the null bucket.
+ auto serialized_str = [null_map](const auto& column_string,
+ DorisVector<StringRef>& stored_keys) {
const auto& offsets = column_string.get_offsets();
const auto* chars = column_string.get_chars().data();
stored_keys.resize(column_string.size());
for (size_t row = 0; row < column_string.size(); row++) {
- stored_keys[row] =
- StringRef(chars + offsets[row - 1], offsets[row] - offsets[row - 1]);
+ stored_keys[row] = (null_map != nullptr && null_map[row])
+ ? StringRef()
+ : StringRef(chars + offsets[row - 1],
+ offsets[row] - offsets[row - 1]);
}
};
if (nested_column.is_column_string64()) {
@@ -400,7 +409,7 @@ struct MethodStringNoCache : public MethodBase<TData> {
const uint8_t* null_map = nullptr, bool is_join = false,
bool is_build = false, uint32_t bucket_size = 0) override {
init_serialized_keys_impl(key_columns, num_rows,
- is_build ? _build_stored_keys : _stored_keys);
+ is_build ? _build_stored_keys : _stored_keys, null_map);
if (is_join) {
Base::init_join_bucket_num(num_rows, bucket_size, null_map);
} else {
diff --git a/be/test/exec/hash_map/hash_table_method_test.cpp b/be/test/exec/hash_map/hash_table_method_test.cpp
index 75543290827..e556e2669a5 100644
--- a/be/test/exec/hash_map/hash_table_method_test.cpp
+++ b/be/test/exec/hash_map/hash_table_method_test.cpp
@@ -131,6 +131,33 @@ TEST(HashTableMethodTest, testMethodStringNoCache) {
{0, 1, -1, 3, -1, 4});
}
+// For join, null keys are routed to a dedicated null bucket and matched by raw key
+// comparison. The nested column of a null row may hold residual bytes left by expression
+// evaluation, so init_serialized_keys must normalize null keys to a canonical empty
+// StringRef to make null keys equal (e.g. single-column null-safe equal join).
+TEST(HashTableMethodTest, testMethodStringNoCacheNullKeyNormalized) {
+ MethodStringNoCache<StringHashMap<IColumn::ColumnIndex>> method;
+
+ // Row 1 is null but its nested data holds residual bytes, row 2 is a real empty string.
+ auto column =
+ ColumnHelper::create_nullable_column<DataTypeString>({"a", "residual", ""}, {0, 1, 0});
+ ColumnRawPtrs key_columns {column.get()};
+ const auto& null_map = assert_cast<const ColumnNullable&>(*column).get_null_map_data();
+
+ const uint32_t bucket_size = 8;
+ method.init_serialized_keys(key_columns, 3, null_map.data(), true, false, bucket_size);
+
+ // Null key is normalized to a canonical empty StringRef instead of the residual bytes.
+ EXPECT_TRUE(method._stored_keys[1] == StringRef());
+ // Non-null rows keep their real bytes, including the real empty string.
+ EXPECT_TRUE(method._stored_keys[0] == StringRef("a", 1));
+ EXPECT_TRUE(method._stored_keys[2] == StringRef("", 0));
+ // Null row is routed to the dedicated null bucket, real rows to normal hash buckets.
+ EXPECT_EQ(method.bucket_nums[1], bucket_size);
+ EXPECT_LT(method.bucket_nums[0], bucket_size);
+ EXPECT_LT(method.bucket_nums[2], bucket_size);
+}
+
static AggregateDataPtr make_mapped(size_t val) {
return reinterpret_cast<AggregateDataPtr>(val);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 7a83e22961d..a027bc4d307 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -546,7 +546,6 @@ import org.apache.doris.nereids.trees.expressions.Default;
import org.apache.doris.nereids.trees.expressions.DefaultValueSlot;
import org.apache.doris.nereids.trees.expressions.DereferenceExpression;
import org.apache.doris.nereids.trees.expressions.Divide;
-import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Exists;
import org.apache.doris.nereids.trees.expressions.Expression;
@@ -4736,7 +4735,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
}
List<Expression> conjuncts = ExpressionUtils.extractConjunction(condition.get());
for (Expression expression : conjuncts) {
- if (!(expression instanceof EqualPredicate)) {
+ if (!(expression instanceof EqualTo)) {
throw new ParseException("ASOF JOIN's ON clause must be one or more EQUAL(=) conjuncts",
join);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
index 345d6715b5e..6e16892e4bc 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
@@ -423,6 +423,19 @@ public class NereidsParserTest extends ParserTestBase {
Assertions.assertEquals(JoinType.CROSS_JOIN, logicalJoin.getJoinType());
}
+ @Test
+ public void testParseAsofJoinRejectNullSafeEquality() {
+ parsePlan("SELECT t1.a FROM t1 ASOF INNER JOIN t2 "
+ + "MATCH_CONDITION(t1.dt < t2.dt) ON t1.id <=> t2.id")
+ .assertThrowsExactly(ParseException.class)
+ .assertMessageContains("ASOF JOIN's ON clause must be one or more EQUAL(=) conjuncts");
+
+ parsePlan("SELECT t1.a FROM t1 ASOF LEFT JOIN t2 "
+ + "MATCH_CONDITION(t1.dt < t2.dt) ON t1.id <=> t2.id")
+ .assertThrowsExactly(ParseException.class)
+ .assertMessageContains("ASOF JOIN's ON clause must be one or more EQUAL(=) conjuncts");
+ }
+
@Test
void parseJoinEmptyConditionError() {
parsePlan("select * from t1 LEFT JOIN t2")
diff --git a/regression-test/data/query_p0/join/test_null_safe_eq_join_string_key.out b/regression-test/data/query_p0/join/test_null_safe_eq_join_string_key.out
new file mode 100644
index 00000000000..43eed813d86
--- /dev/null
+++ b/regression-test/data/query_p0/join/test_null_safe_eq_join_string_key.out
@@ -0,0 +1,66 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !minimal_left_join --
+1 2
+2 1
+
+-- !minimal_inner_join --
+1 2
+2 1
+
+-- !minimal_oracle --
+1 2
+2 1
+
+-- !mixed_inner_join --
+0 0
+1 3
+1 4
+2 1
+3 3
+3 4
+4 2
+5 3
+5 4
+7 3
+7 4
+8 5
+9 3
+9 4
+
+-- !mixed_left_join --
+0 0
+1 3
+1 4
+2 1
+3 3
+3 4
+4 2
+5 3
+5 4
+6 \N
+7 3
+7 4
+8 5
+9 3
+9 4
+
+-- !mixed_oracle --
+0 0
+1 3
+1 4
+2 1
+3 3
+3 4
+4 2
+5 3
+5 4
+7 3
+7 4
+8 5
+9 3
+9 4
+
+-- !empty_string_inner_join --
+1 2
+2 1
+
diff --git a/regression-test/suites/nereids_p0/join/asof/test_asof_join.groovy b/regression-test/suites/nereids_p0/join/asof/test_asof_join.groovy
index 6efc63dbbb6..611ca15a779 100644
--- a/regression-test/suites/nereids_p0/join/asof/test_asof_join.groovy
+++ b/regression-test/suites/nereids_p0/join/asof/test_asof_join.groovy
@@ -1247,6 +1247,30 @@ suite("test_asof_join", "nereids_p0") {
exception "ASOF JOIN's ON clause must be one or more EQUAL(=) conjuncts"
}
+ test {
+ sql """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_precision_left l
+ ASOF INNER JOIN asof_precision_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp <=> r.grp
+ ORDER BY l.id
+ """
+ exception "ASOF JOIN's ON clause must be one or more EQUAL(=) conjuncts"
+ }
+
+ test {
+ sql """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_precision_left l
+ ASOF LEFT JOIN asof_precision_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp <=> r.grp
+ ORDER BY l.id
+ """
+ exception "ASOF JOIN's ON clause must be one or more EQUAL(=) conjuncts"
+ }
+
test {
sql """
SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.value
diff --git a/regression-test/suites/query_p0/join/test_null_safe_eq_join_string_key.groovy b/regression-test/suites/query_p0/join/test_null_safe_eq_join_string_key.groovy
new file mode 100644
index 00000000000..7a87b2b0de7
--- /dev/null
+++ b/regression-test/suites/query_p0/join/test_null_safe_eq_join_string_key.groovy
@@ -0,0 +1,154 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test single-column null-safe equal ( <=> ) hash join on string keys.
+// Null keys produced by expressions must match each other (NULL <=> NULL is true),
+// and must never match a real empty string key.
+suite("test_null_safe_eq_join_string_key") {
+ sql """ DROP TABLE IF EXISTS test_nsej_string_p """
+ sql """ DROP TABLE IF EXISTS test_nsej_string_c """
+
+ sql """
+ CREATE TABLE test_nsej_string_p (
+ pk int,
+ ch char(10) not null,
+ v tinyint null
+ ) duplicate key(pk)
+ distributed by hash(pk) buckets 1
+ properties("replication_num" = "1");
+ """
+ sql """ insert into test_nsej_string_p values (1,'x',3),(2,'x',NULL) """
+
+ sql """
+ CREATE TABLE test_nsej_string_c (
+ pk int,
+ s varchar(100) null
+ ) duplicate key(pk)
+ distributed by hash(pk) buckets 1
+ properties("replication_num" = "1");
+ """
+ sql """ insert into test_nsej_string_c values (1, NULL), (2, 'x3') """
+
+ // Minimal case: NULL key comes from an expression, NULL <=> NULL must match, so (2,1) is expected.
+ order_qt_minimal_left_join """
+ SELECT p.pk, c.pk AS cpk
+ FROM test_nsej_string_p p LEFT JOIN test_nsej_string_c c
+ ON CONCAT(COALESCE(p.ch,''), CAST((p.v % 5) AS STRING)) <=> c.s
+ ORDER BY 1, 2;
+ """
+
+ order_qt_minimal_inner_join """
+ SELECT p.pk, c.pk AS cpk
+ FROM test_nsej_string_p p INNER JOIN test_nsej_string_c c
+ ON CONCAT(COALESCE(p.ch,''), CAST((p.v % 5) AS STRING)) <=> c.s
+ ORDER BY 1, 2;
+ """
+
+ // The null-safe equal join must return the same result as its semantic oracle.
+ order_qt_minimal_oracle """
+ SELECT p.pk, c.pk AS cpk
+ FROM test_nsej_string_p p LEFT JOIN test_nsej_string_c c
+ ON CONCAT(COALESCE(p.ch,''), CAST((p.v % 5) AS STRING)) = c.s
+ OR (CONCAT(COALESCE(p.ch,''), CAST((p.v % 5) AS STRING)) IS NULL AND c.s IS NULL)
+ ORDER BY 1, 2;
+ """
+
+ sql """ DROP TABLE IF EXISTS test_nsej_string_a """
+ sql """ DROP TABLE IF EXISTS test_nsej_string_b """
+
+ sql """
+ CREATE TABLE test_nsej_string_a (
+ pk int,
+ v int null
+ ) duplicate key(pk)
+ distributed by hash(pk) buckets 3
+ properties("replication_num" = "1");
+ """
+ sql """
+ insert into test_nsej_string_a values
+ (0,0),(1,NULL),(2,2),(3,NULL),(4,4),(5,NULL),(6,6),(7,NULL),(8,8),(9,NULL)
+ """
+
+ sql """
+ CREATE TABLE test_nsej_string_b (
+ pk int,
+ s string null
+ ) duplicate key(pk)
+ distributed by hash(pk) buckets 3
+ properties("replication_num" = "1");
+ """
+ sql """
+ insert into test_nsej_string_b values
+ (0,'0'),(1,'2'),(2,'4'),(3,NULL),(4,NULL),(5,'8'),(6,'10')
+ """
+
+ // Mixed blocks: NULL keys from expression may hold residual bytes of evaluated nested values,
+ // they must still match NULL keys on the other side.
+ order_qt_mixed_inner_join """
+ SELECT a.pk, b.pk AS bpk
+ FROM test_nsej_string_a a INNER JOIN test_nsej_string_b b
+ ON CAST(a.v AS STRING) <=> b.s
+ ORDER BY 1, 2;
+ """
+
+ order_qt_mixed_left_join """
+ SELECT a.pk, b.pk AS bpk
+ FROM test_nsej_string_a a LEFT JOIN test_nsej_string_b b
+ ON CAST(a.v AS STRING) <=> b.s
+ ORDER BY 1, 2;
+ """
+
+ order_qt_mixed_oracle """
+ SELECT a.pk, b.pk AS bpk
+ FROM test_nsej_string_a a INNER JOIN test_nsej_string_b b
+ ON CAST(a.v AS STRING) = b.s OR (CAST(a.v AS STRING) IS NULL AND b.s IS NULL)
+ ORDER BY 1, 2;
+ """
+
+ // A normalized null key must not collide with a real empty string key.
+ sql """ DROP TABLE IF EXISTS test_nsej_string_e """
+ sql """ DROP TABLE IF EXISTS test_nsej_string_f """
+
+ sql """
+ CREATE TABLE test_nsej_string_e (
+ pk int,
+ v int null
+ ) duplicate key(pk)
+ distributed by hash(pk) buckets 1
+ properties("replication_num" = "1");
+ """
+ sql """ insert into test_nsej_string_e values (1,10),(2,NULL) """
+
+ sql """
+ CREATE TABLE test_nsej_string_f (
+ pk int,
+ s string null
+ ) duplicate key(pk)
+ distributed by hash(pk) buckets 1
+ properties("replication_num" = "1");
+ """
+ sql """ insert into test_nsej_string_f values (1,NULL),(2,'') """
+
+ // Row (1,10) yields a real empty string key, row (2,NULL) yields a null key.
+ // Expected matches: (1,2) by empty string, (2,1) by NULL <=> NULL.
+ order_qt_empty_string_inner_join """
+ SELECT e.pk, f.pk AS fpk
+ FROM test_nsej_string_e e INNER JOIN test_nsej_string_f f
+ ON IF(e.v IS NOT NULL, '', NULL) <=> f.s
+ ORDER BY 1, 2;
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 19/28: branch-4.1: [fix](expr) Canonicalize logical OR results (#68798)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 66f60c1da509806ca2fa989259df65f4a38141b0
Author: Dongyang Li <li...@selectdb.com>
AuthorDate: Sat Oct 10 09:25:51 2026 +0800
branch-4.1: [fix](expr) Canonicalize logical OR results (#68798)
### What problem does this PR solve?
Backport #68308 / #68401 to branch-4.1. The repair is present on
branch-4.1.4, but branch-4.1 at c8c0ddca4d4d452bcb236f2499c5fed9a3c68780
still has the same three unnormalized OR paths as 4.1.5-rc01
(7fda449b7afb09c266ce2c4fc738a4d5e46fc458).
A nullable Boolean can contain an arbitrary hidden byte. When the other
operand makes OR non-null, preserving the bitwise result can expose a
value other than 0 or 1. Release regression reproduced incorrect Boolean
grouping on the original candidate in both LOCAL and S3 modes.
This is an exact backport of fc344110d103312a6bfafb55e936b76f26177b8e,
preserving its author and all five changed files: normalize FunctionOr
and both compound OR paths, add three BE unit tests, and include the
original regression suite and unchanged golden output. No unrelated
NULL/CASE implementation changes are included.
### Release note
Fix noncanonical logical OR results involving hidden NULL payloads.
### Check List (For Author)
- Validation: patch applies cleanly to the stated branch-4.1 base; all
five resulting files are byte-identical to the original fixed commit;
git diff --check passes.
- Runtime validation: this new backport has not run BE unit tests or
live regression yet. Required validation is FunctionsLogicalTest.* and
test_json_extract_bool_null_payload in LOCAL/S3, with both short-circuit
settings, against a binary built from this PR's exact SHA.
- The existing candidate failure remains a release blocker. This PR does
not replace the candidate artifact or establish a passing release
result.
- Duplicate check: currently open branch-4.1 PR file lists and matching
repair references were checked; no equivalent open backport was found.
Co-authored-by: Mingyu Chen (Rayner) <mo...@gmail.com>
---
be/src/exprs/function/functions_logical.h | 4 +-
be/src/exprs/vcompound_pred.h | 7 +-
be/test/exprs/function/functions_logical_test.cpp | 143 +++++++++++++++++++
.../test_json_extract_bool_null_payload.out | 41 ++++++
.../test_json_extract_bool_null_payload.groovy | 153 +++++++++++++++++++++
5 files changed, 345 insertions(+), 3 deletions(-)
diff --git a/be/src/exprs/function/functions_logical.h b/be/src/exprs/function/functions_logical.h
index 9b684376ac4..783e661422e 100644
--- a/be/src/exprs/function/functions_logical.h
+++ b/be/src/exprs/function/functions_logical.h
@@ -84,7 +84,9 @@ struct AndImpl {
struct OrImpl {
using ResultType = UInt8;
- static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return a | b; }
+ // A NULL row may carry an arbitrary nested byte. Canonicalize logical OR to 0 or 1 so that
+ // such a byte cannot become visible when the other operand makes the result non-NULL.
+ static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return (a | b) != 0; }
static inline constexpr ResultType apply_null(UInt8 a, UInt8 l_null, UInt8 b, UInt8 r_null) {
// (<> || true) is true, (false || NULL) is NULL
return (l_null & r_null) | (r_null & (r_null ^ a)) | (l_null & (l_null ^ b));
diff --git a/be/src/exprs/vcompound_pred.h b/be/src/exprs/vcompound_pred.h
index 77b84a5733e..9fe4386fd30 100644
--- a/be/src/exprs/vcompound_pred.h
+++ b/be/src/exprs/vcompound_pred.h
@@ -625,7 +625,8 @@ private:
if constexpr (is_and) {
lhs[i] &= rhs[i];
} else {
- lhs[i] |= rhs[i];
+ // Logical OR must produce a canonical Boolean instead of preserving input bits.
+ lhs[i] = (lhs[i] | rhs[i]) != 0;
}
}
}
@@ -648,7 +649,9 @@ private:
res_data[i] = lhs_data[i] & rhs_data[i];
} else {
res_null[i] = apply_or_null(lhs_data[i], lhs_null[i], rhs_data[i], rhs_null[i]);
- res_data[i] = lhs_data[i] | rhs_data[i];
+ // A NULL row may carry an arbitrary nested byte. If the result remains NULL the
+ // byte is ignored; otherwise normalization prevents it from becoming visible.
+ res_data[i] = (lhs_data[i] | rhs_data[i]) != 0;
}
}
}
diff --git a/be/test/exprs/function/functions_logical_test.cpp b/be/test/exprs/function/functions_logical_test.cpp
new file mode 100644
index 00000000000..90d7cea69c4
--- /dev/null
+++ b/be/test/exprs/function/functions_logical_test.cpp
@@ -0,0 +1,143 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exprs/function/functions_logical.h"
+
+#include <gen_cpp/Exprs_types.h>
+#include <gtest/gtest.h>
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "core/block/block.h"
+#include "core/column/column_const.h"
+#include "core/column/column_nullable.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/primitive_type.h"
+#include "exprs/vcompound_pred.h"
+#include "exprs/vexpr_context.h"
+#include "testutil/column_helper.h"
+
+namespace doris {
+
+namespace {
+
+ColumnWithTypeAndName nullable_boolean_column(std::vector<UInt8> data, std::vector<UInt8> null_map,
+ std::string name) {
+ return {ColumnHelper::create_nullable_column<DataTypeUInt8>(data, null_map),
+ make_nullable(std::make_shared<DataTypeUInt8>()), std::move(name)};
+}
+
+ColumnPtr execute_or(ColumnWithTypeAndName left, ColumnWithTypeAndName right, size_t rows) {
+ auto result_type = make_nullable(std::make_shared<DataTypeUInt8>());
+ Block block({std::move(left), std::move(right), {nullptr, result_type, "result"}});
+ auto status = FunctionOr::create()->execute_impl(nullptr, block, {0, 1}, 2, rows);
+ EXPECT_TRUE(status.ok()) << status.to_string();
+ return block.get_by_position(2).column;
+}
+
+class ColumnExpr final : public VExpr {
+public:
+ ColumnExpr(ColumnPtr column, DataTypePtr type)
+ : VExpr(std::move(type), false), _column(std::move(column)) {}
+
+ bool is_constant() const override { return false; }
+
+ const std::string& expr_name() const override {
+ static const std::string name = "ColumnExpr";
+ return name;
+ }
+
+ Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t,
+ ColumnPtr& result_column) const override {
+ result_column = _column;
+ return Status::OK();
+ }
+
+private:
+ ColumnPtr _column;
+};
+
+ColumnPtr execute_compound_or(ColumnWithTypeAndName left, ColumnWithTypeAndName right,
+ size_t rows) {
+ TExprNode node;
+ node.__set_type(create_type_desc(TYPE_BOOLEAN));
+ node.__set_node_type(TExprNodeType::COMPOUND_PRED);
+ node.__set_opcode(TExprOpcode::COMPOUND_OR);
+ node.__set_num_children(2);
+ node.__set_is_nullable(true);
+
+ auto compound = VCompoundPred::create_shared(node);
+ compound->add_child(std::make_shared<ColumnExpr>(std::move(left.column), left.type));
+ compound->add_child(std::make_shared<ColumnExpr>(std::move(right.column), right.type));
+ VExprContext context(compound);
+ ColumnPtr result;
+ auto status = compound->execute_column(&context, nullptr, nullptr, rows, result);
+ EXPECT_TRUE(status.ok()) << status.to_string();
+ return result;
+}
+
+void expect_boolean(const ColumnNullable& result, size_t row, UInt8 value) {
+ EXPECT_FALSE(result.is_null_at(row));
+ EXPECT_EQ(assert_cast<const ColumnUInt8&>(result.get_nested_column()).get_data()[row], value);
+}
+
+} // namespace
+
+TEST(FunctionsLogicalTest, NullableOrIgnoresNullPayload) {
+ auto result =
+ execute_or(nullable_boolean_column({65, 0, 65, 1, 65}, {1, 0, 1, 0, 1}, "left"),
+ nullable_boolean_column({1, 0, 0, 65, 127}, {0, 0, 0, 1, 1}, "right"), 5);
+
+ const auto& nullable_result = assert_cast<const ColumnNullable&>(*result);
+ expect_boolean(nullable_result, 0, 1);
+ expect_boolean(nullable_result, 1, 0);
+ EXPECT_TRUE(nullable_result.is_null_at(2));
+ expect_boolean(nullable_result, 3, 1);
+ EXPECT_TRUE(nullable_result.is_null_at(4));
+}
+
+TEST(FunctionsLogicalTest, NullableOrWithTrueConstantProducesCanonicalTrue) {
+ constexpr size_t rows = 4;
+ auto left = nullable_boolean_column({3, 65, 0, 255}, {1, 1, 1, 1}, "left");
+ ColumnWithTypeAndName right {
+ ColumnConst::create(ColumnHelper::create_column<DataTypeUInt8>({1}), rows),
+ std::make_shared<DataTypeUInt8>(), "right"};
+
+ auto result = execute_or(std::move(left), std::move(right), rows);
+ const auto& nullable_result = assert_cast<const ColumnNullable&>(*result);
+ for (size_t row = 0; row < rows; ++row) {
+ expect_boolean(nullable_result, row, 1);
+ }
+}
+
+TEST(FunctionsLogicalTest, CompoundNullableOrIgnoresNullPayload) {
+ auto result = execute_compound_or(
+ nullable_boolean_column({65, 0, 65, 1, 65}, {1, 0, 1, 0, 1}, "left"),
+ nullable_boolean_column({1, 0, 0, 65, 127}, {0, 0, 0, 1, 1}, "right"), 5);
+
+ const auto& nullable_result = assert_cast<const ColumnNullable&>(*result);
+ expect_boolean(nullable_result, 0, 1);
+ expect_boolean(nullable_result, 1, 0);
+ EXPECT_TRUE(nullable_result.is_null_at(2));
+ expect_boolean(nullable_result, 3, 1);
+ EXPECT_TRUE(nullable_result.is_null_at(4));
+}
+
+} // namespace doris
diff --git a/regression-test/data/query_p0/sql_functions/json_functions/test_json_extract_bool_null_payload.out b/regression-test/data/query_p0/sql_functions/json_functions/test_json_extract_bool_null_payload.out
new file mode 100644
index 00000000000..5c37ac4894d
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/json_functions/test_json_extract_bool_null_payload.out
@@ -0,0 +1,41 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !or_payload_no_short_circuit --
+\N 66000
+true 34000
+
+-- !and_or_payload_no_short_circuit --
+\N 66000
+true 34000
+
+-- !case_count_no_short_circuit --
+100000
+
+-- !case_branches_no_short_circuit --
+then_a 66000
+then_b 34000
+
+-- !or_payload_short_circuit --
+\N 66000
+true 34000
+
+-- !and_or_payload_short_circuit --
+\N 66000
+true 34000
+
+-- !case_count_short_circuit --
+100000
+
+-- !case_branches_short_circuit --
+then_a 66000
+then_b 34000
+
+-- !nullable_or_truth_table --
+false false false
+false null \N
+false true true
+null false \N
+null null \N
+null true true
+true false true
+true null true
+true true true
diff --git a/regression-test/suites/query_p0/sql_functions/json_functions/test_json_extract_bool_null_payload.groovy b/regression-test/suites/query_p0/sql_functions/json_functions/test_json_extract_bool_null_payload.groovy
new file mode 100644
index 00000000000..7550fe0d6b6
--- /dev/null
+++ b/regression-test/suites/query_p0/sql_functions/json_functions/test_json_extract_bool_null_payload.groovy
@@ -0,0 +1,153 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_json_extract_bool_null_payload") {
+ sql "DROP TABLE IF EXISTS test_json_extract_bool_null_payload"
+ sql """
+ CREATE TABLE test_json_extract_bool_null_payload (
+ id BIGINT,
+ j STRING,
+ a BIGINT,
+ b BIGINT
+ ) DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO test_json_extract_bool_null_payload
+ SELECT number,
+ CASE WHEN number % 100 = 0
+ THEN '{"flag": true}'
+ ELSE CONCAT('{"n": ', number, '}')
+ END,
+ number + CASE number % 3
+ WHEN 0 THEN 10
+ WHEN 1 THEN -100
+ ELSE -200000
+ END,
+ number
+ FROM numbers("number" = "100000")
+ """
+
+ sql "SET enable_sql_cache = false"
+ sql "SET short_circuit_evaluation = false"
+
+ order_qt_or_payload_no_short_circuit """
+ SELECT v, count(*) FROM (
+ SELECT JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag') OR (a > b) AS v
+ FROM test_json_extract_bool_null_payload
+ ) t
+ GROUP BY v
+ ORDER BY v
+ """
+
+ order_qt_and_or_payload_no_short_circuit """
+ SELECT v, count(*) FROM (
+ SELECT (JSON_PARSE_ERROR_TO_NULL(j) IS NOT NULL
+ AND JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag'))
+ OR (a > b) AS v
+ FROM test_json_extract_bool_null_payload
+ ) t
+ GROUP BY v
+ ORDER BY v
+ """
+
+ qt_case_count_no_short_circuit """
+ SELECT count(*)
+ FROM test_json_extract_bool_null_payload
+ WHERE (CASE
+ WHEN JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag') OR a > b THEN b
+ WHEN a < b THEN a
+ END) IS NOT NULL
+ """
+
+ order_qt_case_branches_no_short_circuit """
+ SELECT CASE
+ WHEN JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag') OR a > b
+ THEN 'then_b'
+ WHEN a < b THEN 'then_a'
+ ELSE 'else'
+ END AS branch,
+ count(*)
+ FROM test_json_extract_bool_null_payload
+ GROUP BY branch
+ ORDER BY branch
+ """
+
+ sql "SET short_circuit_evaluation = true"
+
+ order_qt_or_payload_short_circuit """
+ SELECT v, count(*) FROM (
+ SELECT JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag') OR (a > b) AS v
+ FROM test_json_extract_bool_null_payload
+ ) t
+ GROUP BY v
+ ORDER BY v
+ """
+
+ order_qt_and_or_payload_short_circuit """
+ SELECT v, count(*) FROM (
+ SELECT (JSON_PARSE_ERROR_TO_NULL(j) IS NOT NULL
+ AND JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag'))
+ OR (a > b) AS v
+ FROM test_json_extract_bool_null_payload
+ ) t
+ GROUP BY v
+ ORDER BY v
+ """
+
+ qt_case_count_short_circuit """
+ SELECT count(*)
+ FROM test_json_extract_bool_null_payload
+ WHERE (CASE
+ WHEN JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag') OR a > b THEN b
+ WHEN a < b THEN a
+ END) IS NOT NULL
+ """
+
+ order_qt_case_branches_short_circuit """
+ SELECT CASE
+ WHEN JSON_EXTRACT_BOOL(JSON_PARSE_ERROR_TO_NULL(j), '\$.flag') OR a > b
+ THEN 'then_b'
+ WHEN a < b THEN 'then_a'
+ ELSE 'else'
+ END AS branch,
+ count(*)
+ FROM test_json_extract_bool_null_payload
+ GROUP BY branch
+ ORDER BY branch
+ """
+
+ order_qt_nullable_or_truth_table """
+ SELECT lhs.label, rhs.label, lhs.v OR rhs.v AS result
+ FROM (
+ SELECT 'false' AS label, CAST(false AS BOOLEAN) AS v
+ UNION ALL
+ SELECT 'null', CAST(NULL AS BOOLEAN)
+ UNION ALL
+ SELECT 'true', CAST(true AS BOOLEAN)
+ ) lhs
+ CROSS JOIN (
+ SELECT 'false' AS label, CAST(false AS BOOLEAN) AS v
+ UNION ALL
+ SELECT 'null', CAST(NULL AS BOOLEAN)
+ UNION ALL
+ SELECT 'true', CAST(true AS BOOLEAN)
+ ) rhs
+ ORDER BY lhs.label, rhs.label
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 18/28: branch-4.1: [fix](test)(dynamic-partition) fix some unstable test cases and dynamic-partition logic (#63551) (#68820)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 89fa7feecc2434f3e64e9cf493eaf0e311090fee
Author: daidai <ch...@selectdb.com>
AuthorDate: Sat Oct 10 09:20:27 2026 +0800
branch-4.1: [fix](test)(dynamic-partition) fix some unstable test cases and dynamic-partition logic (#63551) (#68820)
pick #63551
---
.../apache/doris/alter/SchemaChangeHandler.java | 33 +++++++++++++++++++---
.../org/apache/doris/load/loadv2/LoadManager.java | 20 +++++++++++++
.../org/apache/doris/qe/runtime/LoadProcessor.java | 6 ++++
regression-test/pipeline/p0/conf/fe.conf | 1 +
.../jdbc/test_doris_jdbc_catalog.groovy | 4 +++
.../suites/manager/test_manager_interface_1.groovy | 4 +--
6 files changed, 62 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 1431aeae35b..c5e8f3ea976 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -2508,10 +2508,35 @@ public class SchemaChangeHandler extends AlterHandler {
skip = Boolean.parseBoolean(skipWriteIndexOnLoad) ? 1 : 0;
}
- for (Partition partition : partitions) {
- updatePartitionProperties(db, olapTable.getName(), partition.getName(), storagePolicyId, isInMemory,
- null, compactionPolicy, timeSeriesCompactionConfig, skip,
- disableAutoCompaction, verticalCompactionNumColumnsPerGroup);
+ // Only iterate partitions when there are properties that actually need to be
+ // dispatched to each partition's tablets. Pure catalog-level metadata properties
+ // such as partition.retention_count do not require per-partition updates, and
+ // iterating over a stale partition snapshot can race with concurrent partition
+ // drops (e.g., by DynamicPartitionScheduler when retention_count or dynamic_partition
+ // is enabled) and fail with "Partition does not exist".
+ boolean needPerPartitionUpdate = isInMemory >= 0 || storagePolicyId >= 0
+ || compactionPolicy != null || !timeSeriesCompactionConfig.isEmpty()
+ || skip >= 0 || disableAutoCompaction >= 0
+ || verticalCompactionNumColumnsPerGroup >= 0;
+ if (needPerPartitionUpdate) {
+ for (Partition partition : partitions) {
+ try {
+ updatePartitionProperties(db, olapTable.getName(), partition.getName(),
+ storagePolicyId, isInMemory, null, compactionPolicy, timeSeriesCompactionConfig,
+ skip, disableAutoCompaction,
+ verticalCompactionNumColumnsPerGroup);
+ } catch (DdlException e) {
+ // The partition may have been dropped concurrently (e.g., by
+ // DynamicPartitionScheduler). It is safe to skip the meta dispatch
+ // for a partition that no longer exists.
+ if (olapTable.getPartition(partition.getName()) == null) {
+ LOG.info("partition {} of table {} was dropped concurrently, "
+ + "skip updating its properties", partition.getName(), olapTable.getName());
+ continue;
+ }
+ throw e;
+ }
+ }
}
olapTable.writeLockOrDdlException();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
index 019e06893a0..9fc0ca13fdc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
@@ -216,11 +216,25 @@ public class LoadManager implements Writable {
LoadJob loadJob;
if (idToLoadJob.containsKey(jobId)) {
loadJob = idToLoadJob.get(jobId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("recordFinishedLoadJob: reuse existing load job, jobId={}, label={}, dbId={}, jobType={}",
+ jobId, label, db.getId(), jobType);
+ }
if (loadJob instanceof InsertLoadJob) {
((InsertLoadJob) loadJob).setJobProperties(transactionId, tableId, createTimestamp,
failMsg, trackingUrl, firstErrorMsg, userInfo);
}
} else {
+ // The jobId received here does not exist in idToLoadJob. This means the InsertLoadJob
+ // that was registered during executor construction (and that accumulated BE-reported
+ // load statistics via updateJobProgress) is NOT the one we are about to snapshot here.
+ // A brand-new InsertLoadJob will be created below with an empty LoadStatistic, so
+ // SHOW LOAD's JobDetails (ScannedRows / LoadBytes / All backends) will be all zero.
+ // Logging this at WARN so CI failures of the form
+ // "test_insert_statistic: expected:<N> but was:<0>" can be diagnosed directly.
+ LOG.warn("recordFinishedLoadJob: jobId={} not found in idToLoadJob, creating a new "
+ + "{} load job for label={}, dbId={}. JobDetails statistics will be empty.",
+ jobId, jobType, label, db.getId());
switch (jobType) {
case INSERT:
loadJob = new InsertLoadJob(label, transactionId, db.getId(), tableId, createTimestamp, failMsg,
@@ -813,6 +827,12 @@ public class LoadManager implements Writable {
public void updateJobProgress(Long jobId, Long beId, TUniqueId loadId, TUniqueId fragmentId, long scannedRows,
long scannedBytes, boolean isDone) {
LoadJob job = idToLoadJob.get(jobId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("updateJobProgress: jobId={}, beId={}, scannedRows={}, scannedBytes={}, isDone={}, "
+ + "found={}, jobIdMatched={}",
+ jobId, beId, scannedRows, scannedBytes, isDone,
+ idToLoadJob.containsKey(jobId), job == null ? -1L : job.getId());
+ }
if (job != null) {
job.updateProgress(beId, loadId, fragmentId, scannedRows, scannedBytes, isDone);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java
index c2b769196af..b805fea5312 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java
@@ -165,6 +165,12 @@ public class LoadProcessor extends AbstractJobProcessor {
@Override
protected void doProcessReportExecStatus(TReportExecStatusParams params, SingleFragmentPipelineTask fragmentTask) {
if (params.isSetLoadedRows() && jobId != -1) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("doProcessReportExecStatus: forwarding load progress to LoadManager, "
+ + "jobId={}, beId={}, queryId={}, loadedRows={}, loadedBytes={}, isDone={}",
+ jobId, params.getBackendId(), DebugUtil.printId(params.getQueryId()),
+ params.getLoadedRows(), params.getLoadedBytes(), params.isDone());
+ }
if (params.isSetFragmentInstanceReports()) {
for (TFragmentInstanceReport report : params.getFragmentInstanceReports()) {
Env.getCurrentEnv().getLoadManager().updateJobProgress(
diff --git a/regression-test/pipeline/p0/conf/fe.conf b/regression-test/pipeline/p0/conf/fe.conf
index c79ec24eb5f..3d86c5c95e6 100644
--- a/regression-test/pipeline/p0/conf/fe.conf
+++ b/regression-test/pipeline/p0/conf/fe.conf
@@ -93,3 +93,4 @@ max_spilled_profile_num = 2000
check_table_lock_leaky=true
max_bucket_num_per_partition=512
+max_remote_file_system_cache_num=1000
diff --git a/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy
index 0057c58956a..743cc79d5de 100644
--- a/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy
@@ -262,6 +262,10 @@ suite("test_doris_jdbc_catalog", "p0,external,doris,external_docker,external_doc
order_qt_keywork_table_name """ select * from `order` order by test_col; """
+ // cleanup reserved-keyword table so downstream infra suites (e.g. check_hash_bucket_table)
+ // do not trip over an unquoted `order` table name
+ sql """ drop table if exists internal.regression_test_jdbc_catalog_p0.`order` """
+
// //clean
// qt_sql """select current_catalog()"""
// sql "switch internal"
diff --git a/regression-test/suites/manager/test_manager_interface_1.groovy b/regression-test/suites/manager/test_manager_interface_1.groovy
index 1c9aa226200..1426306e9cc 100644
--- a/regression-test/suites/manager/test_manager_interface_1.groovy
+++ b/regression-test/suites/manager/test_manager_interface_1.groovy
@@ -594,7 +594,7 @@ suite('test_manager_interface_1',"p0") {
assertTrue(x == 1);
- sql """ admin set frontend config("query_metadata_name_ids_timeout"= "${val}")"""
+ sql """ admin set all frontends config("query_metadata_name_ids_timeout"= "${val}")"""
result = sql """
admin show frontend config
"""
@@ -613,7 +613,7 @@ suite('test_manager_interface_1',"p0") {
assertTrue(x == 1);
val -= 2
- sql """ admin set frontend config("query_metadata_name_ids_timeout"= "${val}")"""
+ sql """ admin set all frontends config("query_metadata_name_ids_timeout"= "${val}")"""
logger.info("result = ${result}" )
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 14/28: [fix](regression) Bound large Parquet map output batches (#68808)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2a6f1521f03fdbe924667eedf4d66433c8568b41
Author: Gabriel <li...@selectdb.com>
AuthorDate: Fri Oct 9 14:54:24 2026 +0800
[fix](regression) Bound large Parquet map output batches (#68808)
### What problem does this PR solve?
Issue Number: N/A
Related PR: N/A
The large-string-map Parquet regression fixture contains two 1 GiB
string keys in a column chunk larger than 2 GiB, with dictionary and
PLAIN data pages. Materializing both rows into one output column
requires a 4 GiB character buffer and can exceed the BE process memory
limit during ASAN regression runs.
Read this query with `batch_size=1` and disable aggregate pushdown using
a statement-local hint. Both rows still go through full Map and string
decoding, while each output batch contains only one large key. The query
and its expected result remain unchanged. This retains the oversized
Parquet column-chunk and mixed-encoding coverage; it no longer exercises
accumulating both keys in one output ColumnString.
### Release note
None
### Check List (For Author)
- Test
- [ ] Regression test
- [ ] Unit Test
- [x] Manual test
- Compiled the modified suite with Groovy 4.0.19 successfully.
- `git diff --check` passed.
- Reviewed the batch-size validation and propagation to the native
Parquet reader on branch-4.1.
- The HDFS regression was not run locally: the original CI HDFS fixture
environment is not configured. ASAN end-to-end validation remains for
CI.
- Behavior changed:
- [x] No. Production code is unchanged; the existing test uses
statement-local execution settings.
- [ ] Yes.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../suites/external_table_p0/tvf/test_hdfs_parquet_group0.groovy | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group0.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group0.groovy
index 7007ecefd3a..d964cb4a225 100644
--- a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group0.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group0.groovy
@@ -105,7 +105,10 @@ suite("test_hdfs_parquet_group0","external,hive,tvf,external_docker") {
uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group0/large_string_map.brotli.parquet"
- order_qt_test_11 """ select count(arr) from HDFS(
+ // Read both 1 GiB keys one row per batch to avoid a 4 GiB output buffer allocation.
+ // Disable aggregate pushdown to retain full decoding of the >2 GiB column chunk.
+ order_qt_test_11 """ select /*+ SET_VAR(batch_size=1, enable_push_down_no_group_agg=false) */
+ count(arr) from HDFS(
"uri" = "${uri}",
"hadoop.username" = "${hdfsUserName}",
"format" = "parquet"); """
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 22/28: branch-4.1: [fix](function) Sum integers exactly in array_avg #68632 (#68831)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4981796ac3895be74584f9773ebc120a89c3022d
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Sat Oct 10 09:29:40 2026 +0800
branch-4.1: [fix](function) Sum integers exactly in array_avg #68632 (#68831)
Cherry-picked from #68632
Co-authored-by: Chenyang Sun <su...@selectdb.com>
Co-authored-by: Claude Opus 5.5 <no...@anthropic.com>
---
be/src/exprs/aggregate/aggregate_function_avg.cpp | 17 +-----
be/src/exprs/aggregate/aggregate_function_avg.h | 15 ++++++
.../function/array/function_array_aggregation.cpp | 3 +-
.../function/function_array_aggregation_test.cpp | 17 ++++++
.../data/doc/sql-manual/ArrayNullsafe.out | 2 +-
.../nereids_function_p0/scalar_function/Array1.out | 24 ++++-----
.../array_functions/test_array_avg_integer_sum.out | 21 ++++++++
.../test_array_avg_integer_sum.groovy | 60 ++++++++++++++++++++++
8 files changed, 130 insertions(+), 29 deletions(-)
diff --git a/be/src/exprs/aggregate/aggregate_function_avg.cpp b/be/src/exprs/aggregate/aggregate_function_avg.cpp
index acf2cd8d830..7d1117de15e 100644
--- a/be/src/exprs/aggregate/aggregate_function_avg.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_avg.cpp
@@ -28,25 +28,12 @@
namespace doris {
#include "common/compile_check_begin.h"
-constexpr PrimitiveType result_type(PrimitiveType T) {
- if (T == TYPE_LARGEINT || T == TYPE_BIGINT) {
- return TYPE_LARGEINT;
- } else if (is_int_or_bool(T)) {
- return TYPE_BIGINT;
- } else if (is_float_or_double(T) || is_time_type(T)) {
- return TYPE_DOUBLE;
- } else if (is_decimalv3(T) && T != TYPE_DECIMAL256) {
- return TYPE_DECIMAL128I;
- } else {
- return T;
- }
-}
-
// TODO: use result type got from FE plan
template <PrimitiveType T>
struct Avg {
static constexpr PrimitiveType ResultPType = T == TYPE_DECIMALV2 ? T : TYPE_DOUBLE;
- using Function = AggregateFunctionAvg<T, ResultPType, AggregateFunctionAvgData<result_type(T)>>;
+ using Function =
+ AggregateFunctionAvg<T, ResultPType, AggregateFunctionAvgData<avg_sum_type(T)>>;
};
template <PrimitiveType T>
diff --git a/be/src/exprs/aggregate/aggregate_function_avg.h b/be/src/exprs/aggregate/aggregate_function_avg.h
index 665757ab343..0e9152ba1fa 100644
--- a/be/src/exprs/aggregate/aggregate_function_avg.h
+++ b/be/src/exprs/aggregate/aggregate_function_avg.h
@@ -52,6 +52,21 @@ class DataTypeNumber;
template <PrimitiveType T>
class ColumnVector;
+// avg() adds up the values in this type and divides only at the end, so integers keep all digits.
+constexpr PrimitiveType avg_sum_type(PrimitiveType T) {
+ if (T == TYPE_LARGEINT || T == TYPE_BIGINT) {
+ return TYPE_LARGEINT;
+ } else if (is_int_or_bool(T)) {
+ return TYPE_BIGINT;
+ } else if (is_float_or_double(T) || is_time_type(T)) {
+ return TYPE_DOUBLE;
+ } else if (is_decimalv3(T) && T != TYPE_DECIMAL256) {
+ return TYPE_DECIMAL128I;
+ } else {
+ return T;
+ }
+}
+
template <PrimitiveType T>
struct AggregateFunctionAvgData {
using ResultType = typename PrimitiveTypeTraits<T>::CppType;
diff --git a/be/src/exprs/function/array/function_array_aggregation.cpp b/be/src/exprs/function/array/function_array_aggregation.cpp
index cc1ec65c100..d2c6e65452f 100644
--- a/be/src/exprs/function/array/function_array_aggregation.cpp
+++ b/be/src/exprs/function/array/function_array_aggregation.cpp
@@ -90,7 +90,8 @@ struct AggregateFunctionTraits<AggregateOperation::AVERAGE> {
struct TypeTraits {
static constexpr PrimitiveType ResultType =
Element == TYPE_DECIMALV2 ? TYPE_DECIMALV2 : TYPE_DOUBLE;
- using AggregateDataType = AggregateFunctionAvgData<ResultType>;
+ // Add up the values in the same type as avg(). A double sum loses digits of large integers.
+ using AggregateDataType = AggregateFunctionAvgData<avg_sum_type(Element)>;
using Function = AggregateFunctionAvg<Element, ResultType, AggregateDataType>;
static_assert(std::is_same_v<typename PrimitiveTypeTraits<ResultType>::CppType,
typename Function::ResultType>,
diff --git a/be/test/exprs/function/function_array_aggregation_test.cpp b/be/test/exprs/function/function_array_aggregation_test.cpp
index 9a99e482876..2427d053165 100644
--- a/be/test/exprs/function/function_array_aggregation_test.cpp
+++ b/be/test/exprs/function/function_array_aggregation_test.cpp
@@ -16,6 +16,7 @@
// under the License.
#include <cstddef>
+#include <limits>
#include <string>
#include <utility>
#include <vector>
@@ -336,6 +337,22 @@ TEST(VFunctionArrayAggregationTest, TestArrayAverageNullable) {
func_name, data_set, PrimitiveType::TYPE_DOUBLE, PrimitiveType::TYPE_DOUBLE, true));
}
+TEST(VFunctionArrayAggregationTest, TestArrayAverageLargeIntegers) {
+ const std::string func_name = "array_avg";
+ // The sum is 1. A double sum loses the 1 next to the large values and gives 0.
+ std::vector<std::pair<std::vector<AnyValue<Int64>>, AnyValue<double>>> bigint_data_set = {
+ {{std::numeric_limits<Int64>::max(), 1, -std::numeric_limits<Int64>::max()}, 1.0 / 3},
+ };
+ static_cast<void>(check_function_array_wrapper<DataTypeInt64, DataTypeFloat64>(
+ func_name, bigint_data_set, PrimitiveType::TYPE_BIGINT, PrimitiveType::TYPE_DOUBLE));
+ std::vector<std::pair<std::vector<AnyValue<Int128>>, AnyValue<double>>> largeint_data_set = {
+ {{std::numeric_limits<Int128>::max(), 1, -std::numeric_limits<Int128>::max()}, 1.0 / 3},
+ };
+ static_cast<void>(check_function_array_wrapper<DataTypeInt128, DataTypeFloat64>(
+ func_name, largeint_data_set, PrimitiveType::TYPE_LARGEINT,
+ PrimitiveType::TYPE_DOUBLE));
+}
+
TEST(VFunctionArrayAggregationTest, TestArrayProduct) {
const std::string func_name = "array_product";
IntDataSet data_set = {
diff --git a/regression-test/data/doc/sql-manual/ArrayNullsafe.out b/regression-test/data/doc/sql-manual/ArrayNullsafe.out
index 64a056a2cdf..85431c5197e 100644
--- a/regression-test/data/doc/sql-manual/ArrayNullsafe.out
+++ b/regression-test/data/doc/sql-manual/ArrayNullsafe.out
@@ -879,7 +879,7 @@ false
\N
-- !sql_array_avg --
-1.7014118346046923e+38
+5.671372782015641e+37
3
2
\N
diff --git a/regression-test/data/nereids_function_p0/scalar_function/Array1.out b/regression-test/data/nereids_function_p0/scalar_function/Array1.out
index 2efd01b377c..67c41ff960c 100644
--- a/regression-test/data/nereids_function_p0/scalar_function/Array1.out
+++ b/regression-test/data/nereids_function_p0/scalar_function/Array1.out
@@ -123,8 +123,8 @@
\N
\N
\N
-0
-0
+-0.5
+-0.5
0
7.5
7.875
@@ -149,8 +149,8 @@
\N
\N
\N
-0
-0
+-0.5
+-0.5
0
7.5
7.875
@@ -172,10 +172,10 @@
\N
\N
\N
+-0.5
-9.055674363474186e+18
-9.223372036854776e+18
0
-0
3037000499.5
7.5
7.875
@@ -198,10 +198,10 @@
\N
\N
\N
+-0.5
-9.055674363474186e+18
-9.223372036854776e+18
0
-0
3037000499.5
7.5
7.875
@@ -24095,8 +24095,8 @@ true
\N
\N
\N
-0
-0
+-0.5
+-0.5
0
7.5
7.875
@@ -24121,8 +24121,8 @@ true
\N
\N
\N
-0
-0
+-0.5
+-0.5
0
7.5
7.875
@@ -24144,10 +24144,10 @@ true
\N
\N
\N
+-0.5
-9.055674363474186e+18
-9.223372036854776e+18
0
-0
3037000499.5
7.5
7.875
@@ -24170,10 +24170,10 @@ true
\N
\N
\N
+-0.5
-9.055674363474186e+18
-9.223372036854776e+18
0
-0
3037000499.5
7.5
7.875
diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_avg_integer_sum.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_avg_integer_sum.out
new file mode 100644
index 00000000000..7bcb966e22f
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_avg_integer_sum.out
@@ -0,0 +1,21 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !const --
+0.3333333333333333 0.3333333333333333
+
+-- !column --
+1 0.3333333333333333 0.3333333333333333
+2 0.3333333333333333 0.3333333333333333
+3 2 2.3333333333333335
+4 \N \N
+5 \N \N
+
+-- !same_as_avg_bigint --
+1 0.3333333333333333
+2 0.3333333333333333
+3 2
+
+-- !same_as_avg_largeint --
+1 0.3333333333333333
+2 0.3333333333333333
+3 2.3333333333333335
+
diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_avg_integer_sum.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_avg_integer_sum.groovy
new file mode 100644
index 00000000000..f746b31886a
--- /dev/null
+++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_avg_integer_sum.groovy
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_array_avg_integer_sum") {
+ // array_avg adds up integers in the same type as avg(), so the large values cancel out
+ // and the average of the sum 1 is 1/3.
+ order_qt_const """
+ SELECT array_avg(array(9223372036854775807, 1, -9223372036854775807)),
+ array_avg(array(170141183460469231731687303715884105727, 1,
+ -170141183460469231731687303715884105727))
+ """
+
+ sql """ DROP TABLE IF EXISTS test_array_avg_integer_sum """
+ sql """
+ CREATE TABLE test_array_avg_integer_sum (
+ id INT,
+ b ARRAY<BIGINT>,
+ l ARRAY<LARGEINT>
+ ) DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO test_array_avg_integer_sum VALUES
+ (1, array(9223372036854775807, 1, -9223372036854775807),
+ array(170141183460469231731687303715884105727, 1,
+ -170141183460469231731687303715884105727)),
+ (2, array(9223372036854775807, NULL, 1, -9223372036854775807),
+ array(170141183460469231731687303715884105727, NULL, 1,
+ -170141183460469231731687303715884105727)),
+ (3, array(1, 2, 3), array(1, 2, 4)),
+ (4, array(), array()),
+ (5, NULL, NULL)
+ """
+ order_qt_column """
+ SELECT id, array_avg(b), array_avg(l) FROM test_array_avg_integer_sum
+ """
+
+ // avg() over the same elements gives the same result.
+ order_qt_same_as_avg_bigint """
+ SELECT id, avg(x) FROM test_array_avg_integer_sum LATERAL VIEW explode(b) t AS x GROUP BY id
+ """
+ order_qt_same_as_avg_largeint """
+ SELECT id, avg(x) FROM test_array_avg_integer_sum LATERAL VIEW explode(l) t AS x GROUP BY id
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 08/28: branch-4.1: [fix](be) Defer Bloom allocation for IN runtime filters #68763 (#68789)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 68e4aa389150dc42bdcc42c740dc5797be65dd35
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Fri Oct 9 09:26:35 2026 +0800
branch-4.1: [fix](be) Defer Bloom allocation for IN runtime filters #68763 (#68789)
Cherry-picked from #68763
Co-authored-by: Mryange <ya...@selectdb.com>
---
.../exec/runtime_filter/runtime_filter_wrapper.cpp | 2 +-
.../runtime_filter/runtime_filter_wrapper_test.cpp | 42 ++++++++++++++++++++++
2 files changed, 43 insertions(+), 1 deletion(-)
diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
index 4b0a8ca8090..330bcb6895e 100644
--- a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
+++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
@@ -75,7 +75,7 @@ Status RuntimeFilterWrapper::init(const size_t real_size) {
if (get_real_type() == RuntimeFilterType::IN_FILTER && real_size > _max_in_num) {
set_state(RuntimeFilterWrapper::State::DISABLED, "reach max in num");
}
- if (_bloom_filter_func) {
+ if (get_real_type() == RuntimeFilterType::BLOOM_FILTER) {
RETURN_IF_ERROR(_bloom_filter_func->init_with_fixed_length(real_size));
}
return Status::OK();
diff --git a/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp b/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
index d5b2f9d9641..44cde989ab0 100644
--- a/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
+++ b/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
@@ -749,6 +749,46 @@ TEST_F(RuntimeFilterWrapperTest, TestMinMax) {
}
}
+TEST_F(RuntimeFilterWrapperTest, TestInOrBloomLazyInitialization) {
+ for (bool build_by_runtime_size : {false, true}) {
+ RuntimeFilterParams params {.filter_id = 0,
+ .filter_type = RuntimeFilterType::IN_OR_BLOOM_FILTER,
+ .column_return_type = TYPE_INT,
+ .null_aware = false,
+ .max_in_num = 16,
+ .runtime_bloom_filter_min_size = 64,
+ .runtime_bloom_filter_max_size = 128,
+ .bloom_filter_size = 64,
+ .build_bf_by_runtime_size = build_by_runtime_size,
+ .bloom_filter_size_calculated_by_ndv = false};
+ for (size_t runtime_size : {0, 1, 16, 17}) {
+ SCOPED_TRACE(runtime_size);
+ SCOPED_TRACE(build_by_runtime_size);
+ RuntimeFilterWrapper wrapper(¶ms);
+ ASSERT_NE(wrapper.bloom_filter_func(), nullptr);
+ ASSERT_EQ(wrapper.bloom_filter_func()->get_size(), 0);
+ ASSERT_TRUE(wrapper.init(runtime_size).ok());
+ if (runtime_size <= static_cast<size_t>(params.max_in_num)) {
+ EXPECT_EQ(wrapper.get_real_type(), RuntimeFilterType::IN_FILTER);
+ EXPECT_EQ(wrapper.bloom_filter_func()->get_size(), 0);
+ EXPECT_EQ(wrapper.bloom_filter_func()->_bloom_filter_alloced, 0);
+ EXPECT_EQ(wrapper.bloom_filter_func()->_bloom_filter, nullptr);
+ PMergeFilterRequest request;
+ ASSERT_TRUE(wrapper.to_protobuf(request.mutable_in_filter()).ok());
+ } else {
+ EXPECT_EQ(wrapper.get_real_type(), RuntimeFilterType::BLOOM_FILTER);
+ EXPECT_GT(wrapper.bloom_filter_func()->get_size(), 0);
+ auto col = ColumnHelper::create_column<DataTypeInt32>({1, 2, 3});
+ ASSERT_TRUE(wrapper.insert(col, 0).ok());
+ std::vector<uint8_t> matches(3);
+ wrapper.bloom_filter_func()->find_fixed_len(col, matches.data());
+ EXPECT_TRUE(std::all_of(matches.begin(), matches.end(),
+ [](uint8_t match) { return match != 0; }));
+ }
+ }
+ }
+}
+
TEST_F(RuntimeFilterWrapperTest, TestInOrBloom) {
std::vector<int> data_vector(10);
std::iota(data_vector.begin(), data_vector.end(), 0);
@@ -817,6 +857,7 @@ TEST_F(RuntimeFilterWrapperTest, TestInOrBloom) {
// Init (keep in filter)
EXPECT_TRUE(wrapper->init(runtime_size).ok());
EXPECT_EQ(wrapper->get_real_type(), RuntimeFilterType::IN_FILTER);
+ EXPECT_EQ(wrapper->bloom_filter_func()->get_size(), 0);
EXPECT_EQ(wrapper->get_state(), RuntimeFilterWrapper::State::UNINITED);
// Insert
auto col = ColumnHelper::create_column<DataType>(data_vector);
@@ -875,6 +916,7 @@ TEST_F(RuntimeFilterWrapperTest, TestInOrBloom) {
EXPECT_TRUE(wrapper->merge(new_wrapper.get()).ok());
EXPECT_EQ(wrapper->hybrid_set()->size(), col->size() * 2);
EXPECT_EQ(wrapper->get_real_type(), RuntimeFilterType::IN_FILTER);
+ EXPECT_EQ(wrapper->bloom_filter_func()->get_size(), 0);
}
{
// In + In -> Bloom
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org