You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/10/21 02:27:34 UTC
(paimon) branch master updated: [Python] Separate Python and Native execution implementations (#10699)
This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 8c09824c06 [Python] Separate Python and Native execution implementations (#10699)
8c09824c06 is described below
commit 8c09824c060520f3f1f4a03ce2369de371e5c7c6
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Wed Oct 21 10:27:28 2026 +0800
[Python] Separate Python and Native execution implementations (#10699)
---
docs/docs/program-api/file-cache.mdx | 2 +-
.../test/java/org/apache/paimon/JavaPyE2ETest.java | 64 +-
.../java/org/apache/paimon/JavaPyLanceE2ETest.java | 63 ++
paimon-python/README.md | 8 +
paimon-python/conftest.py | 6 +-
paimon-python/dev/run_mixed_tests.sh | 20 +-
paimon-python/dev/test_packaging.py | 69 ++
.../acceptance/incremental_diff_acceptance_test.py | 6 +-
.../pypaimon/benchmark/blob_index_cold_load.py | 2 +-
paimon-python/pypaimon/casting/data_type_casts.py | 12 +-
paimon-python/pypaimon/casting/decimal_cast.py | 5 +-
paimon-python/pypaimon/casting/schema_evolution.py | 43 +-
paimon-python/pypaimon/daft/daft_datasink.py | 36 +-
paimon-python/pypaimon/execution/README.md | 122 +++
.../{read/scanner => execution}/__init__.py | 0
paimon-python/pypaimon/execution/dispatch.py | 266 +++++
.../{write/commit => execution/native}/__init__.py | 0
.../writer => execution/native/read}/__init__.py | 0
.../pypaimon/execution/native/read/builder.py | 68 ++
.../native/read}/native_full_text_search.py | 8 +-
.../native/read}/native_hybrid_search.py | 8 +-
.../native/read}/native_search_plan.py | 86 +-
.../native/read}/native_vector_search.py | 4 +-
.../pypaimon/execution/native/read/plan.py | 326 +++++++
.../native/read/row_iterator.py} | 0
.../execution/native/read/split_adapter.py | 71 ++
.../execution/native/read/streaming_table_scan.py | 105 ++
.../native/read/system_table.py} | 5 +-
.../pypaimon/execution/native/read/table_read.py | 275 ++++++
.../execution/native/read/vindex_reader.py | 79 ++
paimon-python/pypaimon/execution/native/runtime.py | 47 +
.../native_plan.py => execution/native/table.py} | 299 +-----
.../scanner => execution/native/write}/__init__.py | 0
.../native/write/bucket_metadata.py} | 39 +-
.../pypaimon/execution/native/write/commit.py | 45 +
.../native}/write/native_blob_rows.py | 0
.../{ => execution/native}/write/native_commit.py | 9 +-
.../native/write}/native_index_build.py | 13 +-
.../execution/native/write/native_index_drop.py | 42 +
.../native}/write/native_merge_into.py | 22 +-
.../{ => execution/native}/write/native_update.py | 26 +-
.../{ => execution/native}/write/native_write.py | 127 ++-
.../execution/native/write/postpone_bucket.py | 50 +
.../{read/scanner => execution/python}/__init__.py | 0
.../python/globalindex}/__init__.py | 0
.../python/globalindex/bitmap}/__init__.py | 0
.../globalindex/bitmap/bitmap_index_reader.py | 6 +-
.../globalindex/bitmap/bitmap_index_writer.py | 6 +-
.../bitmap/lazy_filtered_bitmap_reader.py | 4 +-
.../python}/globalindex/block_compression.py | 2 +-
.../python/globalindex/btree}/__init__.py | 0
.../globalindex/btree/block_aligned_type.py | 4 +-
.../python}/globalindex/btree/block_entry.py | 10 +-
.../python}/globalindex/btree/block_handle.py | 0
.../python}/globalindex/btree/block_reader.py | 90 +-
.../python}/globalindex/btree/bloom_filter.py | 0
.../python}/globalindex/btree/btree_file_footer.py | 30 +-
.../globalindex/btree/btree_file_meta_selector.py | 2 +-
.../python}/globalindex/btree/btree_index_meta.py | 0
.../globalindex/btree/btree_index_reader.py | 12 +-
.../globalindex/btree/btree_index_writer.py | 8 +-
.../globalindex/btree/btree_posting_list.py | 4 +-
.../btree/lazy_filtered_btree_reader.py | 4 +-
.../python}/globalindex/btree/sst_file_reader.py | 30 +-
.../python}/globalindex/build_plan.py | 0
.../python}/globalindex/create_global_index.py | 295 ++----
.../data_evolution_global_index_coverage.py | 0
.../data_evolution_global_index_scanner.py | 18 +-
.../python/globalindex/drop_global_index.py | 76 ++
.../python}/globalindex/global_index_evaluator.py | 12 +-
.../python}/globalindex/index_file_utils.py | 2 +-
.../python}/globalindex/key_serializer.py | 0
.../python/globalindex/lumina}/__init__.py | 0
.../globalindex/lumina/lumina_index_meta.py | 0
.../lumina/lumina_vector_global_index_reader.py | 14 +-
.../lumina/lumina_vector_index_options.py | 0
.../python}/globalindex/memory_slice_input.py | 0
.../globalindex/offset_global_index_reader.py | 0
.../python}/globalindex/result_entry.py | 0
.../globalindex/sorted_file_global_index_reader.py | 4 +-
.../globalindex/sorted_file_meta_selector.py | 2 +-
.../globalindex/union_global_index_reader.py | 0
.../python/globalindex/vindex}/__init__.py | 0
.../vindex/vindex_vector_global_index_reader.py | 212 ++++
.../vindex/vindex_vector_index_writer.py | 221 +----
.../python}/merge_engine_dispatch.py | 9 +-
.../scanner => execution/python/read}/__init__.py | 0
.../pypaimon/execution/python/read/auth_support.py | 100 ++
.../python/read}/format_table_read.py | 189 ++--
.../python/read}/format_table_scan.py | 53 +-
.../python}/read/merge_engine_support.py | 110 +--
.../read}/native_full_text_global_index_reader.py | 59 +-
.../execution/python/read/object_table_read.py | 131 +++
.../python/read/object_table_scan.py} | 11 +-
.../python/read/reader}/__init__.py | 0
.../python}/read/reader/aggregate/__init__.py | 4 +-
.../python}/read/reader/aggregate/aggregators.py | 6 +-
.../read/reader/aggregate/field_aggregator.py | 0
.../read/reader/aggregation_merge_function.py | 8 +-
.../python}/read/reader/auth_masking_reader.py | 0
.../read/reader/blob_descriptor_convert_reader.py | 0
.../python}/read/reader/blob_view_read_support.py | 4 +-
.../python}/read/reader/concat_batch_reader.py | 0
.../python}/read/reader/concat_record_reader.py | 0
.../python}/read/reader/data_file_batch_reader.py | 11 +-
.../read/reader/deduplicate_merge_function.py | 0
.../read/reader/deferred_blob_resolve_reader.py | 0
.../python}/read/reader/drop_delete_reader.py | 0
.../python}/read/reader/empty_record_reader.py | 0
.../python}/read/reader/field_bunch.py | 0
.../read/reader/filter_record_batch_reader.py | 0
.../python}/read/reader/filter_record_reader.py | 0
.../read/reader/first_row_merge_function.py | 0
.../python}/read/reader/format_avro_reader.py | 0
.../python}/read/reader/format_blob_reader.py | 2 +-
.../python}/read/reader/format_lance_reader.py | 2 +-
.../python}/read/reader/format_mosaic_reader.py | 0
.../python}/read/reader/format_pyarrow_reader.py | 2 +-
.../python}/read/reader/format_row_reader.py | 0
.../python}/read/reader/format_vortex_reader.py | 2 +-
.../python}/read/reader/key_value_unwrap_reader.py | 0
.../python}/read/reader/key_value_wrap_reader.py | 0
.../python}/read/reader/lance_utils.py | 0
.../python}/read/reader/limited_record_reader.py | 0
.../read/reader/nested_leaf_batch_reader.py | 7 +-
.../read/reader/outer_projection_record_reader.py | 0
.../read/reader/parquet_page_index_reader.py | 2 +-
.../read/reader/partial_update_merge_function.py | 0
.../read/reader/row_range_filter_record_reader.py | 0
.../python}/read/reader/shard_batch_reader.py | 2 +-
.../python}/read/reader/sort_merge_reader.py | 27 +-
.../python}/read/reader/video_format_reader.py | 0
.../python}/read/reader/vortex_utils.py | 0
.../python}/read/scan_distribution.py | 16 +-
.../python}/read/scanner/__init__.py | 0
.../read/scanner/append_table_split_generator.py | 4 +-
.../read/scanner/bucket_select_converter.py | 10 +-
.../read/scanner/changelog_follow_up_scanner.py | 2 +-
.../read/scanner/chunk_shuffle_split_generator.py | 2 +-
.../read/scanner/data_evolution_split_generator.py | 2 +-
.../python}/read/scanner/data_evolution_stats.py | 0
.../read/scanner/delta_follow_up_scanner.py | 2 +-
.../python}/read/scanner/file_scanner.py | 154 +--
.../python}/read/scanner/follow_up_scanner.py | 0
.../read/scanner/incremental_diff_scanner.py | 2 +-
.../scanner/primary_key_table_split_generator.py | 4 +-
.../python}/read/scanner/split_generator.py | 3 +-
.../python/read/search}/__init__.py | 0
.../python/read/search}/full_text_read.py | 53 +-
.../python/read/search}/full_text_scan.py | 33 +-
.../read/search}/global_index_live_row_filter.py | 0
.../python/read/search}/global_index_row_filter.py | 2 +-
.../execution/python/read/search/hybrid_search.py | 184 ++++
.../read/search}/primary_key_full_text_read.py | 8 +-
.../read/search}/primary_key_full_text_scan.py | 21 +-
.../read/search}/primary_key_sorted_index_scan.py | 4 +-
.../python/read/search}/primary_key_vector_read.py | 6 +-
.../python/read/search}/primary_key_vector_scan.py | 21 +-
.../python/read/search}/vector_search_read.py | 63 +-
.../python/read/search}/vector_search_scan.py | 32 +-
.../{ => execution/python}/read/split_read.py | 96 +-
.../python}/read/streaming_table_scan.py | 113 +--
.../execution/python/read/system_table_read.py | 235 +++++
.../python/read/system_table_scan.py} | 9 +-
.../pypaimon/execution/python/read/table_read.py | 785 +++++++++++++++
.../pypaimon/execution/python/read/table_scan.py | 129 +++
.../scanner => execution/python/write}/__init__.py | 0
.../python}/write/blob_format_writer.py | 0
.../python/write/bucket_metadata.py} | 38 +-
.../python/write/commit}/__init__.py | 0
.../python}/write/commit/commit_rollback.py | 0
.../python}/write/commit/commit_scanner.py | 2 +-
.../python}/write/commit/conflict_detection.py | 4 +-
.../write/commit/fixed_bucket_commit_check.py | 0
.../write/commit/global_index_source_check.py | 2 +-
.../write/commit/overwrite_changes_provider.py | 2 +-
.../write/commit/row_id_conflict_rewriter.py | 2 +-
.../python}/write/file_store_commit.py | 29 +-
.../python}/write/file_store_write.py | 28 +-
.../python/write}/format_table_commit.py | 4 +-
.../python/write}/format_table_write.py | 25 +-
.../python}/write/global_index_update_checker.py | 0
.../python}/write/map_shared_shredding_writer.py | 2 +-
.../python/write/merge_into.py} | 288 +-----
.../python/write}/native_full_text_index_writer.py | 10 +-
.../python}/write/postpone_batch_table_write.py | 38 +-
.../execution/python/write/postpone_bucket.py | 44 +
.../python}/write/row_id_file_index.py | 4 +-
.../python}/write/row_key_extractor.py | 328 +------
.../python/write/table_commit.py} | 31 +-
.../{ => execution/python}/write/table_delete.py | 6 +-
.../{ => execution/python}/write/table_update.py | 611 ++----------
.../python}/write/table_update_by_row_id.py | 120 +--
.../python}/write/table_upsert_by_key.py | 160 +--
.../{ => execution/python}/write/table_write.py | 94 +-
.../python}/write/video_format_writer.py | 2 +-
.../python/write/writer}/__init__.py | 0
.../write/writer/append_only_data_writer.py | 2 +-
.../python}/write/writer/blob_file_writer.py | 4 +-
.../python}/write/writer/blob_writer.py | 6 +-
.../python}/write/writer/composite_data_writer.py | 4 +-
.../python}/write/writer/data_vector_writer.py | 4 +-
.../python}/write/writer/data_writer.py | 8 +-
.../write/writer/dedicated_format_writer.py | 8 +-
.../python}/write/writer/format_row_writer.py | 0
.../python}/write/writer/key_value_data_writer.py | 7 +-
.../python}/write/writer/mosaic_writer_options.py | 0
.../python}/write/writer/parquet_writer_options.py | 0
.../python}/write/writer/postpone_data_writer.py | 2 +-
.../python}/write/writer/single_file_writer.py | 0
.../python}/write/writer/vector_writer.py | 4 +-
.../python}/write/writer/video_group.py | 0
.../python}/write/writer/write_buffer.py | 0
paimon-python/pypaimon/execution/vindex_input.py | 189 ++++
.../pypaimon/filesystem/hdfs_native_file_io.py | 8 +-
paimon-python/pypaimon/filesystem/local_file_io.py | 8 +-
.../pypaimon/filesystem/pyarrow_file_io.py | 8 +-
paimon-python/pypaimon/globalindex/__init__.py | 28 +-
.../pypaimon/globalindex/btree/__init__.py | 34 -
.../pypaimon/globalindex/btree/key_serializer.py | 34 -
.../pypaimon/globalindex/create_global_index.py | 470 +--------
.../pypaimon/globalindex/drop_global_index.py | 76 +-
.../pypaimon/globalindex/full_text/__init__.py | 32 -
.../pypaimon/globalindex/full_text_options.py | 69 ++
.../global_index_evaluation.py} | 15 +-
.../memory_slice_input.py => index_types.py} | 10 +-
.../pypaimon/globalindex/lumina/__init__.py | 30 -
.../pypaimon/globalindex/vindex/__init__.py | 32 -
.../vindex/vindex_vector_global_index_reader.py | 394 --------
.../pypaimon/globalindex/vindex_options.py | 244 +++++
.../pypaimon/globalindex/vindex_reader.py | 100 ++
paimon-python/pypaimon/index/dynamic_bucket.py | 53 +-
paimon-python/pypaimon/multimodal/temporal.py | 6 +-
paimon-python/pypaimon/ray/batch_vector_search.py | 4 +-
.../pypaimon/ray/data_evolution_merge_join.py | 6 +-
.../pypaimon/ray/data_evolution_merge_transform.py | 98 +-
paimon-python/pypaimon/ray/full_text_search.py | 6 +-
.../pypaimon/ray/row_id_conflict_rewriter.py | 2 +-
paimon-python/pypaimon/ray/shuffle.py | 3 +-
paimon-python/pypaimon/ray/vector_index_build.py | 6 +-
paimon-python/pypaimon/ray/vector_search.py | 6 +-
paimon-python/pypaimon/read/auth_support.py | 66 ++
paimon-python/pypaimon/read/blob_read_support.py | 55 ++
.../memory_slice_input.py => read/constants.py} | 7 +-
.../pypaimon/read/datasource/torch_dataset.py | 3 +-
.../pypaimon/read/merge_engine_support.py | 214 +---
paimon-python/pypaimon/read/scan_config.py | 184 ++++
paimon-python/pypaimon/read/scan_distribution.py | 56 +-
paimon-python/pypaimon/read/scan_request.py | 241 +++++
paimon-python/pypaimon/read/sequence_fields.py | 43 +
paimon-python/pypaimon/read/stream_read_builder.py | 4 +-
.../pypaimon/read/streaming_table_scan.py | 532 +---------
paimon-python/pypaimon/read/table_read.py | 980 +------------------
paimon-python/pypaimon/read/table_scan.py | 692 +++----------
paimon-python/pypaimon/schema/arrow_schema.py | 2 +-
.../pypaimon/table/data_evolution_merge_into.py | 853 +---------------
paimon-python/pypaimon/table/file_store_table.py | 14 +-
.../pypaimon/table/format/format_table_commit.py | 52 +-
.../pypaimon/table/format/format_table_read.py | 243 +----
.../pypaimon/table/format/format_table_scan.py | 101 +-
.../pypaimon/table/format/format_table_write.py | 264 +----
.../pypaimon/table/object/object_table_read.py | 114 +--
.../pypaimon/table/object/object_table_scan.py | 8 +-
.../table/source/batch_vector_search_builder.py | 20 +-
.../table/source/full_text_search_builder.py | 37 +-
.../pypaimon/table/source/hybrid_search_builder.py | 153 +--
.../table/source/primary_key_scored_result.py | 2 +-
.../pypaimon/table/source/search_types.py | 144 +++
.../pypaimon/table/source/vector_search_builder.py | 31 +-
paimon-python/pypaimon/table/system/files_table.py | 2 +-
.../pypaimon/table/system/system_table_read.py | 201 +---
.../pypaimon/table/system/system_table_scan.py | 9 +-
paimon-python/pypaimon/tests/arrow_schema_test.py | 2 +-
.../pypaimon/tests/auth_masking_reader_test.py | 2 +-
.../pypaimon/tests/batch_scan_level_filter_test.py | 2 +-
.../pypaimon/tests/batch_vector_raw_scan_test.py | 6 +-
.../tests/batch_vector_streaming_refine_test.py | 10 +-
paimon-python/pypaimon/tests/binary_row_test.py | 2 +-
.../pypaimon/tests/blob_index_cache_test.py | 2 +-
paimon-python/pypaimon/tests/blob_table_test.py | 30 +-
paimon-python/pypaimon/tests/blob_test.py | 154 +--
.../pypaimon/tests/btree_bloom_filter_test.py | 10 +-
.../pypaimon/tests/btree_index_reader_test.py | 14 +-
.../pypaimon/tests/btree_posting_list_test.py | 2 +-
.../pypaimon/tests/btree_thread_safety_test.py | 12 +-
.../tests/changelog_follow_up_scanner_test.py | 3 +-
.../tests/commit_update_schema_identity_test.py | 4 +-
.../tests/composite_btree_fallback_test.py | 4 +-
.../pypaimon/tests/concat_batch_reader_test.py | 15 +-
.../pypaimon/tests/concurrent_import_test.py | 2 +-
.../tests/contiguous_window_dataset_test.py | 4 +-
.../pypaimon/tests/contiguous_window_plan_test.py | 4 +-
.../tests/daft/daft_pk_distributed_write_test.py | 4 +-
.../pypaimon/tests/daft/daft_sink_test.py | 2 +-
.../tests/data_evolution_deletion_vector_test.py | 4 +-
.../pypaimon/tests/data_evolution_formats_test.py | 2 +-
.../tests/data_evolution_group_stats_test.py | 6 +-
.../tests/data_evolution_row_rolling_test.py | 4 +-
.../tests/data_evolution_split_generator_test.py | 2 +-
.../pypaimon/tests/data_evolution_test.py | 6 +-
.../pypaimon/tests/data_file_creation_time_test.py | 2 +-
.../tests/data_file_path_directory_test.py | 15 +-
.../pypaimon/tests/decimal_key_serializer_test.py | 10 +-
.../pypaimon/tests/decimal_predicate_test.py | 4 +-
.../pypaimon/tests/deferred_blob_resolve_test.py | 2 +-
.../pypaimon/tests/deletion_vector_path_test.py | 3 +-
.../pypaimon/tests/e2e/java_py_read_write_test.py | 29 +-
.../pypaimon/tests/execution_dispatch_test.py | 211 ++++
.../tests/execution_table_read_boundary_test.py | 82 ++
paimon-python/pypaimon/tests/field_bunch_test.py | 2 +-
.../pypaimon/tests/file_store_commit_test.py | 12 +-
.../pypaimon/tests/follow_up_scanner_test.py | 4 +-
.../pypaimon/tests/format_mosaic_reader_test.py | 2 +-
.../format_pyarrow_shared_shredding_map_test.py | 6 +-
.../tests/format_pyarrow_variant_row_group_test.py | 5 +-
.../pypaimon/tests/format_write_execution_test.py | 173 ++++
.../tests/full_text_reader_cleanup_test.py | 4 +-
.../pypaimon/tests/full_text_scalar_filter_test.py | 8 +-
.../pypaimon/tests/full_text_top_k_test.py | 10 +-
.../pypaimon/tests/global_index_build_test.py | 32 +-
.../pypaimon/tests/global_index_evaluator_test.py | 2 +-
.../tests/global_index_execution_boundary_test.py | 174 ++++
.../tests/global_index_family_parity_test.py | 2 +-
.../tests/global_index_live_row_filter_test.py | 2 +-
.../tests/global_index_scalar_search_mode_test.py | 4 +-
paimon-python/pypaimon/tests/global_index_test.py | 50 +-
.../tests/global_index_update_action_test.py | 14 +-
paimon-python/pypaimon/tests/hdfs_native_test.py | 10 +-
.../pypaimon/tests/hdfs_native_write_row_test.py | 2 +-
.../pypaimon/tests/hybrid_search_execution_test.py | 4 +-
.../pypaimon/tests/interval_partition_test.py | 2 +-
paimon-python/pypaimon/tests/lance_utils_test.py | 2 +-
.../pypaimon/tests/lazy_read_execution_test.py | 189 ++++
.../pypaimon/tests/lazy_search_execution_test.py | 219 +++++
.../pypaimon/tests/lumina_vector_index_test.py | 6 +-
.../tests/manifest/manifest_manager_test.py | 2 +-
.../tests/manifest/manifest_sidecar_test.py | 4 +-
.../tests/map_shared_shredding_write_test.py | 22 +-
.../pypaimon/tests/mosaic_writer_options_test.py | 4 +-
.../pypaimon/tests/multimodal_temporal_test.py | 2 +-
.../tests/native_append_index_search_test.py | 4 +-
.../tests/native_authorized_update_test.py | 16 +-
.../tests/native_blob_collection_write_test.py | 4 +-
.../pypaimon/tests/native_blob_consumer_test.py | 4 +-
.../pypaimon/tests/native_blob_row_write_test.py | 6 +-
.../pypaimon/tests/native_blob_update_test.py | 10 +-
.../pypaimon/tests/native_blob_uri_reader_test.py | 4 +-
.../pypaimon/tests/native_blob_write_test.py | 4 +-
.../pypaimon/tests/native_branch_write_test.py | 2 +-
.../pypaimon/tests/native_bucket_selector_test.py | 2 +-
.../pypaimon/tests/native_calendar_mask_test.py | 4 +-
.../pypaimon/tests/native_cast_completion_test.py | 2 +-
.../pypaimon/tests/native_changelog_batch_test.py | 2 +-
.../tests/native_commit_partition_spec_test.py | 2 +-
.../pypaimon/tests/native_commit_semantics_test.py | 2 +-
paimon-python/pypaimon/tests/native_commit_test.py | 40 +-
.../tests/native_cross_partition_write_test.py | 2 +-
.../pypaimon/tests/native_data_directory_test.py | 4 +-
.../tests/native_data_evolution_commit_test.py | 6 +-
.../pypaimon/tests/native_decimal_cast_test.py | 2 +-
.../tests/native_dv_merge_engine_write_test.py | 2 +-
.../pypaimon/tests/native_dynamic_write_test.py | 4 +-
.../pypaimon/tests/native_external_write_test.py | 4 +-
.../tests/native_filtered_write_validation_test.py | 4 +-
.../native_filtered_writer_configuration_test.py | 40 +-
.../pypaimon/tests/native_full_startup_test.py | 14 +-
.../pypaimon/tests/native_full_text_search_test.py | 4 +-
.../tests/native_generic_index_build_test.py | 8 +-
.../tests/native_global_index_drop_test.py | 10 +-
.../tests/native_historical_update_test.py | 8 +-
.../tests/native_historical_write_commit_test.py | 4 +-
.../pypaimon/tests/native_hybrid_search_test.py | 6 +-
.../tests/native_index_result_scan_test.py | 8 +-
.../pypaimon/tests/native_iterator_test.py | 18 +-
.../pypaimon/tests/native_loaded_schema_io_test.py | 4 +-
.../tests/native_managed_blob_write_test.py | 2 +-
.../tests/native_manifest_compression_test.py | 2 +-
.../tests/native_map_shredding_write_test.py | 2 +-
.../tests/native_merge_dictionary_keys_test.py | 2 +-
.../tests/native_merge_incremental_test.py | 4 +-
.../pypaimon/tests/native_merge_into_test.py | 36 +-
.../pypaimon/tests/native_merge_read_test.py | 6 +-
.../tests/native_merge_source_metadata_test.py | 14 +-
.../pypaimon/tests/native_mod_bucket_write_test.py | 6 +-
.../pypaimon/tests/native_nested_predicate_test.py | 2 +-
.../tests/native_nullable_primary_key_test.py | 8 +-
.../tests/native_overwrite_indexes_test.py | 2 +-
.../tests/native_overwrite_restore_test.py | 4 +-
.../tests/native_parallel_index_build_test.py | 4 +-
.../tests/native_parquet_vector_update_test.py | 6 +-
.../tests/native_partition_blob_read_test.py | 2 +-
.../native_partition_delete_changelog_test.py | 10 +-
.../pypaimon/tests/native_partition_filter_test.py | 4 +-
.../tests/native_partition_metadata_test.py | 2 +-
.../pypaimon/tests/native_partition_types_test.py | 14 +-
.../pypaimon/tests/native_pk_blob_limit_test.py | 5 +-
.../tests/native_pk_full_text_search_test.py | 3 +-
.../pypaimon/tests/native_pk_sorted_index_test.py | 3 +-
.../tests/native_plan_capabilities_test.py | 2 +-
.../tests/native_plan_chunk_shuffle_test.py | 2 +-
.../tests/native_plan_distribution_test.py | 4 +-
.../tests/native_plan_dynamic_bucket_test.py | 2 +-
.../pypaimon/tests/native_plan_expanded_test.py | 8 +-
.../pypaimon/tests/native_plan_incremental_test.py | 6 +-
.../pypaimon/tests/native_plan_integration_test.py | 31 +-
.../tests/native_plan_materialized_pk_test.py | 2 +-
.../tests/native_plan_resolved_schema_test.py | 2 +-
.../pypaimon/tests/native_plan_rest_test.py | 15 +-
paimon-python/pypaimon/tests/native_plan_test.py | 131 ++-
.../tests/native_postpone_fixed_write_test.py | 6 +-
.../pypaimon/tests/native_postpone_write_test.py | 4 +-
.../tests/native_precomputed_bucket_test.py | 4 +-
.../pypaimon/tests/native_predicate_delete_test.py | 4 +-
.../tests/native_preplanned_row_id_read_test.py | 2 +-
.../pypaimon/tests/native_query_auth_test.py | 8 +-
.../tests/native_read_path_completion_test.py | 45 +-
.../pypaimon/tests/native_read_snapshot_test.py | 8 +-
paimon-python/pypaimon/tests/native_read_test.py | 90 +-
.../pypaimon/tests/native_row_sidecar_test.py | 4 +-
.../pypaimon/tests/native_scan_stats_test.py | 5 +-
.../pypaimon/tests/native_schema_cast_test.py | 44 +-
.../tests/native_schema_evolution_cast_test.py | 2 +-
.../pypaimon/tests/native_search_plan_test.py | 25 +-
.../pypaimon/tests/native_sequence_test.py | 2 +-
.../tests/native_shard_partition_carrier_test.py | 4 +-
.../pypaimon/tests/native_shard_update_test.py | 2 +-
.../tests/native_sliced_split_read_test.py | 5 +-
.../tests/native_snapshot_properties_test.py | 2 +-
.../tests/native_sorted_index_build_test.py | 10 +-
.../tests/native_stream_consumer_branch_test.py | 6 +-
.../tests/native_stream_table_scan_test.py | 8 +-
.../pypaimon/tests/native_system_columns_test.py | 2 +-
.../pypaimon/tests/native_system_table_test.py | 11 +-
.../tests/native_temporal_zone_cast_test.py | 2 +-
.../tests/native_timestamp_bucket_write_test.py | 6 +-
.../pypaimon/tests/native_transform_filter_test.py | 4 +-
.../tests/native_update_container_casts_test.py | 2 +-
.../native_update_inactive_vector_option_test.py | 6 +-
.../tests/native_update_source_nullability_test.py | 4 +-
paimon-python/pypaimon/tests/native_update_test.py | 20 +-
.../tests/native_upsert_encoded_keys_test.py | 8 +-
.../tests/native_upsert_row_key_identity_test.py | 12 +-
.../tests/native_upsert_row_shapes_test.py | 2 +-
.../tests/native_upsert_temporal_row_keys_test.py | 22 +-
.../pypaimon/tests/native_variant_update_test.py | 4 +-
.../pypaimon/tests/native_vector_context_test.py | 2 +-
.../tests/native_vector_index_build_test.py | 9 +-
.../pypaimon/tests/native_vector_search_test.py | 64 +-
.../pypaimon/tests/native_video_read_test.py | 2 +-
.../pypaimon/tests/native_vindex_reader_test.py | 12 +-
.../tests/native_write_capabilities_test.py | 2 +-
.../tests/native_write_fork_safety_test.py | 21 +-
.../tests/native_write_inactive_options_test.py | 14 +-
.../tests/native_write_path_completion_test.py | 26 +-
.../pypaimon/tests/native_write_routing_test.py | 14 +-
paimon-python/pypaimon/tests/native_write_test.py | 34 +-
.../pypaimon/tests/native_write_type_test.py | 4 +-
.../pypaimon/tests/overwrite_changes_cache_test.py | 2 +-
.../pypaimon/tests/parquet_metadata_cache_test.py | 4 +-
.../pypaimon/tests/parquet_page_index_test.py | 4 +-
.../pypaimon/tests/parquet_row_range_test.py | 18 +-
.../tests/parquet_write_page_index_test.py | 4 +-
.../pypaimon/tests/partition_predicate_test.py | 22 +-
.../tests/primary_key_global_index_golden_test.py | 10 +-
.../tests/primary_key_index_definitions_test.py | 34 +-
.../tests/primary_key_sorted_index_scan_test.py | 2 +-
.../tests/projection_predicate_index_test.py | 2 +-
.../tests/pure_writer_creation_time_test.py | 2 +-
.../pypaimon/tests/pushdown_bucket_test.py | 28 +-
.../pypaimon/tests/pushdown_correctness_test.py | 2 +-
.../tests/py36/filter_record_batch_reader_test.py | 2 +-
.../pypaimon/tests/py36/rest_ao_read_write_test.py | 2 +-
paimon-python/pypaimon/tests/range_test.py | 2 +-
.../pypaimon/tests/ray_batch_vector_search_test.py | 4 +-
.../pypaimon/tests/ray_bucket_join_test.py | 2 +-
.../tests/ray_data_evolution_merge_into_test.py | 2 +-
.../pypaimon/tests/ray_full_text_search_test.py | 4 +-
.../pypaimon/tests/ray_integration_test.py | 2 +-
.../tests/ray_row_id_conflict_rewriter_test.py | 8 +-
paimon-python/pypaimon/tests/ray_sink_test.py | 2 +-
.../tests/ray_vector_filter_exactness_test.py | 7 +-
.../pypaimon/tests/ray_vector_search_test.py | 2 +-
.../pypaimon/tests/reader_append_only_test.py | 2 +-
paimon-python/pypaimon/tests/reader_base_test.py | 2 +-
.../pypaimon/tests/reader_split_generator_test.py | 2 +-
.../pypaimon/tests/resolving_file_io_test.py | 2 +-
.../pypaimon/tests/rest/rest_simple_test.py | 7 +-
.../pypaimon/tests/rest/rollback_protocol_test.py | 8 +-
.../tests/robomind_agilex_pipeline_test.py | 3 +-
.../pypaimon/tests/scan_file_shard_test.py | 4 +-
paimon-python/pypaimon/tests/scan_request_test.py | 154 +++
.../scanner/chunk_shuffle_split_generator_test.py | 16 +-
.../scanner/file_scanner_schema_fields_test.py | 2 +-
.../tests/scanner/incremental_diff_scanner_test.py | 63 +-
.../pypaimon/tests/schema_evolution_cast_test.py | 89 +-
.../tests/schema_evolution_nested_read_test.py | 32 +-
.../pypaimon/tests/search_snapshot_test.py | 4 +-
.../pypaimon/tests/shard_batch_reader_test.py | 2 +-
.../pypaimon/tests/shard_table_updator_test.py | 25 +-
.../pypaimon/tests/sql_permission_test.py | 2 +-
.../pypaimon/tests/sst_file_iterator_test.py | 8 +-
.../pypaimon/tests/stream_read_builder_test.py | 4 +-
.../pypaimon/tests/streaming_table_scan_test.py | 60 +-
.../pypaimon/tests/system/files_table_test.py | 21 +-
.../tests/system/native_system_table_test.py | 14 +-
.../tests/system/system_string_filter_test.py | 2 +-
.../pypaimon/tests/table_query_auth_test.py | 3 +-
.../tests/table_update_by_row_id_chunked_test.py | 6 +-
.../pypaimon/tests/table_upsert_by_key_test.py | 6 +-
.../tests/test_aggregation_merge_function.py | 4 +-
.../tests/test_field_aggregator_registry.py | 7 +-
.../pypaimon/tests/test_field_aggregators.py | 8 +-
.../tests/test_first_row_merge_function.py | 3 +-
.../tests/test_format_mosaic_reader_writer.py | 2 +-
.../tests/test_format_row_reader_writer.py | 8 +-
.../pypaimon/tests/test_limit_pushdown.py | 2 +-
.../pypaimon/tests/test_limited_record_reader.py | 2 +-
.../pypaimon/tests/test_merge_engine_dispatch.py | 7 +-
.../tests/test_optional_datasketches_dependency.py | 2 +-
.../tests/test_outer_projection_record_reader.py | 8 +-
.../tests/test_partial_update_merge_function.py | 3 +-
.../pypaimon/tests/test_ray_shuffle_helper.py | 22 +-
.../pypaimon/tests/test_sequence_field_read.py | 9 +-
.../pypaimon/tests/test_sort_merge_reader.py | 2 +-
.../pypaimon/tests/test_write_merge_buffer.py | 10 +-
.../tests/timestamp_key_serializer_test.py | 10 +-
.../tests/union_global_index_reader_test.py | 2 +-
.../tests/variant_row_sidecar_index_test.py | 2 +-
paimon-python/pypaimon/tests/variant_test.py | 4 +-
.../pypaimon/tests/vector_filter_exactness_test.py | 11 +-
.../pypaimon/tests/vector_index_dv_build_test.py | 4 +-
.../tests/vector_index_parallel_search_test.py | 6 +-
.../tests/vector_metric_consistency_test.py | 13 +-
.../pypaimon/tests/vector_scoring_test.py | 15 +-
.../pypaimon/tests/vector_search_filter_test.py | 215 ++--
paimon-python/pypaimon/tests/video_format_test.py | 6 +-
.../pypaimon/tests/vindex_batch_write_test.py | 4 +-
paimon-python/pypaimon/tests/vindex_input_test.py | 8 +-
.../pypaimon/tests/vindex_training_test.py | 5 +-
.../pypaimon/tests/vindex_vector_index_test.py | 6 +-
.../pypaimon/tests/write/append_row_kind_test.py | 2 +-
.../tests/write/conflict_detection_test.py | 2 +-
.../write/distributed_input_execution_test.py | 178 ++++
.../pypaimon/tests/write/dynamic_bucket_test.py | 2 +-
.../tests/write/execution_lifecycle_test.py | 538 ++++++++++
.../pypaimon/tests/write/object_row_kind_test.py | 2 +-
.../write/postpone_execution_boundary_test.py | 152 +++
.../write/retained_delete_nullability_test.py | 4 +-
.../tests/write/row_kind_nullability_test.py | 14 +-
.../pypaimon/tests/write/row_kind_write_test.py | 2 +-
.../write/simple_hash_bucket_assigner_test.py | 4 +-
.../pypaimon/tests/write/table_write_test.py | 14 +-
.../pypaimon/tests/write/write_buffer_test.py | 10 +-
.../tests/write/writer_file_cleanup_test.py | 4 +-
paimon-python/pypaimon/write/bucket_plan.py | 59 ++
.../{postpone_bucket.py => bucket_planning.py} | 66 +-
.../block_handle.py => write/commit_options.py} | 34 +-
paimon-python/pypaimon/write/input_adapter.py | 84 ++
paimon-python/pypaimon/write/input_routing.py | 335 +++++++
paimon-python/pypaimon/write/merge_inputs.py | 451 +++++++++
paimon-python/pypaimon/write/merge_types.py | 98 ++
.../pypaimon/write/postpone_batch_table_write.py | 197 +---
paimon-python/pypaimon/write/postpone_bucket.py | 342 +------
paimon-python/pypaimon/write/ray_datasink.py | 13 +-
paimon-python/pypaimon/write/row_input.py | 114 +++
paimon-python/pypaimon/write/table_commit.py | 144 +--
paimon-python/pypaimon/write/table_delete.py | 312 +-----
paimon-python/pypaimon/write/table_update.py | 749 ++------------
.../pypaimon/write/table_update_by_row_id.py | 1029 +-------------------
.../pypaimon/write/table_upsert_by_key.py | 567 +----------
paimon-python/pypaimon/write/table_write.py | 382 +-------
.../block_handle.py => write/update_validation.py} | 33 +-
paimon-python/pypaimon/write/write_builder.py | 36 +-
573 files changed, 11464 insertions(+), 13991 deletions(-)
diff --git a/docs/docs/program-api/file-cache.mdx b/docs/docs/program-api/file-cache.mdx
index 67320f7a17..ad6926ee4a 100644
--- a/docs/docs/program-api/file-cache.mdx
+++ b/docs/docs/program-api/file-cache.mdx
@@ -106,7 +106,7 @@ options = {
"local-cache.block-size": "1mb",
}
-catalog = pypaimon.create_catalog(options)
+catalog = pypaimon.CatalogFactory.create(options)
# Eligible file reads from tables loaded by this catalog use the cache
table = catalog.get_table("db.my_table")
diff --git a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
index 0cdcb98791..0d00ebf96b 100644
--- a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
@@ -179,16 +179,76 @@ public class JavaPyE2ETest {
FileStoreTable fileStoreTable = (FileStoreTable) table;
List<Split> splits =
new ArrayList<>(fileStoreTable.newSnapshotReader().read().dataSplits());
- TableRead read = fileStoreTable.newRead();
List<String> res =
getResult(
- read,
+ fileStoreTable.newRead(),
splits,
row -> DataFormatTestUtil.toStringNoRowKind(row, table.rowType()));
+ assertThat(res).hasSize(6);
+ for (String prefix :
+ Arrays.asList(
+ "1, Apple, Fruit, 1.5, ",
+ "2, Banana, Fruit, 0.8, ",
+ "3, Carrot, Vegetable, 0.6, ",
+ "4, Broccoli, Vegetable, 1.2, ",
+ "5, Chicken, Meat, 5.0, ",
+ "6, Beef, Meat, 8.0, ")) {
+ assertThat(res).anySatisfy(row -> assertThat(row).startsWith(prefix));
+ }
LOG.info("Read append table: {} row(s)", res.size());
}
}
+ @Test
+ @EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
+ public void testJavaWriteReadAppendTable() throws Exception {
+ for (String format : Arrays.asList("parquet", "orc", "avro")) {
+ Identifier identifier = identifier("mixed_test_append_tablej_" + format);
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("name", DataTypes.STRING())
+ .column("category", DataTypes.STRING())
+ .column("value", DataTypes.DOUBLE())
+ .partitionKeys("category")
+ .option("file.format", format)
+ .build();
+ catalog.dropTable(identifier, true);
+ catalog.createTable(identifier, schema, false);
+ FileStoreTable table = (FileStoreTable) catalog.getTable(identifier);
+ String[] names = {"Apple", "Banana", "Carrot", "Broccoli", "Chicken", "Beef"};
+ String[] categories = {"Fruit", "Fruit", "Vegetable", "Vegetable", "Meat", "Meat"};
+ double[] values = {1.5, 0.8, 0.6, 1.2, 5.0, 8.0};
+ try (StreamTableWrite write = table.newWrite(commitUser);
+ InnerTableCommit commit = table.newCommit(commitUser)) {
+ for (int i = 0; i < names.length; i++) {
+ write.write(
+ GenericRow.of(
+ i + 1,
+ BinaryString.fromString(names[i]),
+ BinaryString.fromString(categories[i]),
+ values[i]));
+ }
+ commit.commit(0, write.prepareCommit(true, 0));
+ }
+ List<Split> splits = new ArrayList<>(table.newSnapshotReader().read().dataSplits());
+ assertThat(
+ getResult(
+ table.newRead(),
+ splits,
+ row ->
+ DataFormatTestUtil.toStringNoRowKind(
+ row, table.rowType())))
+ .containsExactlyInAnyOrder(
+ "1, Apple, Fruit, 1.5",
+ "2, Banana, Fruit, 0.8",
+ "3, Carrot, Vegetable, 0.6",
+ "4, Broccoli, Vegetable, 1.2",
+ "5, Chicken, Meat, 5.0",
+ "6, Beef, Meat, 8.0");
+ }
+ }
+
@Test
@EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
public void testJavaWriteReadPkTable() throws Exception {
diff --git a/paimon-lance/src/test/java/org/apache/paimon/JavaPyLanceE2ETest.java b/paimon-lance/src/test/java/org/apache/paimon/JavaPyLanceE2ETest.java
index 9581074291..d10b07284c 100644
--- a/paimon-lance/src/test/java/org/apache/paimon/JavaPyLanceE2ETest.java
+++ b/paimon-lance/src/test/java/org/apache/paimon/JavaPyLanceE2ETest.java
@@ -216,6 +216,69 @@ public class JavaPyLanceE2ETest {
}
}
+ @Test
+ @EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
+ public void testJavaWriteReadAppendTableLance() throws Exception {
+ Identifier identifier = identifier("mixed_test_append_tablej_lance");
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("name", DataTypes.STRING())
+ .column("category", DataTypes.STRING())
+ .column("value", DataTypes.DOUBLE())
+ .partitionKeys("category")
+ .option("file.format", "lance")
+ .build();
+ catalog.dropTable(identifier, true);
+ catalog.createTable(identifier, schema, false);
+ FileStoreTable table = (FileStoreTable) catalog.getTable(identifier);
+ try (StreamTableWrite write = table.newWrite(commitUser);
+ InnerTableCommit commit = table.newCommit(commitUser)) {
+ write.write(createRow(1, "Apple", "Fruit", 1.5));
+ write.write(createRow(2, "Banana", "Fruit", 0.8));
+ write.write(createRow(3, "Carrot", "Vegetable", 0.6));
+ write.write(createRow(4, "Broccoli", "Vegetable", 1.2));
+ write.write(createRow(5, "Chicken", "Meat", 5.0));
+ write.write(createRow(6, "Beef", "Meat", 8.0));
+ commit.commit(0, write.prepareCommit(true, 0));
+ }
+ List<Split> splits = new ArrayList<>(table.newSnapshotReader().read().dataSplits());
+ assertThat(
+ getResult(
+ table.newRead(),
+ splits,
+ row -> DataFormatTestUtil.toStringNoRowKind(row, table.rowType())))
+ .containsExactlyInAnyOrder(
+ "1, Apple, Fruit, 1.5",
+ "2, Banana, Fruit, 0.8",
+ "3, Carrot, Vegetable, 0.6",
+ "4, Broccoli, Vegetable, 1.2",
+ "5, Chicken, Meat, 5.0",
+ "6, Beef, Meat, 8.0");
+ }
+
+ @Test
+ @EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
+ public void testReadAppendTableLance() throws Exception {
+ FileStoreTable table =
+ (FileStoreTable) catalog.getTable(identifier("mixed_test_append_tablep_lance"));
+ RowType readType = table.rowType().project("id", "name", "category", "value");
+ ReadBuilder readBuilder = table.newReadBuilder().withReadType(readType);
+ List<Split> splits = readBuilder.newScan().plan().splits();
+ assertThat(
+ getResult(
+ readBuilder.newRead(),
+ splits,
+ row -> DataFormatTestUtil.toStringNoRowKind(row, readType)))
+ .containsExactlyInAnyOrder(
+ "1, Apple, Fruit, 1.5",
+ "2, Banana, Fruit, 0.8",
+ "3, Carrot, Vegetable, 0.6",
+ "4, Broccoli, Vegetable, 1.2",
+ "5, Chicken, Meat, 5.0",
+ "6, Beef, Meat, 8.0");
+ }
+
@Test
@EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
public void testJavaWriteReadPkTableLance() throws Exception {
diff --git a/paimon-python/README.md b/paimon-python/README.md
index d72c46d2c0..9907c43bc7 100644
--- a/paimon-python/README.md
+++ b/paimon-python/README.md
@@ -414,6 +414,14 @@ Native reads return PyArrow batches through the Arrow C Data interface. Native
planner handles are used directly when available; splits refined by Python
index or shuffle logic are serialized through the stable split contract and
reconstructed by Rust. Nested projection and row-kind output are supported.
+Public builders and table APIs delegate to implementations under
+`pypaimon.execution.python` and `pypaimon.execution.native`. Shared plans,
+splits, schemas and commit messages remain independent of either backend.
+Python scanners, search readers and committers are initialized only when Python
+execution is selected. Index execution lives under these implementation
+directories too; public index APIs, results and file formats remain shared.
+See the [execution layout](pypaimon/execution/README.md) for the dependency and
+fallback boundaries.
Authorization captured by an original Native plan is evaluated in Rust;
Python-planned or serialized authorization rules retain the Python reader.
For both materialized
diff --git a/paimon-python/conftest.py b/paimon-python/conftest.py
index eeb1602b72..a52cb7b18e 100644
--- a/paimon-python/conftest.py
+++ b/paimon-python/conftest.py
@@ -141,7 +141,7 @@ def pytest_configure(config):
TableRead._try_native_batches = tracked_read
if _native_write_enabled():
- from pypaimon.write.native_write import NativeTableWrite
+ from pypaimon.execution.native.write.native_write import NativeTableWrite
def track_write(method):
original_write = getattr(NativeTableWrite, method)
@@ -173,8 +173,8 @@ def pytest_configure(config):
TableCommit._prepare_native_commit = tracked_prepare
if _native_update_enabled():
- from pypaimon.write.native_merge_into import NativeTableMergeInto
- from pypaimon.write.native_update import (
+ from pypaimon.execution.native.write.native_merge_into import NativeTableMergeInto
+ from pypaimon.execution.native.write.native_update import (
NativeBatchTableUpdate, NativePredicateTableUpdate,
NativeTableUpdateByRowId, NativeTableUpsert, NativeShardTableUpdator,
)
diff --git a/paimon-python/dev/run_mixed_tests.sh b/paimon-python/dev/run_mixed_tests.sh
index 84a54e21db..07fbb64bad 100755
--- a/paimon-python/dev/run_mixed_tests.sh
+++ b/paimon-python/dev/run_mixed_tests.sh
@@ -98,8 +98,11 @@ run_batched_java_write_tests() {
local result=0
local core_tests="org.apache.paimon.JavaPyE2ETest#testJavaWriteReadPkTable"
+ core_tests="${core_tests}+testJavaWriteReadAppendTable"
core_tests="${core_tests}+testJavaWriteDynamicBucketHashIndex"
core_tests="${core_tests}+testPKDeletionVectorWrite"
+ core_tests="${core_tests}+testPKDeletionVectorWriteMultiBatch"
+ core_tests="${core_tests}+testPKDeletionVectorWriteMultiBatchRawConvertable"
core_tests="${core_tests}+testBtreeIndexWrite"
core_tests="${core_tests}+testBtreeRawFallbackWrite"
core_tests="${core_tests}+testBitmapIndexWrite"
@@ -124,6 +127,7 @@ run_batched_java_write_tests() {
# lance has no Python wheel on <3.8; its readers are all skipped there.
if [[ "$PYTHON_MINOR" -ge 8 ]]; then
local lance_tests="org.apache.paimon.JavaPyLanceE2ETest#testJavaWriteReadPkTableLance"
+ lance_tests="${lance_tests}+testJavaWriteReadAppendTableLance"
lance_tests="${lance_tests}+testDataEvolutionWriteLance"
if ! run_maven_test_batch "paimon-lance Java write tests" "paimon-lance" "$lance_tests" -q; then
result=1
@@ -187,7 +191,7 @@ run_java_write_test() {
echo "Running Maven test for JavaPyE2ETest.testJavaWriteReadPkTable (Parquet/Orc/Avro)..."
echo "Note: Maven may download dependencies on first run, this may take a while..."
local parquet_result=0
- if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testJavaWriteReadPkTable+testJavaWriteDynamicBucketHashIndex -pl paimon-core -Drun.e2e.tests=true; then
+ if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testJavaWriteReadPkTable+testJavaWriteReadAppendTable+testJavaWriteDynamicBucketHashIndex -pl paimon-core -Drun.e2e.tests=true; then
echo -e "${GREEN}✓ Java write Parquet/Orc/Avro test completed successfully${NC}"
else
echo -e "${RED}✗ Java write Parquet/Orc/Avro test failed${NC}"
@@ -201,7 +205,7 @@ run_java_write_test() {
if [[ "$PYTHON_MINOR" -ge 8 ]]; then
echo "Running Maven test for JavaPyLanceE2ETest.testJavaWriteReadPkTableLance (Lance)..."
echo "Note: Maven may download dependencies on first run, this may take a while..."
- if mvn test -Dtest=org.apache.paimon.JavaPyLanceE2ETest#testJavaWriteReadPkTableLance -pl paimon-lance -Drun.e2e.tests=true; then
+ if mvn test -Dtest=org.apache.paimon.JavaPyLanceE2ETest#testJavaWriteReadPkTableLance+testJavaWriteReadAppendTableLance -pl paimon-lance -Drun.e2e.tests=true; then
echo -e "${GREEN}✓ Java write lance test completed successfully${NC}"
else
echo -e "${RED}✗ Java write lance test failed${NC}"
@@ -225,7 +229,7 @@ run_python_read_test() {
# Run the parameterized Python test method (runs for both Parquet/Orc/Avro and Lance)
echo "Running Python test for JavaPyReadWriteTest.test_read_pk_table..."
- if python -m pytest java_py_read_write_test.py::JavaPyReadWriteTest -k "test_read_pk_table or test_read_java_dynamic_bucket_hash_index" -v; then
+ if python -m pytest java_py_read_write_test.py::JavaPyReadWriteTest -k "test_read_pk_table or test_read_append_table or test_read_java_dynamic_bucket_hash_index" -v; then
echo -e "${GREEN}✓ Python test completed successfully${NC}"
# source deactivate
return 0
@@ -244,7 +248,7 @@ run_python_write_test() {
# Run the parameterized Python test method for writing data (pk table, includes bucket num assertion)
echo "Running Python test for JavaPyReadWriteTest (test_py_write_read_pk_table)..."
- if python -m pytest java_py_read_write_test.py::JavaPyReadWriteTest -k "test_py_write_read_pk_table or test_py_write_dynamic_bucket_hash_index or test_py_write_floating_sequence" -v; then
+ if python -m pytest java_py_read_write_test.py::JavaPyReadWriteTest -k "test_py_write_read_pk_table or test_py_write_read_append_table or test_py_write_dynamic_bucket_hash_index or test_py_write_floating_sequence" -v; then
echo -e "${GREEN}✓ Python write test completed successfully${NC}"
return 0
else
@@ -263,7 +267,7 @@ run_java_read_test() {
echo "Running Maven test for JavaPyE2ETest.testReadPkTable (Java Read Parquet/Orc/Avro)..."
echo "Note: Maven may download dependencies on first run, this may take a while..."
local parquet_result=0
- if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testReadPkTable+testReadPythonDynamicBucketHashIndex+testReadPythonFloatingSequence -pl paimon-core -Drun.e2e.tests=true -Dpython.version="$PYTHON_VERSION"; then
+ if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testReadPkTable+testReadAppendTable+testReadPythonDynamicBucketHashIndex+testReadPythonFloatingSequence -pl paimon-core -Drun.e2e.tests=true -Dpython.version="$PYTHON_VERSION"; then
echo -e "${GREEN}✓ Java read Parquet/Orc/Avro test completed successfully${NC}"
else
echo -e "${RED}✗ Java read Parquet/Orc/Avro test failed${NC}"
@@ -277,7 +281,7 @@ run_java_read_test() {
if [[ "$PYTHON_MINOR" -ge 8 ]]; then
echo "Running Maven test for JavaPyLanceE2ETest.testReadPkTableLance (Java Read Lance)..."
echo "Note: Maven may download dependencies on first run, this may take a while..."
- if mvn test -Dtest=org.apache.paimon.JavaPyLanceE2ETest#testReadPkTableLance -pl paimon-lance -Drun.e2e.tests=true -Dpython.version="$PYTHON_VERSION"; then
+ if mvn test -Dtest=org.apache.paimon.JavaPyLanceE2ETest#testReadPkTableLance+testReadAppendTableLance -pl paimon-lance -Drun.e2e.tests=true -Dpython.version="$PYTHON_VERSION"; then
echo -e "${GREEN}✓ Java read lance test completed successfully${NC}"
else
echo -e "${RED}✗ Java read lance test failed${NC}"
@@ -302,7 +306,7 @@ run_pk_dv_test() {
# Run the specific Java test method
echo "Running Maven test for JavaPyE2ETest.testPKDeletionVectorWrite..."
- if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testPKDeletionVectorWrite -pl paimon-core -q -Drun.e2e.tests=true; then
+ if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testPKDeletionVectorWrite+testPKDeletionVectorWriteMultiBatch+testPKDeletionVectorWriteMultiBatchRawConvertable -pl paimon-core -q -Drun.e2e.tests=true; then
echo -e "${GREEN}✓ Java test completed successfully${NC}"
else
echo -e "${RED}✗ Java test failed${NC}"
@@ -312,7 +316,7 @@ run_pk_dv_test() {
cd "$PAIMON_PYTHON_DIR"
# Run the specific Python test method
echo "Running Python test for JavaPyReadWriteTest.test_pk_dv_read..."
- if python -m pytest java_py_read_write_test.py::JavaPyReadWriteTest::test_pk_dv_read -v; then
+ if python -m pytest java_py_read_write_test.py::JavaPyReadWriteTest -k "test_pk_dv_read" -v; then
echo -e "${GREEN}✓ Python test completed successfully${NC}"
return 0
else
diff --git a/paimon-python/dev/test_packaging.py b/paimon-python/dev/test_packaging.py
index 688ad7d1da..7130077c1e 100644
--- a/paimon-python/dev/test_packaging.py
+++ b/paimon-python/dev/test_packaging.py
@@ -122,6 +122,71 @@ class PackagingTest(unittest.TestCase):
for name in ("pypaimon/sample/data/data.jsonl", "pypaimon/_version.py",
"pypaimon/benchmark/act/default_experiment.json"):
self.assertIn(name, names)
+ execution_files = (
+ "pypaimon/execution/dispatch.py",
+ "pypaimon/execution/vindex_input.py",
+ "pypaimon/execution/python/read/table_read.py",
+ "pypaimon/execution/python/read/table_scan.py",
+ "pypaimon/execution/python/read/auth_support.py",
+ "pypaimon/execution/python/read/scan_distribution.py",
+ "pypaimon/execution/python/read/format_table_read.py",
+ "pypaimon/execution/python/read/format_table_scan.py",
+ "pypaimon/execution/python/read/object_table_read.py",
+ "pypaimon/execution/python/read/object_table_scan.py",
+ "pypaimon/execution/python/read/system_table_read.py",
+ "pypaimon/execution/python/read/system_table_scan.py",
+ "pypaimon/execution/python/merge_engine_dispatch.py",
+ "pypaimon/execution/python/read/search/vector_search_read.py",
+ "pypaimon/execution/python/read/search/hybrid_search.py",
+ "pypaimon/execution/python/globalindex/btree/btree_index_reader.py",
+ "pypaimon/execution/python/globalindex/btree/block_handle.py",
+ "pypaimon/execution/python/globalindex/block_compression.py",
+ "pypaimon/execution/python/globalindex/key_serializer.py",
+ "pypaimon/execution/python/globalindex/vindex/vindex_vector_global_index_reader.py",
+ "pypaimon/execution/python/write/table_write.py",
+ "pypaimon/execution/python/write/table_commit.py",
+ "pypaimon/execution/python/write/format_table_write.py",
+ "pypaimon/execution/python/write/format_table_commit.py",
+ "pypaimon/execution/python/write/bucket_metadata.py",
+ "pypaimon/execution/native/read/table_read.py",
+ "pypaimon/execution/native/read/plan.py",
+ "pypaimon/execution/native/read/system_table.py",
+ "pypaimon/execution/native/read/vindex_reader.py",
+ "pypaimon/execution/native/write/native_write.py",
+ "pypaimon/execution/native/write/native_commit.py",
+ "pypaimon/execution/native/write/bucket_metadata.py",
+ "pypaimon/execution/native/write/postpone_bucket.py",
+ "pypaimon/read/blob_read_support.py",
+ "pypaimon/read/scan_request.py",
+ "pypaimon/table/source/search_types.py",
+ "pypaimon/globalindex/vindex_reader.py",
+ "pypaimon/write/bucket_planning.py",
+ "pypaimon/write/commit_options.py",
+ "pypaimon/write/input_routing.py",
+ )
+ for name in execution_files:
+ self.assertIn(name, names)
+ internal_aliases = (
+ "pypaimon/common/merge_engine_dispatch.py",
+ "pypaimon/read/native_plan.py",
+ "pypaimon/read/native_row_iterator.py",
+ "pypaimon/table/source/full_text_read.py",
+ "pypaimon/table/source/vector_search_scan.py",
+ "pypaimon/table/source/native_search_plan.py",
+ "pypaimon/table/system/native_system_table.py",
+ "pypaimon/globalindex/bitmap/__init__.py",
+ "pypaimon/globalindex/btree/__init__.py",
+ "pypaimon/globalindex/full_text/__init__.py",
+ "pypaimon/globalindex/lumina/__init__.py",
+ "pypaimon/globalindex/vindex/__init__.py",
+ "pypaimon/globalindex/btree/block_reader.py",
+ "pypaimon/globalindex/btree_format/block_handle.py",
+ "pypaimon/globalindex/bitmap/bitmap_index_writer.py",
+ "pypaimon/globalindex/block_compression.py",
+ "pypaimon/globalindex/vindex_input.py",
+ )
+ for name in internal_aliases:
+ self.assertNotIn(name, names)
for suffix in ("/LICENSE", "/NOTICE"):
self.assertTrue(any(".dist-info/" in name and name.endswith(suffix)
for name in names))
@@ -130,6 +195,10 @@ class PackagingTest(unittest.TestCase):
prefix = "pypaimon-{}/".format(self.version)
for name in ("pyproject.toml", "setup.py", "LICENSE", "NOTICE", "README.md"):
self.assertIn(prefix + name, names)
+ for name in execution_files:
+ self.assertIn(prefix + name, names)
+ for name in internal_aliases:
+ self.assertNotIn(prefix + name, names)
self.assertFalse(any("requirements.txt" in name or "requirements-dev.txt" in name
or name.startswith((prefix + "pypaimon/tests/",
prefix + "pypaimon/acceptance/"))
diff --git a/paimon-python/pypaimon/acceptance/incremental_diff_acceptance_test.py b/paimon-python/pypaimon/acceptance/incremental_diff_acceptance_test.py
index 6f0695c5fa..b89261ba94 100644
--- a/paimon-python/pypaimon/acceptance/incremental_diff_acceptance_test.py
+++ b/paimon-python/pypaimon/acceptance/incremental_diff_acceptance_test.py
@@ -34,10 +34,8 @@ import pyarrow as pa
from pypaimon import CatalogFactory, Schema
from pypaimon.manifest.manifest_file_manager import ManifestFileManager
from pypaimon.manifest.manifest_list_manager import ManifestListManager
-from pypaimon.read.scanner.append_table_split_generator import \
- AppendTableSplitGenerator
-from pypaimon.read.scanner.incremental_diff_scanner import \
- IncrementalDiffScanner
+from pypaimon.execution.python.read.scanner.append_table_split_generator import (AppendTableSplitGenerator)
+from pypaimon.execution.python.read.scanner.incremental_diff_scanner import (IncrementalDiffScanner)
class IncrementalDiffAcceptanceTest(unittest.TestCase):
diff --git a/paimon-python/pypaimon/benchmark/blob_index_cold_load.py b/paimon-python/pypaimon/benchmark/blob_index_cold_load.py
index be06d78028..3b33be9429 100644
--- a/paimon-python/pypaimon/benchmark/blob_index_cold_load.py
+++ b/paimon-python/pypaimon/benchmark/blob_index_cold_load.py
@@ -39,7 +39,7 @@ from unittest.mock import patch
import pyarrow as pa
from pypaimon import CatalogFactory, Schema
-from pypaimon.read.reader import format_blob_reader
+from pypaimon.execution.python.read.reader import format_blob_reader
class _IndexProbe:
diff --git a/paimon-python/pypaimon/casting/data_type_casts.py b/paimon-python/pypaimon/casting/data_type_casts.py
index 027f3293c7..1086ffefc1 100644
--- a/paimon-python/pypaimon/casting/data_type_casts.py
+++ b/paimon-python/pypaimon/casting/data_type_casts.py
@@ -246,14 +246,10 @@ def _schema_evolution_cast_supported(source_type, target_type) -> bool:
cached = _EXECUTABLE_CAST_CACHE.get(cache_key)
if cached is not None:
return cached
- # Probe a one-row (null-valued) array rather than an empty one. An empty
- # array only resolves the cast kernel; some kernels additionally reject the
- # target type parameters on any non-empty input -- e.g. INT -> DECIMAL(10,2)
- # has a kernel but needs precision >= 12 to hold an int's range at scale 2,
- # so an empty probe passes yet the read later fails with ArrowInvalid. A
- # single null row triggers that static type-parameter validation while
- # avoiding per-value parse/overflow errors (which ``safe=False`` -- matching
- # the read path -- tolerates anyway).
+ # Probe a one-row (null-valued) array rather than an empty one. Some Arrow
+ # kernels validate target parameters only for a non-empty input. Using the
+ # read executor also includes Java's per-value overflow-to-NULL rules, such
+ # as integer-to-DECIMAL targets narrower than the source type's full range.
try:
cast_array_for_schema_evolution(
pa.nulls(1, type=source_pa), target_pa, source_type, target_type)
diff --git a/paimon-python/pypaimon/casting/decimal_cast.py b/paimon-python/pypaimon/casting/decimal_cast.py
index eb7170dbea..eaa0b3a6d4 100644
--- a/paimon-python/pypaimon/casting/decimal_cast.py
+++ b/paimon-python/pypaimon/casting/decimal_cast.py
@@ -37,8 +37,9 @@ def cast_decimal(array, target):
fails the conversion instead of silently becoming NULL.
"""
floating = pa.types.is_floating(array.type)
+ integer = pa.types.is_signed_integer(array.type)
text = pa.types.is_string(array.type)
- if not pa.types.is_decimal(target) or not (floating or text or pa.types.is_decimal(array.type)):
+ if not pa.types.is_decimal(target) or not (integer or floating or text or pa.types.is_decimal(array.type)):
return None
values = []
# The caller's rounding traps and exponent limits must not affect reads.
@@ -53,6 +54,8 @@ def cast_decimal(array, target):
continue
if floating:
value = java_double_decimal_text(value)
+ if integer:
+ value = Decimal(value)
if floating or text:
if _DECIMAL_TEXT.fullmatch(value) is None:
raise pa.ArrowInvalid('Invalid decimal schema-evolution value: %s' % value)
diff --git a/paimon-python/pypaimon/casting/schema_evolution.py b/paimon-python/pypaimon/casting/schema_evolution.py
index 1a624cb6fa..4615ee7574 100644
--- a/paimon-python/pypaimon/casting/schema_evolution.py
+++ b/paimon-python/pypaimon/casting/schema_evolution.py
@@ -18,13 +18,13 @@
"""Java cast rules shared by old-file readers."""
import math
-import json
import pyarrow as pa
from pypaimon.casting.decimal_cast import cast_decimal
from pypaimon.casting.temporal_cast import cast_temporal
from pypaimon.casting.temporal_string import temporal_to_string
+from pypaimon.schema.data_types import ArrayType, MapType, PyarrowFieldParser, RowType
from pypaimon.table.row.generic_row import _parse_type_precision_scale
@@ -122,31 +122,25 @@ def _integer_to_timestamp(array, target_type):
return pa.array(values, type=pa.int64()).cast(target_type)
-def _native_schema_cast(array, target_type, source_paimon_type, target_paimon_type):
- if source_paimon_type is None or target_paimon_type is None:
- return None
- from pypaimon.read.native_plan import native_runtime_available
- if not native_runtime_available():
- return None
- from pypaimon_rust.data import cast_array_for_schema_evolution as native_cast
- source = json.dumps(source_paimon_type.to_dict())
- target = json.dumps(target_paimon_type.to_dict())
- if isinstance(array, pa.ChunkedArray):
- chunks = array.chunks or [pa.array([], type=array.type)]
- # Runtime cast results may be NULL despite a logical NOT NULL child.
- return pa.chunked_array([native_cast(chunk, source, target) for chunk in chunks])
- return native_cast(array, source, target)
+def _read_arrow_type(data_type):
+ if isinstance(data_type, RowType):
+ return pa.struct([pa.field(field.name, _read_arrow_type(field.type)) for field in data_type.fields])
+ if isinstance(data_type, ArrayType):
+ return pa.list_(pa.field('element', _read_arrow_type(data_type.element)))
+ if isinstance(data_type, MapType):
+ return pa.map_(pa.field('key', _read_arrow_type(data_type.key), nullable=False),
+ pa.field('value', _read_arrow_type(data_type.value)))
+ return PyarrowFieldParser.from_paimon_type(data_type)
def read_arrow_schema(fields):
- """Use core runtime nullability while retaining logical write constraints."""
- from pypaimon.read.native_plan import native_runtime_available
- if not native_runtime_available():
- from pypaimon.schema.data_types import PyarrowFieldParser
- return PyarrowFieldParser.from_paimon_schema(fields)
- from pypaimon_rust.data import read_arrow_schema as native_schema
- schema = native_schema(json.dumps([field.to_dict() for field in fields]))
- return pa.schema([field.remove_metadata() for field in schema])
+ """Keep runtime NULLs representable without changing logical write constraints.
+
+ Old files can lack a requested field, and Java casts can produce NULL on
+ overflow. ROW fields, ARRAY elements and MAP values therefore stay nullable
+ in read results, including declared NOT NULL children.
+ """
+ return pa.schema([pa.field(field.name, _read_arrow_type(field.type)) for field in fields])
def cast_array_for_schema_evolution(array, target_type, source_paimon_type=None, target_paimon_type=None):
@@ -161,9 +155,6 @@ def cast_array_for_schema_evolution(array, target_type, source_paimon_type=None,
precision overflow after rounding.
LTZ conversions use the process local zone, including Java gap/overlap rules.
"""
- native = _native_schema_cast(array, target_type, source_paimon_type, target_paimon_type)
- if native is not None:
- return native
if pa.types.is_string(target_type):
strings = temporal_to_string(array, source_paimon_type, target_paimon_type)
if strings is not None:
diff --git a/paimon-python/pypaimon/daft/daft_datasink.py b/paimon-python/pypaimon/daft/daft_datasink.py
index 9bf0dbe8ad..675276efff 100644
--- a/paimon-python/pypaimon/daft/daft_datasink.py
+++ b/paimon-python/pypaimon/daft/daft_datasink.py
@@ -438,10 +438,10 @@ def make_fixed_bucket_udf(table: FileStoreTable):
from daft import DataType, Series
from pypaimon.schema.data_types import PyarrowFieldParser
- from pypaimon.write.row_key_extractor import FixedBucketRowKeyExtractor
+ from pypaimon.write.input_routing import FixedBucketInputRouter
schema = PyarrowFieldParser.from_paimon_schema(table.fields)
- extractor = FixedBucketRowKeyExtractor(table.table_schema)
+ extractor = FixedBucketInputRouter(table.table_schema)
@daft.func.batch(return_dtype=DataType.int32())
def _fixed_bucket(*columns) -> Series:
@@ -468,10 +468,10 @@ def make_dynamic_routing_udf(
from pypaimon.index.dynamic_bucket import compute_assigner, to_signed_int32
from pypaimon.schema.data_types import PyarrowFieldParser
- from pypaimon.write.row_key_extractor import DynamicBucketRowKeyExtractor
+ from pypaimon.write.input_routing import DynamicBucketInputRouter
schema = PyarrowFieldParser.from_paimon_schema(table.fields)
- extractor = DynamicBucketRowKeyExtractor(table.table_schema)
+ extractor = DynamicBucketInputRouter(table.table_schema)
return_fields = {
assigner_column: DataType.int32(),
key_hash_column: DataType.int32(),
@@ -528,7 +528,7 @@ def make_dynamic_bucket_assignment_udf(
from daft import DataType, Series
from pypaimon.schema.data_types import PyarrowFieldParser
- from pypaimon.write.row_key_extractor import DynamicBucketRowKeyExtractor
+ from pypaimon.write.input_routing import DynamicBucketInputRouter
catalog_options = _extract_catalog_options(table)
table_identifier = _extract_identifier(table)
@@ -582,26 +582,28 @@ def make_dynamic_bucket_assignment_udf(
table_path,
schema_token,
)
- extractor = DynamicBucketRowKeyExtractor(
- worker_table.table_schema,
- table=worker_table,
+ from pypaimon.index.dynamic_bucket import HashBucketAssigner, resolve_bucket_snapshot
+
+ router = DynamicBucketInputRouter(worker_table.table_schema)
+ snapshot = resolve_bucket_snapshot(worker_table, base_snapshot_id)
+ options = worker_table.options
+ assigner = HashBucketAssigner(
+ worker_table,
num_channels=num_channels,
num_assigners=num_assigners,
assign_id=assign_id,
ignore_existing=ignore_existing,
- maintain_index=False,
- base_snapshot_id=base_snapshot_id,
+ target_bucket_row_number=options.dynamic_bucket_target_row_num(),
+ max_buckets_num=options.dynamic_bucket_max_buckets(),
+ snapshot=snapshot,
)
buckets = []
offset = 0
for batch in arrow_table.to_batches():
batch_hashes = key_hash_values[offset:offset + batch.num_rows]
- _, batch_buckets, _ = (
- extractor.extract_partition_bucket_status_from_hashes_batch(
- batch, batch_hashes
- )
- )
- buckets.extend(batch_buckets)
+ partitions, partition_hashes = router.extract_partition_hashes_batch(batch)
+ buckets.extend(bucket for bucket, _ in assigner.assign_batch(
+ partitions, partition_hashes, batch_hashes))
offset += batch.num_rows
arrays = [
arrow_table.column(field.name).combine_chunks()
@@ -609,7 +611,7 @@ def make_dynamic_bucket_assignment_udf(
]
arrays.append(pa.array(buckets, type=pa.int32()))
arrays.append(pa.array(
- [extractor.base_snapshot_id] * arrow_table.num_rows,
+ [snapshot.id if snapshot is not None else 0] * arrow_table.num_rows,
type=pa.int64(),
))
return Series.from_arrow(
diff --git a/paimon-python/pypaimon/execution/README.md b/paimon-python/pypaimon/execution/README.md
new file mode 100644
index 0000000000..995d90eb75
--- /dev/null
+++ b/paimon-python/pypaimon/execution/README.md
@@ -0,0 +1,122 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Execution implementations
+
+PyPaimon's public builders and table operations remain in `pypaimon.read`,
+`pypaimon.write` and `pypaimon.table`. Their signatures and configuration are
+independent of the implementation that executes an operation.
+
+```text
+pypaimon/
+ read/ Public read/scan APIs and shared read contracts
+ write/ Public write/commit/update APIs and shared inputs
+ execution/
+ dispatch.py Preparation and safe backend selection
+ python/
+ read/ Python scanning, file readers and merge execution
+ Format/object file I/O and system-table filtering
+ search/ Vector, text, hybrid search and candidate filtering
+ write/ Python writers, committers and update execution
+ Format-table writing and explicit cleanup
+ globalindex/ Python index readers, writers, build and evaluation
+ merge_engine_dispatch.py Python merge-function selection for read and write
+ native/
+ runtime.py Lazy runtime discovery and process safety
+ table.py Rust table/configuration bridge
+ read/ Native scan/read/search adapters
+ write/ Native write/commit/update/index adapters
+ vindex_input.py Shared vector-index streams and I/O callbacks
+```
+
+## Dependencies
+
+Both implementations consume shared requests, predicates, schemas, plans, splits,
+commit messages and input adapters. These contracts stay outside either backend.
+Neither implementation imports the other. Public APIs retain configuration and
+select an executor when needed. Python scanners, readers, writers and committers
+are created only when Python execution is selected. Direct access to a Python
+scanner, writer router or committer through an existing public property explicitly
+selects that implementation. Attribute inspection and configuration getters do
+not select Python. Public index entry points, interfaces, options, results and
+shared metadata remain in `globalindex`; shared search plans and splits remain in
+`table/source`. Python index file codecs, compression and reader/writer helpers
+live in `execution/python/globalindex`. Vector-index stream ownership and I/O
+callbacks are shared by both backends through `execution/vindex_input.py`.
+Public Format, Object and System table classes keep their existing paths and
+delegate file enumeration, decoding and filtering to Python read execution.
+Native system-table planning uses `execution/native/read/system_table.py`.
+Scan authorization and request validation remain shared; Python file-scanner
+pruning, sharding and slicing execute under `execution/python/read`.
+
+Native adapters convert requests and results and invoke `pypaimon_rust`. General
+execution algorithms belong in Rust Core. The optional extension is loaded only
+when a Native operation is requested. Importing the Python implementation does
+not require or initialize it.
+
+Python schema evolution uses Python cast rules and builds Arrow read schemas
+directly, including runtime child nullability. It does not select a cast engine
+based on whether the Native extension is installed.
+
+Ray and Daft input grouping share bucket arithmetic, binary-row size estimation
+and fixed/dynamic hash routing in `write/bucket_planning.py` and
+`write/input_routing.py`. These operators hold input configuration and partition
+counts, with no file scanner, reader, writer or index maintainer. Native postpone
+and dynamic bucket coordinators load existing buckets from Rust's complete file
+plan, pinned to the restore snapshot and partition. Dynamic assignment still
+requires a complete HASH index for existing data buckets. Python coordinators
+use their Python metadata loader. Selection finishes before assigning input
+keys. Python's Arrow grouping capability probe runs only when its grouping
+implementation is used.
+
+## Selection and failure boundaries
+
+Planning, reading, writing and committing retain their independent options. A
+Native plan can feed a Python reader, and Python writers can prepare messages for
+a Native committer. Transportable `Plan`, `Split` and `CommitMessage` contracts
+connect these stages.
+
+`dispatch.py` owns preparation fallbacks for writers, committers and updates.
+MERGE selects Python only for an explicit unsupported result, because preparing
+it may consume user input. Native read/scan adapters keep their existing
+capability checks and preparation boundaries. An empty successful result is distinct from an
+unsupported operation. Fork safety errors always propagate.
+
+Reader iteration errors end that read. Once a writer has accepted its first
+mutation, it cannot switch to a Python writer. Actual update, MERGE and snapshot
+publication calls run outside preparation fallback handlers. A failure may occur
+after a write or publication succeeded, so execution is never replayed through
+another implementation and prepared files are preserved.
+
+Native search keeps a lazy Python factory for transported plans or Ray split
+workers. Creating a Native scan or reader, executing its Native plan, and
+serializing it do not construct that Python executor. Search execution errors
+propagate without a second search through Python. Explicit split reads and Ray
+split workers select their Python factory when invoked, not when their methods
+or optional attributes are inspected.
+
+## Tests
+
+Existing tests continue through the public APIs. Backend-specific tests patch
+the implementation module where an operation executes. `execution_dispatch_test`
+checks backend dependencies, lazy imports, empty results, preparation failures,
+fork errors, public updater types and prevention of write replay.
+The lazy read/search and execution lifecycle tests also reject Python executor
+construction on Native paths. Global index boundary tests verify those imports
+in fresh processes and preserve the exported public index contracts.
diff --git a/paimon-python/pypaimon/read/scanner/__init__.py b/paimon-python/pypaimon/execution/__init__.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/__init__.py
copy to paimon-python/pypaimon/execution/__init__.py
diff --git a/paimon-python/pypaimon/execution/dispatch.py b/paimon-python/pypaimon/execution/dispatch.py
new file mode 100644
index 0000000000..7386fd087d
--- /dev/null
+++ b/paimon-python/pypaimon/execution/dispatch.py
@@ -0,0 +1,266 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Select execution implementations before data is read or published.
+
+Only preparation belongs in the fallback boundary. An execution failure must
+reach the caller: a writer or committer may already have produced side effects.
+The independent plan/read/write/commit options keep their existing meaning.
+"""
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def propagate_fork_safety_error(error):
+ from pypaimon.execution.native.runtime import _raise_if_native_fork_safety_error
+ _raise_if_native_fork_safety_error(error)
+
+
+def try_prepare_native(factory, *args, **kwargs):
+ """Return a prepared executor, or None when preparation permits fallback."""
+ try:
+ return factory(*args, **kwargs)
+ except Exception as error:
+ propagate_fork_safety_error(error)
+ logger.debug('Native execution preparation failed; using Python: %s', error)
+ return None
+
+
+def create_python_write(table, commit_user, static_partition=None, stream=False,
+ restore_snapshot_id=None, fixed_bucket=False, bucket_plan=None):
+ if fixed_bucket:
+ from pypaimon.write.postpone_batch_table_write import PostponeFixedBucketBatchTableWrite
+ return PostponeFixedBucketBatchTableWrite(table, commit_user, static_partition, bucket_plan)
+ from pypaimon.write.table_write import BatchTableWrite, StreamTableWrite
+ if stream:
+ return StreamTableWrite(table, commit_user, restore_snapshot_id=restore_snapshot_id)
+ return BatchTableWrite(table, commit_user, static_partition, restore_snapshot_id=restore_snapshot_id)
+
+
+def create_write(table, commit_user, static_partition=None, stream=False,
+ restore_snapshot_id=None, fixed_bucket=False, bucket_plan=None):
+ from pypaimon.read.merge_engine_support import check_sequence_field_supported
+ check_sequence_field_supported(table)
+ if table.options.native_write_enabled():
+ def prepare():
+ from pypaimon.execution.native.write.native_write import create_native_write
+ return create_native_write(
+ table, commit_user, static_partition, stream,
+ restore_snapshot_id=restore_snapshot_id, fixed_bucket=fixed_bucket, bucket_plan=bucket_plan)
+ native = try_prepare_native(prepare)
+ if native is not None:
+ return native
+ return create_python_write(
+ table, commit_user, static_partition, stream, restore_snapshot_id, fixed_bucket, bucket_plan)
+
+
+def switch_writer_to_python(owner):
+ """Switch only before the native writer has accepted its first mutation."""
+ if owner._python_writer is not None:
+ return owner._python_writer
+ if owner._written:
+ raise RuntimeError('Cannot switch to the Python writer after native data was written')
+ owner._native_writer.close()
+ owner._native_writer = None
+ owner._python_writer = owner._new_python_writer()
+ return owner._python_writer
+
+
+def create_postpone_bucket_planner(table, known_num_buckets=None):
+ """Resolve metadata before Ray materializes or writes any input."""
+ if table.options.native_write_enabled():
+ def prepare():
+ from pypaimon.execution.native.write.postpone_bucket import create_native_postpone_bucket_planner
+ return create_native_postpone_bucket_planner(table, known_num_buckets)
+ native = try_prepare_native(prepare)
+ if native is not None:
+ return native
+ from pypaimon.write.postpone_bucket import PostponeBucketPlanner
+ return PostponeBucketPlanner(table, known_num_buckets=known_num_buckets)
+
+
+def dynamic_bucket_data_buckets(table, partition, snapshot):
+ """Resolve complete bucket metadata before assigning any input keys."""
+ if table.options.native_write_enabled():
+ def prepare():
+ from pypaimon.execution.native.write.bucket_metadata import dynamic_bucket_data_buckets as native_buckets
+ return native_buckets(table, partition, snapshot)
+ native = try_prepare_native(prepare)
+ if native is not None:
+ return native
+ from pypaimon.execution.python.write.bucket_metadata import dynamic_bucket_data_buckets as python_buckets
+ return python_buckets(table, partition, snapshot)
+
+
+def prepare_native_commit(owner, messages, *, batch=False):
+ if not owner.table.options.native_commit_enabled():
+ return None
+
+ def prepare():
+ from pypaimon.execution.native.write.commit import prepare_native_commit as prepare_commit
+ return prepare_commit(owner, messages, batch=batch)
+ return try_prepare_native(prepare)
+
+
+def prepare_native_write_operation(operation, table, commit_user, *args, **kwargs):
+ names = {
+ 'update': 'create_native_update',
+ 'update_by_row_id': 'create_native_update_by_row_id',
+ 'shard_update': 'create_native_shard_updator',
+ 'upsert': 'create_native_upsert',
+ 'predicate_update': 'create_native_predicate_update',
+ 'predicate_delete': 'create_native_predicate_delete',
+ 'delete': 'create_native_delete',
+ }
+ name = names[operation]
+
+ def prepare():
+ from pypaimon.execution.native.write import native_update
+ native = getattr(native_update, name)(table, commit_user, *args, **kwargs)
+ if native is not None and operation == 'update_by_row_id':
+ from pypaimon.write.table_update_by_row_id import TableUpdateByRowId
+ # Preserve the public return type without inheriting Python's file
+ # rewriting implementation in the Native adapter.
+ TableUpdateByRowId.register(native_update.NativeTableUpdateByRowId)
+ return native
+ return try_prepare_native(prepare)
+
+
+def try_native_predicate_row_ids(scan_table, predicate, splits):
+ def prepare():
+ from pypaimon.execution.native.write.native_update import native_predicate_row_ids
+ return native_predicate_row_ids(scan_table, predicate, splits)
+ return try_prepare_native(prepare)
+
+
+def prepare_native_merge_into(*args, **kwargs):
+ # MERGE preparation can consume user input. Preserve its existing boundary:
+ # only an explicit unsupported result selects Python; failures propagate.
+ from pypaimon.execution.native.write.native_merge_into import create_native_merge_into
+ return create_native_merge_into(*args, **kwargs)
+
+
+def python_commit(owner):
+ """Create Python commit resources only after that backend is selected."""
+ if owner._python_commit is None:
+ from pypaimon.execution.python.write.table_commit import create_python_commit
+ owner._python_commit = create_python_commit(owner)
+ return owner._python_commit
+
+
+def commit_table_messages(
+ owner,
+ commit_messages,
+ commit_identifier,
+ snapshot_properties=None):
+ """Publish prepared messages while preserving files on any failure."""
+ non_empty_messages = [msg for msg in commit_messages if not msg.is_empty()]
+ commit_kwargs = {
+ "commit_messages": non_empty_messages,
+ "commit_identifier": commit_identifier,
+ }
+ if snapshot_properties is not None:
+ commit_kwargs["snapshot_properties"] = snapshot_properties
+ native_kwargs = ({} if snapshot_properties is None else
+ {"snapshot_properties": snapshot_properties})
+
+ # Never abort files in response to a commit exception. Preserving
+ # possible orphan files is safer than deleting files which another
+ # attempt may still commit or which a snapshot may already reference.
+ if owner.overwrite_partition is not None:
+ # Always call overwrite() even with empty messages, so that
+ # FileStoreCommit.overwrite can handle the empty case properly
+ # (e.g. static overwrite with empty data should delete the partition).
+ logger.info(
+ "Committing overwrite to table %s, %d non-empty messages",
+ owner.table.identifier, len(non_empty_messages)
+ )
+ prepared = owner._prepare_native_commit(non_empty_messages)
+ if prepared is not None:
+ native, messages = prepared
+ # Keep publication failures outside the preparation fallback.
+ native.commit(messages, **native_kwargs)
+ return
+ python_commit(owner).overwrite(
+ overwrite_partition=owner.overwrite_partition,
+ **commit_kwargs)
+ else:
+ if (not non_empty_messages
+ and owner.table.options.snapshot_ignore_empty_commit()):
+ return
+ logger.info(
+ "Committing table %s, %d non-empty messages",
+ owner.table.identifier, len(non_empty_messages)
+ )
+ prepared = owner._prepare_native_commit(non_empty_messages)
+ if prepared is not None:
+ native, messages = prepared
+ # Mutation is deliberately outside the fallback boundary:
+ # an exception can mean the snapshot was already published.
+ if owner._native_commit_is_batch:
+ native.commit(messages, **native_kwargs)
+ else:
+ native.commit(commit_identifier, messages, **native_kwargs)
+ return
+ python_commit(owner).commit(**commit_kwargs)
+
+
+def abort_table_commit(owner, messages):
+ prepared = owner._prepare_native_commit(messages)
+ if prepared is not None:
+ native, wire_messages = prepared
+ native.abort(wire_messages)
+ return
+ python_commit(owner).abort(messages)
+
+
+def truncate_table_commit(owner, commit_identifier):
+ prepared = owner._prepare_native_commit([], batch=True)
+ if prepared is not None:
+ prepared[0].truncate_table()
+ return
+ python_commit(owner).truncate_table(commit_identifier)
+
+
+def truncate_partitions_commit(owner, partitions, commit_identifier):
+ prepared = owner._prepare_native_commit([], batch=True)
+ if prepared is not None:
+ prepared[0].truncate_partitions(partitions)
+ return
+ python_commit(owner).drop_partitions(partitions, commit_identifier)
+
+
+def close_table_commit(owner):
+ """Close only initialized backends and each registered callback once."""
+ if owner._closed:
+ return
+ owner._closed = True
+ try:
+ if owner._native_commit is not None:
+ owner._native_commit.close()
+ finally:
+ try:
+ if owner._python_commit is not None:
+ owner._python_commit.close()
+ finally:
+ for callback in owner._commit_callbacks:
+ try:
+ callback.close()
+ except Exception:
+ pass
diff --git a/paimon-python/pypaimon/write/commit/__init__.py b/paimon-python/pypaimon/execution/native/__init__.py
similarity index 100%
rename from paimon-python/pypaimon/write/commit/__init__.py
rename to paimon-python/pypaimon/execution/native/__init__.py
diff --git a/paimon-python/pypaimon/write/writer/__init__.py b/paimon-python/pypaimon/execution/native/read/__init__.py
similarity index 100%
rename from paimon-python/pypaimon/write/writer/__init__.py
rename to paimon-python/pypaimon/execution/native/read/__init__.py
diff --git a/paimon-python/pypaimon/execution/native/read/builder.py b/paimon-python/pypaimon/execution/native/read/builder.py
new file mode 100644
index 0000000000..c8bcabe902
--- /dev/null
+++ b/paimon-python/pypaimon/execution/native/read/builder.py
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Translate canonical read requests into Native builders."""
+
+import json
+from typing import List, Optional
+
+from pypaimon.common.predicate import Predicate
+from pypaimon.execution.native import table as native_table
+from pypaimon.schema.data_types import DataField, RowType
+
+
+def _predicate_to_native(predicate: Predicate) -> dict:
+ """Convert PyPaimon's predicate tree to pypaimon-rust's dict API."""
+ if predicate.method in ('and', 'or'):
+ children = predicate.literals or []
+ if not children:
+ raise ValueError("Native compound predicate requires children")
+ return {
+ 'method': predicate.method,
+ 'children': [_predicate_to_native(child) for child in children],
+ }
+ result = {
+ 'method': predicate.method,
+ 'field': predicate.field,
+ 'literals': list(predicate.literals or []),
+ }
+ if predicate.field_path:
+ result['field'] = predicate.field_path[0]
+ if len(predicate.field_path) > 1:
+ result['nested_path'] = list(predicate.field_path[1:])
+ return result
+
+
+def _native_read_builder(table):
+ return native_table._native_table(table).new_read_builder()
+
+
+def _configure_native_read_builder(builder, predicate, limit, projection,
+ read_type: Optional[List[DataField]] = None,
+ include_row_kind=False):
+ if read_type is not None:
+ builder = builder.with_read_type(
+ json.dumps(RowType(True, read_type).to_dict()))
+ if projection is not None and read_type is None:
+ builder = builder.with_projection(projection)
+ if predicate is not None:
+ builder = builder.with_filter(_predicate_to_native(predicate))
+ if limit is not None:
+ builder = builder.with_limit(limit)
+ if include_row_kind:
+ builder = builder.with_include_row_kind(True)
+ return builder
diff --git a/paimon-python/pypaimon/table/source/native_full_text_search.py b/paimon-python/pypaimon/execution/native/read/native_full_text_search.py
similarity index 91%
rename from paimon-python/pypaimon/table/source/native_full_text_search.py
rename to paimon-python/pypaimon/execution/native/read/native_full_text_search.py
index 74b869e734..9ba1505c62 100644
--- a/paimon-python/pypaimon/table/source/native_full_text_search.py
+++ b/paimon-python/pypaimon/execution/native/read/native_full_text_search.py
@@ -19,12 +19,14 @@
from pypaimon.catalog.table_query_auth import reject_search_under_query_auth
from pypaimon.common.options.core_options import GlobalIndexSearchMode
-from pypaimon.globalindex.full_text.native_full_text_global_index_reader import NativeFullTextIndexOptions
+from pypaimon.globalindex.full_text_options import NativeFullTextIndexOptions
from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult
-from pypaimon.read.native_plan import _native_table, _predicate_to_native, native_runtime_available
+from pypaimon.execution.native.table import (_native_table)
+from pypaimon.execution.native.read.builder import (_predicate_to_native)
+from pypaimon.execution.native.runtime import (native_runtime_available)
from pypaimon.table.file_store_table import FileStoreTable
from pypaimon.table.source.primary_key_scored_result import PrimaryKeyScoredResult
-from pypaimon.write.native_commit import _rest_catalog_supported
+from pypaimon.execution.native.write.native_commit import _rest_catalog_supported
def _native_full_text_builder(builder, require_limit=True):
diff --git a/paimon-python/pypaimon/table/source/native_hybrid_search.py b/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py
similarity index 89%
rename from paimon-python/pypaimon/table/source/native_hybrid_search.py
rename to paimon-python/pypaimon/execution/native/read/native_hybrid_search.py
index 863030bb0f..b40a7676d2 100644
--- a/paimon-python/pypaimon/table/source/native_hybrid_search.py
+++ b/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py
@@ -18,11 +18,13 @@
"""Transport an existing local hybrid query to one Rust core operation."""
from pypaimon.catalog.table_query_auth import reject_search_under_query_auth
-from pypaimon.globalindex.full_text.native_full_text_global_index_reader import NativeFullTextIndexOptions
+from pypaimon.globalindex.full_text_options import NativeFullTextIndexOptions
from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult
-from pypaimon.read.native_plan import _native_table, _predicate_to_native, native_runtime_available
+from pypaimon.execution.native.table import (_native_table)
+from pypaimon.execution.native.read.builder import (_predicate_to_native)
+from pypaimon.execution.native.runtime import (native_runtime_available)
from pypaimon.table.file_store_table import FileStoreTable
-from pypaimon.write.native_commit import _rest_catalog_supported
+from pypaimon.execution.native.write.native_commit import _rest_catalog_supported
def try_native_hybrid_search(builder):
diff --git a/paimon-python/pypaimon/table/source/native_search_plan.py b/paimon-python/pypaimon/execution/native/read/native_search_plan.py
similarity index 75%
rename from paimon-python/pypaimon/table/source/native_search_plan.py
rename to paimon-python/pypaimon/execution/native/read/native_search_plan.py
index 993ed22861..ddb3eec8a3 100644
--- a/paimon-python/pypaimon/table/source/native_search_plan.py
+++ b/paimon-python/pypaimon/execution/native/read/native_search_plan.py
@@ -26,15 +26,15 @@ from pypaimon.common.json_util import JSON
from pypaimon.common.options.core_options import CoreOptions
from pypaimon.common.options.options import Options
from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult
-from pypaimon.read.native_plan import deserialize_split_v1, _partition_fields
+from pypaimon.read.split_serializer import deserialize_split_v1
+from pypaimon.execution.native.table import (_partition_fields)
from pypaimon.snapshot.snapshot import Snapshot
-from pypaimon.table.source.full_text_scan import FullTextScanPlan
-from pypaimon.table.source.primary_key_full_text_scan import (
+from pypaimon.table.source.search_types import (
+ FullTextScanPlan, VectorSearchScanPlan,
PrimaryKeyFullTextScanPlan, PrimaryKeyFullTextSearchSplit,
+ PrimaryKeyVectorScanPlan, PrimaryKeyVectorSearchSplit,
)
from pypaimon.table.source.primary_key_scored_result import PrimaryKeyScoredResult
-from pypaimon.table.source.primary_key_vector_scan import PrimaryKeyVectorScanPlan, PrimaryKeyVectorSearchSplit
-from pypaimon.table.source.vector_search_scan import VectorSearchScanPlan
from pypaimon.utils.range import Range
from pypaimon.write.commit_message_serializer import _index_from_row
@@ -128,12 +128,13 @@ def _result(table, value, primary_key):
class _NativeScan:
- def __init__(self, table, scan, full_text, primary_key, python, source_modes):
+ def __init__(self, table, scan, full_text, primary_key, python_factory, source_modes):
self._table = table
self._scan = scan
self._full_text = full_text
self._primary_key = primary_key
- self._python = python
+ self._python_factory = python_factory
+ self._python_executor = None
self._source_modes = source_modes
def __getstate__(self):
@@ -143,24 +144,26 @@ class _NativeScan:
def scan(self):
if self._scan is None:
- return self._python.scan()
+ if self._python_executor is None:
+ self._python_executor = self._python_factory()
+ return self._python_executor.scan()
return _plan_from_native(self._table, self._scan.scan(), self._full_text, self._primary_key, self._source_modes)
class _NativeRead:
- def __init__(self, table, native, python, primary_key, source_modes):
+ def __init__(self, table, native, python_factory, primary_key, source_modes):
self._table = table
self._native = native
- self._python = python
+ self._python_factory = python_factory
+ self._python_executor = None
self._primary_key = primary_key
self._source_modes = source_modes
self._table_options = dict(table.options.options.to_map())
- def __getattr__(self, name):
- python = self.__dict__.get('_python')
- if python is None:
- raise AttributeError(name)
- return getattr(python, name)
+ def _python_reader(self):
+ if self._python_executor is None:
+ self._python_executor = self._python_factory()
+ return self._python_executor
def __getstate__(self):
state = self.__dict__.copy()
@@ -172,7 +175,7 @@ class _NativeRead:
if native is not None and self._native is not None:
# Once execution starts, propagate errors without a second search.
return _result(self._table, self._native.read_plan(native), self._primary_key)
- reader = copy(self._python)
+ reader = copy(self._python_reader())
if plan.snapshot() is not None:
reader._table = reader._table._copy_with_snapshot(plan.snapshot())
return reader.read_plan(plan)
@@ -181,7 +184,7 @@ class _NativeRead:
native = self._native_plan(plan)
if native is not None and self._native is not None:
return [_result(self._table, result, False) for result in self._native.read_batch_plan(native)]
- reader = copy(self._python)
+ reader = copy(self._python_reader())
if plan.snapshot() is not None:
reader._table = reader._table._copy_with_snapshot(plan.snapshot())
return reader.read_batch_plan(plan)
@@ -195,6 +198,18 @@ class _NativeRead:
return None
+class _NativeSplitRead(_NativeRead):
+ def read(self, splits):
+ # Explicit split reads select Python only when invoked. Attribute probes
+ # and method inspection must not construct a fallback executor.
+ return self._python_reader().read(splits)
+
+
+class _NativeBatchRead(_NativeRead):
+ def read_batch(self, splits):
+ return self._python_reader().read_batch(splits)
+
+
def _source_modes(builder, full_text=False):
options = dict(builder._table.options.options.to_map())
options.update(getattr(builder, '_options', {}))
@@ -205,42 +220,45 @@ def _source_modes(builder, full_text=False):
def python_search_reader(reader):
"""Keep the existing Ray split workers independent of a live native reader."""
- return reader._python if isinstance(reader, _NativeRead) else reader
+ return reader._python_reader() if isinstance(reader, _NativeRead) else reader
-def native_vector_scan(builder, python, batch=False):
- from pypaimon.table.source.native_vector_search import _native_vector_builder
+def native_vector_scan(builder, python_factory, batch=False):
+ from pypaimon.execution.native.read.native_vector_search import _native_vector_builder
native = _native_vector_builder(builder, batch)
- return python if native is None else _NativeScan(
- builder._table, native.new_vector_search_scan(), False, builder._is_primary_key_vector_search(), python,
+ return python_factory() if native is None else _NativeScan(
+ builder._table, native.new_vector_search_scan(), False, builder._is_primary_key_vector_search(), python_factory,
_source_modes(builder))
-def native_vector_read(builder, python, batch=False):
- from pypaimon.table.source.native_vector_search import _native_vector_builder
+def native_vector_read(builder, python_factory, batch=False):
+ from pypaimon.execution.native.read.native_vector_search import _native_vector_builder
native = _native_vector_builder(builder, batch)
if native is None:
- return python
+ return python_factory()
if batch:
native.with_query_vectors(builder._query_vectors)
reader = native.new_batch_vector_search_read()
else:
native.with_query_vector(builder._query_vector)
reader = native.new_vector_search_read()
- return _NativeRead(builder._table, reader, python, builder._is_primary_key_vector_search(), _source_modes(builder))
+ read_class = _NativeBatchRead if batch else _NativeSplitRead
+ return read_class(
+ builder._table, reader, python_factory, builder._is_primary_key_vector_search(), _source_modes(builder))
-def native_full_text_scan(builder, python):
- from pypaimon.table.source.native_full_text_search import _native_full_text_builder
+def native_full_text_scan(builder, python_factory):
+ from pypaimon.execution.native.read.native_full_text_search import _native_full_text_builder
native = _native_full_text_builder(builder, require_limit=False)
- return python if native is None else _NativeScan(
+ return python_factory() if native is None else _NativeScan(
builder._table, native.new_full_text_scan(), True,
- builder._primary_key_full_text_definition() is not None, python, _source_modes(builder, True))
+ builder._primary_key_full_text_definition() is not None, python_factory, _source_modes(builder, True))
-def native_full_text_read(builder, python):
- from pypaimon.table.source.native_full_text_search import _native_full_text_builder
+def native_full_text_read(builder, python_factory):
+ from pypaimon.execution.native.read.native_full_text_search import _native_full_text_builder
native = _native_full_text_builder(builder)
- return python if native is None else _NativeRead(
- builder._table, native.new_full_text_read(), python, builder._primary_key_full_text_definition() is not None,
+ return python_factory() if native is None else _NativeSplitRead(
+ builder._table, native.new_full_text_read(), python_factory,
+ builder._primary_key_full_text_definition() is not None,
_source_modes(builder, True))
diff --git a/paimon-python/pypaimon/table/source/native_vector_search.py b/paimon-python/pypaimon/execution/native/read/native_vector_search.py
similarity index 93%
rename from paimon-python/pypaimon/table/source/native_vector_search.py
rename to paimon-python/pypaimon/execution/native/read/native_vector_search.py
index 01db9cbd2a..83ea544fc5 100644
--- a/paimon-python/pypaimon/table/source/native_vector_search.py
+++ b/paimon-python/pypaimon/execution/native/read/native_vector_search.py
@@ -19,7 +19,9 @@
from pypaimon.catalog.table_query_auth import reject_search_under_query_auth
from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult
-from pypaimon.read.native_plan import _catalog_metastore, _native_table, _predicate_to_native, native_runtime_available
+from pypaimon.execution.native.table import (_catalog_metastore, _native_table)
+from pypaimon.execution.native.read.builder import (_predicate_to_native)
+from pypaimon.execution.native.runtime import (native_runtime_available)
from pypaimon.table.source.primary_key_scored_result import PrimaryKeyScoredResult
diff --git a/paimon-python/pypaimon/execution/native/read/plan.py b/paimon-python/pypaimon/execution/native/read/plan.py
new file mode 100644
index 0000000000..e8430d08b8
--- /dev/null
+++ b/paimon-python/pypaimon/execution/native/read/plan.py
@@ -0,0 +1,326 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Native batch and streaming planning behind the public scan facade."""
+
+import logging
+from dataclasses import replace
+from typing import List, Optional, Tuple
+
+from pypaimon.common.identifier import UNKNOWN_DATABASE
+from pypaimon.common.options.core_options import CoreOptions, ChangelogProducer
+from pypaimon.common.predicate import Predicate
+from pypaimon.execution.native import runtime
+from pypaimon.execution.native import table as native_table
+from pypaimon.execution.native.read import builder as native_builder
+from pypaimon.read.plan import Plan
+from pypaimon.read.query_auth_split import QueryAuthSplit
+from pypaimon.read.scan_stats import ScanStats
+from pypaimon.read.scan_request import ScanRequest
+from pypaimon.read import split_serializer
+
+logger = logging.getLogger(__name__)
+
+_NATIVE_TIME_TRAVEL_OPTIONS = frozenset({
+ CoreOptions.SCAN_VERSION.key(), CoreOptions.SCAN_SNAPSHOT_ID.key(),
+ CoreOptions.SCAN_TAG_NAME.key(), CoreOptions.SCAN_TIMESTAMP.key(),
+ CoreOptions.SCAN_TIMESTAMP_MILLIS.key(), CoreOptions.SCAN_WATERMARK.key(),
+})
+
+
+def native_plan(
+ table,
+ predicate: Optional[Predicate] = None,
+ limit: Optional[int] = None,
+ projection: Optional[List[str]] = None,
+ row_ranges: Optional[List[Tuple[int, int]]] = None,
+ incremental_range: Optional[Tuple[int, int]] = None,
+ incremental_mode: str = 'delta',
+ row_position_slice: Optional[Tuple[int, int]] = None,
+ row_position_shard: Optional[Tuple[int, int]] = None,
+ chunk_shuffle: Optional[Tuple[int, int]] = None,
+ shard: Optional[Tuple[int, int]] = None,
+ partition_predicate: Optional[Predicate] = None,
+ global_index_result=None,
+ all_files: bool = False) -> Plan:
+ """Plan with pypaimon_rust, preserving snapshot metadata.
+
+ Native conversion or planning failures are handled by TableScan, which
+ falls back to the Python planner.
+ """
+ if all_files and incremental_range is not None:
+ raise ValueError('planning all files does not accept incremental ranges')
+ if incremental_range is None and incremental_mode != 'delta':
+ raise ValueError('incremental_mode requires incremental_range')
+ if not runtime.native_runtime_available():
+ raise RuntimeError(
+ "scan.native-plan.enabled needs the pypaimon-rust extension")
+ builder = native_builder._configure_native_read_builder(
+ native_builder._native_read_builder(table), predicate, limit, projection)
+ if partition_predicate is not None:
+ builder = builder.with_partition_filter(native_builder._predicate_to_native(partition_predicate))
+ if row_ranges is not None:
+ builder = builder.with_row_ranges(row_ranges)
+ from pypaimon.table.source.global_index_split_result import GlobalIndexSplitResult
+ supplied_pk_splits = (isinstance(global_index_result, GlobalIndexSplitResult)
+ and not table.options.query_auth_enabled)
+ if supplied_pk_splits or incremental_range is None:
+ scan = builder.new_scan()
+ elif incremental_mode == 'delta':
+ scan = builder.new_incremental_scan(*incremental_range)
+ else:
+ scan = builder.new_incremental_scan(*incremental_range, incremental_mode)
+ if global_index_result is not None and not table.options.query_auth_enabled:
+ # A PK result already selected a snapshot and physical files. Java's
+ # PrimaryKeyBatchScan returns those selections before ordinary planning.
+ from pypaimon_rust.datafusion import GlobalIndexResult, Split as NativeSplit
+ from pypaimon.globalindex.vector_search_result import ScoredGlobalIndexResult
+ from pypaimon.table.source.primary_key_scored_result import PrimaryKeyScoredResult
+ if isinstance(global_index_result, GlobalIndexSplitResult):
+ # Searches and scans share the same core GlobalIndexResult.
+ result = (getattr(global_index_result, '_native_result', None)
+ if type(global_index_result) is PrimaryKeyScoredResult else None)
+ # Java's PK result uses snapshot 0 for an uncommitted table,
+ # whereas the core search uses None. Keep the Python result's ID.
+ if result is not None and result.snapshot_id() is None:
+ result = None
+ if result is None:
+ result = GlobalIndexResult.from_splits(
+ global_index_result.snapshot_id,
+ [NativeSplit.deserialize(split_serializer.serialize_split_v1(split))
+ for split in global_index_result.splits])
+ elif isinstance(global_index_result, ScoredGlobalIndexResult):
+ score = global_index_result.score_getter()
+ result = GlobalIndexResult.from_scored_row_ids({
+ row_id: score(row_id) for row_id in global_index_result.results()})
+ else:
+ result = GlobalIndexResult.from_row_ranges([
+ (range_.from_, range_.to) for range_ in global_index_result.results().to_range_list()])
+ scan = scan.with_global_index_result(result)
+ if row_position_slice is not None:
+ scan = scan.with_row_position_slice(*row_position_slice)
+ if row_position_shard is not None:
+ scan = scan.with_row_position_shard(*row_position_shard)
+ if chunk_shuffle is not None:
+ seed, chunk_size = chunk_shuffle
+ scan = scan.with_chunk_shuffle(str(seed), chunk_size)
+ if shard is not None:
+ scan = scan.with_shard(*shard)
+ rust_plan = scan.plan_all_files() if all_files else scan.plan()
+ return _from_native_plan(table, rust_plan)
+
+
+def native_stream_scan(table, predicate=None, read_type=None, bucket_filter=None, consumer_id=None):
+ """Create Rust's stateful StreamTableScan with the canonical reader type."""
+ builder = native_builder._configure_native_read_builder(
+ native_builder._native_read_builder(table), predicate, None, None, read_type=read_type)
+ scan = builder.new_stream_scan()
+ if bucket_filter is not None:
+ scan.with_bucket_filter(bucket_filter)
+ if consumer_id is not None:
+ scan.with_consumer_id(consumer_id)
+ return scan
+
+
+def _from_native_plan(table, rust_plan) -> Plan:
+ """Keep one split/metadata bridge for batch and streaming planning."""
+ rust_splits = rust_plan.splits()
+ pfields = native_table._partition_fields(table)
+ # Trimmed primary keys decode per-file min/max keys (PK merge-on-read).
+ kfields = table.trimmed_primary_keys_fields
+ splits = [
+ split_serializer.deserialize_split_v1(split.serialize(), pfields, kfields)
+ for split in rust_splits
+ ]
+ if table.options.native_read_enabled():
+ # Retain the opaque Rust split next to the Python metadata view so an
+ # unchanged native plan can be read without reserializing each split.
+ # A caller that needs different metadata (such as an endpoint DV)
+ # passes a new Python split, which the native reader converts at read
+ # time from its current fields.
+ for split, rust_split in zip(splits, rust_splits):
+ split._native_split = rust_split
+ snapshot_id = rust_plan.snapshot_id()
+ return Plan(splits, snapshot_id=snapshot_id, _native_stats=rust_plan.stats())
+
+
+class NativeTableScanMixin:
+ def _native_requested(self) -> bool:
+ if self.table.options.native_plan_enabled():
+ return True
+ if not self.table.options.native_read_enabled():
+ return False
+ return runtime.native_runtime_available()
+
+ def _native_plan_supported(self) -> bool:
+ # Any probe failure (e.g. a remote schema/metadata read) must fall back, not fail the scan.
+ try:
+ return self._native_plan_supported_impl()
+ except Exception as e:
+ logger.warning("Native-plan capability probe failed, falling back: %s", e)
+ return False
+
+ def _native_plan_supported_impl(self) -> bool:
+ """Fall back to the Python scanner for scans native can't carry:
+ a primary-key table whose trimmed PK is empty (PK equals the partition
+ key; Rust rejects this schema), unsupported time travel selectors,
+ custom authorization, a missing pypaimon-rust, or a catalog / identifier Rust
+ cannot reconstruct. Keep this capability gate in sync when adding scan
+ features."""
+ from pypaimon.execution.native.table import _resolved_schema_file_io_options
+ if not runtime.native_runtime_available():
+ return False
+ request = self._scan_request()
+ if not self._native_global_index_result_supported(request):
+ return False
+ if request.chunk_shuffle is not None:
+ request.validate_chunk_shuffle_compat()
+ if _resolved_schema_file_io_options(self.table) is None:
+ loader = getattr(
+ getattr(self.table, 'catalog_environment', None),
+ 'catalog_loader',
+ None,
+ )
+ context_fn = getattr(loader, 'context', None)
+ if not callable(context_fn):
+ return False
+ from pypaimon.execution.native.table import _catalog_metastore
+ if _catalog_metastore(loader) is None:
+ return False
+ context = context_fn()
+ catalog_options = getattr(context, 'options', None)
+ if catalog_options is None:
+ return False
+ database_name = self.table.identifier.get_database_name()
+ if not database_name or database_name == UNKNOWN_DATABASE:
+ return False
+ if self.table.options.query_auth_enabled:
+ # The filesystem path has no catalog-issued authorization session.
+ loader = getattr(getattr(self.table, 'catalog_environment', None), 'catalog_loader', None)
+ from pypaimon.execution.native.table import _catalog_metastore
+ if loader is None or _catalog_metastore(loader) != 'rest':
+ return False
+ from pypaimon.execution.native.table import _native_query_auth_supported
+ if not _native_query_auth_supported(self.table):
+ return False
+ # Rust rejects schemas whose primary keys are all partition keys.
+ if getattr(self.table, 'is_primary_key_table', False) \
+ and not self.table.trimmed_primary_keys:
+ return False
+ options = self.table.options.options
+ from pypaimon.snapshot.time_travel_util import SCAN_KEYS
+ unsupported_scan_keys = set(SCAN_KEYS) - _NATIVE_TIME_TRAVEL_OPTIONS
+ return not any(options.contains_key(k) for k in unsupported_scan_keys)
+
+ def _native_global_index_result_supported(self, request: Optional[ScanRequest] = None) -> bool:
+ request = self._scan_request() if request is None else request
+ result = request.global_index_result
+ if result is None:
+ return True
+ if request.is_primary_key_table:
+ from pypaimon.table.source.global_index_split_result import GlobalIndexSplitResult
+ return isinstance(result, GlobalIndexSplitResult)
+ if not request.data_evolution:
+ return False
+ from pypaimon.globalindex.global_index_result import GlobalIndexResult
+ return isinstance(result, GlobalIndexResult)
+
+ def _native_row_ranges(self) -> Optional[List[Tuple[int, int]]]:
+ row_ranges = self._scan_request().row_ranges
+ return list(row_ranges) if row_ranges is not None else None
+
+ def _try_native_plan(self, all_files=False) -> Optional[Plan]:
+ """Plan via pypaimon_rust, then drop partitions the predicate rejects.
+
+ Predicate and limit are pushed into Rust planning and are still enforced
+ by the reader. Snapshot metadata is preserved even when pruning removes
+ every split.
+ """
+
+ try:
+ request = self._scan_request()
+ has_distribution = (request.idx_of_this_subtask is not None
+ or request.start_pos_of_this_subtask is not None)
+ extra_options = {}
+ if self.table.options.options.contains(CoreOptions.INCREMENTAL_BETWEEN_TIMESTAMP):
+ if request.incremental_snapshot_range is None:
+ return Plan([], _scan_stats=ScanStats.empty_native(request.read_limit))
+ extra_options['incremental_range'] = request.incremental_snapshot_range
+ if self.table.options.changelog_producer() != ChangelogProducer.NONE:
+ extra_options['incremental_mode'] = 'changelog'
+ chunk_shuffle = request.chunk_shuffle
+ if chunk_shuffle is not None:
+ extra_options['chunk_shuffle'] = chunk_shuffle
+ if (request.idx_of_this_subtask is not None
+ and (chunk_shuffle is not None or request.is_primary_key_table)):
+ extra_options['shard'] = (
+ request.idx_of_this_subtask, request.number_of_para_subtasks)
+ if (has_distribution and not request.is_primary_key_table
+ and chunk_shuffle is None):
+ if request.idx_of_this_subtask is not None:
+ extra_options['row_position_shard'] = (
+ request.idx_of_this_subtask, request.number_of_para_subtasks)
+ else:
+ extra_options['row_position_slice'] = (
+ request.start_pos_of_this_subtask, request.end_pos_of_this_subtask)
+ row_ranges = list(request.row_ranges) if request.row_ranges is not None else None
+ if request.global_index_result is not None:
+ extra_options['global_index_result'] = request.global_index_result
+ if request.partition_filter is not None:
+ # Explicit partition selectors are scan-only. Keep the ordinary
+ # row predicate separate so core can apply Java's override and
+ # masked-partition validation semantics.
+ extra_options['partition_predicate'] = request.partition_key_predicate
+ # Append positions still precede LIMIT. Rust owns PK shards and
+ # source-backed sorted-index refinement before applying LIMIT.
+ defer_limit = has_distribution and not request.is_primary_key_table
+ if all_files:
+ extra_options['all_files'] = True
+ plan = native_plan(
+ self.table,
+ predicate=request.read_predicate,
+ limit=None if defer_limit else request.read_limit,
+ projection=(
+ [field.name for field in request.read_type]
+ if request.read_type is not None and chunk_shuffle is None else None),
+ row_ranges=row_ranges,
+ **extra_options,
+ )
+ splits = plan.splits()
+ partition_predicate = request.partition_key_predicate
+ if (partition_predicate is not None and not (
+ request.is_primary_key_table and request.global_index_result is not None)
+ and not any(isinstance(split, QueryAuthSplit) for split in splits)):
+ # Authorized user predicates observe masked partition values;
+ # Rust has already applied only the safe raw partition tests.
+ splits = [s for s in splits
+ if getattr(s, 'partition', None) is None
+ or partition_predicate.test(s.partition)]
+ if chunk_shuffle is None and defer_limit:
+ # A partial IndexedSplit plus a file-wide DV cardinality cannot
+ # reveal how many deleted rows lie inside the selected range.
+ # Keep every selected split and let the reader enforce LIMIT.
+ if not (request.deletion_vectors_enabled
+ and not request.is_primary_key_table):
+ splits = request.apply_push_down_limit(splits)
+ return replace(plan, _splits=splits, _stats_limit=request.read_limit if defer_limit else None)
+ except Exception as e:
+ runtime._raise_if_native_fork_safety_error(e)
+ # Any native construction/planning/pruning failure -> fall back.
+ logger.warning(
+ "Native plan failed, falling back to the Python scanner: %s", e)
+ return None
diff --git a/paimon-python/pypaimon/read/native_row_iterator.py b/paimon-python/pypaimon/execution/native/read/row_iterator.py
similarity index 100%
rename from paimon-python/pypaimon/read/native_row_iterator.py
rename to paimon-python/pypaimon/execution/native/read/row_iterator.py
diff --git a/paimon-python/pypaimon/execution/native/read/split_adapter.py b/paimon-python/pypaimon/execution/native/read/split_adapter.py
new file mode 100644
index 0000000000..01452db508
--- /dev/null
+++ b/paimon-python/pypaimon/execution/native/read/split_adapter.py
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Convert public Python split values for Native execution."""
+
+from pypaimon.read import split_serializer
+
+
+def _native_read_split(split, ranges_use_row_ids):
+ """Express Python's file slices with the existing indexed-split selector."""
+ from pypaimon.globalindex.indexed_split import IndexedSplit
+ from pypaimon.read.query_auth_split import QueryAuthSplit
+ from pypaimon.read.sliced_split import SlicedSplit
+ from pypaimon.utils.range import Range
+
+ def select_rows(base, ranges):
+ if isinstance(base, QueryAuthSplit):
+ return QueryAuthSplit(select_rows(base.split, ranges), base.auth_result)
+ if isinstance(base, IndexedSplit):
+ return select_rows(base.data_split(), Range.and_(base.row_ranges(), ranges))
+ return IndexedSplit(base, ranges)
+
+ if isinstance(split, QueryAuthSplit):
+ base = _native_read_split(split.split, ranges_use_row_ids)
+ return split if base is split.split else QueryAuthSplit(base, split.auth_result)
+ if isinstance(split, IndexedSplit):
+ base = _native_read_split(split.data_split(), ranges_use_row_ids)
+ return split if base is split.data_split() else select_rows(base, split.row_ranges())
+ if not isinstance(split, SlicedSplit):
+ return split
+ if ranges_use_row_ids is None:
+ raise ValueError('Native file-slice conversion requires the reader row-tracking mode')
+ ranges = []
+ offset = 0
+ slices = split.shard_file_idx_map()
+ for file in split.files:
+ start, end = slices.get(file.file_name, (0, file.row_count))
+ start, end = max(0, start), min(file.row_count, end)
+ if start < end:
+ base = file.first_row_id if ranges_use_row_ids else offset
+ if base is None:
+ raise ValueError("Row-tracked file '%s' is missing first_row_id" % file.file_name)
+ ranges.append(Range(base + start, base + end - 1))
+ offset += file.row_count
+ return select_rows(_native_read_split(split.data_split(), ranges_use_row_ids), ranges)
+
+
+def native_split_from_python(split, *, ranges_use_row_ids=None):
+ """Convert a Python-planned split for the Rust reader.
+
+ Vector scores intentionally stay on the Python IndexedSplit. Rust needs
+ only its row ranges to perform the physical read. File slices need the
+ reader's tracking mode to map their local offsets to those ranges.
+ """
+ from pypaimon_rust.datafusion import Split as NativeSplit
+ return NativeSplit.deserialize(split_serializer.serialize_split_v1(
+ _native_read_split(split, ranges_use_row_ids), include_scores=False))
diff --git a/paimon-python/pypaimon/execution/native/read/streaming_table_scan.py b/paimon-python/pypaimon/execution/native/read/streaming_table_scan.py
new file mode 100644
index 0000000000..ec92defbeb
--- /dev/null
+++ b/paimon-python/pypaimon/execution/native/read/streaming_table_scan.py
@@ -0,0 +1,105 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Polling adapters for Native stateful streaming scans."""
+
+import asyncio
+
+from pypaimon.execution.native.read.plan import native_stream_scan, _from_native_plan
+
+
+class StreamTableScan:
+ """Python iterator adapter for Rust's stateful StreamTableScan.
+
+ Rust owns snapshot selection, split planning, checkpoints and consumer
+ persistence. Python only polls, converts splits and acknowledges a yielded
+ plan when the caller resumes the iterator after processing it.
+ """
+
+ def __init__(self, table, predicate=None, read_type=None, poll_interval_ms=1000,
+ bucket_filter=None, consumer_id=None):
+ self.table = table
+ self.poll_interval = poll_interval_ms / 1000.0
+ self._scan = native_stream_scan(
+ table, predicate=predicate, read_type=read_type,
+ bucket_filter=bucket_filter, consumer_id=consumer_id)
+ self._pending_consumer_snapshot = None
+
+ @property
+ def next_snapshot_id(self):
+ return self.checkpoint()
+
+ @next_snapshot_id.setter
+ def next_snapshot_id(self, next_snapshot_id):
+ self.restore(next_snapshot_id)
+
+ def checkpoint(self):
+ return self._scan.checkpoint()
+
+ def watermark(self):
+ return self._scan.watermark()
+
+ def restore(self, next_snapshot_id=None):
+ self._scan.restore(next_snapshot_id)
+ self._pending_consumer_snapshot = None
+
+ def notify_checkpoint_complete(self, next_snapshot):
+ self._scan.notify_checkpoint_complete(next_snapshot)
+
+ def plan(self):
+ # Decode before returning or acknowledging progress. A failed conversion
+ # must leave the selected frame available for retry.
+ checkpoint = self.checkpoint()
+ native_plan = self._scan.plan()
+ if native_plan is None:
+ return None
+ try:
+ return _from_native_plan(self.table, native_plan)
+ except Exception:
+ self._scan.restore(checkpoint)
+ raise
+
+ def _flush_pending_consumer(self):
+ if self._pending_consumer_snapshot is not None:
+ self.notify_checkpoint_complete(self._pending_consumer_snapshot)
+ self._pending_consumer_snapshot = None
+
+ async def stream(self):
+ while True:
+ self._flush_pending_consumer()
+ checkpoint = self.checkpoint()
+ plan = self.plan()
+ if plan is None:
+ next_snapshot = self.checkpoint()
+ if next_snapshot is not None and next_snapshot != checkpoint:
+ # Skipped frames have no rows awaiting caller processing.
+ self._pending_consumer_snapshot = next_snapshot
+ self._flush_pending_consumer()
+ await asyncio.sleep(self.poll_interval)
+ continue
+ self._pending_consumer_snapshot = self.checkpoint()
+ yield plan
+
+ def stream_sync(self):
+ loop = asyncio.new_event_loop()
+ iterator = self.stream()
+ try:
+ while True:
+ yield loop.run_until_complete(iterator.__anext__())
+ finally:
+ loop.run_until_complete(iterator.aclose())
+ loop.close()
diff --git a/paimon-python/pypaimon/table/system/native_system_table.py b/paimon-python/pypaimon/execution/native/read/system_table.py
similarity index 89%
rename from paimon-python/pypaimon/table/system/native_system_table.py
rename to paimon-python/pypaimon/execution/native/read/system_table.py
index 5eb15ac1f9..638e4b532d 100644
--- a/paimon-python/pypaimon/table/system/native_system_table.py
+++ b/paimon-python/pypaimon/execution/native/read/system_table.py
@@ -20,8 +20,9 @@
import pyarrow
from pypaimon.schema.data_types import PyarrowFieldParser
-from pypaimon.read.native_plan import _native_table, native_runtime_available
-from pypaimon.write.native_commit import _rest_catalog_supported
+from pypaimon.execution.native.table import (_native_table)
+from pypaimon.execution.native.runtime import (native_runtime_available)
+from pypaimon.execution.native.write.native_commit import _rest_catalog_supported
_NATIVE_TABLES = frozenset((
'snapshots', 'schemas', 'options', 'manifests', 'files', 'buckets',
diff --git a/paimon-python/pypaimon/execution/native/read/table_read.py b/paimon-python/pypaimon/execution/native/read/table_read.py
new file mode 100644
index 0000000000..b5590e18d4
--- /dev/null
+++ b/paimon-python/pypaimon/execution/native/read/table_read.py
@@ -0,0 +1,275 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Native reads and adaptation to the public Arrow and row contracts."""
+
+import logging
+from typing import List, Optional
+
+import pyarrow
+
+from pypaimon.casting.schema_evolution import read_arrow_schema
+from pypaimon.common.options.core_options import CoreOptions
+from pypaimon.common.predicate import Predicate
+from pypaimon.execution.native import runtime
+from pypaimon.execution.native import table as native_table
+from pypaimon.execution.native.read import builder as native_builder
+from pypaimon.read.read_type import OutputProjection, adapter_output_paths, extract_array, output_schema
+from pypaimon.read.split import Split
+from pypaimon.read.variant_read_type import has_variant_extractions
+from pypaimon.schema.data_types import DataField, is_blob_file_field
+
+logger = logging.getLogger(__name__)
+_NATIVE_READ_FILE_FORMATS = frozenset({
+ CoreOptions.FILE_FORMAT_PARQUET, CoreOptions.FILE_FORMAT_ORC,
+ CoreOptions.FILE_FORMAT_AVRO, CoreOptions.FILE_FORMAT_ROW,
+ CoreOptions.FILE_FORMAT_MOSAIC,
+})
+_NATIVE_READ_FILE_SUFFIXES = tuple('.%s' % file_format for file_format in _NATIVE_READ_FILE_FORMATS)
+_NATIVE_BLOB_FILE_SUFFIXES = ('.blob', '.video')
+
+
+def _prepare_native_read(table, predicate: Optional[Predicate] = None,
+ limit: Optional[int] = None,
+ projection: Optional[List[str]] = None,
+ blob_parallelism: Optional[int] = None,
+ read_type: Optional[List[DataField]] = None,
+ include_row_kind: bool = False):
+ """Create the configured Core reader entrypoint."""
+ if not runtime.native_runtime_available():
+ raise RuntimeError(
+ "read.native.enabled needs the pypaimon-rust extension")
+ if not native_table._native_query_auth_supported(table):
+ raise RuntimeError('Native reads cannot reconstruct custom query authorization')
+ if read_type is None:
+ from pypaimon.read.read_builder import ReadBuilder
+ resolved = ReadBuilder(table)
+ if projection is not None:
+ resolved.with_projection(projection)
+ read_type = resolved.read_type()
+ builder = native_builder._configure_native_read_builder(
+ native_builder._native_read_builder(table), predicate, limit, projection,
+ read_type=read_type,
+ include_row_kind=include_row_kind)
+ if blob_parallelism is not None:
+ builder = builder.with_blob_parallelism(blob_parallelism)
+ reader = builder.new_read()
+ return reader.read_arrow
+
+
+def native_read(table, splits, predicate: Optional[Predicate] = None,
+ limit: Optional[int] = None,
+ projection: Optional[List[str]] = None,
+ blob_parallelism: Optional[int] = None,
+ read_type: Optional[List[DataField]] = None,
+ include_row_kind: bool = False,
+ parallelism: int = 1,
+ ordered: bool = False):
+ """Run one Core read; ``ordered`` retains the supplied split order."""
+ read_splits = _prepare_native_read(
+ table,
+ predicate=predicate,
+ limit=limit,
+ projection=projection,
+ blob_parallelism=blob_parallelism,
+ read_type=read_type,
+ include_row_kind=include_row_kind)
+ return read_splits(splits, parallelism=parallelism, ordered=ordered)
+
+
+class NativeTableReadMixin:
+ def _native_fallback(
+ self,
+ reason: str,
+ error: Optional[Exception] = None,
+ log: bool = False,
+ ):
+ if has_variant_extractions(self.read_type):
+ raise RuntimeError(
+ "Variant extraction requires a compatible native reader: %s"
+ % reason) from error
+ if log:
+ logger.warning(
+ "Native read failed, falling back to the Python reader: %s",
+ reason)
+ return None
+
+ def _try_native_batches(
+ self,
+ splits: List[Split],
+ schema: pyarrow.Schema,
+ parallelism: Optional[int] = None,
+ blob_parallelism: Optional[int] = None,
+ ordered: bool = False):
+ """Return Rust-read batches, or ``None`` when this read must fall back."""
+ if not self.table.options.native_read_enabled():
+ return self._native_fallback("read.native.enabled is false")
+ if self.table.options.file_format() not in _NATIVE_READ_FILE_FORMATS:
+ return self._native_fallback(
+ "the table file format is not supported")
+ if not splits:
+ return []
+ if not self._native_blob_view_supported():
+ return self._native_fallback(
+ "the table uses an unsupported BLOB view")
+ try:
+ from pypaimon.execution.native.read.split_adapter import native_split_from_python
+ except Exception as e:
+ return self._native_fallback(str(e), e, log=True)
+ rust_splits = []
+ for split in splits:
+ if not self._native_split_files_supported(split):
+ return self._native_fallback(
+ "a split contains a file unsupported by the native reader")
+ rust_split = getattr(split, '_native_split', None)
+ if rust_split is None:
+ try:
+ rust_split = native_split_from_python(split, ranges_use_row_ids=(
+ self.table.options.row_tracking_enabled() or self.table.options.data_evolution_enabled()))
+ except Exception as e:
+ return self._native_fallback(
+ "native split conversion failed: %s" % e,
+ e,
+ log=True)
+ rust_splits.append(rust_split)
+ # Python adapts the public configuration; Core owns split grouping,
+ # bounded concurrency, the row quota and cancellation of every stream.
+ effective = 1 if parallelism is None else parallelism
+ try:
+ batches = native_read(
+ self.table, rust_splits,
+ parallelism=effective,
+ ordered=ordered,
+ **self._native_read_kwargs(blob_parallelism))
+ except Exception as error:
+ runtime._raise_if_native_fork_safety_error(error)
+ return self._native_fallback(str(error), error, log=True)
+ return self._convert_native_batches(batches, schema)
+
+ def _native_read_kwargs(self, blob_parallelism=None):
+ kwargs = {
+ 'predicate': self.predicate,
+ 'limit': self.limit,
+ }
+ if blob_parallelism is not None:
+ kwargs['blob_parallelism'] = blob_parallelism
+ kwargs['read_type'] = self.read_type
+ if self.include_row_kind:
+ kwargs['include_row_kind'] = True
+ return kwargs
+
+ @staticmethod
+ def _native_split_files_supported(split):
+ for data_file in split.files:
+ file_name = data_file.file_name.lower()
+ if (not file_name.endswith(_NATIVE_READ_FILE_SUFFIXES)
+ and not file_name.endswith(_NATIVE_BLOB_FILE_SUFFIXES)):
+ return False
+ return True
+
+ def _convert_native_batches(self, batches, schema):
+ try:
+ for batch in batches:
+ if batch.num_rows == 0:
+ continue
+ batch = self._prepare_native_batch(batch, schema)
+ batch = self._project_batch_to_output(batch)
+ yield self._try_to_pad_batch_by_schema(batch, schema)
+ finally:
+ close = getattr(batches, 'close', None)
+ if close is not None:
+ close()
+
+ def _prepare_native_batch(self, batch, schema):
+ if self.include_row_kind and 'rowkind' in batch.schema.names:
+ index = batch.schema.get_field_index('rowkind')
+ batch = batch.rename_columns([
+ self._row_kind_column if position == index else name
+ for position, name in enumerate(batch.schema.names)
+ ])
+ if self.nested_name_paths:
+ batch = self._flatten_native_nested_batch(batch, schema)
+ return batch
+
+ def _flatten_native_nested_batch(self, batch, schema):
+ _, paths = self._reader_adapter()
+ # Rust already evaluated the complete predicate and projected away
+ # filter-only fields. Flatten just the user output dependencies; the
+ # Python split reader retains the additional adapter inputs internally.
+ outputs = adapter_output_paths(self.output_projection, self._adapter_read_type, paths)
+ needed = {path[0] for _, path in outputs}
+ fields_and_paths = [(field, path) for field, path in zip(self._adapter_read_type, paths)
+ if field.name in needed]
+ adapter_fields = [field for field, _ in fields_and_paths]
+ paths = [path for _, path in fields_and_paths]
+ arrays = [extract_array(batch, path) for path in paths]
+ fields = list(read_arrow_schema(adapter_fields))
+ # Authorization can mask a NOT NULL field. Keep the captured source
+ # nullability when flattening, including whole-parent/MAP projections.
+ projected = output_schema(batch.schema, OutputProjection([
+ (field.name, path) for field, path in zip(fields, paths)]))
+ fields = [field.with_nullable(field.nullable or projected[i].nullable) for i, field in enumerate(fields)]
+ if self.include_row_kind:
+ arrays.insert(0, batch.column(self._row_kind_column))
+ fields.insert(0, batch.schema.field(self._row_kind_column))
+ return pyarrow.RecordBatch.from_arrays(arrays, schema=pyarrow.schema(fields))
+
+ def _native_blob_view_supported(self) -> bool:
+ from pypaimon.execution.native.table import _native_blob_view_supported
+ return _native_blob_view_supported(self.table, (field.name for field in self._scan_read_type))
+
+ def _native_resolved_blob_fields(self) -> set:
+ """BLOB fields Rust resolves before emitting a batch."""
+ options = self.table.options
+ if options.blob_as_descriptor():
+ return set()
+ if self.table.is_primary_key_table:
+ return {field.name for field in self._scan_read_type if is_blob_file_field(field)}
+ inline_fields = (
+ options.blob_descriptor_fields()
+ | options.blob_view_fields()
+ )
+ read_names = {
+ field.name
+ for field in getattr(self, '_scan_read_type', self._adapter_read_type)
+ }
+ return inline_fields & read_names
+
+ def _try_native_rows(self, splits):
+ from copy import copy
+ from pypaimon.execution.native.read.row_iterator import native_rows
+ from pypaimon.read.read_type import output_fields
+
+ # InternalRow carries row kind separately from projected fields.
+ read = copy(self)
+ read.include_row_kind = True
+ read._row_kind_column = 'rowkind'
+ schema = read._add_row_kind_to_schema(read._output_arrow_schema(splits), read._row_kind_column)
+ fields = output_fields(self.read_type, self.output_projection)
+ batches = read._try_native_batches(splits, schema, parallelism=1, ordered=True)
+ if batches is None:
+ return None
+
+ def rows():
+ try:
+ yield from native_rows(batches, self.table, fields)
+ finally:
+ close = getattr(batches, 'close', None)
+ if close is not None:
+ close()
+
+ return rows()
diff --git a/paimon-python/pypaimon/execution/native/read/vindex_reader.py b/paimon-python/pypaimon/execution/native/read/vindex_reader.py
new file mode 100644
index 0000000000..8057d3eed5
--- /dev/null
+++ b/paimon-python/pypaimon/execution/native/read/vindex_reader.py
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Vector index queries delegated to the Rust reader."""
+
+from pypaimon.globalindex.global_index_reader import GlobalIndexReader, _completed_future
+from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult
+from pypaimon.execution.vindex_input import VindexReaderResources, _filter_bytes
+
+
+def _native_reader_type():
+ from pypaimon.execution.native.runtime import native_runtime_available
+ if not native_runtime_available():
+ return None
+ from pypaimon_rust.datafusion import VindexVectorGlobalIndexReader
+ return VindexVectorGlobalIndexReader
+
+
+def create_native_vindex_reader(file_io, index_path, io_metas, options=None):
+ reader_type = _native_reader_type()
+ if reader_type is None:
+ return None
+ return NativeVindexVectorGlobalIndexReader(file_io, index_path, io_metas, options, reader_type)
+
+
+class NativeVindexVectorGlobalIndexReader(VindexReaderResources, GlobalIndexReader):
+ """Rust owns vector validation, filtering, query options and scoring."""
+
+ def __init__(self, file_io, index_path, io_metas, options, reader_type):
+ super().__init__(file_io, index_path, io_metas, options)
+ self._reader_type = reader_type
+
+ def _create_reader(self, index_input, file_path):
+ return self._reader_type(
+ index_input, file_path, self._io_meta.file_size,
+ {str(key): str(value) for key, value in self._options.items()})
+
+ def visit_vector_search(self, vector_search):
+ results = self._run_search(
+ [vector_search.vector],
+ vector_search.limit,
+ vector_search.include_row_ids,
+ vector_search.options,
+ )
+ return _completed_future(results[0])
+
+ def visit_batch_vector_search(self, batch_vector_search):
+ results = self._run_search(
+ batch_vector_search.vectors,
+ batch_vector_search.limit,
+ batch_vector_search.include_row_ids,
+ batch_vector_search.options,
+ )
+ return _completed_future(results)
+
+ def _run_search(self, vectors, limit, include_row_ids, query_options):
+ self._ensure_loaded()
+ results = self._reader.search(
+ vectors, limit, filter_bytes=_filter_bytes(include_row_ids),
+ options={str(key): str(value) for key, value in (query_options or {}).items()})
+ return [DictBasedScoredIndexResult(scores) if scores else None for scores in results]
+
+ def vector_metric(self):
+ self._ensure_loaded()
+ return self._reader.metric()
diff --git a/paimon-python/pypaimon/execution/native/runtime.py b/paimon-python/pypaimon/execution/native/runtime.py
new file mode 100644
index 0000000000..6e4d0460ff
--- /dev/null
+++ b/paimon-python/pypaimon/execution/native/runtime.py
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Lazy discovery and process safety for the optional Native runtime."""
+
+from importlib import import_module
+
+
+def _raise_if_native_fork_safety_error(error):
+ """Propagate native fork errors instead of using inherited SDK state."""
+ if not native_runtime_available():
+ return
+ from pypaimon_rust import ForkSafetyError
+
+ current = error
+ seen = set()
+ while current is not None and id(current) not in seen:
+ seen.add(id(current))
+ if isinstance(current, ForkSafetyError):
+ raise current
+ current = current.__cause__ or current.__context__
+
+
+def native_runtime_available() -> bool:
+ """Whether the optional companion Rust extension is installed.
+
+ Native paths require this release's APIs; they do not probe older builds.
+ """
+ try:
+ import_module('pypaimon_rust.datafusion')
+ except ImportError:
+ return False
+ return True
diff --git a/paimon-python/pypaimon/read/native_plan.py b/paimon-python/pypaimon/execution/native/table.py
similarity index 56%
rename from paimon-python/pypaimon/read/native_plan.py
rename to paimon-python/pypaimon/execution/native/table.py
index 336c807dfb..a88de786af 100644
--- a/paimon-python/pypaimon/read/native_plan.py
+++ b/paimon-python/pypaimon/execution/native/table.py
@@ -15,54 +15,16 @@
# specific language governing permissions and limitations
# under the License.
-"""Plan splits with pypaimon_rust, decoded for the normal pypaimon reader.
-
-Optional, lazily-imported dependency; enabled by ``scan.native-plan.enabled``.
-Predicates and limits are pushed into Rust planning. The normal pypaimon reader
-still applies them while reading, so pushdown remains an optimization.
-"""
+"""Adapt a resolved public table to the optional Native execution runtime."""
import json
import os
-from importlib import import_module
from threading import Lock
from types import SimpleNamespace
-from typing import List, Optional, Tuple
+from typing import Optional
from pypaimon.common.options.config import CatalogOptions, FuseOptions, OssOptions
from pypaimon.common.options.options_utils import OptionsUtils
-from pypaimon.common.predicate import Predicate
-from pypaimon.read.plan import Plan
-from pypaimon.read.split_serializer import (
- deserialize_split_v1, serialize_split_v1)
-from pypaimon.schema.data_types import DataField, RowType
-
-
-def _raise_if_native_fork_safety_error(error):
- """Propagate native fork errors instead of using inherited SDK state."""
- if not native_runtime_available():
- return
- from pypaimon_rust import ForkSafetyError
-
- current = error
- seen = set()
- while current is not None and id(current) not in seen:
- seen.add(id(current))
- if isinstance(current, ForkSafetyError):
- raise current
- current = current.__cause__ or current.__context__
-
-
-def native_runtime_available() -> bool:
- """Whether the optional companion Rust extension is installed.
-
- Native paths require this release's APIs; they do not probe older builds.
- """
- try:
- import_module('pypaimon_rust.datafusion')
- except ImportError:
- return False
- return True
def _native_blob_view_supported(table, read_names) -> bool:
@@ -75,57 +37,6 @@ def _native_blob_view_supported(table, read_names) -> bool:
return loader is None or _catalog_metastore(loader) == 'rest'
-def _native_read_split(split, ranges_use_row_ids):
- """Express Python's file slices with the existing indexed-split selector."""
- from pypaimon.globalindex.indexed_split import IndexedSplit
- from pypaimon.read.query_auth_split import QueryAuthSplit
- from pypaimon.read.sliced_split import SlicedSplit
- from pypaimon.utils.range import Range
-
- def select_rows(base, ranges):
- if isinstance(base, QueryAuthSplit):
- return QueryAuthSplit(select_rows(base.split, ranges), base.auth_result)
- if isinstance(base, IndexedSplit):
- return select_rows(base.data_split(), Range.and_(base.row_ranges(), ranges))
- return IndexedSplit(base, ranges)
-
- if isinstance(split, QueryAuthSplit):
- base = _native_read_split(split.split, ranges_use_row_ids)
- return split if base is split.split else QueryAuthSplit(base, split.auth_result)
- if isinstance(split, IndexedSplit):
- base = _native_read_split(split.data_split(), ranges_use_row_ids)
- return split if base is split.data_split() else select_rows(base, split.row_ranges())
- if not isinstance(split, SlicedSplit):
- return split
- if ranges_use_row_ids is None:
- raise ValueError('Native file-slice conversion requires the reader row-tracking mode')
- ranges = []
- offset = 0
- slices = split.shard_file_idx_map()
- for file in split.files:
- start, end = slices.get(file.file_name, (0, file.row_count))
- start, end = max(0, start), min(file.row_count, end)
- if start < end:
- base = file.first_row_id if ranges_use_row_ids else offset
- if base is None:
- raise ValueError("Row-tracked file '%s' is missing first_row_id" % file.file_name)
- ranges.append(Range(base + start, base + end - 1))
- offset += file.row_count
- return select_rows(_native_read_split(split.data_split(), ranges_use_row_ids), ranges)
-
-
-def native_split_from_python(split, *, ranges_use_row_ids=None):
- """Convert a Python-planned split for the Rust reader.
-
- Vector scores intentionally stay on the Python IndexedSplit. Rust needs
- only its row ranges to perform the physical read. File slices need the
- reader's tracking mode to map their local offsets to those ranges.
- """
- from pypaimon_rust.datafusion import Split as NativeSplit
- return NativeSplit.deserialize(serialize_split_v1(
- _native_read_split(split, ranges_use_row_ids), include_scores=False))
-
-
def _partition_fields(table):
"""Ordered partition DataFields, used to decode the split partition bytes."""
schema = table.table_schema
@@ -236,28 +147,6 @@ def _resolved_schema_json(table) -> str:
return JSON.to_json(table.table_schema.copy(new_options=options))
-def _predicate_to_native(predicate: Predicate) -> dict:
- """Convert PyPaimon's predicate tree to pypaimon-rust's dict API."""
- if predicate.method in ('and', 'or'):
- children = predicate.literals or []
- if not children:
- raise ValueError("Native compound predicate requires children")
- return {
- 'method': predicate.method,
- 'children': [_predicate_to_native(child) for child in children],
- }
- result = {
- 'method': predicate.method,
- 'field': predicate.field,
- 'literals': list(predicate.literals or []),
- }
- if predicate.field_path:
- result['field'] = predicate.field_path[0]
- if len(predicate.field_path) > 1:
- result['nested_path'] = list(predicate.field_path[1:])
- return result
-
-
def _resolved_rest_table_response(table, identifier=None):
"""Reuse REST metadata only when the standard loader can be reproduced."""
from pypaimon.catalog.catalog_environment import CatalogEnvironment
@@ -473,156 +362,6 @@ def _with_native_read_snapshot(table, native_table):
return native_table
-def _native_read_builder(table):
- return _native_table(table).new_read_builder()
-
-
-def _configure_native_read_builder(builder, predicate, limit, projection,
- read_type: Optional[List[DataField]] = None,
- include_row_kind=False):
- if read_type is not None:
- builder = builder.with_read_type(
- json.dumps(RowType(True, read_type).to_dict()))
- if projection is not None and read_type is None:
- builder = builder.with_projection(projection)
- if predicate is not None:
- builder = builder.with_filter(_predicate_to_native(predicate))
- if limit is not None:
- builder = builder.with_limit(limit)
- if include_row_kind:
- builder = builder.with_include_row_kind(True)
- return builder
-
-
-def _prepare_native_read(table, predicate: Optional[Predicate] = None,
- limit: Optional[int] = None,
- projection: Optional[List[str]] = None,
- blob_parallelism: Optional[int] = None,
- read_type: Optional[List[DataField]] = None,
- include_row_kind: bool = False):
- """Create the configured Core reader entrypoint."""
- if not native_runtime_available():
- raise RuntimeError(
- "read.native.enabled needs the pypaimon-rust extension")
- if not _native_query_auth_supported(table):
- raise RuntimeError('Native reads cannot reconstruct custom query authorization')
- if read_type is None:
- from pypaimon.read.read_builder import ReadBuilder
- resolved = ReadBuilder(table)
- if projection is not None:
- resolved.with_projection(projection)
- read_type = resolved.read_type()
- builder = _configure_native_read_builder(
- _native_read_builder(table), predicate, limit, projection,
- read_type=read_type,
- include_row_kind=include_row_kind)
- if blob_parallelism is not None:
- builder = builder.with_blob_parallelism(blob_parallelism)
- reader = builder.new_read()
- return reader.read_arrow
-
-
-def native_read(table, splits, predicate: Optional[Predicate] = None,
- limit: Optional[int] = None,
- projection: Optional[List[str]] = None,
- blob_parallelism: Optional[int] = None,
- read_type: Optional[List[DataField]] = None,
- include_row_kind: bool = False,
- parallelism: int = 1,
- ordered: bool = False):
- """Run one Core read; ``ordered`` retains the supplied split order."""
- read_splits = _prepare_native_read(
- table,
- predicate=predicate,
- limit=limit,
- projection=projection,
- blob_parallelism=blob_parallelism,
- read_type=read_type,
- include_row_kind=include_row_kind)
- return read_splits(splits, parallelism=parallelism, ordered=ordered)
-
-
-def native_plan(
- table,
- predicate: Optional[Predicate] = None,
- limit: Optional[int] = None,
- projection: Optional[List[str]] = None,
- row_ranges: Optional[List[Tuple[int, int]]] = None,
- incremental_range: Optional[Tuple[int, int]] = None,
- incremental_mode: str = 'delta',
- row_position_slice: Optional[Tuple[int, int]] = None,
- row_position_shard: Optional[Tuple[int, int]] = None,
- chunk_shuffle: Optional[Tuple[int, int]] = None,
- shard: Optional[Tuple[int, int]] = None,
- partition_predicate: Optional[Predicate] = None,
- global_index_result=None,
- all_files: bool = False) -> Plan:
- """Plan with pypaimon_rust, preserving snapshot metadata.
-
- Native conversion or planning failures are handled by TableScan, which
- falls back to the Python planner.
- """
- if all_files and incremental_range is not None:
- raise ValueError('planning all files does not accept incremental ranges')
- if incremental_range is None and incremental_mode != 'delta':
- raise ValueError('incremental_mode requires incremental_range')
- if not native_runtime_available():
- raise RuntimeError(
- "scan.native-plan.enabled needs the pypaimon-rust extension")
- builder = _configure_native_read_builder(
- _native_read_builder(table), predicate, limit, projection)
- if partition_predicate is not None:
- builder = builder.with_partition_filter(_predicate_to_native(partition_predicate))
- if row_ranges is not None:
- builder = builder.with_row_ranges(row_ranges)
- from pypaimon.table.source.global_index_split_result import GlobalIndexSplitResult
- supplied_pk_splits = (isinstance(global_index_result, GlobalIndexSplitResult)
- and not table.options.query_auth_enabled)
- if supplied_pk_splits or incremental_range is None:
- scan = builder.new_scan()
- elif incremental_mode == 'delta':
- scan = builder.new_incremental_scan(*incremental_range)
- else:
- scan = builder.new_incremental_scan(*incremental_range, incremental_mode)
- if global_index_result is not None and not table.options.query_auth_enabled:
- # A PK result already selected a snapshot and physical files. Java's
- # PrimaryKeyBatchScan returns those selections before ordinary planning.
- from pypaimon_rust.datafusion import GlobalIndexResult, Split as NativeSplit
- from pypaimon.globalindex.vector_search_result import ScoredGlobalIndexResult
- from pypaimon.table.source.primary_key_scored_result import PrimaryKeyScoredResult
- if isinstance(global_index_result, GlobalIndexSplitResult):
- # Searches and scans share the same core GlobalIndexResult.
- result = (getattr(global_index_result, '_native_result', None)
- if type(global_index_result) is PrimaryKeyScoredResult else None)
- # Java's PK result uses snapshot 0 for an uncommitted table,
- # whereas the core search uses None. Keep the Python result's ID.
- if result is not None and result.snapshot_id() is None:
- result = None
- if result is None:
- result = GlobalIndexResult.from_splits(
- global_index_result.snapshot_id,
- [NativeSplit.deserialize(serialize_split_v1(split)) for split in global_index_result.splits])
- elif isinstance(global_index_result, ScoredGlobalIndexResult):
- score = global_index_result.score_getter()
- result = GlobalIndexResult.from_scored_row_ids({
- row_id: score(row_id) for row_id in global_index_result.results()})
- else:
- result = GlobalIndexResult.from_row_ranges([
- (range_.from_, range_.to) for range_ in global_index_result.results().to_range_list()])
- scan = scan.with_global_index_result(result)
- if row_position_slice is not None:
- scan = scan.with_row_position_slice(*row_position_slice)
- if row_position_shard is not None:
- scan = scan.with_row_position_shard(*row_position_shard)
- if chunk_shuffle is not None:
- seed, chunk_size = chunk_shuffle
- scan = scan.with_chunk_shuffle(str(seed), chunk_size)
- if shard is not None:
- scan = scan.with_shard(*shard)
- rust_plan = scan.plan_all_files() if all_files else scan.plan()
- return _from_native_plan(table, rust_plan)
-
-
def _native_query_auth_supported(table):
"""Only reproduce authorization owned by the standard REST environment."""
from pypaimon.catalog.catalog_environment import CatalogEnvironment, _TableQueryAuthFn
@@ -639,37 +378,3 @@ def _native_query_auth_supported(table):
or _catalog_metastore(environment.catalog_loader) != 'rest'):
return False
return True
-
-
-def native_stream_scan(table, predicate=None, read_type=None, bucket_filter=None, consumer_id=None):
- """Create Rust's stateful StreamTableScan with the canonical reader type."""
- builder = _configure_native_read_builder(
- _native_read_builder(table), predicate, None, None, read_type=read_type)
- scan = builder.new_stream_scan()
- if bucket_filter is not None:
- scan.with_bucket_filter(bucket_filter)
- if consumer_id is not None:
- scan.with_consumer_id(consumer_id)
- return scan
-
-
-def _from_native_plan(table, rust_plan) -> Plan:
- """Keep one split/metadata bridge for batch and streaming planning."""
- rust_splits = rust_plan.splits()
- pfields = _partition_fields(table)
- # Trimmed primary keys decode per-file min/max keys (PK merge-on-read).
- kfields = table.trimmed_primary_keys_fields
- splits = [
- deserialize_split_v1(split.serialize(), pfields, kfields)
- for split in rust_splits
- ]
- if table.options.native_read_enabled():
- # Retain the opaque Rust split next to the Python metadata view so an
- # unchanged native plan can be read without reserializing each split.
- # A caller that needs different metadata (such as an endpoint DV)
- # passes a new Python split, which the native reader converts at read
- # time from its current fields.
- for split, rust_split in zip(splits, rust_splits):
- split._native_split = rust_split
- snapshot_id = rust_plan.snapshot_id()
- return Plan(splits, snapshot_id=snapshot_id, _native_stats=rust_plan.stats())
diff --git a/paimon-python/pypaimon/read/scanner/__init__.py b/paimon-python/pypaimon/execution/native/write/__init__.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/__init__.py
copy to paimon-python/pypaimon/execution/native/write/__init__.py
diff --git a/paimon-python/pypaimon/read/reader/deduplicate_merge_function.py b/paimon-python/pypaimon/execution/native/write/bucket_metadata.py
similarity index 50%
copy from paimon-python/pypaimon/read/reader/deduplicate_merge_function.py
copy to paimon-python/pypaimon/execution/native/write/bucket_metadata.py
index 5b669aae55..1dec9016e5 100644
--- a/paimon-python/pypaimon/read/reader/deduplicate_merge_function.py
+++ b/paimon-python/pypaimon/execution/native/write/bucket_metadata.py
@@ -16,35 +16,16 @@
# limitations under the License.
################################################################################
-"""Default merge function for primary-key tables.
+"""Rust file planning for distributed input-assignment metadata."""
-Mirrors Java ``DeduplicateMergeFunction`` -- for a run of KVs sharing
-the same primary key, keep only the one with the highest sequence
-number (by virtue of ``add`` being called in sequence-number order).
-"""
-from typing import Optional
+def dynamic_bucket_data_buckets(table, partition, snapshot):
+ from pypaimon.execution.native.read.plan import native_plan
+ from pypaimon.index.dynamic_bucket import bucket_read_builder
-from pypaimon.table.row.key_value import KeyValue
-
-
-class DeduplicateMergeFunction:
- """Keep only the latest KV per primary key.
-
- Used by both the read path (``SortMergeReaderWithMinHeap``) and the
- write path (``KeyValueDataWriter`` in-memory merge buffer) -- the
- latter is what enforces the LSM "PK unique within a file"
- invariant on flush.
- """
-
- def __init__(self):
- self.latest_kv: Optional[KeyValue] = None
-
- def reset(self) -> None:
- self.latest_kv = None
-
- def add(self, kv: KeyValue) -> None:
- self.latest_kv = kv
-
- def get_result(self) -> Optional[KeyValue]:
- return self.latest_kv
+ read_builder = bucket_read_builder(table, partition, snapshot)
+ # plan_all_files includes level zero and requires unrestricted query
+ # authorization. A policy which cannot supply complete metadata must fail
+ # preparation before input assignment, never omit buckets from validation.
+ plan = native_plan(read_builder.table, predicate=read_builder._predicate, all_files=True)
+ return {split.bucket for split in plan.splits()}
diff --git a/paimon-python/pypaimon/execution/native/write/commit.py b/paimon-python/pypaimon/execution/native/write/commit.py
new file mode 100644
index 0000000000..4f27022b23
--- /dev/null
+++ b/paimon-python/pypaimon/execution/native/write/commit.py
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+def prepare_native_commit(owner, messages, *, batch=False):
+ """Prepare the wire bridge before any snapshot mutation starts."""
+ from pypaimon.execution.native.write.native_commit import (
+ create_native_commit, native_commit_callback, native_messages_supported,
+ to_native_commit_messages,
+ )
+ if not native_messages_supported(owner.table, messages):
+ return None
+ if owner._native_commit is None or (batch and not owner._native_commit_is_batch):
+ native = create_native_commit(
+ owner.table, owner.commit_user, owner.overwrite_partition,
+ **({'batch': True} if batch else {}))
+ if native is None:
+ return None
+ if owner._native_commit is not None:
+ owner._native_commit.close()
+ owner._native_commit = native
+ owner._native_commit_is_batch = batch or owner.overwrite_partition is not None
+ owner._native_callbacks_registered = False
+ if owner._native_commit is None:
+ return None
+ native_messages = to_native_commit_messages(owner.table, messages)
+ # Callbacks may be registered after the native committer is created.
+ # The public commit lifecycle owns callback closure in either execution path.
+ if owner._commit_callbacks and not owner._native_callbacks_registered:
+ owner._native_commit.add_commit_callback(native_commit_callback(owner.table, owner._commit_callbacks))
+ owner._native_callbacks_registered = True
+ return owner._native_commit, native_messages
diff --git a/paimon-python/pypaimon/write/native_blob_rows.py b/paimon-python/pypaimon/execution/native/write/native_blob_rows.py
similarity index 100%
rename from paimon-python/pypaimon/write/native_blob_rows.py
rename to paimon-python/pypaimon/execution/native/write/native_blob_rows.py
diff --git a/paimon-python/pypaimon/write/native_commit.py b/paimon-python/pypaimon/execution/native/write/native_commit.py
similarity index 96%
rename from paimon-python/pypaimon/write/native_commit.py
rename to paimon-python/pypaimon/execution/native/write/native_commit.py
index 886baac77b..75b6a1a5a8 100644
--- a/paimon-python/pypaimon/write/native_commit.py
+++ b/paimon-python/pypaimon/execution/native/write/native_commit.py
@@ -20,8 +20,9 @@
import json
from pypaimon.common.json_util import JSON
-from pypaimon.read.native_plan import (
- native_runtime_available, _catalog_context_options, _catalog_metastore, _option_value_to_string,
+from pypaimon.execution.native.runtime import native_runtime_available
+from pypaimon.execution.native.table import (
+ _catalog_context_options, _catalog_metastore, _option_value_to_string,
_resolved_schema_file_io_options)
from pypaimon.utils.file_store_path_factory import canonical_data_file_path
from pypaimon.write.commit_message_serializer import (
@@ -94,7 +95,7 @@ def _rest_source_uuid(table):
"""Identity of the catalog table used to reconstruct a copied FUSE view."""
if table.catalog_environment.uuid is not None:
return table.catalog_environment.uuid
- from pypaimon.read.native_plan import _resolved_rest_table_response, _validate_native_fuse_binding
+ from pypaimon.execution.native.table import _resolved_rest_table_response, _validate_native_fuse_binding
try:
original = _validate_native_fuse_binding(table)
if original is None or original == table.identifier:
@@ -108,7 +109,7 @@ def _rest_source_uuid(table):
def _native_rest_table(table, schema_json):
from pypaimon_rust.datafusion import PaimonCatalog as NativeCatalog
- from pypaimon.read.native_plan import _native_table, _validate_native_fuse_binding
+ from pypaimon.execution.native.table import _native_table, _validate_native_fuse_binding
identifier = _validate_native_fuse_binding(table) or table.identifier
catalog_options = _catalog_context_options(table)
diff --git a/paimon-python/pypaimon/globalindex/native_index_build.py b/paimon-python/pypaimon/execution/native/write/native_index_build.py
similarity index 87%
rename from paimon-python/pypaimon/globalindex/native_index_build.py
rename to paimon-python/pypaimon/execution/native/write/native_index_build.py
index a3d1cc71b1..4eeeb63c2e 100644
--- a/paimon-python/pypaimon/globalindex/native_index_build.py
+++ b/paimon-python/pypaimon/execution/native/write/native_index_build.py
@@ -17,9 +17,10 @@
"""Local global-index builds delegated to Rust, retaining Python's commit API."""
-from pypaimon.read.native_plan import (
- _native_table, _option_value_to_string, _predicate_to_native, native_runtime_available)
-from pypaimon.write.native_commit import _rest_catalog_supported, from_native_commit_messages
+from pypaimon.execution.native.table import (_native_table, _option_value_to_string)
+from pypaimon.execution.native.read.builder import (_predicate_to_native)
+from pypaimon.execution.native.runtime import (native_runtime_available)
+from pypaimon.execution.native.write.native_commit import _rest_catalog_supported, from_native_commit_messages
def build_native_global_index(builder, partition_filter):
@@ -29,7 +30,7 @@ def build_native_global_index(builder, partition_filter):
Build errors propagate: a failed native build must not start a second build.
"""
table = builder._table
- from pypaimon.globalindex.vindex.vindex_vector_global_index_reader import VINDEX_IDENTIFIERS
+ from pypaimon.globalindex.vindex_options import VINDEX_IDENTIFIERS
if (builder._index_type not in ('btree', 'bitmap', 'full-text') + tuple(VINDEX_IDENTIFIERS)
or not table.options.native_write_enabled()
or (table.options.deletion_vectors_enabled() and builder._index_type == 'full-text')
@@ -47,11 +48,11 @@ def build_native_global_index(builder, partition_filter):
options = {str(key): _option_value_to_string(value)
for key, value in builder._user_options.items() if value is not None}
if builder._index_type == 'full-text':
- from pypaimon.globalindex.full_text.native_full_text_global_index_reader import NativeFullTextIndexOptions
+ from pypaimon.globalindex.full_text_options import NativeFullTextIndexOptions
options.update({'full-text.' + key: value for key, value in
NativeFullTextIndexOptions.from_options(builder._options.to_map()).to_native_options().items()})
elif builder._index_type in VINDEX_IDENTIFIERS:
- from pypaimon.globalindex.vindex.vindex_vector_index_writer import native_options, train_sample_ratio
+ from pypaimon.globalindex.vindex_options import native_options, train_sample_ratio
column = builder._index_columns[0]
# Reuse the Java-compatible option mapping; Rust's low-level builder also
# accepts native aliases which Python's public API deliberately ignores.
diff --git a/paimon-python/pypaimon/execution/native/write/native_index_drop.py b/paimon-python/pypaimon/execution/native/write/native_index_drop.py
new file mode 100644
index 0000000000..ef8bdb0819
--- /dev/null
+++ b/paimon-python/pypaimon/execution/native/write/native_index_drop.py
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Native global index deletion message preparation."""
+
+
+def build_native_global_index_drop(builder, partition_filter):
+ from pypaimon.execution.native.table import (_native_table)
+ from pypaimon.execution.native.read.builder import (_predicate_to_native)
+ from pypaimon.execution.native.runtime import (native_runtime_available)
+ from pypaimon.execution.native.write.native_commit import _rest_catalog_supported, from_native_commit_messages
+
+ table = builder._table
+ if (not table.options.native_write_enabled()
+ or table.options.query_auth_enabled
+ or not native_runtime_available()
+ or not _rest_catalog_supported(table)):
+ return None
+ from pypaimon_rust.datafusion import GlobalIndexDropBuilder
+ if not GlobalIndexDropBuilder.supports_index_type(builder._index_type):
+ return None
+ native = _native_table(table).new_global_index_drop_builder()
+ native.with_index_columns(builder._index_columns).with_index_type(builder._index_type)
+ if partition_filter is not None:
+ native.with_partition_filter(_predicate_to_native(partition_filter))
+ # Preparing deletion messages is read-only. Python retains explicit
+ # commit and dry-run ownership; failures must not remove index files.
+ return from_native_commit_messages(table, native.build())
diff --git a/paimon-python/pypaimon/write/native_merge_into.py b/paimon-python/pypaimon/execution/native/write/native_merge_into.py
similarity index 88%
rename from paimon-python/pypaimon/write/native_merge_into.py
rename to paimon-python/pypaimon/execution/native/write/native_merge_into.py
index 97c431d32b..49ce8381eb 100644
--- a/paimon-python/pypaimon/write/native_merge_into.py
+++ b/paimon-python/pypaimon/execution/native/write/native_merge_into.py
@@ -20,15 +20,15 @@
import pyarrow as pa
from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER
-from pypaimon.write.native_commit import create_native_write_table, from_native_commit_messages
-from pypaimon.write.native_update import (
+from pypaimon.execution.native.write.native_commit import create_native_write_table, from_native_commit_messages
+from pypaimon.execution.native.write.native_update import (
_dictionary_value_type, _native_row_id_table, _native_update_columns_supported,
_native_update_formats_supported, _supported_upsert_key_type,
)
def _callable_assignment(function, on_pairs):
- from pypaimon.ray.data_evolution_merge_transform import _merge_callable_input
+ from pypaimon.write.merge_inputs import _merge_callable_input
def evaluate(batch):
return function(_merge_callable_input(batch, on_pairs))
@@ -37,9 +37,9 @@ def _callable_assignment(function, on_pairs):
def create_native_merge_into(table, source, on, matched, not_matched, commit_user, commit_identifier):
- from pypaimon.table.data_evolution_merge_into import _is_self_merge, _is_table_like, _prepare
- from pypaimon.ray.data_evolution_merge_into import _normalize_on, _union_update_cols
- from pypaimon.ray.data_evolution_merge_transform import SourceColumnRef, TargetColumnRef, LiteralValue
+ from pypaimon.write.merge_inputs import _is_self_merge, _is_table_like, _prepare
+ from pypaimon.write.merge_inputs import _normalize_on, _union_update_cols
+ from pypaimon.write.merge_types import SourceColumnRef, TargetColumnRef, LiteralValue
from pypaimon.ray.merge_condition import rewrite_condition
target_keys, source_keys = _normalize_on(on)
@@ -53,14 +53,14 @@ def create_native_merge_into(table, source, on, matched, not_matched, commit_use
native_source = None
if _is_table_like(source) and not self_merge:
# The source can be an append or PK table; only the target requires DE.
- from pypaimon.read.native_plan import _native_blob_view_supported
+ from pypaimon.execution.native.table import (_native_blob_view_supported)
from pypaimon.table.file_store_table import FileStoreTable
if (type(source) is not FileStoreTable
or not _native_update_formats_supported(source)
or source.options.video_frame_fields()
or (source.is_primary_key_table and not source.trimmed_primary_keys)):
return None
- from pypaimon.read.table_scan import _validate_scan_mode
+ from pypaimon.read.scan_config import _validate_scan_mode
_validate_scan_mode(source)
if not _native_blob_view_supported(source, source.field_names):
return None
@@ -76,8 +76,8 @@ def create_native_merge_into(table, source, on, matched, not_matched, commit_use
source_fields = source.new_read_builder().with_projection(projection).read_type()
source_schema = PyarrowFieldParser.from_paimon_schema(source_fields)
if source.options.query_auth_enabled:
- from pypaimon.read.native_plan import _native_query_auth_supported, _native_table
- from pypaimon.write.native_commit import _rest_catalog_supported
+ from pypaimon.execution.native.table import (_native_query_auth_supported, _native_table)
+ from pypaimon.execution.native.write.native_commit import _rest_catalog_supported
if not _rest_catalog_supported(source) or not _native_query_auth_supported(source):
return None
# A source is a read, not a write. Core obtains a fresh REST grant
@@ -87,7 +87,7 @@ def create_native_merge_into(table, source, on, matched, not_matched, commit_use
native_source = create_native_write_table(source)
if native_source is None:
return None
- from pypaimon.read.native_plan import _with_native_read_snapshot
+ from pypaimon.execution.native.table import (_with_native_read_snapshot)
native_source = _with_native_read_snapshot(source, native_source)
source, matched, not_matched, context = _prepare(
table, source, list(matched), list(not_matched), on, source_schema=source_schema)
diff --git a/paimon-python/pypaimon/write/native_update.py b/paimon-python/pypaimon/execution/native/write/native_update.py
similarity index 96%
rename from paimon-python/pypaimon/write/native_update.py
rename to paimon-python/pypaimon/execution/native/write/native_update.py
index 0a8b6601ed..0d21285428 100644
--- a/paimon-python/pypaimon/write/native_update.py
+++ b/paimon-python/pypaimon/execution/native/write/native_update.py
@@ -23,16 +23,17 @@ from datetime import date, datetime, time
import pyarrow as pa
import pyarrow.compute as pc
-from pypaimon.read.native_plan import _native_query_auth_supported, native_runtime_available
+from pypaimon.execution.native.table import _native_query_auth_supported
+from pypaimon.execution.native.runtime import native_runtime_available
from pypaimon.schema.arrow_schema import arrow_schemas_compatible
from pypaimon.schema.data_types import MapType, PyarrowFieldParser, RowType, VectorType, is_blob_file_field
from pypaimon.common.options.core_options import ChangelogProducer
from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER
from pypaimon.table.file_store_table import FileStoreTable
-from pypaimon.write.native_commit import (
+from pypaimon.execution.native.write.native_commit import (
create_native_write_table, from_native_commit_messages,
)
-from pypaimon.write.table_update_by_row_id import TableUpdateByRowId
+from pypaimon.write.row_input import RowIdUpdateInputMixin
from pypaimon.write.row_utils import _contains_blob_value, value_for_arrow
@@ -53,7 +54,7 @@ def _native_update_table(table, *, allow_changelog=False):
return None
native_table = create_native_write_table(table)
if native_table is not None:
- from pypaimon.read.native_plan import _with_native_read_snapshot
+ from pypaimon.execution.native.table import _with_native_read_snapshot
native_table = _with_native_read_snapshot(table, native_table)
return native_table
@@ -352,7 +353,7 @@ def create_native_upsert(table, commit_user, data, keys, columns):
if any(_contains_blob_value(value) for row in data for value in row.values()) or any(
is_blob_file_field(field) and isinstance(field.type, MapType) and field.name in row
for field in fields for row in data):
- from pypaimon.write.native_blob_rows import NativeBlobRows
+ from pypaimon.execution.native.write.native_blob_rows import NativeBlobRows
blob_rows = NativeBlobRows()
data = _upsert_row_batches(
data, fields, schema, table, blob_rows, list(dict.fromkeys(keys + table.partition_keys)))
@@ -393,7 +394,7 @@ def create_native_predicate_update(table, commit_user, predicate, columns=None):
native_table = _native_row_id_table(table)
if native_table is None:
return None
- from pypaimon.read.native_plan import _predicate_to_native
+ from pypaimon.execution.native.read.builder import _predicate_to_native
native_predicate = None if predicate is None else _predicate_to_native(predicate)
if native_predicate is not None:
# Check predicate translation while fallback is still safe. Planning,
@@ -406,7 +407,8 @@ def create_native_predicate_update(table, commit_user, predicate, columns=None):
def native_predicate_row_ids(scan_table, predicate, splits):
"""Match a batch delete predicate in Rust and return its row IDs."""
- from pypaimon.read.native_plan import _prepare_native_read, native_split_from_python
+ from pypaimon.execution.native.read.table_read import _prepare_native_read
+ from pypaimon.execution.native.read.split_adapter import native_split_from_python
if not native_runtime_available():
return None
reader = _prepare_native_read(
@@ -423,7 +425,7 @@ def native_predicate_row_ids(scan_table, predicate, splits):
def create_native_predicate_delete(table, commit_user, predicate):
"""Delegate matching and partition metadata deletion to the core updater."""
- from pypaimon.read.native_plan import _predicate_to_native
+ from pypaimon.execution.native.read.builder import _predicate_to_native
if type(table) is not FileStoreTable:
return None
@@ -435,8 +437,8 @@ def create_native_predicate_delete(table, commit_user, predicate):
# Core selects whole-partition metadata deletes for append/PK tables too;
# only the row-level branch requires row tracking and deletion vectors.
- from pypaimon.write.table_update import TableUpdate
- fields = set() if predicate is None else TableUpdate._predicate_fields(predicate)
+ from pypaimon.write.update_validation import predicate_fields
+ fields = set() if predicate is None else predicate_fields(predicate)
partition_delete = (table.is_primary_key_table and bool(fields)
and fields.issubset(table.partition_keys) and canonical_paths(predicate))
# Partition deletion removes complete file/index metadata. It does not
@@ -511,7 +513,7 @@ class NativeBatchTableUpdate:
return from_native_commit_messages(self.table, messages)
-class NativeTableUpdateByRowId(TableUpdateByRowId):
+class NativeTableUpdateByRowId(RowIdUpdateInputMixin):
"""Adapt Python row inputs around the core incremental Arrow updater."""
def __init__(self, table, commit_user, commit_identifier, writer):
@@ -536,7 +538,7 @@ class NativeTableUpdateByRowId(TableUpdateByRowId):
# field layouts and Blob references are handled by the core updater.
if not blob_object_columns:
return self.update_columns(data, column_names)
- from pypaimon.write.native_blob_rows import NativeBlobRows
+ from pypaimon.execution.native.write.native_blob_rows import NativeBlobRows
blobs = NativeBlobRows()
for name, values in blob_object_columns.items():
field = self.table.field_dict[name]
diff --git a/paimon-python/pypaimon/write/native_write.py b/paimon-python/pypaimon/execution/native/write/native_write.py
similarity index 79%
rename from paimon-python/pypaimon/write/native_write.py
rename to paimon-python/pypaimon/execution/native/write/native_write.py
index 43f2235492..11a3d86321 100644
--- a/paimon-python/pypaimon/write/native_write.py
+++ b/paimon-python/pypaimon/execution/native/write/native_write.py
@@ -19,11 +19,11 @@
import pyarrow as pa
from pypaimon.common.options.core_options import ChangelogProducer
-from pypaimon.read.native_plan import native_runtime_available
+from pypaimon.execution.native.runtime import native_runtime_available
from pypaimon.schema.arrow_schema import arrow_schemas_compatible
from pypaimon.schema.data_types import PyarrowFieldParser, is_blob_file_field
from pypaimon.table.bucket_mode import BucketMode
-from pypaimon.write.native_commit import (
+from pypaimon.execution.native.write.native_commit import (
create_native_write_table, from_native_commit_messages,
)
from pypaimon.write.row_utils import require_columns, row_to_named_values, row_values_to_arrow_table
@@ -87,6 +87,7 @@ class NativeTableWrite:
self._native_writer = native_writer
self._python_writer = None
self._written = False
+ self._batch_committed = False
self._table_schema = PyarrowFieldParser.from_paimon_schema(table.table_schema.fields)
self._schema = self._table_schema
self._write_cols = None
@@ -96,32 +97,26 @@ class NativeTableWrite:
self._blob_rows = None
self._defer_blob_input = False
if any(is_blob_file_field(field) for field in table.table_schema.fields):
- from pypaimon.write.native_blob_rows import NativeBlobRows
+ from pypaimon.execution.native.write.native_blob_rows import NativeBlobRows
self._blob_rows = NativeBlobRows()
self._native_writer.with_blob_uri_reader_factory(self._blob_rows)
def _switch_to_python(self):
- if self._python_writer is not None:
- return self._python_writer
- if self._written:
- raise RuntimeError(
- 'Cannot switch to the Python writer after native data was written')
- self._native_writer.close()
- self._native_writer = None
- self._python_writer = self._new_python_writer()
- return self._python_writer
+ from pypaimon.execution.dispatch import switch_writer_to_python
+ return switch_writer_to_python(self)
def _new_python_writer(self):
- from pypaimon.write.table_write import BatchTableWrite, StreamTableWrite
- if self.stream:
- writer = StreamTableWrite(self.table, self.commit_user,
- restore_snapshot_id=self._restore_snapshot_id)
- else:
- writer = BatchTableWrite(self.table, self.commit_user, self.static_partition,
- restore_snapshot_id=self._restore_snapshot_id)
+ from pypaimon.execution.dispatch import create_python_write
+ writer = create_python_write(
+ self.table, self.commit_user, self.static_partition, self.stream,
+ restore_snapshot_id=self._restore_snapshot_id)
return self._configure_python_writer(writer)
def _configure_python_writer(self, writer):
+ if not self.stream:
+ # Empty batch preparation can leave _written false. Explicit
+ # diagnostics must not reset the one-time preparation guard.
+ writer.batch_committed = self._batch_committed
if self._write_cols is not None:
writer.with_write_type(self._write_cols)
if self._blob_consumer is not None:
@@ -130,10 +125,53 @@ class NativeTableWrite:
writer.with_blob_uri_reader_factory(self._blob_uri_reader_factory)
return writer
- def __getattr__(self, name):
- if name.startswith('_'):
- raise AttributeError(name)
- return getattr(self._switch_to_python(), name)
+ @property
+ def file_store_write(self):
+ """Explicitly request the Python writer's diagnostic implementation."""
+ return self._switch_to_python().file_store_write
+
+ @file_store_write.setter
+ def file_store_write(self, writer):
+ self._switch_to_python().file_store_write = writer
+
+ @property
+ def row_key_extractor(self):
+ """Explicitly request the Python writer's routing implementation."""
+ return self._switch_to_python().row_key_extractor
+
+ @row_key_extractor.setter
+ def row_key_extractor(self, extractor):
+ self._switch_to_python().row_key_extractor = extractor
+
+ @property
+ def table_pyarrow_schema(self):
+ return (self._python_writer.table_pyarrow_schema
+ if self._python_writer is not None else self._table_schema)
+
+ @table_pyarrow_schema.setter
+ def table_pyarrow_schema(self, schema):
+ self._switch_to_python().table_pyarrow_schema = schema
+
+ @property
+ def restore_snapshot_id(self):
+ return (self._python_writer.restore_snapshot_id
+ if self._python_writer is not None else self._restore_snapshot_id)
+
+ @restore_snapshot_id.setter
+ def restore_snapshot_id(self, snapshot_id):
+ self._switch_to_python().restore_snapshot_id = snapshot_id
+
+ @property
+ def batch_committed(self):
+ if self._python_writer is not None:
+ return self._python_writer.batch_committed
+ if self.stream:
+ raise AttributeError('batch_committed')
+ return self._batch_committed
+
+ @batch_committed.setter
+ def batch_committed(self, value):
+ self._switch_to_python().batch_committed = value
def with_write_type(self, write_cols):
if self._python_writer is not None:
@@ -156,7 +194,7 @@ class NativeTableWrite:
if self._python_writer is not None:
return self._python_writer.with_blob_consumer(blob_consumer)
if blob_consumer is None:
- self._native_writer.with_blob_consumer(None)
+ self._configure_native_blob('with_blob_consumer', None)
self._blob_consumer = None
return self
if not callable(blob_consumer):
@@ -167,7 +205,7 @@ class NativeTableWrite:
descriptor = None if encoded is None else BlobDescriptor.deserialize(encoded)
return blob_consumer(field_name, descriptor)
- self._native_writer.with_blob_consumer(consume)
+ self._configure_native_blob('with_blob_consumer', consume)
self._blob_consumer = blob_consumer
return self
@@ -180,13 +218,25 @@ class NativeTableWrite:
# Only core knows whether RowKind filtering accepted any rows or a
# postpone writer buffered data. Validate before changing the live
# bridge's fallback, including when clearing the application factory.
- self._native_writer.with_blob_uri_reader_factory(self._blob_rows)
+ self._configure_native_blob('with_blob_uri_reader_factory', self._blob_rows)
self._blob_rows.fallback = uri_reader_factory
else:
- self._native_writer.with_blob_uri_reader_factory(uri_reader_factory)
+ self._configure_native_blob('with_blob_uri_reader_factory', uri_reader_factory)
self._blob_uri_reader_factory = uri_reader_factory
return self
+ def _configure_native_blob(self, method, value):
+ try:
+ getattr(self._native_writer, method)(value)
+ except ValueError as error:
+ # Core owns written state: an ignored RowKind can leave it writable
+ # even after this facade accepted a batch. Preserve the existing
+ # Python state-error type without changing unrelated input errors.
+ message = method + ' must be called before any write operation'
+ if message in str(error):
+ raise RuntimeError(message) from error
+ raise
+
def write_arrow(self, data, bucket=None):
if self._python_writer is not None:
return self._python_writer.write_arrow(data, bucket)
@@ -228,7 +278,7 @@ class NativeTableWrite:
def write_pandas(self, dataframe):
if self._python_writer is not None:
return self._python_writer.write_pandas(dataframe)
- from pypaimon.write.table_write import _pandas_write_batch
+ from pypaimon.write.input_adapter import _pandas_write_batch
self.write_arrow_batch(_pandas_write_batch(self.table, dataframe, self._schema, self._table_schema))
def write_ray(self, dataset, overwrite=False, concurrency=None, ray_remote_args=None,
@@ -236,7 +286,7 @@ class NativeTableWrite:
if self._python_writer is not None:
return self._python_writer.write_ray(
dataset, overwrite, concurrency, ray_remote_args, hash_fixed_precluster, static_partition)
- from pypaimon.write.table_write import _write_ray
+ from pypaimon.write.input_adapter import _write_ray
_write_ray(self, dataset, overwrite, concurrency, ray_remote_args,
hash_fixed_precluster, static_partition)
@@ -302,6 +352,7 @@ class NativeTableWrite:
else:
if commit_identifier is not None:
raise TypeError('BatchTableWrite.prepare_commit accepts no identifier')
+ self._batch_committed = True
try:
messages = self._native_writer.prepare_commit()
finally:
@@ -331,8 +382,8 @@ class NativeTableWrite:
class NativePostponeFixedBucketTableWrite(NativeTableWrite):
"""Fixed-bucket batches retain the standard native ownership contract.
- A distributed coordinator can use Python planning before receiving data;
- workers use the same shared plan through the Rust fixed-bucket writer.
+ A distributed coordinator reads bucket metadata through Rust and groups
+ inputs with shared arithmetic before workers use the fixed-bucket writer.
"""
def __init__(self, table, commit_user, static_partition, bucket_plan, native_writer):
@@ -343,13 +394,11 @@ class NativePostponeFixedBucketTableWrite(NativeTableWrite):
and table.options.postpone_default_bucket_num() is None)
def _new_python_writer(self):
- from pypaimon.write.postpone_batch_table_write import PostponeFixedBucketBatchTableWrite
- return self._configure_python_writer(PostponeFixedBucketBatchTableWrite(
- self.table, self.commit_user, self.static_partition, self._bucket_plan))
+ from pypaimon.execution.dispatch import create_python_write
+ return self._configure_python_writer(create_python_write(
+ self.table, self.commit_user, self.static_partition,
+ fixed_bucket=True, bucket_plan=self._bucket_plan))
def _distributed_write_options(self, *, overwrite=False):
- from pypaimon.write.postpone_bucket import PostponeBucketPlanner
- return {'postpone_bucket_planner': PostponeBucketPlanner(
- self.table, known_num_buckets=(
- self._bucket_plan.as_dict() if self._bucket_plan is not None
- else {} if overwrite else None))}
+ from pypaimon.write.input_adapter import postpone_distributed_write_options
+ return postpone_distributed_write_options(self.table, self._bucket_plan, overwrite)
diff --git a/paimon-python/pypaimon/execution/native/write/postpone_bucket.py b/paimon-python/pypaimon/execution/native/write/postpone_bucket.py
new file mode 100644
index 0000000000..ed07f07c9e
--- /dev/null
+++ b/paimon-python/pypaimon/execution/native/write/postpone_bucket.py
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Read existing bucket layouts through Rust before distributed input work."""
+
+from pypaimon.execution.native.read.plan import native_plan
+from pypaimon.execution.native.runtime import native_runtime_available
+from pypaimon.write.bucket_planning import PostponeBucketPlanning
+
+
+def create_native_postpone_bucket_planner(table, known_num_buckets=None):
+ if not native_runtime_available():
+ return None
+ return NativePostponeBucketPlanner(table, known_num_buckets)
+
+
+class NativePostponeBucketPlanner(PostponeBucketPlanning):
+ """Use Rust planning metadata and shared input-only bucket arithmetic."""
+
+ @staticmethod
+ def _load_bucket_metadata(table):
+ # PartitionStat does not carry total_buckets. Complete file planning
+ # preserves that layout in the shared Split v1 metadata without ever
+ # creating a Python scanner, reader or data writer.
+ known = {}
+ for split in native_plan(table, all_files=True).splits():
+ if split.bucket < 0 or split.total_buckets is None or split.total_buckets <= 0:
+ continue
+ partition = tuple(split.partition.values)
+ previous = known.get(partition)
+ if previous is not None and previous != split.total_buckets:
+ raise RuntimeError(
+ "Partition {} has different total buckets {} and {}"
+ .format(partition, previous, split.total_buckets))
+ known[partition] = split.total_buckets
+ return known
diff --git a/paimon-python/pypaimon/read/scanner/__init__.py b/paimon-python/pypaimon/execution/python/__init__.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/__init__.py
copy to paimon-python/pypaimon/execution/python/__init__.py
diff --git a/paimon-python/pypaimon/read/scanner/__init__.py b/paimon-python/pypaimon/execution/python/globalindex/__init__.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/__init__.py
copy to paimon-python/pypaimon/execution/python/globalindex/__init__.py
diff --git a/paimon-python/pypaimon/read/scanner/__init__.py b/paimon-python/pypaimon/execution/python/globalindex/bitmap/__init__.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/__init__.py
copy to paimon-python/pypaimon/execution/python/globalindex/bitmap/__init__.py
diff --git a/paimon-python/pypaimon/globalindex/bitmap/bitmap_index_reader.py b/paimon-python/pypaimon/execution/python/globalindex/bitmap/bitmap_index_reader.py
similarity index 98%
rename from paimon-python/pypaimon/globalindex/bitmap/bitmap_index_reader.py
rename to paimon-python/pypaimon/execution/python/globalindex/bitmap/bitmap_index_reader.py
index 0f182014c0..566bc8afc7 100644
--- a/paimon-python/pypaimon/globalindex/bitmap/bitmap_index_reader.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/bitmap/bitmap_index_reader.py
@@ -23,14 +23,14 @@ import threading
from typing import Callable, Dict, List, Optional
from pypaimon.common.file_io import FileIO, pread, supports_pread
-from pypaimon.globalindex.block_compression import (
+from pypaimon.execution.python.globalindex.block_compression import (
BLOCK_TRAILER_LENGTH,
decompress_block_with_trailer,
)
from pypaimon.globalindex.global_index_meta import GlobalIndexIOMeta
from pypaimon.globalindex.global_index_result import GlobalIndexResult
-from pypaimon.globalindex.key_serializer import KeySerializer
-from pypaimon.globalindex.memory_slice_input import MemorySliceInput
+from pypaimon.execution.python.globalindex.key_serializer import KeySerializer
+from pypaimon.execution.python.globalindex.memory_slice_input import MemorySliceInput
from pypaimon.utils.roaring_bitmap import RoaringBitmap64
diff --git a/paimon-python/pypaimon/globalindex/bitmap/bitmap_index_writer.py b/paimon-python/pypaimon/execution/python/globalindex/bitmap/bitmap_index_writer.py
similarity index 97%
rename from paimon-python/pypaimon/globalindex/bitmap/bitmap_index_writer.py
rename to paimon-python/pypaimon/execution/python/globalindex/bitmap/bitmap_index_writer.py
index c2cd24d7ec..fc3d1e1eb0 100644
--- a/paimon-python/pypaimon/globalindex/bitmap/bitmap_index_writer.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/bitmap/bitmap_index_writer.py
@@ -21,7 +21,7 @@ from dataclasses import dataclass
import struct
from typing import Dict, List
-from pypaimon.globalindex.index_file_utils import (
+from pypaimon.execution.python.globalindex.index_file_utils import (
BlockInfo,
PositionOutput,
new_global_index_file_name,
@@ -31,12 +31,12 @@ from pypaimon.globalindex.index_file_utils import (
write_var_len_int,
write_var_len_long,
)
-from pypaimon.globalindex.result_entry import ResultEntry
+from pypaimon.execution.python.globalindex.result_entry import ResultEntry
from pypaimon.globalindex.sorted_index_file_meta import SortedIndexFileMeta
from pypaimon.utils.roaring_bitmap import RoaringBitmap64
-BITMAP_IDENTIFIER = "bitmap"
+from pypaimon.globalindex.index_types import BITMAP_IDENTIFIER
_BITMAP_FOOTER_LENGTH = 48
_BITMAP_MAGIC = 0x42474958
_BITMAP_VERSION = 1
diff --git a/paimon-python/pypaimon/globalindex/bitmap/lazy_filtered_bitmap_reader.py b/paimon-python/pypaimon/execution/python/globalindex/bitmap/lazy_filtered_bitmap_reader.py
similarity index 86%
rename from paimon-python/pypaimon/globalindex/bitmap/lazy_filtered_bitmap_reader.py
rename to paimon-python/pypaimon/execution/python/globalindex/bitmap/lazy_filtered_bitmap_reader.py
index 024f719cbd..e7e5bad1ae 100644
--- a/paimon-python/pypaimon/globalindex/bitmap/lazy_filtered_bitmap_reader.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/bitmap/lazy_filtered_bitmap_reader.py
@@ -17,9 +17,9 @@
"""Bitmap global index reader for sorted index files."""
-from pypaimon.globalindex.bitmap.bitmap_index_reader import BitmapIndexReader
+from pypaimon.execution.python.globalindex.bitmap.bitmap_index_reader import BitmapIndexReader
from pypaimon.globalindex.global_index_meta import GlobalIndexIOMeta
-from pypaimon.globalindex.sorted_file_global_index_reader import SortedFileGlobalIndexReader
+from pypaimon.execution.python.globalindex.sorted_file_global_index_reader import SortedFileGlobalIndexReader
class LazyFilteredBitmapReader(SortedFileGlobalIndexReader):
diff --git a/paimon-python/pypaimon/globalindex/block_compression.py b/paimon-python/pypaimon/execution/python/globalindex/block_compression.py
similarity index 99%
rename from paimon-python/pypaimon/globalindex/block_compression.py
rename to paimon-python/pypaimon/execution/python/globalindex/block_compression.py
index 68931af6e3..72439239a2 100644
--- a/paimon-python/pypaimon/globalindex/block_compression.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/block_compression.py
@@ -20,7 +20,7 @@
import struct
import zlib
-from pypaimon.globalindex.memory_slice_input import MemorySliceInput
+from pypaimon.execution.python.globalindex.memory_slice_input import MemorySliceInput
COMPRESSION_NONE = 0
diff --git a/paimon-python/pypaimon/read/scanner/__init__.py b/paimon-python/pypaimon/execution/python/globalindex/btree/__init__.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/__init__.py
copy to paimon-python/pypaimon/execution/python/globalindex/btree/__init__.py
diff --git a/paimon-python/pypaimon/globalindex/btree/block_aligned_type.py b/paimon-python/pypaimon/execution/python/globalindex/btree/block_aligned_type.py
similarity index 99%
rename from paimon-python/pypaimon/globalindex/btree/block_aligned_type.py
rename to paimon-python/pypaimon/execution/python/globalindex/btree/block_aligned_type.py
index bb86baaff1..a5e0c73e39 100644
--- a/paimon-python/pypaimon/globalindex/btree/block_aligned_type.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/btree/block_aligned_type.py
@@ -22,10 +22,10 @@ from enum import Enum
class BlockAlignedType(Enum):
"""Aligned type for block."""
-
+
ALIGNED = 0
UNALIGNED = 1
-
+
@classmethod
def from_byte(cls, b: int) -> 'BlockAlignedType':
"""Create BlockAlignedType from byte value."""
diff --git a/paimon-python/pypaimon/globalindex/btree/block_entry.py b/paimon-python/pypaimon/execution/python/globalindex/btree/block_entry.py
similarity index 98%
rename from paimon-python/pypaimon/globalindex/btree/block_entry.py
rename to paimon-python/pypaimon/execution/python/globalindex/btree/block_entry.py
index e625aad047..42e03dede3 100644
--- a/paimon-python/pypaimon/globalindex/btree/block_entry.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/btree/block_entry.py
@@ -20,10 +20,10 @@
class BlockEntry:
"""Entry represents a key value."""
-
+
def __init__(self, key: bytes, value: bytes):
"""Initialize BlockEntry.
-
+
Args:
key: The key bytes
value: The value bytes
@@ -34,7 +34,7 @@ class BlockEntry:
raise ValueError("value is null")
self.key = key
self.value = value
-
+
def __eq__(self, other) -> bool:
"""Check equality with another BlockEntry."""
if self is other:
@@ -42,13 +42,13 @@ class BlockEntry:
if other is None or not isinstance(other, BlockEntry):
return False
return self.key == other.key and self.value == other.value
-
+
def __hash__(self) -> int:
"""Return hash of this BlockEntry."""
result = hash(self.key)
result = 31 * result + hash(self.value)
return result
-
+
def __repr__(self) -> str:
"""Return string representation."""
return f"BlockEntry(key={self.key!r}, value={self.value!r})"
diff --git a/paimon-python/pypaimon/globalindex/btree/block_handle.py b/paimon-python/pypaimon/execution/python/globalindex/btree/block_handle.py
similarity index 100%
copy from paimon-python/pypaimon/globalindex/btree/block_handle.py
copy to paimon-python/pypaimon/execution/python/globalindex/btree/block_handle.py
diff --git a/paimon-python/pypaimon/globalindex/btree/block_reader.py b/paimon-python/pypaimon/execution/python/globalindex/btree/block_reader.py
similarity index 93%
rename from paimon-python/pypaimon/globalindex/btree/block_reader.py
rename to paimon-python/pypaimon/execution/python/globalindex/btree/block_reader.py
index 1702696943..d757078da9 100644
--- a/paimon-python/pypaimon/globalindex/btree/block_reader.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/btree/block_reader.py
@@ -20,14 +20,14 @@
import struct
from typing import Callable, Optional
-from pypaimon.globalindex.btree.block_aligned_type import BlockAlignedType
-from pypaimon.globalindex.btree.block_entry import BlockEntry
-from pypaimon.globalindex.memory_slice_input import MemorySliceInput
+from pypaimon.execution.python.globalindex.btree.block_aligned_type import BlockAlignedType
+from pypaimon.execution.python.globalindex.btree.block_entry import BlockEntry
+from pypaimon.execution.python.globalindex.memory_slice_input import MemorySliceInput
class BlockReader:
"""Reader for a block."""
-
+
def __init__(
self,
block: bytes,
@@ -35,7 +35,7 @@ class BlockReader:
comparator: Callable[[bytes, bytes], int]
):
"""Initialize BlockReader.
-
+
Args:
block: The block data bytes
record_count: Number of records in the block
@@ -44,40 +44,40 @@ class BlockReader:
self.block = block
self.record_count = record_count
self.comparator = comparator
-
+
def block_input(self) -> MemorySliceInput:
"""Create a MemorySliceInput for this block."""
return MemorySliceInput(self.block)
-
+
def iterator(self) -> 'BlockIterator':
"""Create a BlockIterator for this reader."""
return BlockIterator(self)
-
+
def seek_to(self, record_position: int) -> int:
"""Seek to slice position from record position.
-
+
Args:
record_position: The record position to seek to
-
+
Returns:
The slice position
-
+
Raises:
NotImplementedError: If not implemented in subclass
"""
raise NotImplementedError("seekTo must be implemented in subclass")
-
+
@staticmethod
def create(
block: bytes,
comparator: Optional[Callable[[bytes, bytes], int]] = None
) -> 'BlockReader':
"""Create a BlockReader from block data.
-
+
Args:
block: The block data bytes
comparator: Optional comparator function for keys
-
+
Returns:
A BlockReader instance (AlignedBlockReader or UnalignedBlockReader)
"""
@@ -85,7 +85,7 @@ class BlockReader:
aligned_type_byte = block[-1]
aligned_type = BlockAlignedType.from_byte(aligned_type_byte)
int_value = struct.unpack('<I', block[-5:-1])[0]
-
+
if aligned_type == BlockAlignedType.ALIGNED:
# Aligned block: records have fixed size
data = block[:-5]
@@ -101,7 +101,7 @@ class BlockReader:
class AlignedBlockReader(BlockReader):
"""Block reader for aligned blocks (fixed record size)."""
-
+
def __init__(
self,
data: bytes,
@@ -109,7 +109,7 @@ class AlignedBlockReader(BlockReader):
comparator: Optional[Callable[[bytes, bytes], int]] = None
):
"""Initialize AlignedBlockReader.
-
+
Args:
data: The block data bytes
record_size: The fixed size of each record
@@ -118,13 +118,13 @@ class AlignedBlockReader(BlockReader):
record_count = len(data) // record_size
super().__init__(data, record_count, comparator)
self.record_size = record_size
-
+
def seek_to(self, record_position: int) -> int:
"""Seek to slice position from record position.
-
+
Args:
record_position: The record position to seek to
-
+
Returns:
The slice position
"""
@@ -133,7 +133,7 @@ class AlignedBlockReader(BlockReader):
class UnalignedBlockReader(BlockReader):
"""Block reader for unaligned blocks (uses index)."""
-
+
def __init__(
self,
data: bytes,
@@ -141,7 +141,7 @@ class UnalignedBlockReader(BlockReader):
comparator: Optional[Callable[[bytes, bytes], int]] = None
):
"""Initialize UnalignedBlockReader.
-
+
Args:
data: The block data bytes
index: The index bytes (4 bytes per record)
@@ -150,13 +150,13 @@ class UnalignedBlockReader(BlockReader):
record_count = len(index) // 4
super().__init__(data, record_count, comparator)
self.index = index
-
+
def seek_to(self, record_position: int) -> int:
"""Seek to slice position from record position.
-
+
Args:
record_position: The record position to seek to
-
+
Returns:
The slice position
"""
@@ -167,63 +167,63 @@ class UnalignedBlockReader(BlockReader):
class BlockIterator:
"""Iterator for block entries."""
-
+
def __init__(self, reader: BlockReader):
"""Initialize BlockIterator.
-
+
Args:
reader: The BlockReader to iterate over
"""
self.reader = reader
self.input = reader.block_input()
self.polled: Optional[BlockEntry] = None
-
+
def __iter__(self):
"""Return self as iterator."""
return self
-
+
def __next__(self) -> BlockEntry:
"""Get next entry.
-
+
Returns:
The next BlockEntry
-
+
Raises:
StopIteration: If no more entries
"""
if not self.has_next():
raise StopIteration
-
+
if self.polled is not None:
result = self.polled
self.polled = None
return result
-
+
return self.read_entry()
-
+
def has_next(self) -> bool:
"""Check if there are more entries."""
return self.polled is not None or self.input.is_readable()
-
+
def seek_to(self, target_key: bytes) -> bool:
"""Seek to the first key >= target_key using binary search.
-
+
Args:
target_key: The target key to seek to
-
+
Returns:
True if exact match found, False otherwise
"""
left = 0
right = self.reader.record_count - 1
-
+
while left <= right:
mid = left + (right - left) // 2
-
+
self.input.set_position(self.reader.seek_to(mid))
mid_entry = self.read_entry()
compare = self.reader.comparator(mid_entry.key, target_key)
-
+
if compare == 0:
self.polled = mid_entry
return True
@@ -233,21 +233,21 @@ class BlockIterator:
else:
self.polled = None
left = mid + 1
-
+
return False
-
+
def read_entry(self) -> BlockEntry:
"""Read a key-value entry.
-
+
Returns:
A BlockEntry containing key and value
"""
# Read key
key_length = self.input.read_var_len_int()
key = self.input.read_slice(key_length)
-
+
# Read value
value_length = self.input.read_var_len_int()
value = self.input.read_slice(value_length)
-
+
return BlockEntry(key, value)
diff --git a/paimon-python/pypaimon/globalindex/btree/bloom_filter.py b/paimon-python/pypaimon/execution/python/globalindex/btree/bloom_filter.py
similarity index 100%
rename from paimon-python/pypaimon/globalindex/btree/bloom_filter.py
rename to paimon-python/pypaimon/execution/python/globalindex/btree/bloom_filter.py
diff --git a/paimon-python/pypaimon/globalindex/btree/btree_file_footer.py b/paimon-python/pypaimon/execution/python/globalindex/btree/btree_file_footer.py
similarity index 95%
rename from paimon-python/pypaimon/globalindex/btree/btree_file_footer.py
rename to paimon-python/pypaimon/execution/python/globalindex/btree/btree_file_footer.py
index 4b6940b0f6..493f3d200d 100644
--- a/paimon-python/pypaimon/globalindex/btree/btree_file_footer.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/btree/btree_file_footer.py
@@ -21,17 +21,17 @@ from dataclasses import dataclass
from typing import Optional
import struct
-from pypaimon.globalindex.btree.block_handle import BlockHandle
+from pypaimon.execution.python.globalindex.btree.block_handle import BlockHandle
@dataclass
class BloomFilterHandle:
"""Handle for bloom filter block."""
-
+
offset: int
size: int
expected_entries: int
-
+
def is_null(self) -> bool:
"""Check if this handle represents a null bloom filter."""
return self.offset == 0 and self.size == 0 and self.expected_entries == 0
@@ -40,11 +40,11 @@ class BloomFilterHandle:
class BTreeFileFooter:
"""
The Footer for BTree file.
-
+
This footer contains the handles to the bloom filter, index block, and null bitmap,
allowing efficient navigation of the B-tree index file.
"""
-
+
MAGIC_NUMBER = 0x50425449
VERSION_1 = 1
VERSION_2 = 2
@@ -71,18 +71,18 @@ class BTreeFileFooter:
self.bloom_filter_handle = bloom_filter_handle
self.index_block_handle = index_block_handle
self.null_bitmap_handle = null_bitmap_handle
-
+
@classmethod
def read_footer(cls, data: bytes) -> 'BTreeFileFooter':
"""
Read footer from byte data.
-
+
Args:
data: Byte data containing the footer
-
+
Returns:
BTreeFileFooter instance
-
+
Raises:
ValueError: If magic number doesn't match
"""
@@ -96,7 +96,7 @@ class BTreeFileFooter:
raise ValueError("File is not a btree index file (bad magic number)")
offset = 0
-
+
# Read bloom filter handle
bf_offset = struct.unpack('<Q', data[offset:offset + 8])[0]
offset += 8
@@ -104,25 +104,25 @@ class BTreeFileFooter:
offset += 4
bf_expected = struct.unpack('<Q', data[offset:offset + 8])[0]
offset += 8
-
+
bloom_filter_handle = None
if not (bf_offset == 0 and bf_size == 0 and bf_expected == 0):
bloom_filter_handle = BloomFilterHandle(bf_offset, bf_size, bf_expected)
-
+
# Read index block handle
index_offset = struct.unpack('<Q', data[offset:offset + 8])[0]
offset += 8
index_size = struct.unpack('<I', data[offset:offset + 4])[0]
offset += 4
-
+
index_block_handle = BlockHandle(index_offset, index_size)
-
+
# Read null bitmap handle
nb_offset = struct.unpack('<Q', data[offset:offset + 8])[0]
offset += 8
nb_size = struct.unpack('<I', data[offset:offset + 4])[0]
offset += 4
-
+
null_bitmap_handle = None
if not (nb_offset == 0 and nb_size == 0):
null_bitmap_handle = BlockHandle(nb_offset, nb_size)
diff --git a/paimon-python/pypaimon/globalindex/btree/btree_file_meta_selector.py b/paimon-python/pypaimon/execution/python/globalindex/btree/btree_file_meta_selector.py
similarity index 90%
rename from paimon-python/pypaimon/globalindex/btree/btree_file_meta_selector.py
rename to paimon-python/pypaimon/execution/python/globalindex/btree/btree_file_meta_selector.py
index 7b00b59079..83ee86f975 100644
--- a/paimon-python/pypaimon/globalindex/btree/btree_file_meta_selector.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/btree/btree_file_meta_selector.py
@@ -17,7 +17,7 @@
"""Compatibility alias for sorted global index file pruning."""
-from pypaimon.globalindex.sorted_file_meta_selector import SortedFileMetaSelector
+from pypaimon.execution.python.globalindex.sorted_file_meta_selector import SortedFileMetaSelector
BTreeFileMetaSelector = SortedFileMetaSelector
diff --git a/paimon-python/pypaimon/globalindex/btree/btree_index_meta.py b/paimon-python/pypaimon/execution/python/globalindex/btree/btree_index_meta.py
similarity index 100%
rename from paimon-python/pypaimon/globalindex/btree/btree_index_meta.py
rename to paimon-python/pypaimon/execution/python/globalindex/btree/btree_index_meta.py
diff --git a/paimon-python/pypaimon/globalindex/btree/btree_index_reader.py b/paimon-python/pypaimon/execution/python/globalindex/btree/btree_index_reader.py
similarity index 95%
rename from paimon-python/pypaimon/globalindex/btree/btree_index_reader.py
rename to paimon-python/pypaimon/execution/python/globalindex/btree/btree_index_reader.py
index 007ccf4361..f447aaa137 100644
--- a/paimon-python/pypaimon/globalindex/btree/btree_index_reader.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/btree/btree_index_reader.py
@@ -28,15 +28,15 @@ from contextlib import suppress
from typing import List, Optional
from pypaimon.common.file_io import FileIO, supports_pread, pread
-from pypaimon.globalindex.btree.btree_index_meta import BTreeIndexMeta
+from pypaimon.execution.python.globalindex.btree.btree_index_meta import BTreeIndexMeta
from pypaimon.globalindex.global_index_meta import GlobalIndexIOMeta
from pypaimon.globalindex.global_index_result import GlobalIndexResult
-from pypaimon.globalindex.key_serializer import KeySerializer
+from pypaimon.execution.python.globalindex.key_serializer import KeySerializer
from pypaimon.utils.roaring_bitmap import RoaringBitmap64
-from pypaimon.globalindex.btree.btree_file_footer import BTreeFileFooter
-from pypaimon.globalindex.btree.sst_file_reader import SstFileReader
-from pypaimon.globalindex.btree.btree_posting_list import add_row_ids
-from pypaimon.globalindex.sorted_file_meta_selector import SortedFileMetaSelector
+from pypaimon.execution.python.globalindex.btree.btree_file_footer import BTreeFileFooter
+from pypaimon.execution.python.globalindex.btree.sst_file_reader import SstFileReader
+from pypaimon.execution.python.globalindex.btree.btree_posting_list import add_row_ids
+from pypaimon.execution.python.globalindex.sorted_file_meta_selector import SortedFileMetaSelector
class BTreeIndexReader:
diff --git a/paimon-python/pypaimon/globalindex/btree/btree_index_writer.py b/paimon-python/pypaimon/execution/python/globalindex/btree/btree_index_writer.py
similarity index 97%
rename from paimon-python/pypaimon/globalindex/btree/btree_index_writer.py
rename to paimon-python/pypaimon/execution/python/globalindex/btree/btree_index_writer.py
index ad244d0ce5..a0f3696857 100644
--- a/paimon-python/pypaimon/globalindex/btree/btree_index_writer.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/btree/btree_index_writer.py
@@ -22,23 +22,23 @@ import struct
import zlib
from typing import List, Optional
-from pypaimon.globalindex.index_file_utils import (
+from pypaimon.execution.python.globalindex.index_file_utils import (
PositionOutput,
new_global_index_file_name,
write_uncompressed_block,
write_var_len_int,
write_var_len_long,
)
-from pypaimon.globalindex.btree.bloom_filter import (
+from pypaimon.execution.python.globalindex.btree.bloom_filter import (
BloomFilter,
murmur_hash_bytes,
)
-from pypaimon.globalindex.result_entry import ResultEntry
+from pypaimon.execution.python.globalindex.result_entry import ResultEntry
from pypaimon.globalindex.sorted_index_file_meta import SortedIndexFileMeta
from pypaimon.utils.roaring_bitmap import RoaringBitmap64
-BTREE_IDENTIFIER = "btree"
+from pypaimon.globalindex.index_types import BTREE_IDENTIFIER
_BTREE_FOOTER_LENGTH = 52
_BTREE_MAGIC_NUMBER = 0x50425449
_BTREE_CURRENT_VERSION = 1
diff --git a/paimon-python/pypaimon/globalindex/btree/btree_posting_list.py b/paimon-python/pypaimon/execution/python/globalindex/btree/btree_posting_list.py
similarity index 94%
rename from paimon-python/pypaimon/globalindex/btree/btree_posting_list.py
rename to paimon-python/pypaimon/execution/python/globalindex/btree/btree_posting_list.py
index b9ba140ae8..c3b7ebac6b 100644
--- a/paimon-python/pypaimon/globalindex/btree/btree_posting_list.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/btree/btree_posting_list.py
@@ -17,8 +17,8 @@
"""Decode BTree postings into a query result bitmap."""
-from pypaimon.globalindex.btree.btree_file_footer import BTreeFileFooter
-from pypaimon.globalindex.memory_slice_input import MemorySliceInput
+from pypaimon.execution.python.globalindex.btree.btree_file_footer import BTreeFileFooter
+from pypaimon.execution.python.globalindex.memory_slice_input import MemorySliceInput
from pypaimon.utils.roaring_bitmap import RoaringBitmap64
diff --git a/paimon-python/pypaimon/globalindex/btree/lazy_filtered_btree_reader.py b/paimon-python/pypaimon/execution/python/globalindex/btree/lazy_filtered_btree_reader.py
similarity index 86%
rename from paimon-python/pypaimon/globalindex/btree/lazy_filtered_btree_reader.py
rename to paimon-python/pypaimon/execution/python/globalindex/btree/lazy_filtered_btree_reader.py
index 23bc1e1d98..27b11a39f0 100644
--- a/paimon-python/pypaimon/globalindex/btree/lazy_filtered_btree_reader.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/btree/lazy_filtered_btree_reader.py
@@ -17,9 +17,9 @@
"""BTree global index reader for sorted index files."""
-from pypaimon.globalindex.btree.btree_index_reader import BTreeIndexReader
+from pypaimon.execution.python.globalindex.btree.btree_index_reader import BTreeIndexReader
from pypaimon.globalindex.global_index_meta import GlobalIndexIOMeta
-from pypaimon.globalindex.sorted_file_global_index_reader import SortedFileGlobalIndexReader
+from pypaimon.execution.python.globalindex.sorted_file_global_index_reader import SortedFileGlobalIndexReader
class LazyFilteredBTreeReader(SortedFileGlobalIndexReader):
diff --git a/paimon-python/pypaimon/globalindex/btree/sst_file_reader.py b/paimon-python/pypaimon/execution/python/globalindex/btree/sst_file_reader.py
similarity index 93%
rename from paimon-python/pypaimon/globalindex/btree/sst_file_reader.py
rename to paimon-python/pypaimon/execution/python/globalindex/btree/sst_file_reader.py
index 44a27dbaf9..2c82855e6b 100644
--- a/paimon-python/pypaimon/globalindex/btree/sst_file_reader.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/btree/sst_file_reader.py
@@ -31,34 +31,34 @@ from typing import Optional, Callable
from typing import BinaryIO
from pypaimon.common.file_io import pread
-from pypaimon.globalindex.block_compression import (
+from pypaimon.execution.python.globalindex.block_compression import (
BLOCK_TRAILER_LENGTH,
crc32c,
decompress_block,
)
-from pypaimon.globalindex.btree.block_handle import BlockHandle
-from pypaimon.globalindex.btree.block_entry import BlockEntry
-from pypaimon.globalindex.btree.block_reader import BlockReader, BlockIterator
-from pypaimon.globalindex.btree.bloom_filter import (
+from pypaimon.execution.python.globalindex.btree.block_handle import BlockHandle
+from pypaimon.execution.python.globalindex.btree.block_entry import BlockEntry
+from pypaimon.execution.python.globalindex.btree.block_reader import BlockReader, BlockIterator
+from pypaimon.execution.python.globalindex.btree.bloom_filter import (
BloomFilter,
murmur_hash_bytes,
)
-from pypaimon.globalindex.btree.btree_file_footer import BloomFilterHandle
-from pypaimon.globalindex.memory_slice_input import MemorySliceInput
+from pypaimon.execution.python.globalindex.btree.btree_file_footer import BloomFilterHandle
+from pypaimon.execution.python.globalindex.memory_slice_input import MemorySliceInput
class SstFileIterator:
"""
Iterator for range queries on SST file.
-
+
Allows seeking to a position and reading batches of records.
"""
-
+
def __init__(self, read_block: Callable[[BlockHandle], BlockReader], index_block_iterator: BlockIterator):
self.read_block = read_block
self.index_iterator = index_block_iterator
self.sought_data_block: Optional[BlockIterator] = None
-
+
@staticmethod
def _parse_block_handle(block_handle_bytes: bytes) -> BlockHandle:
handle_input = MemorySliceInput(block_handle_bytes)
@@ -87,12 +87,12 @@ class SstFileIterator:
self.sought_data_block.seek_to(key)
else:
self.sought_data_block = None
-
+
def read_batch(self) -> Optional[BlockIterator]:
"""
Read a batch of records from this SST File and move current record
position to the next batch.
-
+
Returns:
BlockIterator for the current batch, or None if at file end
"""
@@ -100,13 +100,13 @@ class SstFileIterator:
result = self.sought_data_block
self.sought_data_block = None
return result
-
+
if not self.index_iterator.has_next():
return None
-
+
index_entry = self.index_iterator.__next__()
block_handle = self._parse_block_handle(index_entry.value)
-
+
# Create data block reader
data_block_reader = self.read_block(block_handle)
return data_block_reader.iterator()
diff --git a/paimon-python/pypaimon/globalindex/build_plan.py b/paimon-python/pypaimon/execution/python/globalindex/build_plan.py
similarity index 100%
rename from paimon-python/pypaimon/globalindex/build_plan.py
rename to paimon-python/pypaimon/execution/python/globalindex/build_plan.py
diff --git a/paimon-python/pypaimon/globalindex/create_global_index.py b/paimon-python/pypaimon/execution/python/globalindex/create_global_index.py
similarity index 63%
copy from paimon-python/pypaimon/globalindex/create_global_index.py
copy to paimon-python/pypaimon/execution/python/globalindex/create_global_index.py
index 8be65d5e1e..c98c9898eb 100644
--- a/paimon-python/pypaimon/globalindex/create_global_index.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/create_global_index.py
@@ -15,46 +15,36 @@
# specific language governing permissions and limitations
# under the License.
-"""Build global index files from Python."""
+"""Python global index file construction."""
import threading
from concurrent.futures import FIRST_EXCEPTION, ThreadPoolExecutor, wait
from functools import cmp_to_key
-from typing import Dict, List, Optional, Sequence, Union
+from typing import List, Optional, Union
import pyarrow as pa
import pyarrow.compute as pc
-from pypaimon.common.options.core_options import CoreOptions
-from pypaimon.common.options.options import Options
-from pypaimon.common.predicate import Predicate
-from pypaimon.common.predicate_builder import PredicateBuilder
-from pypaimon.globalindex.btree.btree_index_writer import (
+from pypaimon.execution.python.globalindex.btree.btree_index_writer import (
BTREE_IDENTIFIER,
BTreeIndexWriter,
)
-from pypaimon.globalindex.bitmap.bitmap_index_writer import (
+from pypaimon.execution.python.globalindex.bitmap.bitmap_index_writer import (
BITMAP_IDENTIFIER,
BitmapIndexWriter,
)
-from pypaimon.globalindex.build_plan import (
+from pypaimon.execution.python.globalindex.build_plan import (
filter_non_indexable_splits as _filter_non_indexable_splits,
split_by_contiguous_unindexed_row_range as _split_by_contiguous_unindexed_row_range,
split_by_global_index_shard as _split_by_global_index_shard,
unindexed_row_ranges as _unindexed_row_ranges,
)
from pypaimon.globalindex.global_index_meta import GlobalIndexMeta
-from pypaimon.globalindex.key_serializer import create_serializer
-from pypaimon.globalindex.full_text.native_full_text_global_index_reader import (
- FULL_TEXT_IDENTIFIER,
-)
-from pypaimon.globalindex.full_text.native_full_text_index_writer import (
- NativeFullTextIndexWriter,
-)
-from pypaimon.globalindex.vindex.vindex_vector_global_index_reader import (
- VINDEX_IDENTIFIERS,
-)
-from pypaimon.globalindex.vindex.vindex_vector_index_writer import (
+from pypaimon.execution.python.globalindex.key_serializer import create_serializer
+from pypaimon.globalindex.full_text_options import FULL_TEXT_IDENTIFIER
+from pypaimon.execution.python.write.native_full_text_index_writer import NativeFullTextIndexWriter
+from pypaimon.globalindex.vindex_options import VINDEX_IDENTIFIERS
+from pypaimon.execution.python.globalindex.vindex.vindex_vector_index_writer import (
ADD_BATCH_SIZE,
VindexVectorIndexWriter,
)
@@ -66,169 +56,65 @@ from pypaimon.utils.range import Range
from pypaimon.write.commit_message import CommitMessage
-def create_global_index(
- table,
- index_column: Union[str, Sequence[str]],
- index_type: str = BTREE_IDENTIFIER,
- partition_filter: Optional[Predicate] = None,
- partitions: Optional[Union[Dict[str, object], Sequence[Dict[str, object]]]] = None,
- options: Optional[Dict[str, object]] = None,
- *,
- execution="local",
- concurrency=None,
- ray_remote_args=None,
-) -> int:
- """Build and commit global index files for a table.
-
- Returns the number of index files added to the table snapshot.
- """
-
- builder = GlobalIndexBuilder(
- table,
- index_column,
- index_type=index_type,
- partition_filter=partition_filter,
- partitions=partitions,
- options=options,
- )
- messages = builder.build(
- execution=execution, concurrency=concurrency, ray_remote_args=ray_remote_args)
- if not messages:
- return 0
-
- write_builder = table.new_batch_write_builder()
- commit = write_builder.new_commit()
- try:
- commit.commit(messages)
- finally:
- commit.close()
- return sum(len(message.index_adds) for message in messages)
-
-
_SORTED_INDEX_IDENTIFIERS = (BTREE_IDENTIFIER, BITMAP_IDENTIFIER)
-_GENERIC_INDEX_IDENTIFIERS = tuple(VINDEX_IDENTIFIERS) + (
- FULL_TEXT_IDENTIFIER,
-)
+_GENERIC_INDEX_IDENTIFIERS = tuple(VINDEX_IDENTIFIERS) + (FULL_TEXT_IDENTIFIER,)
_SORTED_INDEX_RECORDS_PER_FILE_FLOATING = 1.2
-class GlobalIndexBuilder:
- """Small Python builder for global indexes."""
-
- def __init__(
- self,
- table,
- index_column: Union[str, Sequence[str]],
- index_type: str = BTREE_IDENTIFIER,
- partition_filter: Optional[Predicate] = None,
- partitions: Optional[Union[Dict[str, object], Sequence[Dict[str, object]]]] = None,
- options: Optional[Dict[str, object]] = None,
- ):
- self._table = table
- self._index_columns = _normalize_index_columns(index_column)
- self._index_type = index_type.lower().strip()
- self._partition_filter = partition_filter
- self._partitions = partitions
- self._user_options = dict(options or {})
- self._options = _merged_options(table, options)
- self._core_options = CoreOptions(self._options)
-
- if (
- self._index_type not in _SORTED_INDEX_IDENTIFIERS
- and self._index_type not in _GENERIC_INDEX_IDENTIFIERS
- ):
- raise ValueError(
- "Python global index build currently supports %s and %s, got '%s'."
- % (
- _SORTED_INDEX_IDENTIFIERS,
- _GENERIC_INDEX_IDENTIFIERS,
- index_type,
- )
- )
- if len(self._index_columns) != 1:
- raise ValueError(
- "Python global index build currently supports one column, got %s."
- % self._index_columns
- )
- if not self._table.options.row_tracking_enabled():
- raise ValueError(
- "Table '%s' must enable 'row-tracking.enabled=true' before "
- "creating global index." % self._table.identifier
- )
- for column in self._index_columns:
- if column not in self._table.field_dict:
- raise ValueError(
- "Column '%s' does not exist in table '%s'."
- % (column, self._table.identifier)
- )
- self._validate_unaware_bucket_table()
- if self._index_type in _GENERIC_INDEX_IDENTIFIERS:
- self._validate_generic_index_table()
-
- def build(self, *, execution="local", concurrency=None, ray_remote_args=None) -> List[CommitMessage]:
- if execution not in ("local", "ray"):
- raise ValueError("execution must be 'local' or 'ray'.")
- if execution == "local" and (concurrency is not None or ray_remote_args is not None):
- raise ValueError("Ray options require execution='ray'.")
- if execution == "ray":
- from pypaimon.ray.vector_index_build import validate_build_options
- concurrency, ray_remote_args = validate_build_options(
- self, concurrency, ray_remote_args)
- partition_filter = self._resolve_partition_filter()
- if execution == "local":
- from pypaimon.globalindex.native_index_build import build_native_global_index
- messages = build_native_global_index(self, partition_filter)
- if messages is not None:
- return messages
- read_builder = self._table.new_read_builder()
- if partition_filter is not None:
- read_builder = read_builder.with_partition_filter(partition_filter)
+def build_global_index(self, partition_filter, *, execution="local", concurrency=None, ray_remote_args=None):
+ read_builder = self._table.new_read_builder()
+ if partition_filter is not None:
+ read_builder = read_builder.with_partition_filter(partition_filter)
+
+ scan = read_builder.new_scan()
+ plan = scan.plan_all_files()
+ splits = plan.splits()
+ if not splits:
+ return []
+
+ index_field = self._table.field_dict[self._index_columns[0]]
+ snapshot = self._snapshot_for_plan(plan)
+ unindexed_ranges = _unindexed_row_ranges(
+ self._table,
+ snapshot,
+ partition_filter,
+ index_field.id,
+ self._index_type,
+ )
+ if not unindexed_ranges:
+ return []
- scan = read_builder.new_scan()
- plan = scan.plan_all_files()
- splits = plan.splits()
+ if self._index_type in _GENERIC_INDEX_IDENTIFIERS:
+ splits = _filter_non_indexable_splits(
+ self._table, splits, self._index_columns)
if not splits:
return []
- index_field = self._table.field_dict[self._index_columns[0]]
- snapshot = self._snapshot_for_plan(plan)
- unindexed_ranges = _unindexed_row_ranges(
- self._table,
- snapshot,
- partition_filter,
- index_field.id,
- self._index_type,
- )
- if not unindexed_ranges:
- return []
-
- if self._index_type in _GENERIC_INDEX_IDENTIFIERS:
- splits = _filter_non_indexable_splits(
- self._table, splits, self._index_columns)
- if not splits:
- return []
+ read_type = [index_field, SpecialFields.ROW_ID]
+ from pypaimon.read.table_read import TableRead
- read_type = [index_field, SpecialFields.ROW_ID]
- from pypaimon.read.table_read import TableRead
+ table_read = TableRead(
+ table=self._table,
+ predicate=None,
+ read_type=read_type,
+ )
+ index_path_factory = self._table.path_factory().global_index_path_factory()
+ index_path = index_path_factory.global_index_root_path()
- table_read = TableRead(
- table=self._table,
- predicate=None,
- read_type=read_type,
- )
- index_path_factory = self._table.path_factory().global_index_path_factory()
- index_path = index_path_factory.global_index_root_path()
-
- if self._index_type in _SORTED_INDEX_IDENTIFIERS:
- return self._build_sorted_index(
- splits, unindexed_ranges, index_field, table_read, index_path)
- if execution == "ray":
- from pypaimon.ray.vector_index_build import build_vector_index
- return build_vector_index(
- self, splits, unindexed_ranges, index_field, table_read,
- index_path, concurrency, ray_remote_args)
- return self._build_generic_index(
+ if self._index_type in _SORTED_INDEX_IDENTIFIERS:
+ return self._build_sorted_index(
splits, unindexed_ranges, index_field, table_read, index_path)
+ if execution == "ray":
+ from pypaimon.ray.vector_index_build import build_vector_index
+ return build_vector_index(
+ self, splits, unindexed_ranges, index_field, table_read,
+ index_path, concurrency, ray_remote_args)
+ return self._build_generic_index(
+ splits, unindexed_ranges, index_field, table_read, index_path)
+
+
+class PythonGlobalIndexBuilder:
+ """Algorithms invoked only by the Python index build path."""
def _snapshot_for_plan(self, plan):
snapshot_id = getattr(plan, "snapshot_id", None)
@@ -505,71 +391,6 @@ class GlobalIndexBuilder:
)
raise ValueError("Unsupported generic global index type: %s" % self._index_type)
- def _validate_unaware_bucket_table(self) -> None:
- # Global index files live in bucket 0 and are keyed by the table-wide
- # _ROW_ID, so every index type needs an unaware-bucket table. Check the
- # table options: caller options only tune the index build.
- bucket = self._table.options.bucket()
- if bucket != -1:
- raise ValueError(
- "Global index only supports unaware-bucket tables "
- "(bucket = -1), but table '%s' has bucket = %s."
- % (self._table.identifier, bucket)
- )
-
- def _validate_generic_index_table(self) -> None:
- if (self._table.options.deletion_vectors_enabled()
- and not (self._index_type in VINDEX_IDENTIFIERS
- and self._table.options.data_evolution_enabled())):
- raise ValueError(
- "Global index build with deletion vectors requires a native "
- "vector index on a data-evolution table. Table '%s', index type '%s'."
- % (self._table.identifier, self._index_type)
- )
-
- def _resolve_partition_filter(self) -> Optional[Predicate]:
- if self._partition_filter is not None:
- return self._partition_filter
- if self._partitions is None:
- return None
-
- partitions = self._partitions
- if isinstance(partitions, dict):
- partitions = [partitions]
-
- predicate_builder = PredicateBuilder(self._table.partition_keys_fields)
- partition_predicates = []
- for partition in partitions:
- sub_predicates = []
- for key, value in partition.items():
- if key not in self._table.partition_keys:
- raise ValueError(
- "Partition spec key '%s' is not a partition column. "
- "Partition keys are: %s"
- % (key, list(self._table.partition_keys))
- )
- if value is None:
- sub_predicates.append(predicate_builder.is_null(key))
- else:
- sub_predicates.append(predicate_builder.equal(key, value))
- if sub_predicates:
- partition_predicates.append(
- predicate_builder.and_predicates(sub_predicates))
- return predicate_builder.or_predicates(partition_predicates)
-
-
-def _normalize_index_columns(index_column: Union[str, Sequence[str]]) -> List[str]:
- if isinstance(index_column, str):
- return [c.strip() for c in index_column.split(",") if c.strip()]
- return [str(c).strip() for c in index_column if str(c).strip()]
-
-
-def _merged_options(table, options: Optional[Dict[str, object]]) -> Options:
- merged = dict(table.options.options.to_map())
- if options:
- merged.update(options)
- return Options(merged)
-
def _extract_sorted_rows(
table: pa.Table,
diff --git a/paimon-python/pypaimon/globalindex/data_evolution_global_index_coverage.py b/paimon-python/pypaimon/execution/python/globalindex/data_evolution_global_index_coverage.py
similarity index 100%
rename from paimon-python/pypaimon/globalindex/data_evolution_global_index_coverage.py
rename to paimon-python/pypaimon/execution/python/globalindex/data_evolution_global_index_coverage.py
diff --git a/paimon-python/pypaimon/globalindex/data_evolution_global_index_scanner.py b/paimon-python/pypaimon/execution/python/globalindex/data_evolution_global_index_scanner.py
similarity index 94%
rename from paimon-python/pypaimon/globalindex/data_evolution_global_index_scanner.py
rename to paimon-python/pypaimon/execution/python/globalindex/data_evolution_global_index_scanner.py
index bfb54354f8..6e9d562332 100644
--- a/paimon-python/pypaimon/globalindex/data_evolution_global_index_scanner.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/data_evolution_global_index_scanner.py
@@ -20,7 +20,7 @@
from concurrent.futures import ThreadPoolExecutor
from typing import Collection, List, Optional
-from pypaimon.globalindex.global_index_evaluator import (
+from pypaimon.execution.python.globalindex.global_index_evaluator import (
GlobalIndexEvaluation,
GlobalIndexEvaluator,
)
@@ -30,7 +30,7 @@ from pypaimon.globalindex.global_index_result import GlobalIndexResult
from pypaimon.common.options.core_options import CoreOptions
from pypaimon.common.options.options import Options
from pypaimon.common.predicate import Predicate
-from pypaimon.globalindex.data_evolution_global_index_coverage import DataEvolutionGlobalIndexCoverage
+from pypaimon.execution.python.globalindex.data_evolution_global_index_coverage import DataEvolutionGlobalIndexCoverage
from pypaimon.read.push_down_utils import _get_all_fields
from pypaimon.schema.data_types import DataField
from pypaimon.utils.range import Range
@@ -110,7 +110,7 @@ class DataEvolutionGlobalIndexScanner:
options = self._options
def readers_function(field: DataField) -> Collection[GlobalIndexReader]:
- from pypaimon.globalindex.union_global_index_reader import UnionGlobalIndexReader
+ from pypaimon.execution.python.globalindex.union_global_index_reader import UnionGlobalIndexReader
groups = []
group = index_metas.get(field.id)
@@ -330,10 +330,10 @@ def _create_readers(
if index_type_metas is None:
return []
- from pypaimon.globalindex.offset_global_index_reader import (
+ from pypaimon.execution.python.globalindex.offset_global_index_reader import (
OffsetGlobalIndexReader,
)
- from pypaimon.globalindex.union_global_index_reader import (
+ from pypaimon.execution.python.globalindex.union_global_index_reader import (
UnionGlobalIndexReader,
)
@@ -357,8 +357,8 @@ def _create_inner_readers(
"""Build per-file (or per-shard) readers for a single indexType/range."""
core_options = options or CoreOptions(Options.from_none())
if index_type == 'btree':
- from pypaimon.globalindex.btree.lazy_filtered_btree_reader import LazyFilteredBTreeReader
- from pypaimon.globalindex.key_serializer import create_serializer
+ from pypaimon.execution.python.globalindex.btree.lazy_filtered_btree_reader import LazyFilteredBTreeReader
+ from pypaimon.execution.python.globalindex.key_serializer import create_serializer
key_serializer = create_serializer(field.type)
return [LazyFilteredBTreeReader(
key_serializer=key_serializer,
@@ -370,8 +370,8 @@ def _create_inner_readers(
)]
if index_type == 'bitmap':
- from pypaimon.globalindex.bitmap.lazy_filtered_bitmap_reader import LazyFilteredBitmapReader
- from pypaimon.globalindex.key_serializer import create_serializer
+ from pypaimon.execution.python.globalindex.bitmap.lazy_filtered_bitmap_reader import LazyFilteredBitmapReader
+ from pypaimon.execution.python.globalindex.key_serializer import create_serializer
key_serializer = create_serializer(field.type)
return [LazyFilteredBitmapReader(
key_serializer=key_serializer,
diff --git a/paimon-python/pypaimon/execution/python/globalindex/drop_global_index.py b/paimon-python/pypaimon/execution/python/globalindex/drop_global_index.py
new file mode 100644
index 0000000000..77c4790dbf
--- /dev/null
+++ b/paimon-python/pypaimon/execution/python/globalindex/drop_global_index.py
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Python global index deletion message preparation."""
+
+from typing import List
+
+from pypaimon.index.index_file_handler import IndexFileHandler
+from pypaimon.manifest.index_manifest_entry import IndexManifestEntry
+from pypaimon.write.commit_message import CommitMessage
+
+
+def build_global_index_drop(self, partition_filter):
+ snapshot = self._table.snapshot_manager().get_latest_snapshot()
+ if snapshot is None:
+ return []
+
+ target_field_ids = [
+ self._table.field_dict[column].id for column in self._index_columns
+ ]
+
+ def should_delete(entry: IndexManifestEntry) -> bool:
+ index_file = entry.index_file
+ global_meta = index_file.global_index_meta
+ if index_file.index_type.lower().strip() != self._index_type or global_meta is None:
+ return False
+ if _indexed_field_ids(global_meta) != target_field_ids:
+ return False
+ return partition_filter is None or partition_filter.test(entry.partition)
+
+ entries = IndexFileHandler(self._table).scan(snapshot, should_delete)
+ if not entries:
+ return []
+
+ by_partition = {}
+ for entry in entries:
+ key = tuple(entry.partition.values)
+ by_partition.setdefault(key, []).append(
+ IndexManifestEntry(
+ kind=1,
+ partition=entry.partition,
+ bucket=entry.bucket,
+ index_file=entry.index_file,
+ )
+ )
+
+ return [
+ CommitMessage(
+ partition=partition,
+ bucket=0,
+ new_files=[],
+ index_deletes=deletes,
+ )
+ for partition, deletes in by_partition.items()
+ ]
+
+
+def _indexed_field_ids(global_meta) -> List[int]:
+ field_ids = [global_meta.index_field_id]
+ if global_meta.extra_field_ids:
+ field_ids.extend(global_meta.extra_field_ids)
+ return field_ids
diff --git a/paimon-python/pypaimon/globalindex/global_index_evaluator.py b/paimon-python/pypaimon/execution/python/globalindex/global_index_evaluator.py
similarity index 97%
rename from paimon-python/pypaimon/globalindex/global_index_evaluator.py
rename to paimon-python/pypaimon/execution/python/globalindex/global_index_evaluator.py
index 7bb5e4eb65..436e77204d 100644
--- a/paimon-python/pypaimon/globalindex/global_index_evaluator.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/global_index_evaluator.py
@@ -21,22 +21,16 @@ import threading
import struct
from collections import deque
from concurrent.futures import Future
-from typing import Callable, Collection, Dict, FrozenSet, List, NamedTuple, Optional
+from typing import Callable, Collection, Dict, List, Optional
from pypaimon.globalindex.global_index_reader import GlobalIndexReader, FieldRef, _completed_future
from pypaimon.globalindex.global_index_result import GlobalIndexResult
-from pypaimon.globalindex.key_serializer import create_serializer
+from pypaimon.globalindex.global_index_evaluation import GlobalIndexEvaluation
+from pypaimon.execution.python.globalindex.key_serializer import create_serializer
from pypaimon.common.predicate import Predicate
from pypaimon.schema.data_types import AtomicType, DataField
-class GlobalIndexEvaluation(NamedTuple):
- """Matches and fields whose supported indexes contributed."""
-
- result: GlobalIndexResult
- contributing_field_ids: FrozenSet[int]
-
-
class GlobalIndexEvaluator:
"""Predicate evaluator for filtering data using global indexes.
diff --git a/paimon-python/pypaimon/globalindex/index_file_utils.py b/paimon-python/pypaimon/execution/python/globalindex/index_file_utils.py
similarity index 96%
rename from paimon-python/pypaimon/globalindex/index_file_utils.py
rename to paimon-python/pypaimon/execution/python/globalindex/index_file_utils.py
index a27a923b04..a4b9dab33f 100644
--- a/paimon-python/pypaimon/globalindex/index_file_utils.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/index_file_utils.py
@@ -21,7 +21,7 @@ from dataclasses import dataclass
import struct
import uuid
-from pypaimon.globalindex.block_compression import COMPRESSION_NONE, crc32c
+from pypaimon.execution.python.globalindex.block_compression import COMPRESSION_NONE, crc32c
@dataclass(frozen=True)
diff --git a/paimon-python/pypaimon/globalindex/key_serializer.py b/paimon-python/pypaimon/execution/python/globalindex/key_serializer.py
similarity index 100%
rename from paimon-python/pypaimon/globalindex/key_serializer.py
rename to paimon-python/pypaimon/execution/python/globalindex/key_serializer.py
diff --git a/paimon-python/pypaimon/read/scanner/__init__.py b/paimon-python/pypaimon/execution/python/globalindex/lumina/__init__.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/__init__.py
copy to paimon-python/pypaimon/execution/python/globalindex/lumina/__init__.py
diff --git a/paimon-python/pypaimon/globalindex/lumina/lumina_index_meta.py b/paimon-python/pypaimon/execution/python/globalindex/lumina/lumina_index_meta.py
similarity index 100%
rename from paimon-python/pypaimon/globalindex/lumina/lumina_index_meta.py
rename to paimon-python/pypaimon/execution/python/globalindex/lumina/lumina_index_meta.py
diff --git a/paimon-python/pypaimon/globalindex/lumina/lumina_vector_global_index_reader.py b/paimon-python/pypaimon/execution/python/globalindex/lumina/lumina_vector_global_index_reader.py
similarity index 93%
rename from paimon-python/pypaimon/globalindex/lumina/lumina_vector_global_index_reader.py
rename to paimon-python/pypaimon/execution/python/globalindex/lumina/lumina_vector_global_index_reader.py
index 0ddd1e3325..62a966c765 100644
--- a/paimon-python/pypaimon/globalindex/lumina/lumina_vector_global_index_reader.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/lumina/lumina_vector_global_index_reader.py
@@ -28,10 +28,14 @@ import numpy as np
from pypaimon.globalindex.global_index_reader import GlobalIndexReader, _completed_future
from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult
+from pypaimon.globalindex.index_types import (
+ LUMINA_IDENTIFIER, LUMINA_VECTOR_ANN_IDENTIFIER, LUMINA_IDENTIFIERS,
+)
+
+__all__ = [
+ "LuminaVectorGlobalIndexReader", "LUMINA_IDENTIFIER", "LUMINA_VECTOR_ANN_IDENTIFIER", "LUMINA_IDENTIFIERS",
+]
-LUMINA_IDENTIFIER = "lumina"
-LUMINA_VECTOR_ANN_IDENTIFIER = "lumina-vector-ann"
-LUMINA_IDENTIFIERS = (LUMINA_IDENTIFIER, LUMINA_VECTOR_ANN_IDENTIFIER)
MIN_SEARCH_LIST_SIZE = 16
@@ -154,8 +158,8 @@ class LuminaVectorGlobalIndexReader(GlobalIndexReader):
return
from lumina_data import LuminaSearcher
- from pypaimon.globalindex.lumina.lumina_index_meta import LuminaIndexMeta
- from pypaimon.globalindex.lumina.lumina_vector_index_options import (
+ from pypaimon.execution.python.globalindex.lumina.lumina_index_meta import LuminaIndexMeta
+ from pypaimon.execution.python.globalindex.lumina.lumina_vector_index_options import (
strip_lumina_options,
)
diff --git a/paimon-python/pypaimon/globalindex/lumina/lumina_vector_index_options.py b/paimon-python/pypaimon/execution/python/globalindex/lumina/lumina_vector_index_options.py
similarity index 100%
rename from paimon-python/pypaimon/globalindex/lumina/lumina_vector_index_options.py
rename to paimon-python/pypaimon/execution/python/globalindex/lumina/lumina_vector_index_options.py
diff --git a/paimon-python/pypaimon/globalindex/memory_slice_input.py b/paimon-python/pypaimon/execution/python/globalindex/memory_slice_input.py
similarity index 100%
rename from paimon-python/pypaimon/globalindex/memory_slice_input.py
rename to paimon-python/pypaimon/execution/python/globalindex/memory_slice_input.py
diff --git a/paimon-python/pypaimon/globalindex/offset_global_index_reader.py b/paimon-python/pypaimon/execution/python/globalindex/offset_global_index_reader.py
similarity index 100%
rename from paimon-python/pypaimon/globalindex/offset_global_index_reader.py
rename to paimon-python/pypaimon/execution/python/globalindex/offset_global_index_reader.py
diff --git a/paimon-python/pypaimon/globalindex/result_entry.py b/paimon-python/pypaimon/execution/python/globalindex/result_entry.py
similarity index 100%
rename from paimon-python/pypaimon/globalindex/result_entry.py
rename to paimon-python/pypaimon/execution/python/globalindex/result_entry.py
diff --git a/paimon-python/pypaimon/globalindex/sorted_file_global_index_reader.py b/paimon-python/pypaimon/execution/python/globalindex/sorted_file_global_index_reader.py
similarity index 98%
rename from paimon-python/pypaimon/globalindex/sorted_file_global_index_reader.py
rename to paimon-python/pypaimon/execution/python/globalindex/sorted_file_global_index_reader.py
index 77a9594cb9..e308fd94aa 100644
--- a/paimon-python/pypaimon/globalindex/sorted_file_global_index_reader.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/sorted_file_global_index_reader.py
@@ -25,8 +25,8 @@ from pypaimon.common.like_optimization import try_optimize_like
from pypaimon.globalindex.global_index_meta import GlobalIndexIOMeta
from pypaimon.globalindex.global_index_reader import FieldRef, GlobalIndexReader, _completed_future
from pypaimon.globalindex.global_index_result import GlobalIndexResult
-from pypaimon.globalindex.key_serializer import KeySerializer
-from pypaimon.globalindex.sorted_file_meta_selector import SortedFileMetaSelector
+from pypaimon.execution.python.globalindex.key_serializer import KeySerializer
+from pypaimon.execution.python.globalindex.sorted_file_meta_selector import SortedFileMetaSelector
from pypaimon.globalindex.sorted_index_file_meta import SortedIndexFileMeta
from pypaimon.utils.roaring_bitmap import RoaringBitmap64
diff --git a/paimon-python/pypaimon/globalindex/sorted_file_meta_selector.py b/paimon-python/pypaimon/execution/python/globalindex/sorted_file_meta_selector.py
similarity index 98%
rename from paimon-python/pypaimon/globalindex/sorted_file_meta_selector.py
rename to paimon-python/pypaimon/execution/python/globalindex/sorted_file_meta_selector.py
index 4fdc776bbd..592b30dd7f 100644
--- a/paimon-python/pypaimon/globalindex/sorted_file_meta_selector.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/sorted_file_meta_selector.py
@@ -20,7 +20,7 @@
from typing import List, Optional, Tuple
from pypaimon.globalindex.global_index_meta import GlobalIndexIOMeta
-from pypaimon.globalindex.key_serializer import KeySerializer
+from pypaimon.execution.python.globalindex.key_serializer import KeySerializer
from pypaimon.globalindex.sorted_index_file_meta import SortedIndexFileMeta
diff --git a/paimon-python/pypaimon/globalindex/union_global_index_reader.py b/paimon-python/pypaimon/execution/python/globalindex/union_global_index_reader.py
similarity index 100%
rename from paimon-python/pypaimon/globalindex/union_global_index_reader.py
rename to paimon-python/pypaimon/execution/python/globalindex/union_global_index_reader.py
diff --git a/paimon-python/pypaimon/read/scanner/__init__.py b/paimon-python/pypaimon/execution/python/globalindex/vindex/__init__.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/__init__.py
copy to paimon-python/pypaimon/execution/python/globalindex/vindex/__init__.py
diff --git a/paimon-python/pypaimon/execution/python/globalindex/vindex/vindex_vector_global_index_reader.py b/paimon-python/pypaimon/execution/python/globalindex/vindex/vindex_vector_global_index_reader.py
new file mode 100644
index 0000000000..4aadde5a1f
--- /dev/null
+++ b/paimon-python/pypaimon/execution/python/globalindex/vindex/vindex_vector_global_index_reader.py
@@ -0,0 +1,212 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Vector index queries using the paimon-vindex Python SDK."""
+
+import numpy as np
+
+from pypaimon.globalindex.global_index_reader import GlobalIndexReader, _completed_future
+from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult
+from pypaimon.execution.vindex_input import VindexReaderResources, _filter_bytes
+from pypaimon.globalindex.vindex_options import NPROBE_PARAMETER, L_SEARCH_PARAMETER
+
+
+class PythonVindexVectorGlobalIndexReader(VindexReaderResources, GlobalIndexReader):
+ """Python SDK implementation selected only when Native is unavailable."""
+
+ def visit_vector_search(self, vector_search):
+ results = self._run_search(
+ [vector_search.vector],
+ vector_search.limit,
+ vector_search.include_row_ids,
+ vector_search.options,
+ )
+ return _completed_future(results[0])
+
+ def visit_batch_vector_search(self, batch_vector_search):
+ results = self._run_search(
+ batch_vector_search.vectors,
+ batch_vector_search.limit,
+ batch_vector_search.include_row_ids,
+ batch_vector_search.options,
+ )
+ return _completed_future(results)
+
+ def _run_search(self, vectors, limit, include_row_ids, query_options):
+ self._ensure_loaded()
+
+ queries = self._validate_queries(vectors)
+ n = len(queries)
+ effective_k = self._effective_k(limit, include_row_ids)
+ if effective_k <= 0:
+ return [None] * n
+
+ options = query_options or {}
+ params = _search_params(self._search_params_type, options, effective_k)
+ filter_bytes = _filter_bytes(include_row_ids)
+
+ if n == 1:
+ ids, distances = self._reader.search(
+ queries[0], params, filter_bytes=filter_bytes)
+ return [_result_from_scores(ids, distances, self._metadata.metric)]
+
+ if hasattr(self._reader, "search_batch"):
+ batch_result = self._reader.search_batch(
+ np.ascontiguousarray(queries),
+ params,
+ filter_bytes=filter_bytes,
+ )
+ return _batch_results(batch_result, n, effective_k, self._metadata.metric)
+
+ results = []
+ for query in queries:
+ ids, distances = self._reader.search(
+ query, params, filter_bytes=filter_bytes)
+ results.append(_result_from_scores(ids, distances, self._metadata.metric))
+ return results
+
+ def _validate_queries(self, vectors):
+ queries = []
+ expected_dim = self._metadata.dimension
+ for vector in vectors:
+ query = np.asarray(vector, dtype=np.float32)
+ if query.ndim != 1:
+ raise ValueError("Query vector must be a one-dimensional float32 array")
+ if query.shape[0] != expected_dim:
+ raise ValueError(
+ "Query vector dimension mismatch: expected %d, got %d"
+ % (expected_dim, query.shape[0]))
+ queries.append(query)
+ return np.asarray(queries, dtype=np.float32)
+
+ def vector_metric(self):
+ self._ensure_loaded()
+ return self._metadata.metric
+
+ def _effective_k(self, limit, include_row_ids):
+ total_vectors = getattr(self._metadata, "total_vectors", limit)
+ effective_k = min(limit, int(total_vectors))
+ if include_row_ids is not None:
+ cardinality = include_row_ids.cardinality()
+ if cardinality == 0:
+ return 0
+ effective_k = min(effective_k, cardinality)
+ return effective_k
+
+ def _create_reader(self, index_input, file_path):
+ try:
+ from paimon_vindex import SearchParams, VectorIndexReader
+ except ImportError as error:
+ raise ImportError(
+ "paimon-vindex is required to read vindex vector indexes. "
+ "Install paimon-vindex==0.5.0 or pypaimon[vindex].") from error
+ self._search_params_type = SearchParams
+ return VectorIndexReader(index_input)
+
+ def _initialize_reader(self, reader):
+ self._metadata = reader.metadata()
+ reader.optimize_for_search()
+
+
+def _build_scores(ids, distances, metric):
+ id_to_scores = {}
+ for row_id, distance in zip(ids, distances):
+ row_id = int(row_id)
+ if row_id < 0:
+ continue
+ id_to_scores[row_id] = _convert_distance_to_score(float(distance), metric)
+ return id_to_scores
+
+
+def _result_from_scores(ids, distances, metric):
+ id_to_scores = _build_scores(ids, distances, metric)
+ if not id_to_scores:
+ return None
+ return DictBasedScoredIndexResult(id_to_scores)
+
+
+def _batch_results(batch_result, vector_count, effective_k, metric):
+ if hasattr(batch_result, "ids_for_query"):
+ return [
+ _result_from_scores(
+ batch_result.ids_for_query(i),
+ batch_result.distances_for_query(i),
+ metric,
+ )
+ for i in range(vector_count)
+ ]
+
+ ids, distances = batch_result
+ return [
+ _result_from_scores(
+ _values_for_query(ids, i, effective_k),
+ _values_for_query(distances, i, effective_k),
+ metric,
+ )
+ for i in range(vector_count)
+ ]
+
+
+def _values_for_query(values, query_index, effective_k):
+ array = np.asarray(values)
+ if array.ndim >= 2:
+ return array[query_index]
+ start = query_index * effective_k
+ return array[start:start + effective_k]
+
+
+def _convert_distance_to_score(distance, metric):
+ if metric == "l2":
+ return 1.0 / (1.0 + distance)
+ if metric == "cosine":
+ return 1.0 - distance
+ if metric == "inner_product":
+ return -distance
+ raise ValueError("Unknown vector search metric: %s" % metric)
+
+
+def _search_params(search_params_type, options, top_k):
+ nprobe = _int_parameter(options, NPROBE_PARAMETER)
+ l_search = _int_parameter(options, L_SEARCH_PARAMETER)
+ if nprobe is not None and l_search is not None:
+ raise ValueError(
+ "Cannot set both '%s' and '%s'."
+ % (NPROBE_PARAMETER, L_SEARCH_PARAMETER)
+ )
+ if nprobe is not None:
+ return search_params_type.ivf(top_k=top_k, nprobe=nprobe)
+ if l_search is not None:
+ return search_params_type.diskann(top_k=top_k, l_search=l_search)
+ return search_params_type.automatic(top_k=top_k)
+
+
+def _int_parameter(options, key):
+ value = options.get(key)
+ if value is None:
+ return None
+ try:
+ text = str(value)
+ digits = text[1:] if text[:1] in ('+', '-') else text
+ if not digits or any(ord(char) > 0xffff or not char.isdecimal() for char in digits):
+ raise ValueError('Invalid Java integer text')
+ number = int(text)
+ if not -(1 << 31) <= number < (1 << 31):
+ raise ValueError('Java integer overflow')
+ return number
+ except (TypeError, ValueError) as e:
+ raise ValueError(
+ "Invalid value for '%s': %s. Must be an integer." % (key, value)) from e
diff --git a/paimon-python/pypaimon/globalindex/vindex/vindex_vector_index_writer.py b/paimon-python/pypaimon/execution/python/globalindex/vindex/vindex_vector_index_writer.py
similarity index 63%
rename from paimon-python/pypaimon/globalindex/vindex/vindex_vector_index_writer.py
rename to paimon-python/pypaimon/execution/python/globalindex/vindex/vindex_vector_index_writer.py
index eb15565c49..60bb94b5f2 100644
--- a/paimon-python/pypaimon/globalindex/vindex/vindex_vector_index_writer.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/vindex/vindex_vector_index_writer.py
@@ -24,12 +24,12 @@ import uuid
from array import array
from typing import BinaryIO, Dict, List, Mapping, Optional
-from pypaimon.globalindex.result_entry import ResultEntry
-from pypaimon.schema.data_types import ArrayType, AtomicType, DataType, VectorType
+from pypaimon.execution.python.globalindex.result_entry import ResultEntry
+from pypaimon.schema.data_types import DataType
+from pypaimon.globalindex.vindex_options import native_options, train_sample_ratio, validate_vector_type
FILE_NAME_PREFIX = "vector"
-DEFAULT_DIMENSION = 128
ADD_BATCH_SIZE = 10000
@@ -248,221 +248,6 @@ class VindexVectorIndexWriter:
remaining -= batch_size
-def native_options(
- data_type: DataType,
- table_options: Mapping[str, object],
- index_type: str,
- field_name: str,
- user_options: Optional[Mapping[str, object]] = None,
-) -> Dict[str, str]:
- if user_options is None:
- user_options, table_options = table_options, {}
-
- result: Dict[str, str] = {}
- _collect_native_options(
- result, table_options, index_type, field_name, validate=False)
- _collect_native_options(
- result, user_options, index_type, field_name, validate=True)
-
- result["index.type"] = index_type.replace('-', '_')
- result["dimension"] = str(_dimension(data_type, result, index_type))
- result.setdefault("metric", "inner_product")
- return result
-
-
-def train_sample_ratio(
- table_options: Mapping[str, object],
- index_type: str,
- field_name: str,
- user_options: Optional[Mapping[str, object]] = None,
-) -> float:
- sources = (table_options,) if user_options is None else (user_options, table_options)
- field_key = "fields.%s.train.sample-ratio" % field_name
- index_key = "%s.train.sample-ratio" % index_type
- for options in sources:
- key = field_key if field_key in options else index_key
- if key in options:
- break
- else:
- return 1.0
-
- value = options[key]
- try:
- ratio = float(value)
- except (TypeError, ValueError) as e:
- raise ValueError(
- "Invalid value for '%s': %s. Must be greater than 0 and less "
- "than or equal to 1." % (key, value)
- ) from e
- if not math.isfinite(ratio) or ratio <= 0 or ratio > 1:
- raise ValueError(
- "Invalid value for '%s': %s. Must be greater than 0 and less "
- "than or equal to 1." % (key, value)
- )
- return ratio
-
-
-def validate_vector_type(data_type: DataType) -> None:
- if isinstance(data_type, VectorType):
- element_type = data_type.element
- if _is_float_type(element_type):
- return
- raise ValueError(
- "Vector index requires float vector, but got: %s" % element_type)
-
- if isinstance(data_type, ArrayType):
- element_type = data_type.element
- if _is_float_type(element_type):
- return
- raise ValueError(
- "Vector index requires float array, but got: %s" % element_type)
-
- raise ValueError(
- "Vector index requires VectorType or ArrayType<FLOAT>, but got: %s"
- % data_type)
-
-
-def _collect_native_options(
- result, options, index_type, field_name, validate
-):
- option_prefix = "%s." % index_type
- field_prefix = "fields.%s." % field_name
-
- for key, value in options.items():
- key = str(key)
- native_key = _native_option_key(key)
- if key == native_key and _is_050_build_option(native_key):
- _put_native_option(
- result, key, key, value, index_type, validate)
- for key, value in options.items():
- key = str(key)
- if key.startswith(option_prefix):
- _put_native_option(
- result,
- key,
- key[len(option_prefix):],
- value,
- index_type,
- validate,
- )
- for key, value in options.items():
- key = str(key)
- if key.startswith(field_prefix):
- _put_native_option(
- result,
- key,
- key[len(field_prefix):],
- value,
- index_type,
- validate,
- )
-
-
-def _native_option_key(option_key: str) -> Optional[str]:
- if option_key in ("index.dimension", "dimension"):
- return "dimension"
- if option_key in ("distance.metric", "metric"):
- return "metric"
- if option_key in (
- "nlist",
- "expected-vector-count",
- "ivf.coarse-assignment",
- "ivf.pq-encoding",
- "ivf.train.max-points-per-centroid",
- "pq.train.max-points-per-centroid",
- "pq.m",
- "pq.code-ratio",
- "pq.bits",
- "rq.bits",
- "target-recall",
- "max-bytes-per-vector",
- "deployment-profile",
- ):
- return option_key
- if option_key in ("pq.use-opq", "use-opq"):
- return "use-opq"
- diskann_options = {
- "build-preset": "diskann.build-preset",
- "diskann.build-preset": "diskann.build-preset",
- "max-degree": "diskann.max-degree",
- "diskann.max-degree": "diskann.max-degree",
- "build-search-list-size": "diskann.build-search-list-size",
- "diskann.build-search-list-size": "diskann.build-search-list-size",
- "alpha": "diskann.alpha",
- "diskann.alpha": "diskann.alpha",
- "seed": "diskann.seed",
- "diskann.seed": "diskann.seed",
- "memory-budget-bytes": "diskann.memory-budget-bytes",
- "diskann.memory-budget-bytes": "diskann.memory-budget-bytes",
- "storage-layout": "diskann.storage-layout",
- "diskann.storage-layout": "diskann.storage-layout",
- "raw-vector-encoding": "diskann.raw-vector-encoding",
- "diskann.raw-vector-encoding": "diskann.raw-vector-encoding",
- "build-distance": "diskann.build-distance",
- "diskann.build-distance": "diskann.build-distance",
- }
- if option_key in diskann_options:
- return diskann_options[option_key]
- return None
-
-
-def _put_native_option(
- result, option_key, option_suffix, value, index_type, validate
-):
- native_key = _native_option_key(option_suffix)
- if native_key is None:
- return
- if (_is_050_build_option(native_key)
- and not _is_allowed_050_build_option(native_key, index_type)):
- if validate:
- raise ValueError(
- "Option '%s' is not supported for index type '%s'."
- % (option_key, index_type)
- )
- return
- result[native_key] = str(value)
-
-
-def _is_050_build_option(key):
- return key in (
- "ivf.coarse-assignment",
- "ivf.pq-encoding",
- "ivf.train.max-points-per-centroid",
- "pq.train.max-points-per-centroid",
- )
-
-
-def _is_allowed_050_build_option(key, index_type):
- if key == "ivf.pq-encoding":
- return index_type == "ivf-pq"
- if key == "pq.train.max-points-per-centroid":
- return index_type in ("ivf-pq", "diskann")
- return index_type != "diskann"
-
-
-def _dimension(
- data_type: DataType, native_options_map: Mapping[str, str], index_type: str
-) -> int:
- if isinstance(data_type, VectorType):
- return data_type.length
-
- dimension = native_options_map.get("dimension")
- value = DEFAULT_DIMENSION if dimension is None else int(dimension)
- if value <= 0:
- raise ValueError(
- "Invalid value for '%s.dimension': %s. Must be a positive integer."
- % (index_type, value)
- )
- return value
-
-
-def _is_float_type(data_type: DataType) -> bool:
- return (
- isinstance(data_type, AtomicType)
- and data_type.type.upper() == "FLOAT"
- )
-
-
def _iter_training_batches(
np, vector_file, vector_count: int, dimension: int, sample_ratio: float,
batch_size: int = ADD_BATCH_SIZE,
diff --git a/paimon-python/pypaimon/common/merge_engine_dispatch.py b/paimon-python/pypaimon/execution/python/merge_engine_dispatch.py
similarity index 95%
rename from paimon-python/pypaimon/common/merge_engine_dispatch.py
rename to paimon-python/pypaimon/execution/python/merge_engine_dispatch.py
index a9096fd492..7ec9da03ef 100644
--- a/paimon-python/pypaimon/common/merge_engine_dispatch.py
+++ b/paimon-python/pypaimon/execution/python/merge_engine_dispatch.py
@@ -27,12 +27,9 @@ module is the single source of truth so the two sides cannot drift.
from typing import List, Optional
from pypaimon.common.options.core_options import MergeEngine
-from pypaimon.read.reader.deduplicate_merge_function import \
- DeduplicateMergeFunction
-from pypaimon.read.reader.first_row_merge_function import \
- FirstRowMergeFunction
-from pypaimon.read.reader.partial_update_merge_function import \
- PartialUpdateMergeFunction
+from pypaimon.execution.python.read.reader.deduplicate_merge_function import (DeduplicateMergeFunction)
+from pypaimon.execution.python.read.reader.first_row_merge_function import (FirstRowMergeFunction)
+from pypaimon.execution.python.read.reader.partial_update_merge_function import (PartialUpdateMergeFunction)
# Boolean-valued options that, when truthy, opt the table into
diff --git a/paimon-python/pypaimon/read/scanner/__init__.py b/paimon-python/pypaimon/execution/python/read/__init__.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/__init__.py
copy to paimon-python/pypaimon/execution/python/read/__init__.py
diff --git a/paimon-python/pypaimon/execution/python/read/auth_support.py b/paimon-python/pypaimon/execution/python/read/auth_support.py
new file mode 100644
index 0000000000..f414da2cf6
--- /dev/null
+++ b/paimon-python/pypaimon/execution/python/read/auth_support.py
@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Authorization-based pruning of the Python file scanner."""
+
+import json as _json
+
+from pypaimon.common.predicate import Predicate
+from pypaimon.common.predicate_builder import PredicateBuilder
+from pypaimon.read.push_down_utils import exclude_predicate_with_fields, predicate_field_names
+
+
+def prune_scanner_by_auth(table, scanner, auth_result):
+ masked = set(auth_result.column_masking or {}) if auth_result is not None else set()
+ predicate = scanner._user_predicate
+ effective = exclude_predicate_with_fields(predicate, masked)
+ if effective != scanner.predicate:
+ scanner._set_predicate(effective)
+ scanner.auth_partition_predicate = None
+ scanner.auth_has_non_partition_filter = bool(
+ predicate is not None and masked & predicate_field_names(predicate))
+ if auth_result is None or not auth_result.filter:
+ return
+ partition_preds, has_non_partition = __split_auth_filter(table, auth_result)
+ if partition_preds:
+ combined = PredicateBuilder.and_predicates(partition_preds)
+ scanner.auth_partition_predicate = combined
+ scanner.auth_has_non_partition_filter |= has_non_partition
+
+
+def __split_auth_filter(table, auth_result):
+ partition_keys = list(table.partition_keys or [])
+ if not partition_keys:
+ return [], bool(auth_result.filter)
+
+ partition_preds = []
+ has_non_partition = False
+ partition_key_set = set(partition_keys)
+ partition_index_map = {name: i for i, name in enumerate(partition_keys)}
+
+ for json_str in (auth_result.filter or []):
+ pred = __try_parse_partition_predicate(table, json_str, partition_key_set, partition_index_map)
+ if pred is not None:
+ partition_preds.append(pred)
+ else:
+ has_non_partition = True
+ return partition_preds, has_non_partition
+
+
+def __try_parse_partition_predicate(table, json_str, partition_keys, partition_index_map):
+ data = _json.loads(json_str)
+ if data is None or data.get("kind") != "LEAF":
+ return None
+ transform = data.get("transform", {})
+ if transform.get("name") != "FIELD_REF":
+ return None
+ field_name = transform.get("fieldRef", {}).get("name")
+ if field_name is None or field_name not in partition_keys:
+ return None
+ field_index = partition_index_map.get(field_name)
+ if field_index is None:
+ return None
+
+ partition_field_type = None
+ for f in table.fields:
+ if f.name == field_name:
+ partition_field_type = getattr(f.type, 'type', '')
+ break
+ base_type = partition_field_type.split('(')[0] if partition_field_type else ''
+ safe_types = {'INT', 'BIGINT', 'SMALLINT', 'TINYINT', 'STRING', 'VARCHAR', 'CHAR'}
+ if base_type not in safe_types:
+ return None
+
+ function = data.get("function", "")
+ literals = data.get("literals", [])
+ method_map = {
+ "EQUAL": "equal", "NOT_EQUAL": "notEqual",
+ "LESS_THAN": "lessThan", "LESS_OR_EQUAL": "lessOrEqual",
+ "GREATER_THAN": "greaterThan", "GREATER_OR_EQUAL": "greaterOrEqual",
+ "IS_NULL": "isNull", "IS_NOT_NULL": "isNotNull",
+ "IN": "in", "NOT_IN": "notIn",
+ }
+ method = method_map.get(function)
+ if method is None:
+ return None
+ return Predicate(method=method, index=field_index, field=field_name, literals=literals)
diff --git a/paimon-python/pypaimon/table/format/format_table_read.py b/paimon-python/pypaimon/execution/python/read/format_table_read.py
similarity index 62%
copy from paimon-python/pypaimon/table/format/format_table_read.py
copy to paimon-python/pypaimon/execution/python/read/format_table_read.py
index e4e37b530a..2d2b6e9863 100644
--- a/paimon-python/pypaimon/table/format/format_table_read.py
+++ b/paimon-python/pypaimon/execution/python/read/format_table_read.py
@@ -17,7 +17,6 @@
from typing import Any, Dict, Iterator, List, Optional
-import pandas
import pyarrow
from pypaimon.schema.data_types import PyarrowFieldParser
@@ -166,109 +165,95 @@ def _partition_key_types(
return result if result else None
-class FormatTableRead:
-
- def __init__(
- self,
- table: FormatTable,
- projection: Optional[List[str]] = None,
- limit: Optional[int] = None,
- ):
- self.table = table
- self.projection = projection
- self.limit = limit
-
- def to_arrow(
- self,
- splits: List[FormatDataSplit],
- ) -> pyarrow.Table:
- read_fields = self.projection
- fmt = self.table.format()
- partition_key_types = _partition_key_types(self.table)
- text_col = (
- _text_format_schema_column(self.table)
- if fmt == Format.TEXT
- else None
+def to_arrow(
+ self,
+ splits: List[FormatDataSplit],
+) -> pyarrow.Table:
+ read_fields = self.projection
+ fmt = self.table.format()
+ partition_key_types = _partition_key_types(self.table)
+ text_col = (
+ _text_format_schema_column(self.table)
+ if fmt == Format.TEXT
+ else None
+ )
+ text_delim = (
+ self.table.options().get("text.line-delimiter", "\n")
+ if fmt == Format.TEXT
+ else "\n"
+ )
+ tables = []
+ nrows = 0
+ for split in splits:
+ t = _read_file_to_arrow(
+ self.table.file_io,
+ split,
+ fmt,
+ split.partition,
+ read_fields,
+ partition_key_types,
+ text_column_name=text_col,
+ text_line_delimiter=text_delim,
)
- text_delim = (
- self.table.options().get("text.line-delimiter", "\n")
- if fmt == Format.TEXT
- else "\n"
+ if t.num_rows > 0:
+ tables.append(t)
+ nrows += t.num_rows
+ if self.limit is not None and nrows >= self.limit:
+ if nrows > self.limit:
+ excess = nrows - self.limit
+ last = tables[-1]
+ tables[-1] = last.slice(0, last.num_rows - excess)
+ break
+ if not tables:
+ fields = self.table.fields
+ if read_fields:
+ fields = [
+ f for f in self.table.fields if f.name in read_fields
+ ]
+ schema = PyarrowFieldParser.from_paimon_schema(fields)
+ return pyarrow.Table.from_pydict(
+ {n: [] for n in schema.names},
+ schema=schema,
)
- tables = []
- nrows = 0
- for split in splits:
- t = _read_file_to_arrow(
- self.table.file_io,
- split,
- fmt,
- split.partition,
- read_fields,
- partition_key_types,
- text_column_name=text_col,
- text_line_delimiter=text_delim,
- )
- if t.num_rows > 0:
- tables.append(t)
- nrows += t.num_rows
- if self.limit is not None and nrows >= self.limit:
- if nrows > self.limit:
- excess = nrows - self.limit
- last = tables[-1]
- tables[-1] = last.slice(0, last.num_rows - excess)
- break
- if not tables:
- fields = self.table.fields
- if read_fields:
- fields = [
- f for f in self.table.fields if f.name in read_fields
- ]
- schema = PyarrowFieldParser.from_paimon_schema(fields)
- return pyarrow.Table.from_pydict(
- {n: [] for n in schema.names},
- schema=schema,
- )
- out = pyarrow.concat_tables(tables)
- if self.limit is not None and out.num_rows > self.limit:
- out = out.slice(0, self.limit)
- return out
+ out = pyarrow.concat_tables(tables)
+ if self.limit is not None and out.num_rows > self.limit:
+ out = out.slice(0, self.limit)
+ return out
- def to_pandas(self, splits: List[FormatDataSplit]) -> pandas.DataFrame:
- return self.to_arrow(splits).to_pandas()
- def to_iterator(
- self,
- splits: List[FormatDataSplit],
- ) -> Iterator[Any]:
- partition_key_types = _partition_key_types(self.table)
- fmt = self.table.format()
- text_col = (
- _text_format_schema_column(self.table)
- if fmt == Format.TEXT
- else None
- )
- text_delim = (
- self.table.options().get("text.line-delimiter", "\n")
- if fmt == Format.TEXT
- else "\n"
+def to_iterator(
+ self,
+ splits: List[FormatDataSplit],
+) -> Iterator[Any]:
+ partition_key_types = _partition_key_types(self.table)
+ fmt = self.table.format()
+ text_col = (
+ _text_format_schema_column(self.table)
+ if fmt == Format.TEXT
+ else None
+ )
+ text_delim = (
+ self.table.options().get("text.line-delimiter", "\n")
+ if fmt == Format.TEXT
+ else "\n"
+ )
+ n_yielded = 0
+ for split in splits:
+ if self.limit is not None and n_yielded >= self.limit:
+ break
+ t = _read_file_to_arrow(
+ self.table.file_io,
+ split,
+ fmt,
+ split.partition,
+ self.projection,
+ partition_key_types,
+ text_column_name=text_col,
+ text_line_delimiter=text_delim,
)
- n_yielded = 0
- for split in splits:
- if self.limit is not None and n_yielded >= self.limit:
- break
- t = _read_file_to_arrow(
- self.table.file_io,
- split,
- fmt,
- split.partition,
- self.projection,
- partition_key_types,
- text_column_name=text_col,
- text_line_delimiter=text_delim,
- )
- for batch in t.to_batches():
- for i in range(batch.num_rows):
- if self.limit is not None and n_yielded >= self.limit:
- return
- yield batch.slice(i, 1)
- n_yielded += 1
+ for batch in t.to_batches():
+ for i in range(batch.num_rows):
+ if self.limit is not None and n_yielded >= self.limit:
+ return
+ yield batch.slice(i, 1)
+ n_yielded += 1
diff --git a/paimon-python/pypaimon/table/format/format_table_scan.py b/paimon-python/pypaimon/execution/python/read/format_table_scan.py
similarity index 76%
copy from paimon-python/pypaimon/table/format/format_table_scan.py
copy to paimon-python/pypaimon/execution/python/read/format_table_scan.py
index f2b8cdcbab..71c8dbba5a 100644
--- a/paimon-python/pypaimon/table/format/format_table_scan.py
+++ b/paimon-python/pypaimon/execution/python/read/format_table_scan.py
@@ -22,7 +22,6 @@ import pyarrow.fs as pafs
from pypaimon.common.file_io import FileIO
from pypaimon.read.plan import Plan
from pypaimon.table.format.format_data_split import FormatDataSplit
-from pypaimon.table.format.format_table import FormatTable
def _is_data_file_name(name: str) -> bool:
@@ -98,34 +97,24 @@ def _list_data_files_recursive(
return splits
-class FormatTableScan:
-
- def __init__(
- self,
- table: FormatTable,
- partition_filter: Optional[Dict[str, str]] = None,
- ):
- self.table = table
- self.partition_filter = partition_filter # optional equality filter
-
- def plan(self) -> Plan:
- partition_only_value = self.table.options().get(
- "format-table.partition-path-only-value", "false"
- ).lower() == "true"
- splits = _list_data_files_recursive(
- self.table.file_io,
- self.table.location(),
- self.table.partition_keys,
- partition_only_value,
- )
- if self.partition_filter:
- filtered = []
- for s in splits:
- match = s.partition and all(
- str(s.partition.get(k)) == str(v)
- for k, v in self.partition_filter.items()
- )
- if match:
- filtered.append(s)
- splits = filtered
- return Plan(_splits=splits)
+def plan(self) -> Plan:
+ partition_only_value = self.table.options().get(
+ "format-table.partition-path-only-value", "false"
+ ).lower() == "true"
+ splits = _list_data_files_recursive(
+ self.table.file_io,
+ self.table.location(),
+ self.table.partition_keys,
+ partition_only_value,
+ )
+ if self.partition_filter:
+ filtered = []
+ for s in splits:
+ match = s.partition and all(
+ str(s.partition.get(k)) == str(v)
+ for k, v in self.partition_filter.items()
+ )
+ if match:
+ filtered.append(s)
+ splits = filtered
+ return Plan(_splits=splits)
diff --git a/paimon-python/pypaimon/read/merge_engine_support.py b/paimon-python/pypaimon/execution/python/read/merge_engine_support.py
similarity index 64%
copy from paimon-python/pypaimon/read/merge_engine_support.py
copy to paimon-python/pypaimon/execution/python/read/merge_engine_support.py
index 51025a0df5..a47dcbda31 100644
--- a/paimon-python/pypaimon/read/merge_engine_support.py
+++ b/paimon-python/pypaimon/execution/python/read/merge_engine_support.py
@@ -34,6 +34,8 @@ results when the table configures, e.g.,
from typing import Set
from pypaimon.common.options.core_options import MergeEngine
+from pypaimon.read.merge_engine_support import (
+ check_sequence_field_valid, check_sequence_field_supported)
# Boolean-valued options that, when truthy, opt the table into behaviour
# the Python ``PartialUpdateMergeFunction`` does not implement.
@@ -79,109 +81,6 @@ _FIELD_IGNORE_RETRACT_SUFFIX = ".ignore-retract"
_DEFAULT_AGGREGATE_FUNCTION_KEY = "fields.default-aggregate-function"
-def _unsupported_sequence_fields(table) -> Set[str]:
- """Configured ``sequence.field`` names whose type pypaimon cannot order.
- Java's ``UserDefinedSeqComparator`` delegates to ``RecordComparator``
- and supports ARRAY / VECTOR / MAP / MULTISET / ROW, but pypaimon's
- ``builtin_seq_comparator`` only compares orderable atomic types. This
- flags both complex (non-atomic) types and the atomic-but-unorderable
- VARIANT, so a raw-convertible split (which skips the merge reader) can't
- silently bypass the limitation.
- """
- from pypaimon.read.reader.sort_merge_reader import is_comparable_seq_field
- flagged: Set[str] = set()
- for field in table.options.sequence_field():
- data_field = table.field_dict.get(field)
- if data_field is not None and not is_comparable_seq_field(data_field):
- flagged.add(field)
- return flagged
-
-
-def check_sequence_field_valid(table) -> None:
- """Reject ``sequence.field`` configurations Java forbids at schema
- validation (``SchemaValidation.validateSequenceField``), raising
- ``ValueError`` to mirror Java's ``IllegalArgumentException``.
-
- These are invalid configurations, not deferred features, so they are
- rejected on every merge engine regardless of pypaimon's read-path
- coverage. Mirrors all of Java's checks:
-
- 1. Every sequence field must exist in the table schema.
- 2. No sequence field may be declared more than once.
- 3. ``fields.<seq>.aggregate-function`` on a sequence column: Java
- forbids aggregating the sequence column outright. pypaimon's
- aggregation engine otherwise silently overrides it with
- ``last_value``, hiding the misconfiguration.
- 4. ``sequence.field`` together with ``merge-engine=first-row``:
- first-row keeps the earliest-written row and never honors a
- sequence ordering.
- 5. ``sequence.field`` together with cross-partition update (the PK
- does not include all partition fields).
- """
- sequence_fields = table.options.sequence_field()
- if not sequence_fields:
- return
-
- field_names = set(table.field_names)
- seen: Set[str] = set()
- options_map = table.options.options.to_map()
- for field in sequence_fields:
- if field not in field_names:
- raise ValueError(
- "Sequence field: '{}' can not be found in table "
- "schema.".format(field)
- )
- if field in seen:
- raise ValueError(
- "Sequence field '{}' is defined repeatedly.".format(field)
- )
- seen.add(field)
- agg_key = "fields.{}.aggregate-function".format(field)
- if options_map.get(agg_key) is not None:
- raise ValueError(
- "Should not define aggregation on sequence field: '{}' "
- "({}).".format(field, agg_key)
- )
-
- if table.options.merge_engine() == MergeEngine.FIRST_ROW:
- raise ValueError(
- "Do not support use sequence.field on FIRST_ROW merge engine."
- )
-
- if table.cross_partition_update:
- raise ValueError(
- "You can not use sequence.field in cross partition update case "
- "(primary keys {} do not include all partition fields "
- "{}).".format(table.primary_keys, table.partition_keys)
- )
-
-
-def check_sequence_field_supported(table) -> None:
- """Validate sequence configuration and types before reading or writing."""
- if not table.is_primary_key_table:
- return
- # ``sequence.field`` validity is engine-independent in Java
- # (SchemaValidation.validateSequenceField). pypaimon has no
- # schema-creation validation, so enforce the same invariants here on
- # both paths, before per-engine dispatch.
- check_sequence_field_valid(table)
- # ``sequence.field`` validity (above) is Java-aligned and engine
- # independent. Some field *types* are valid in Java but unimplemented in
- # pypaimon's orderable-atomic-only comparator (complex types, plus the
- # atomic-but-unorderable VARIANT), so reject them as NotImplementedError
- # here -- before per-engine dispatch, so a raw-convertible split can't
- # bypass the merge reader and skip the check.
- unsupported_seq = _unsupported_sequence_fields(table)
- if unsupported_seq:
- raise NotImplementedError(
- "sequence.field with unsupported type is not implemented in "
- "pypaimon yet: {}. pypaimon only supports orderable atomic "
- "sequence-field types; complex types (ARRAY / MAP / ROW etc., "
- "handled by Java via RecordComparator) and VARIANT are not "
- "supported. Open an issue to track support.".format(
- ", ".join(sorted(unsupported_seq))))
-
-
def check_supported(table) -> None:
"""Raise ``NotImplementedError`` if the table's merge-engine
configuration is outside what pypaimon's read path implements, or
@@ -305,3 +204,8 @@ def _option_is_truthy(raw) -> bool:
if isinstance(raw, str):
return raw.strip().lower() in ("true", "1", "yes", "on")
return bool(raw)
+
+__all__ = [
+ "check_supported", "check_sequence_field_valid", "check_sequence_field_supported",
+ "partial_update_unsupported_options", "aggregation_unsupported_options",
+]
diff --git a/paimon-python/pypaimon/globalindex/full_text/native_full_text_global_index_reader.py b/paimon-python/pypaimon/execution/python/read/native_full_text_global_index_reader.py
similarity index 83%
rename from paimon-python/pypaimon/globalindex/full_text/native_full_text_global_index_reader.py
rename to paimon-python/pypaimon/execution/python/read/native_full_text_global_index_reader.py
index 5d621e0efe..a061facc08 100644
--- a/paimon-python/pypaimon/globalindex/full_text/native_full_text_global_index_reader.py
+++ b/paimon-python/pypaimon/execution/python/read/native_full_text_global_index_reader.py
@@ -15,12 +15,11 @@
# specific language governing permissions and limitations
# under the License.
-"""Full-text global index reader backed by paimon-full-text."""
+"""Full-text global index reader using the paimon-ftindex Python SDK."""
-import json
import os
import threading
-from typing import List, Mapping
+from typing import List
from pypaimon.globalindex.global_index_reader import (
FieldRef,
@@ -31,43 +30,19 @@ from pypaimon.globalindex.vector_search_result import (
DictBasedScoredIndexResult,
)
from pypaimon.globalindex.global_index_meta import GlobalIndexIOMeta
+from pypaimon.globalindex.full_text_options import (
+ FULL_TEXT_IDENTIFIER,
+ _FULL_TEXT_OPTION_PREFIX,
+ NativeFullTextIndexOptions,
+ _option_value,
+)
-FULL_TEXT_IDENTIFIER = "full-text"
-_FULL_TEXT_OPTION_PREFIX = "full-text."
-
-
-class NativeFullTextIndexOptions:
- """Native full-text options with the public ``full-text.`` prefix removed."""
-
- def __init__(self, options=None):
- self._options = dict(options or {})
-
- @staticmethod
- def deserialize(data):
- if not data:
- return NativeFullTextIndexOptions()
- return NativeFullTextIndexOptions(json.loads(data.decode("utf-8")))
-
- @staticmethod
- def from_options(options: Mapping[str, object]):
- config = {}
- for key, value in options.items():
- key = str(key)
- if key.startswith(_FULL_TEXT_OPTION_PREFIX):
- key = key[len(_FULL_TEXT_OPTION_PREFIX):]
- config[key] = _option_value(value)
- return NativeFullTextIndexOptions._from_config(config)
-
- @staticmethod
- def _from_config(config):
- return NativeFullTextIndexOptions(
- {str(key): _option_value(value) for key, value in config.items()})
-
- def serialize(self) -> bytes:
- return json.dumps(self._options, separators=(",", ":")).encode("utf-8")
- def to_native_options(self):
- return dict(self._options)
+__all__ = [
+ 'FULL_TEXT_IDENTIFIER', '_FULL_TEXT_OPTION_PREFIX', 'NativeFullTextIndexOptions',
+ 'PaimonFullTextInput', 'NativeFullTextGlobalIndexReader',
+ '_search_top_k', '_read_fully', '_option_value',
+]
class PaimonFullTextInput:
@@ -234,11 +209,3 @@ def _read_fully(stream, length: int) -> bytes:
buf.extend(chunk)
remaining -= len(chunk)
return bytes(buf)
-
-
-def _option_value(value) -> str:
- if isinstance(value, bool):
- return str(value).lower()
- if isinstance(value, list):
- return ";".join(str(item) for item in value)
- return str(value)
diff --git a/paimon-python/pypaimon/execution/python/read/object_table_read.py b/paimon-python/pypaimon/execution/python/read/object_table_read.py
new file mode 100644
index 0000000000..e092cab428
--- /dev/null
+++ b/paimon-python/pypaimon/execution/python/read/object_table_read.py
@@ -0,0 +1,131 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import List
+from urllib.parse import urlparse
+
+import pyarrow
+import pyarrow.fs as pafs
+
+from pypaimon.table.object.object_split import ObjectSplit
+
+from pypaimon.table.object.object_table_read import OBJECT_TABLE_ARROW_SCHEMA
+
+
+def _normalize_path(path_str: str) -> str:
+ """Extract the filesystem path from a URI or plain path."""
+ parsed = urlparse(path_str)
+ if parsed.scheme:
+ return parsed.path
+ return path_str
+
+
+def _list_files_recursive(file_io, directory: str) -> list:
+ """Recursively list all files under a directory using FileIO.
+
+ Returns a list of (file_path, file_info) tuples.
+ """
+ results = []
+ try:
+ infos = file_io.list_status(directory)
+ except Exception:
+ return results
+
+ directory_rstrip = directory.rstrip("/")
+ for info in infos:
+ file_name = info.path.split("/")[-1] if "/" in info.path else info.path
+ full_path = info.path
+ if not full_path.startswith("/") and "://" not in full_path:
+ full_path = "{}/{}".format(directory_rstrip, file_name)
+
+ if info.type == pafs.FileType.Directory:
+ sub_results = _list_files_recursive(file_io, full_path)
+ results.extend(sub_results)
+ elif info.type == pafs.FileType.File:
+ results.append((full_path, info))
+
+ return results
+
+
+def _compute_relative_path(prefix: str, file_path: str) -> str:
+ """Compute relative path by removing the location prefix.
+
+ Matches Java ObjectTableImpl.toRow() logic.
+ """
+ norm_prefix = _normalize_path(prefix).rstrip("/")
+ norm_file = _normalize_path(file_path)
+
+ if norm_file.startswith(norm_prefix):
+ relative = norm_file[len(norm_prefix):]
+ if relative.startswith("/"):
+ relative = relative[1:]
+ return relative
+
+ return norm_file.split("/")[-1] if "/" in norm_file else norm_file
+
+
+def to_arrow(self, splits: List[ObjectSplit]) -> pyarrow.Table:
+ paths = []
+ names = []
+ lengths = []
+ mtimes = []
+ atimes = []
+ owners = []
+
+ for split in splits:
+ file_entries = _list_files_recursive(split.file_io, split.location)
+ for file_path, file_info in file_entries:
+ if self.limit is not None and len(paths) >= self.limit:
+ break
+
+ relative_path = _compute_relative_path(split.location, file_path)
+ file_name = file_path.split("/")[-1] if "/" in file_path else file_path
+ file_size = getattr(file_info, "size", 0) or 0
+ modification_time = getattr(file_info, "mtime_ns", 0) or 0
+ if modification_time > 0:
+ modification_time = modification_time // 1_000_000
+ access_time = 0
+ owner = None
+
+ paths.append(relative_path)
+ names.append(file_name)
+ lengths.append(file_size)
+ mtimes.append(modification_time)
+ atimes.append(access_time)
+ owners.append(owner)
+
+ if self.limit is not None and len(paths) >= self.limit:
+ break
+
+ result = pyarrow.table(
+ {
+ "path": pyarrow.array(paths, type=pyarrow.string()),
+ "name": pyarrow.array(names, type=pyarrow.string()),
+ "length": pyarrow.array(lengths, type=pyarrow.int64()),
+ "mtime": pyarrow.array(mtimes, type=pyarrow.int64()),
+ "atime": pyarrow.array(atimes, type=pyarrow.int64()),
+ "owner": pyarrow.array(owners, type=pyarrow.string()),
+ },
+ schema=OBJECT_TABLE_ARROW_SCHEMA,
+ )
+
+ if self.projection:
+ existing = [c for c in self.projection if c in result.column_names]
+ if existing:
+ result = result.select(existing)
+
+ return result
diff --git a/paimon-python/pypaimon/globalindex/btree/memory_slice_input.py b/paimon-python/pypaimon/execution/python/read/object_table_scan.py
similarity index 75%
copy from paimon-python/pypaimon/globalindex/btree/memory_slice_input.py
copy to paimon-python/pypaimon/execution/python/read/object_table_scan.py
index 4cf97a10e7..38a8696f77 100644
--- a/paimon-python/pypaimon/globalindex/btree/memory_slice_input.py
+++ b/paimon-python/pypaimon/execution/python/read/object_table_scan.py
@@ -15,8 +15,13 @@
# specific language governing permissions and limitations
# under the License.
-"""Compatibility re-export for global index byte-slice input."""
+from pypaimon.read.plan import Plan
+from pypaimon.table.object.object_split import ObjectSplit
-from pypaimon.globalindex.memory_slice_input import MemorySliceInput
-__all__ = ['MemorySliceInput']
+def plan(self) -> Plan:
+ split = ObjectSplit(
+ file_io=self.table.file_io,
+ location=self.table.location(),
+ )
+ return Plan(_splits=[split])
diff --git a/paimon-python/pypaimon/read/scanner/__init__.py b/paimon-python/pypaimon/execution/python/read/reader/__init__.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/__init__.py
copy to paimon-python/pypaimon/execution/python/read/reader/__init__.py
diff --git a/paimon-python/pypaimon/read/reader/aggregate/__init__.py b/paimon-python/pypaimon/execution/python/read/reader/aggregate/__init__.py
similarity index 94%
rename from paimon-python/pypaimon/read/reader/aggregate/__init__.py
rename to paimon-python/pypaimon/execution/python/read/reader/aggregate/__init__.py
index 8d5fe75d48..3e9162a7a6 100644
--- a/paimon-python/pypaimon/read/reader/aggregate/__init__.py
+++ b/paimon-python/pypaimon/execution/python/read/reader/aggregate/__init__.py
@@ -28,7 +28,7 @@ regardless of which call site triggers the first lookup.
from typing import Callable, Dict, TYPE_CHECKING
-from pypaimon.read.reader.aggregate.field_aggregator import FieldAggregator
+from pypaimon.execution.python.read.reader.aggregate.field_aggregator import FieldAggregator
from pypaimon.schema.data_types import DataType
if TYPE_CHECKING:
@@ -81,4 +81,4 @@ def create_field_aggregator(
# caller looks anything up. Placed at the bottom of the module so the
# names ``register_aggregator`` / ``FieldAggregator`` aggregators
# imports back from here are already defined when its import runs.
-from pypaimon.read.reader.aggregate import aggregators # noqa: E402, F401
+from pypaimon.execution.python.read.reader.aggregate import aggregators # noqa: E402, F401
diff --git a/paimon-python/pypaimon/read/reader/aggregate/aggregators.py b/paimon-python/pypaimon/execution/python/read/reader/aggregate/aggregators.py
similarity index 99%
rename from paimon-python/pypaimon/read/reader/aggregate/aggregators.py
rename to paimon-python/pypaimon/execution/python/read/reader/aggregate/aggregators.py
index 7d3ff90256..05d73b765a 100644
--- a/paimon-python/pypaimon/read/reader/aggregate/aggregators.py
+++ b/paimon-python/pypaimon/execution/python/read/reader/aggregate/aggregators.py
@@ -20,7 +20,7 @@
Each class registers itself with the global registry at import time
via :func:`register_aggregator`, so importing
-``pypaimon.read.reader.aggregate`` makes all of them discoverable.
+``pypaimon.execution.python.read.reader.aggregate`` makes all of them discoverable.
This module ships 20 aggregators — the primary-key placeholder plus
the 19 most commonly-used value aggregators: ``primary_key`` /
@@ -40,8 +40,8 @@ from typing import Any, List, Dict, Optional, Tuple, Union, Set
from pypaimon.common.options import CoreOptions
from pypaimon.common.options.core_options import NestedKeyNullStrategy
from pypaimon.data.decimal import Decimal
-from pypaimon.read.reader.aggregate import register_aggregator
-from pypaimon.read.reader.aggregate.field_aggregator import FieldAggregator
+from pypaimon.execution.python.read.reader.aggregate import register_aggregator
+from pypaimon.execution.python.read.reader.aggregate.field_aggregator import FieldAggregator
from pypaimon.schema.data_types import AtomicType, DataType, ArrayType, RowType, MapType
from pypaimon.table.row.generic_row import GenericRow
from pypaimon.table.row.internal_row import InternalRow
diff --git a/paimon-python/pypaimon/read/reader/aggregate/field_aggregator.py b/paimon-python/pypaimon/execution/python/read/reader/aggregate/field_aggregator.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/aggregate/field_aggregator.py
rename to paimon-python/pypaimon/execution/python/read/reader/aggregate/field_aggregator.py
diff --git a/paimon-python/pypaimon/read/reader/aggregation_merge_function.py b/paimon-python/pypaimon/execution/python/read/reader/aggregation_merge_function.py
similarity index 96%
rename from paimon-python/pypaimon/read/reader/aggregation_merge_function.py
rename to paimon-python/pypaimon/execution/python/read/reader/aggregation_merge_function.py
index d5437be78e..b94ed3e296 100644
--- a/paimon-python/pypaimon/read/reader/aggregation_merge_function.py
+++ b/paimon-python/pypaimon/execution/python/read/reader/aggregation_merge_function.py
@@ -33,18 +33,18 @@ are intentionally deferred. Non-INSERT row
kinds raise ``NotImplementedError`` at :meth:`add` time so we never
silently corrupt data with a half-implemented contract, and
out-of-scope aggregator identifiers / options are rejected up-front in
-:mod:`pypaimon.read.merge_engine_support`.
+:mod:`pypaimon.execution.python.read.merge_engine_support`.
"""
from typing import Any, List, Optional
-from pypaimon.read.reader.aggregate import create_field_aggregator
-from pypaimon.read.reader.aggregate.aggregators import (
+from pypaimon.execution.python.read.reader.aggregate import create_field_aggregator
+from pypaimon.execution.python.read.reader.aggregate.aggregators import (
NAME_LAST_NON_NULL_VALUE,
NAME_LAST_VALUE,
NAME_PRIMARY_KEY,
)
-from pypaimon.read.reader.aggregate.field_aggregator import FieldAggregator
+from pypaimon.execution.python.read.reader.aggregate.field_aggregator import FieldAggregator
from pypaimon.schema.data_types import DataField
from pypaimon.table.row.key_value import KeyValue
from pypaimon.table.row.row_kind import RowKind
diff --git a/paimon-python/pypaimon/read/reader/auth_masking_reader.py b/paimon-python/pypaimon/execution/python/read/reader/auth_masking_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/auth_masking_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/auth_masking_reader.py
diff --git a/paimon-python/pypaimon/read/reader/blob_descriptor_convert_reader.py b/paimon-python/pypaimon/execution/python/read/reader/blob_descriptor_convert_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/blob_descriptor_convert_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/blob_descriptor_convert_reader.py
diff --git a/paimon-python/pypaimon/read/reader/blob_view_read_support.py b/paimon-python/pypaimon/execution/python/read/reader/blob_view_read_support.py
similarity index 94%
rename from paimon-python/pypaimon/read/reader/blob_view_read_support.py
rename to paimon-python/pypaimon/execution/python/read/reader/blob_view_read_support.py
index 98a7881b87..54055b592d 100644
--- a/paimon-python/pypaimon/read/reader/blob_view_read_support.py
+++ b/paimon-python/pypaimon/execution/python/read/reader/blob_view_read_support.py
@@ -43,9 +43,9 @@ def wrap_record_reader_with_blob_inline_convert(
split_read,
read_fields: List[DataField],
) -> RecordReader:
- from pypaimon.read.reader.auth_masking_reader import (
+ from pypaimon.execution.python.read.reader.auth_masking_reader import (
BatchToRecordReaderAdapter, RecordReaderToBatchAdapter)
- from pypaimon.read.reader.blob_descriptor_convert_reader import BlobInlineConvertReader
+ from pypaimon.execution.python.read.reader.blob_descriptor_convert_reader import BlobInlineConvertReader
from pypaimon.read.reader.field_indices import (
blob_field_indices, descriptor_field_indices_for_table, vector_field_indices)
diff --git a/paimon-python/pypaimon/read/reader/concat_batch_reader.py b/paimon-python/pypaimon/execution/python/read/reader/concat_batch_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/concat_batch_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/concat_batch_reader.py
diff --git a/paimon-python/pypaimon/read/reader/concat_record_reader.py b/paimon-python/pypaimon/execution/python/read/reader/concat_record_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/concat_record_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/concat_record_reader.py
diff --git a/paimon-python/pypaimon/read/reader/data_file_batch_reader.py b/paimon-python/pypaimon/execution/python/read/reader/data_file_batch_reader.py
similarity index 97%
rename from paimon-python/pypaimon/read/reader/data_file_batch_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/data_file_batch_reader.py
index 408c5a2de8..b51ed5aee1 100644
--- a/paimon-python/pypaimon/read/reader/data_file_batch_reader.py
+++ b/paimon-python/pypaimon/execution/python/read/reader/data_file_batch_reader.py
@@ -30,7 +30,7 @@ from pypaimon.data.map_shared_shredding import (
map_selected_keys,
)
from pypaimon.read.partition_info import PartitionInfo
-from pypaimon.read.reader.format_blob_reader import FormatBlobReader
+from pypaimon.execution.python.read.reader.format_blob_reader import FormatBlobReader
from pypaimon.read.reader.iface.record_batch_reader import RecordBatchReader
from pypaimon.schema.data_types import (ArrayType, AtomicType, DataField,
MapType, PyarrowFieldParser, RowType)
@@ -68,10 +68,6 @@ def _constructed_to_string_array(array, file_type):
ROW -> ``{v1, v2}``, ARRAY -> ``[e1, e2]``, MAP -> ``{k1 -> v1, k2 -> v2}``.
Sub-values are rendered recursively; a NULL sub-value renders as the
literal ``null`` while a NULL container row stays NULL."""
- from pypaimon.casting.schema_evolution import _native_schema_cast
- native = _native_schema_cast(array, pa.string(), file_type, AtomicType('STRING'))
- if native is not None:
- return native
array = _unslice(array)
valid = pc.is_valid(array).to_pylist()
out = []
@@ -268,11 +264,6 @@ class DataFileBatchReader(RecordBatchReader):
"""Return *array* converted to *target_type*, matching ROW sub-fields by
field id (reorder, pad missing with NULL, follow renames, cast changed
types) recursively, transparently through ARRAY/MAP wrappers."""
- from pypaimon.casting.schema_evolution import _native_schema_cast
- native = _native_schema_cast(
- array, PyarrowFieldParser.from_paimon_type(target_type), file_type, target_type)
- if native is not None:
- return native
if isinstance(target_type, RowType) and isinstance(file_type, RowType):
n = len(array)
file_id_to_pos = {f.id: i for i, f in enumerate(file_type.fields)}
diff --git a/paimon-python/pypaimon/read/reader/deduplicate_merge_function.py b/paimon-python/pypaimon/execution/python/read/reader/deduplicate_merge_function.py
similarity index 100%
copy from paimon-python/pypaimon/read/reader/deduplicate_merge_function.py
copy to paimon-python/pypaimon/execution/python/read/reader/deduplicate_merge_function.py
diff --git a/paimon-python/pypaimon/read/reader/deferred_blob_resolve_reader.py b/paimon-python/pypaimon/execution/python/read/reader/deferred_blob_resolve_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/deferred_blob_resolve_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/deferred_blob_resolve_reader.py
diff --git a/paimon-python/pypaimon/read/reader/drop_delete_reader.py b/paimon-python/pypaimon/execution/python/read/reader/drop_delete_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/drop_delete_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/drop_delete_reader.py
diff --git a/paimon-python/pypaimon/read/reader/empty_record_reader.py b/paimon-python/pypaimon/execution/python/read/reader/empty_record_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/empty_record_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/empty_record_reader.py
diff --git a/paimon-python/pypaimon/read/reader/field_bunch.py b/paimon-python/pypaimon/execution/python/read/reader/field_bunch.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/field_bunch.py
rename to paimon-python/pypaimon/execution/python/read/reader/field_bunch.py
diff --git a/paimon-python/pypaimon/read/reader/filter_record_batch_reader.py b/paimon-python/pypaimon/execution/python/read/reader/filter_record_batch_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/filter_record_batch_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/filter_record_batch_reader.py
diff --git a/paimon-python/pypaimon/read/reader/filter_record_reader.py b/paimon-python/pypaimon/execution/python/read/reader/filter_record_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/filter_record_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/filter_record_reader.py
diff --git a/paimon-python/pypaimon/read/reader/first_row_merge_function.py b/paimon-python/pypaimon/execution/python/read/reader/first_row_merge_function.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/first_row_merge_function.py
rename to paimon-python/pypaimon/execution/python/read/reader/first_row_merge_function.py
diff --git a/paimon-python/pypaimon/read/reader/format_avro_reader.py b/paimon-python/pypaimon/execution/python/read/reader/format_avro_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/format_avro_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/format_avro_reader.py
diff --git a/paimon-python/pypaimon/read/reader/format_blob_reader.py b/paimon-python/pypaimon/execution/python/read/reader/format_blob_reader.py
similarity index 99%
rename from paimon-python/pypaimon/read/reader/format_blob_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/format_blob_reader.py
index 35f04dcb2b..020523d1b2 100644
--- a/paimon-python/pypaimon/read/reader/format_blob_reader.py
+++ b/paimon-python/pypaimon/execution/python/read/reader/format_blob_reader.py
@@ -28,7 +28,7 @@ from pypaimon.common.file_io import FileIO
from pypaimon.common.map_blob_key_serializer import create_map_blob_key_serializer
from pypaimon.common.memory_size import MemorySize
from pypaimon.read.reader.iface.record_batch_reader import RecordBatchReader
-from pypaimon.read.reader.video_format_reader import (
+from pypaimon.execution.python.read.reader.video_format_reader import (
VideoFileMeta,
VideoFrameRecordIterator,
)
diff --git a/paimon-python/pypaimon/read/reader/format_lance_reader.py b/paimon-python/pypaimon/execution/python/read/reader/format_lance_reader.py
similarity index 98%
rename from paimon-python/pypaimon/read/reader/format_lance_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/format_lance_reader.py
index 67edfbd025..9e3d30074d 100644
--- a/paimon-python/pypaimon/read/reader/format_lance_reader.py
+++ b/paimon-python/pypaimon/execution/python/read/reader/format_lance_reader.py
@@ -23,7 +23,7 @@ from pyarrow import RecordBatch
from pypaimon.common.file_io import FileIO
from pypaimon.read.reader.iface.record_batch_reader import RecordBatchReader
-from pypaimon.read.reader.lance_utils import to_lance_specified
+from pypaimon.execution.python.read.reader.lance_utils import to_lance_specified
from pypaimon.schema.data_types import DataField, PyarrowFieldParser
from pypaimon.table.special_fields import SpecialFields
diff --git a/paimon-python/pypaimon/read/reader/format_mosaic_reader.py b/paimon-python/pypaimon/execution/python/read/reader/format_mosaic_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/format_mosaic_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/format_mosaic_reader.py
diff --git a/paimon-python/pypaimon/read/reader/format_pyarrow_reader.py b/paimon-python/pypaimon/execution/python/read/reader/format_pyarrow_reader.py
similarity index 99%
rename from paimon-python/pypaimon/read/reader/format_pyarrow_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/format_pyarrow_reader.py
index ff9517892d..0f8bf0399d 100644
--- a/paimon-python/pypaimon/read/reader/format_pyarrow_reader.py
+++ b/paimon-python/pypaimon/execution/python/read/reader/format_pyarrow_reader.py
@@ -567,7 +567,7 @@ class FormatPyArrowReader(RecordBatchReader):
and options is not None
and options.parquet_column_index_enabled()
and self._row_group_cache is None):
- from pypaimon.read.reader.parquet_page_index_reader import (
+ from pypaimon.execution.python.read.reader.parquet_page_index_reader import (
ParquetPageIndexReader,
)
self._page_index_reader = ParquetPageIndexReader.create(
diff --git a/paimon-python/pypaimon/read/reader/format_row_reader.py b/paimon-python/pypaimon/execution/python/read/reader/format_row_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/format_row_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/format_row_reader.py
diff --git a/paimon-python/pypaimon/read/reader/format_vortex_reader.py b/paimon-python/pypaimon/execution/python/read/reader/format_vortex_reader.py
similarity index 98%
rename from paimon-python/pypaimon/read/reader/format_vortex_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/format_vortex_reader.py
index fcd50a24f0..33ca6b474e 100644
--- a/paimon-python/pypaimon/read/reader/format_vortex_reader.py
+++ b/paimon-python/pypaimon/execution/python/read/reader/format_vortex_reader.py
@@ -41,7 +41,7 @@ class FormatVortexReader(RecordBatchReader):
predicate_fields: Optional[Set[str]] = None):
import vortex
- from pypaimon.read.reader.vortex_utils import to_vortex_specified
+ from pypaimon.execution.python.read.reader.vortex_utils import to_vortex_specified
file_path_for_vortex, store_kwargs = to_vortex_specified(file_io, file_path)
if store_kwargs:
diff --git a/paimon-python/pypaimon/read/reader/key_value_unwrap_reader.py b/paimon-python/pypaimon/execution/python/read/reader/key_value_unwrap_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/key_value_unwrap_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/key_value_unwrap_reader.py
diff --git a/paimon-python/pypaimon/read/reader/key_value_wrap_reader.py b/paimon-python/pypaimon/execution/python/read/reader/key_value_wrap_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/key_value_wrap_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/key_value_wrap_reader.py
diff --git a/paimon-python/pypaimon/read/reader/lance_utils.py b/paimon-python/pypaimon/execution/python/read/reader/lance_utils.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/lance_utils.py
rename to paimon-python/pypaimon/execution/python/read/reader/lance_utils.py
diff --git a/paimon-python/pypaimon/read/reader/limited_record_reader.py b/paimon-python/pypaimon/execution/python/read/reader/limited_record_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/limited_record_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/limited_record_reader.py
diff --git a/paimon-python/pypaimon/read/reader/nested_leaf_batch_reader.py b/paimon-python/pypaimon/execution/python/read/reader/nested_leaf_batch_reader.py
similarity index 94%
rename from paimon-python/pypaimon/read/reader/nested_leaf_batch_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/nested_leaf_batch_reader.py
index cb4c4f81c4..9523a11214 100644
--- a/paimon-python/pypaimon/read/reader/nested_leaf_batch_reader.py
+++ b/paimon-python/pypaimon/execution/python/read/reader/nested_leaf_batch_reader.py
@@ -21,14 +21,15 @@ import pyarrow as pa
import pyarrow.compute as pc
from pyarrow import RecordBatch
+from pypaimon.casting.schema_evolution import read_arrow_schema
from pypaimon.data.map_shared_shredding import \
assemble_normal_map_selected_keys
-from pypaimon.read.reader.data_file_batch_reader import \
+from pypaimon.execution.python.read.reader.data_file_batch_reader import \
cast_array_for_schema_evolution
from pypaimon.read.reader.field_indices import (
blob_field_indices, descriptor_field_indices, vector_field_indices)
from pypaimon.read.reader.iface.record_batch_reader import RecordBatchReader
-from pypaimon.schema.data_types import DataField, PyarrowFieldParser
+from pypaimon.schema.data_types import DataField
from pypaimon.utils.arrow_utils import as_blob_data, is_blob_data
@@ -61,7 +62,7 @@ class NestedLeafBatchReader(RecordBatchReader):
len(name_paths), len(output_fields)))
self._inner = inner
self._paths = name_paths
- self._schema = PyarrowFieldParser.from_paimon_schema(output_fields)
+ self._schema = read_arrow_schema(output_fields)
self.file_io = inner.file_io
self.blob_field_indices = blob_field_indices(output_fields)
self.descriptor_field_indices = descriptor_field_indices(
diff --git a/paimon-python/pypaimon/read/reader/outer_projection_record_reader.py b/paimon-python/pypaimon/execution/python/read/reader/outer_projection_record_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/outer_projection_record_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/outer_projection_record_reader.py
diff --git a/paimon-python/pypaimon/read/reader/parquet_page_index_reader.py b/paimon-python/pypaimon/execution/python/read/reader/parquet_page_index_reader.py
similarity index 99%
rename from paimon-python/pypaimon/read/reader/parquet_page_index_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/parquet_page_index_reader.py
index 6ca5ca25ca..53f2839248 100644
--- a/paimon-python/pypaimon/read/reader/parquet_page_index_reader.py
+++ b/paimon-python/pypaimon/execution/python/read/reader/parquet_page_index_reader.py
@@ -648,7 +648,7 @@ class ParquetPageIndexReader:
return payload, patched_column, uncompressed_size
def _column_batches(self, plan, runs):
- from pypaimon.read.reader.format_pyarrow_reader import _RowRunSlicer
+ from pypaimon.execution.python.read.reader.format_pyarrow_reader import _RowRunSlicer
index, column_plans, infos = plan
payloads, chunks = [], []
diff --git a/paimon-python/pypaimon/read/reader/partial_update_merge_function.py b/paimon-python/pypaimon/execution/python/read/reader/partial_update_merge_function.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/partial_update_merge_function.py
rename to paimon-python/pypaimon/execution/python/read/reader/partial_update_merge_function.py
diff --git a/paimon-python/pypaimon/read/reader/row_range_filter_record_reader.py b/paimon-python/pypaimon/execution/python/read/reader/row_range_filter_record_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/row_range_filter_record_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/row_range_filter_record_reader.py
diff --git a/paimon-python/pypaimon/read/reader/shard_batch_reader.py b/paimon-python/pypaimon/execution/python/read/reader/shard_batch_reader.py
similarity index 97%
rename from paimon-python/pypaimon/read/reader/shard_batch_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/shard_batch_reader.py
index 445ff5f802..a9da2ad177 100644
--- a/paimon-python/pypaimon/read/reader/shard_batch_reader.py
+++ b/paimon-python/pypaimon/execution/python/read/reader/shard_batch_reader.py
@@ -19,7 +19,7 @@ from typing import Optional
from pyarrow import RecordBatch
from pypaimon.read.reader.iface.record_batch_reader import RecordBatchReader
-from pypaimon.read.reader.format_blob_reader import FormatBlobReader
+from pypaimon.execution.python.read.reader.format_blob_reader import FormatBlobReader
class ShardBatchReader(RecordBatchReader):
diff --git a/paimon-python/pypaimon/read/reader/sort_merge_reader.py b/paimon-python/pypaimon/execution/python/read/reader/sort_merge_reader.py
similarity index 92%
rename from paimon-python/pypaimon/read/reader/sort_merge_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/sort_merge_reader.py
index 0a1479feff..f02abab936 100644
--- a/paimon-python/pypaimon/read/reader/sort_merge_reader.py
+++ b/paimon-python/pypaimon/execution/python/read/reader/sort_merge_reader.py
@@ -19,14 +19,16 @@ import heapq
import math
from typing import Any, Callable, List, Optional
-from pypaimon.read.reader.deduplicate_merge_function import \
+from pypaimon.execution.python.read.reader.deduplicate_merge_function import \
DeduplicateMergeFunction
from pypaimon.read.reader.iface.record_iterator import RecordIterator
from pypaimon.read.reader.iface.record_reader import RecordReader
-from pypaimon.schema.data_types import AtomicType, DataField, Keyword
+from pypaimon.schema.data_types import DataField
from pypaimon.schema.table_schema import TableSchema
from pypaimon.table.row.internal_row import InternalRow
from pypaimon.table.row.key_value import KeyValue
+from pypaimon.read.sequence_fields import (
+ _base_type_name, _COMPARABLE_TYPE_NAMES, is_comparable_seq_field)
class SortMergeReaderWithMinHeap(RecordReader):
@@ -183,30 +185,9 @@ class HeapEntry:
return result < 0
-def _base_type_name(field: DataField) -> str:
- """Base type keyword of a field, stripping any ``(precision[, scale])``
- parameters and the ``NOT NULL`` suffix. E.g. ``DECIMAL(10, 2)`` and
- ``TIMESTAMP(6)`` map to ``DECIMAL`` / ``TIMESTAMP``.
- """
- return field.type.type.split('(')[0].split(' ')[0]
-
-
# Atomic type keywords pypaimon can order with Python's native comparison
# operators. VARIANT is atomic but has no ordering, so it is excluded --
# matching Java, which has no VARIANT sequence-field support.
-_COMPARABLE_TYPE_NAMES = frozenset(
- member.value for member in Keyword if member is not Keyword.VARIANT)
-
-
-def is_comparable_seq_field(field: DataField) -> bool:
- """Whether ``field`` can serve as a ``sequence.field`` for pypaimon's
- atomic comparator: it must be an ``AtomicType`` whose base type name is
- orderable. Complex types (ARRAY / MAP / ROW / ...) and the atomic-but-
- unorderable VARIANT both return ``False``. Used by the read-builder
- guard to reject unsupported sequence fields up front.
- """
- return (isinstance(field.type, AtomicType)
- and _base_type_name(field) in _COMPARABLE_TYPE_NAMES)
def _row_field_comparator(
diff --git a/paimon-python/pypaimon/read/reader/video_format_reader.py b/paimon-python/pypaimon/execution/python/read/reader/video_format_reader.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/video_format_reader.py
rename to paimon-python/pypaimon/execution/python/read/reader/video_format_reader.py
diff --git a/paimon-python/pypaimon/read/reader/vortex_utils.py b/paimon-python/pypaimon/execution/python/read/reader/vortex_utils.py
similarity index 100%
rename from paimon-python/pypaimon/read/reader/vortex_utils.py
rename to paimon-python/pypaimon/execution/python/read/reader/vortex_utils.py
diff --git a/paimon-python/pypaimon/read/scan_distribution.py b/paimon-python/pypaimon/execution/python/read/scan_distribution.py
similarity index 79%
copy from paimon-python/pypaimon/read/scan_distribution.py
copy to paimon-python/pypaimon/execution/python/read/scan_distribution.py
index 87d1815340..0637c19c55 100644
--- a/paimon-python/pypaimon/read/scan_distribution.py
+++ b/paimon-python/pypaimon/execution/python/read/scan_distribution.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-"""Worker sharding and row-position selection for Python scan planning."""
+"""File and row distribution for the Python scan planner."""
from typing import List, Tuple
@@ -23,20 +23,6 @@ from pypaimon.read.sliced_split import SlicedSplit
from pypaimon.read.split import Split
-def validate_shard(index: int, count: int) -> None:
- if not isinstance(count, int) or count <= 0:
- raise ValueError("number_of_para_subtasks must be a positive integer")
- if not isinstance(index, int) or index < 0:
- raise ValueError("idx_of_this_subtask must be a non-negative integer")
- if index >= count:
- raise ValueError("idx_of_this_subtask must be less than number_of_para_subtasks")
-
-
-def validate_slice(start: int, end: int) -> None:
- if not isinstance(start, int) or not isinstance(end, int) or start < 0 or start >= end:
- raise ValueError("start_pos must be non-negative and less than end_pos; both must be integers")
-
-
def shard_range(total: int, index: int, count: int) -> Tuple[int, int]:
base, remainder = divmod(total, count)
start = index * base + min(index, remainder)
diff --git a/paimon-python/pypaimon/read/scanner/__init__.py b/paimon-python/pypaimon/execution/python/read/scanner/__init__.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/__init__.py
copy to paimon-python/pypaimon/execution/python/read/scanner/__init__.py
diff --git a/paimon-python/pypaimon/read/scanner/append_table_split_generator.py b/paimon-python/pypaimon/execution/python/read/scanner/append_table_split_generator.py
similarity index 93%
rename from paimon-python/pypaimon/read/scanner/append_table_split_generator.py
rename to paimon-python/pypaimon/execution/python/read/scanner/append_table_split_generator.py
index 44c984b580..66bc31d60d 100644
--- a/paimon-python/pypaimon/read/scanner/append_table_split_generator.py
+++ b/paimon-python/pypaimon/execution/python/read/scanner/append_table_split_generator.py
@@ -20,9 +20,9 @@ from typing import List
from pypaimon.manifest.schema.data_file_meta import DataFileMeta
from pypaimon.manifest.schema.manifest_entry import ManifestEntry
-from pypaimon.read.scanner.split_generator import AbstractSplitGenerator
+from pypaimon.execution.python.read.scanner.split_generator import AbstractSplitGenerator
from pypaimon.read.split import Split
-from pypaimon.read.scan_distribution import slice_append_splits
+from pypaimon.execution.python.read.scan_distribution import slice_append_splits
class AppendTableSplitGenerator(AbstractSplitGenerator):
diff --git a/paimon-python/pypaimon/read/scanner/bucket_select_converter.py b/paimon-python/pypaimon/execution/python/read/scanner/bucket_select_converter.py
similarity index 98%
rename from paimon-python/pypaimon/read/scanner/bucket_select_converter.py
rename to paimon-python/pypaimon/execution/python/read/scanner/bucket_select_converter.py
index 6a9cd060f5..279adbc709 100644
--- a/paimon-python/pypaimon/read/scanner/bucket_select_converter.py
+++ b/paimon-python/pypaimon/execution/python/read/scanner/bucket_select_converter.py
@@ -29,8 +29,8 @@ the buckets that contain any matching rows. False-positive (over-keep)
allowed; false-negative (drop a bucket that has matching rows) MUST never
happen — that would be silent data loss.
-The hashing routine reuses ``RowKeyExtractor._hash_bytes_by_words`` /
-``_bucket_from_hash`` from ``pypaimon.write.row_key_extractor`` — the same
+The hashing routine reuses ``_hash_bytes_by_words`` /
+``_bucket_from_hash`` from ``pypaimon.write.input_routing`` — the same
code path the writer uses to assign rows to buckets. Reusing it (rather
than copying) is what guarantees read/write hash agreement in the face of
future routine changes.
@@ -70,8 +70,10 @@ from pypaimon.common.predicate import Predicate
from pypaimon.schema.data_types import DataField
from pypaimon.table.row.generic_row import GenericRow, GenericRowSerializer
from pypaimon.table.row.internal_row import InternalRow, RowKind
-from pypaimon.write.row_key_extractor import (_bucket_from_hash,
- _hash_bytes_by_words)
+from pypaimon.write.input_routing import (
+ _bucket_from_hash,
+ _hash_bytes_by_words,
+)
MAX_VALUES = 1000
diff --git a/paimon-python/pypaimon/read/scanner/changelog_follow_up_scanner.py b/paimon-python/pypaimon/execution/python/read/scanner/changelog_follow_up_scanner.py
similarity index 93%
rename from paimon-python/pypaimon/read/scanner/changelog_follow_up_scanner.py
rename to paimon-python/pypaimon/execution/python/read/scanner/changelog_follow_up_scanner.py
index c56edad267..cd1b76fd6b 100644
--- a/paimon-python/pypaimon/read/scanner/changelog_follow_up_scanner.py
+++ b/paimon-python/pypaimon/execution/python/read/scanner/changelog_follow_up_scanner.py
@@ -17,7 +17,7 @@
"""ChangelogFollowUpScanner for tables with changelog-producer settings."""
-from pypaimon.read.scanner.follow_up_scanner import FollowUpScanner
+from pypaimon.execution.python.read.scanner.follow_up_scanner import FollowUpScanner
from pypaimon.snapshot.snapshot import Snapshot
diff --git a/paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py b/paimon-python/pypaimon/execution/python/read/scanner/chunk_shuffle_split_generator.py
similarity index 99%
rename from paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py
rename to paimon-python/pypaimon/execution/python/read/scanner/chunk_shuffle_split_generator.py
index b34d6c4a77..01410f501f 100644
--- a/paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py
+++ b/paimon-python/pypaimon/execution/python/read/scanner/chunk_shuffle_split_generator.py
@@ -25,7 +25,7 @@ from pypaimon.deletionvectors.deletion_vector import DeletionVector
from pypaimon.globalindex.indexed_split import IndexedSplit
from pypaimon.manifest.schema.data_file_meta import DataFileMeta
from pypaimon.manifest.schema.manifest_entry import ManifestEntry
-from pypaimon.read.scanner.split_generator import AbstractSplitGenerator
+from pypaimon.execution.python.read.scanner.split_generator import AbstractSplitGenerator
from pypaimon.read.split import DataSplit, Split
from pypaimon.table.row.generic_row import GenericRow
from pypaimon.table.source.deletion_file import DeletionFile
diff --git a/paimon-python/pypaimon/read/scanner/data_evolution_split_generator.py b/paimon-python/pypaimon/execution/python/read/scanner/data_evolution_split_generator.py
similarity index 99%
rename from paimon-python/pypaimon/read/scanner/data_evolution_split_generator.py
rename to paimon-python/pypaimon/execution/python/read/scanner/data_evolution_split_generator.py
index be995c5acd..b789b6dff0 100644
--- a/paimon-python/pypaimon/read/scanner/data_evolution_split_generator.py
+++ b/paimon-python/pypaimon/execution/python/read/scanner/data_evolution_split_generator.py
@@ -23,7 +23,7 @@ from pypaimon.utils.range import Range
from pypaimon.utils.range_helper import RangeHelper
from pypaimon.manifest.schema.data_file_meta import DataFileMeta
from pypaimon.manifest.schema.manifest_entry import ManifestEntry
-from pypaimon.read.scanner.split_generator import AbstractSplitGenerator
+from pypaimon.execution.python.read.scanner.split_generator import AbstractSplitGenerator
from pypaimon.read.split import DataSplit, Split
diff --git a/paimon-python/pypaimon/read/scanner/data_evolution_stats.py b/paimon-python/pypaimon/execution/python/read/scanner/data_evolution_stats.py
similarity index 100%
rename from paimon-python/pypaimon/read/scanner/data_evolution_stats.py
rename to paimon-python/pypaimon/execution/python/read/scanner/data_evolution_stats.py
diff --git a/paimon-python/pypaimon/read/scanner/delta_follow_up_scanner.py b/paimon-python/pypaimon/execution/python/read/scanner/delta_follow_up_scanner.py
similarity index 92%
rename from paimon-python/pypaimon/read/scanner/delta_follow_up_scanner.py
rename to paimon-python/pypaimon/execution/python/read/scanner/delta_follow_up_scanner.py
index ce8148bcb3..c6360cb247 100644
--- a/paimon-python/pypaimon/read/scanner/delta_follow_up_scanner.py
+++ b/paimon-python/pypaimon/execution/python/read/scanner/delta_follow_up_scanner.py
@@ -17,7 +17,7 @@
"""DeltaFollowUpScanner for append-only streaming reads."""
-from pypaimon.read.scanner.follow_up_scanner import FollowUpScanner
+from pypaimon.execution.python.read.scanner.follow_up_scanner import FollowUpScanner
from pypaimon.snapshot.snapshot import Snapshot
diff --git a/paimon-python/pypaimon/read/scanner/file_scanner.py b/paimon-python/pypaimon/execution/python/read/scanner/file_scanner.py
similarity index 86%
rename from paimon-python/pypaimon/read/scanner/file_scanner.py
rename to paimon-python/pypaimon/execution/python/read/scanner/file_scanner.py
index cc9e18acf4..78399214b7 100755
--- a/paimon-python/pypaimon/read/scanner/file_scanner.py
+++ b/paimon-python/pypaimon/execution/python/read/scanner/file_scanner.py
@@ -33,29 +33,29 @@ from pypaimon.manifest.schema.manifest_file_meta import ManifestFileMeta
from pypaimon.manifest.simple_stats_evolutions import SimpleStatsEvolutions
from pypaimon.schema.data_types import DataField
from pypaimon.read.plan import Plan
-from pypaimon.read.scan_distribution import validate_shard, validate_slice
-from pypaimon.read.push_down_utils import (_get_all_fields,
- exclude_predicate_with_fields,
- predicate_for_physical_fields,
- predicate_matches_physical_names,
- remove_row_id_filter,
- rewrite_predicate_indices,
- trim_and_transform_predicate,
- trim_predicate_by_fields)
+from pypaimon.read.push_down_utils import (
+ exclude_predicate_with_fields,
+ predicate_for_physical_fields,
+ remove_row_id_filter,
+ trim_and_transform_predicate)
from pypaimon.read.scan_stats import ScanStats
-from pypaimon.read.scanner.append_table_split_generator import \
+from pypaimon.read.scan_request import (
+ apply_push_down_limit, has_non_partition_filter, validate_chunk_shuffle_compat,
+ normalize_row_ranges, scan_partition_predicate, validate_chunk_shuffle,
+ validate_global_index_selection, validate_shard_selection, validate_slice_selection)
+from pypaimon.execution.python.read.scanner.append_table_split_generator import \
AppendTableSplitGenerator
-from pypaimon.read.scanner.bucket_select_converter import \
+from pypaimon.execution.python.read.scanner.bucket_select_converter import \
create_bucket_selector
-from pypaimon.read.scanner.chunk_shuffle_split_generator import (
+from pypaimon.execution.python.read.scanner.chunk_shuffle_split_generator import (
AppendChunkShuffleSplitGenerator,
DataEvolutionChunkShuffleSplitGenerator,
)
-from pypaimon.read.scanner.data_evolution_split_generator import \
+from pypaimon.execution.python.read.scanner.data_evolution_split_generator import \
DataEvolutionSplitGenerator
-from pypaimon.read.scanner.data_evolution_stats import \
+from pypaimon.execution.python.read.scanner.data_evolution_stats import \
DataEvolutionGroupStatsFilter
-from pypaimon.read.scanner.primary_key_table_split_generator import \
+from pypaimon.execution.python.read.scanner.primary_key_table_split_generator import \
PrimaryKeyTableSplitGenerator
from pypaimon.read.split import DataSplit
from pypaimon.snapshot.snapshot import Snapshot
@@ -285,13 +285,7 @@ class FileScanner:
self.predicate_for_stats, SpecialFields.SYSTEM_FIELD_NAMES | set(self.table.partition_keys))
self.primary_key_predicate = trim_and_transform_predicate(
physical, self.table.field_names, self.table.trimmed_primary_keys)
- if self._partition_predicate is None:
- self.partition_key_predicate = trim_and_transform_predicate(
- physical, self.table.field_names, self.table.partition_keys)
- else:
- self.partition_key_predicate = rewrite_predicate_indices(
- trim_predicate_by_fields(self._partition_predicate, self.table.partition_keys),
- self.table.partition_keys_fields)
+ self.partition_key_predicate = scan_partition_predicate(self.table, predicate, self._partition_predicate)
self._bucket_selector = self._init_bucket_selector()
def _schema_fields(self, schema_id: int):
@@ -433,7 +427,7 @@ class FileScanner:
from pypaimon.table.source.primary_key_sorted_index_result import (
PrimaryKeySortedIndexResult,
)
- from pypaimon.table.source import primary_key_sorted_index_scan
+ from pypaimon.execution.python.read.search import primary_key_sorted_index_scan
definitions = PrimaryKeyIndexDefinitions.create(self.table.table_schema).definitions
if not definitions:
@@ -558,7 +552,8 @@ class FileScanner:
if not self.table.options.global_index_enabled():
return None
- from pypaimon.globalindex.data_evolution_global_index_scanner import DataEvolutionGlobalIndexScanner
+ from pypaimon.execution.python.globalindex.data_evolution_global_index_scanner import (
+ DataEvolutionGlobalIndexScanner)
try:
scanner = DataEvolutionGlobalIndexScanner.create(
@@ -661,39 +656,24 @@ class FileScanner:
return self
def with_shard(self, idx_of_this_subtask: int, number_of_para_subtasks: int) -> 'FileScanner':
- validate_shard(idx_of_this_subtask, number_of_para_subtasks)
- if self.start_pos_of_this_subtask is not None:
- raise ValueError("with_shard and with_slice cannot be used simultaneously")
+ validate_shard_selection(idx_of_this_subtask, number_of_para_subtasks, self.start_pos_of_this_subtask)
self.idx_of_this_subtask = idx_of_this_subtask
self.number_of_para_subtasks = number_of_para_subtasks
return self
def with_slice(self, start_pos: int, end_pos: int) -> 'FileScanner':
- validate_slice(start_pos, end_pos)
- if self.table.is_primary_key_table:
- raise NotImplementedError("Primary key tables do not support with_slice(); use with_shard instead")
- if self.idx_of_this_subtask is not None:
- raise ValueError("with_slice and with_shard cannot be used simultaneously")
+ validate_slice_selection(start_pos, end_pos, self.table.is_primary_key_table, self.idx_of_this_subtask)
self.start_pos_of_this_subtask = start_pos
self.end_pos_of_this_subtask = end_pos
return self
def with_global_index_result(self, result) -> 'FileScanner':
- if self._row_ranges is not None:
- raise ValueError(
- "with_global_index_result and with_row_ranges are mutually exclusive")
+ validate_global_index_selection(self._row_ranges)
self._global_index_result = result
return self
def with_row_ranges(self, row_ranges) -> 'FileScanner':
- if not self.data_evolution:
- raise ValueError("Row ranges are only supported for data evolution tables")
- if row_ranges is None:
- raise ValueError("row_ranges cannot be None")
- if self._global_index_result is not None:
- raise ValueError(
- "with_row_ranges and with_global_index_result are mutually exclusive")
- self._row_ranges = Range.sort_and_merge_overlap(list(row_ranges), True)
+ self._row_ranges = normalize_row_ranges(row_ranges, self.data_evolution, self._global_index_result)
return self
def scan_with_stats(self) -> Tuple[Plan, ScanStats]:
@@ -711,75 +691,41 @@ class FileScanner:
return plan, self.scan_stats
def with_chunk_shuffle(self, seed: int, chunk_size: int) -> 'FileScanner':
- if not isinstance(seed, int):
- raise ValueError("chunk_shuffle seed must be an int")
- if not isinstance(chunk_size, int) or chunk_size <= 0:
- raise ValueError("chunk_shuffle chunk_size must be a positive int")
+ validate_chunk_shuffle(seed, chunk_size)
self.chunk_shuffle = (seed, chunk_size)
return self
def _validate_chunk_shuffle_compat(self) -> None:
- if self.table.is_primary_key_table:
- raise ValueError("chunk_shuffle only supports append tables")
- if self.start_pos_of_this_subtask is not None:
- raise ValueError("chunk_shuffle cannot combine with with_slice")
- if self.limit is not None:
- raise ValueError("chunk_shuffle cannot combine with limit")
- if self._global_index_result is not None:
- raise ValueError("chunk_shuffle cannot combine with global index")
- if self._row_ranges is not None:
- raise ValueError("chunk_shuffle cannot combine with row ranges")
- if self.auth_has_non_partition_filter:
- raise ValueError("chunk_shuffle cannot combine with authorization filters evaluated during reading")
- # Only partition predicates are allowed: row-level / column-level
- # predicates would silently shrink each chunk's effective row count,
- # breaking the chunk_size contract DataLoader callers expect.
- if self._user_predicate is not None:
- partition_keys = set(self.table.partition_keys or [])
- non_partition_fields = _get_all_fields(self._user_predicate) - partition_keys
- if non_partition_fields or not predicate_matches_physical_names(self._user_predicate):
- raise ValueError(
- "chunk_shuffle predicate must reference only partition keys; "
- "got non-partition fields: "
- f"{sorted(non_partition_fields)}"
- )
+ validate_chunk_shuffle_compat(
+ is_primary_key_table=self.table.is_primary_key_table,
+ start_pos_of_this_subtask=self.start_pos_of_this_subtask,
+ limit=self.limit,
+ global_index_result=self._global_index_result,
+ row_ranges=self._row_ranges,
+ auth_has_non_partition_filter=self.auth_has_non_partition_filter,
+ user_predicate=self._user_predicate,
+ partition_keys=self.table.partition_keys,
+ )
def _apply_push_down_limit(self, splits: List[DataSplit]) -> List[DataSplit]:
- """Mirror Java ``DataTableBatchScan.applyPushDownLimit``: sum the
- DV-aware ``merged_row_count`` (== Java ``Split.mergedRowCount()``)
- until the limit is met. Splits with unknown merged count fall
- through to the reader unchanged.
- """
- if self.limit is None:
- return splits
- if self.data_evolution and self.deletion_vectors_enabled:
- return splits
- if (self._has_non_partition_filter() or self.auth_has_non_partition_filter
- # An explicit selector replaces ordinary partition pruning.
- # Count rows only after the reader has checked the full filter.
- or (self._explicit_partition_filter and self.predicate is not None)):
- return splits
-
- scanned_row_count = 0
- limited_splits: List[DataSplit] = []
- for split in splits:
- merged = split.merged_row_count()
- if merged is None:
- return splits
- limited_splits.append(split)
- scanned_row_count += merged
- if scanned_row_count >= self.limit:
- return limited_splits
- return splits
+ # Preserve short-circuit validation and the existing private method hook.
+ non_partition_filter = (
+ self.limit is not None
+ and not (self.data_evolution and self.deletion_vectors_enabled)
+ and self._has_non_partition_filter())
+ return apply_push_down_limit(
+ splits,
+ limit=self.limit,
+ data_evolution=self.data_evolution,
+ deletion_vectors_enabled=self.deletion_vectors_enabled,
+ non_partition_filter=non_partition_filter,
+ auth_has_non_partition_filter=self.auth_has_non_partition_filter,
+ explicit_partition_filter=self._explicit_partition_filter,
+ predicate=self.predicate,
+ )
def _has_non_partition_filter(self) -> bool:
- """Mirror Java ``SnapshotReaderImpl.hasNonPartitionFilter``."""
- if self.predicate is None:
- return False
- if not predicate_matches_physical_names(self.predicate):
- return True
- partition_keys = set(self.table.partition_keys or [])
- return not _get_all_fields(self.predicate).issubset(partition_keys)
+ return has_non_partition_filter(self.predicate, self.table.partition_keys)
def _filter_manifest_file(self, file: ManifestFileMeta) -> bool:
# explain() counts bucket rejections at entry level, as with the early
diff --git a/paimon-python/pypaimon/read/scanner/follow_up_scanner.py b/paimon-python/pypaimon/execution/python/read/scanner/follow_up_scanner.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/follow_up_scanner.py
copy to paimon-python/pypaimon/execution/python/read/scanner/follow_up_scanner.py
diff --git a/paimon-python/pypaimon/read/scanner/incremental_diff_scanner.py b/paimon-python/pypaimon/execution/python/read/scanner/incremental_diff_scanner.py
similarity index 97%
rename from paimon-python/pypaimon/read/scanner/incremental_diff_scanner.py
rename to paimon-python/pypaimon/execution/python/read/scanner/incremental_diff_scanner.py
index ac0670feab..8ad9fc000f 100644
--- a/paimon-python/pypaimon/read/scanner/incremental_diff_scanner.py
+++ b/paimon-python/pypaimon/execution/python/read/scanner/incremental_diff_scanner.py
@@ -22,7 +22,7 @@ from pypaimon.manifest.manifest_file_manager import ManifestFileManager
from pypaimon.manifest.manifest_list_manager import ManifestListManager
from pypaimon.manifest.schema.manifest_entry import ManifestEntry
from pypaimon.read.plan import Plan
-from pypaimon.read.scanner.append_table_split_generator import \
+from pypaimon.execution.python.read.scanner.append_table_split_generator import \
AppendTableSplitGenerator
from pypaimon.snapshot.snapshot import Snapshot
diff --git a/paimon-python/pypaimon/read/scanner/primary_key_table_split_generator.py b/paimon-python/pypaimon/execution/python/read/scanner/primary_key_table_split_generator.py
similarity index 96%
rename from paimon-python/pypaimon/read/scanner/primary_key_table_split_generator.py
rename to paimon-python/pypaimon/execution/python/read/scanner/primary_key_table_split_generator.py
index 01032eba53..1d9172ee28 100644
--- a/paimon-python/pypaimon/read/scanner/primary_key_table_split_generator.py
+++ b/paimon-python/pypaimon/execution/python/read/scanner/primary_key_table_split_generator.py
@@ -22,8 +22,8 @@ from pypaimon.common.options.core_options import MergeEngine
from pypaimon.manifest.schema.data_file_meta import DataFileMeta
from pypaimon.manifest.schema.manifest_entry import ManifestEntry
from pypaimon.read.interval_partition import IntervalPartition
-from pypaimon.read.scan_distribution import java_file_name_shard
-from pypaimon.read.scanner.split_generator import AbstractSplitGenerator
+from pypaimon.execution.python.read.scan_distribution import java_file_name_shard
+from pypaimon.execution.python.read.scanner.split_generator import AbstractSplitGenerator
from pypaimon.read.split import Split
diff --git a/paimon-python/pypaimon/read/scanner/split_generator.py b/paimon-python/pypaimon/execution/python/read/scanner/split_generator.py
similarity index 98%
rename from paimon-python/pypaimon/read/scanner/split_generator.py
rename to paimon-python/pypaimon/execution/python/read/scanner/split_generator.py
index 2aa8f2ffb9..65136fb7ed 100644
--- a/paimon-python/pypaimon/read/scanner/split_generator.py
+++ b/paimon-python/pypaimon/execution/python/read/scanner/split_generator.py
@@ -21,7 +21,8 @@ from typing import Callable, List, Optional, Dict, Tuple
from pypaimon.common.options.core_options import CoreOptions
from pypaimon.manifest.schema.data_file_meta import DataFileMeta
from pypaimon.manifest.schema.manifest_entry import ManifestEntry
-from pypaimon.read.scan_distribution import shard_range, validate_shard, validate_slice
+from pypaimon.execution.python.read.scan_distribution import shard_range
+from pypaimon.read.scan_distribution import validate_shard, validate_slice
from pypaimon.read.split import Split
from pypaimon.read.split import DataSplit
from pypaimon.table.row.generic_row import GenericRow
diff --git a/paimon-python/pypaimon/read/scanner/__init__.py b/paimon-python/pypaimon/execution/python/read/search/__init__.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/__init__.py
copy to paimon-python/pypaimon/execution/python/read/search/__init__.py
diff --git a/paimon-python/pypaimon/table/source/full_text_read.py b/paimon-python/pypaimon/execution/python/read/search/full_text_read.py
similarity index 91%
rename from paimon-python/pypaimon/table/source/full_text_read.py
rename to paimon-python/pypaimon/execution/python/read/search/full_text_read.py
index def8fc754e..a7ce89a484 100644
--- a/paimon-python/pypaimon/table/source/full_text_read.py
+++ b/paimon-python/pypaimon/execution/python/read/search/full_text_read.py
@@ -15,9 +15,10 @@
# specific language governing permissions and limitations
# under the License.
+
+from pypaimon.table.source.search_types import (FullTextRead)
"""Full-text read to read index files."""
-from abc import ABC, abstractmethod
from copy import copy
from concurrent.futures import wait
from io import BytesIO
@@ -26,35 +27,24 @@ from typing import Dict, List
from pypaimon.globalindex.full_text_search import FullTextSearch
from pypaimon.globalindex.global_index_meta import GlobalIndexIOMeta
from pypaimon.globalindex.global_index_result import GlobalIndexResult
-from pypaimon.globalindex.offset_global_index_reader import OffsetGlobalIndexReader
+from pypaimon.execution.python.globalindex.offset_global_index_reader import OffsetGlobalIndexReader
from pypaimon.globalindex.vector_search_result import (
DictBasedScoredIndexResult,
ScoredGlobalIndexResult,
)
-from pypaimon.table.source import global_index_live_row_filter
+from pypaimon.execution.python.read.search import global_index_live_row_filter
from pypaimon.table.source.full_text_search_split import (
FullTextSearchSplit,
IndexFullTextSearchSplit,
RawFullTextSearchSplit,
)
-from pypaimon.table.source.full_text_scan import FullTextScanPlan
+from pypaimon.execution.python.read.search.full_text_scan import FullTextScanPlan
from pypaimon.utils.range import Range
from pypaimon.utils.roaring_bitmap import RoaringBitmap64
_UNSET = object()
-class FullTextRead(ABC):
- """Full-text read to read index files."""
-
- def read_plan(self, plan: FullTextScanPlan) -> GlobalIndexResult:
- return self.read(plan.splits())
-
- @abstractmethod
- def read(self, splits: List[FullTextSearchSplit]) -> GlobalIndexResult:
- pass
-
-
class DataEvolutionFullTextRead(FullTextRead):
"""Implementation for FullTextRead."""
@@ -108,7 +98,7 @@ class DataEvolutionFullTextRead(FullTextRead):
return indexed_result.or_(raw_result).top_k(self._limit)
def _matching_rows(self, candidates):
- from pypaimon.table.source.global_index_row_filter import matching_rows
+ from pypaimon.execution.python.read.search.global_index_row_filter import matching_rows
table = self._table.copy({'scalar-index.search-mode': 'full'})
return matching_rows(table, self._filter, candidates, self._partition_filter)
@@ -117,9 +107,11 @@ class DataEvolutionFullTextRead(FullTextRead):
if self._filter is None:
return live_rows
from pypaimon.common.options.core_options import GlobalIndexSearchMode
- from pypaimon.globalindex.data_evolution_global_index_coverage import DataEvolutionGlobalIndexCoverage
+ from pypaimon.execution.python.globalindex.data_evolution_global_index_coverage import (
+ DataEvolutionGlobalIndexCoverage,
+ )
from pypaimon.snapshot.time_travel_util import TimeTravelUtil
- from pypaimon.table.source.global_index_row_filter import is_exact
+ from pypaimon.execution.python.read.search.global_index_row_filter import is_exact
covered = RoaringBitmap64()
files = {}
@@ -154,7 +146,9 @@ class DataEvolutionFullTextRead(FullTextRead):
return RoaringBitmap64.or_(matched, self._matching_rows(unindexed))
def _scalar_index_result(self, scalar_files, snapshot):
- from pypaimon.globalindex.data_evolution_global_index_scanner import DataEvolutionGlobalIndexScanner
+ from pypaimon.execution.python.globalindex.data_evolution_global_index_scanner import (
+ DataEvolutionGlobalIndexScanner,
+ )
scanner = DataEvolutionGlobalIndexScanner.create(
self._table, index_files=scalar_files, snapshot=snapshot, partition_filter=self._partition_filter)
@@ -305,9 +299,7 @@ class DataEvolutionFullTextRead(FullTextRead):
"pypaimon[full-text]."
) from e
- from pypaimon.globalindex.full_text.native_full_text_global_index_reader import (
- NativeFullTextIndexOptions,
- )
+ from pypaimon.globalindex.full_text_options import NativeFullTextIndexOptions
writer = FullTextIndexWriter(
NativeFullTextIndexOptions.from_options(
@@ -342,21 +334,15 @@ def _index_candidate_limit(splits, limit):
def _create_full_text_reader(index_type, file_io, index_path, index_io_meta_list, max_candidates=None):
"""Create a global index reader for full-text search."""
- from pypaimon.globalindex.full_text.native_full_text_global_index_reader import (
- FULL_TEXT_IDENTIFIER,
- )
+ from pypaimon.globalindex.full_text_options import FULL_TEXT_IDENTIFIER
if index_type == FULL_TEXT_IDENTIFIER:
- from pypaimon.globalindex.full_text.native_full_text_global_index_reader import (
- NativeFullTextGlobalIndexReader,
- )
+ from pypaimon.execution.python.read.native_full_text_global_index_reader import NativeFullTextGlobalIndexReader
return NativeFullTextGlobalIndexReader(file_io, index_path, index_io_meta_list, max_candidates)
raise ValueError(f"Unsupported full-text index type: '{index_type}'")
def _supports_full_text_search(index_type):
- from pypaimon.globalindex.full_text.native_full_text_global_index_reader import (
- FULL_TEXT_IDENTIFIER,
- )
+ from pypaimon.globalindex.full_text_options import FULL_TEXT_IDENTIFIER
return index_type == FULL_TEXT_IDENTIFIER
@@ -391,8 +377,9 @@ def _index_type(index_splits):
def _search_raw_full_text(index_bytes, row_range_start, query, limit, include_row_ids=None, max_candidates=None):
from paimon_ftindex import FullTextIndexReader
- from pypaimon.globalindex.full_text.native_full_text_global_index_reader import (
- PaimonFullTextInput, _search_top_k,
+ from pypaimon.execution.python.read.native_full_text_global_index_reader import (
+ PaimonFullTextInput,
+ _search_top_k,
)
reader = FullTextIndexReader(PaimonFullTextInput(BytesIO(index_bytes)))
diff --git a/paimon-python/pypaimon/table/source/full_text_scan.py b/paimon-python/pypaimon/execution/python/read/search/full_text_scan.py
similarity index 87%
rename from paimon-python/pypaimon/table/source/full_text_scan.py
rename to paimon-python/pypaimon/execution/python/read/search/full_text_scan.py
index 57b7d3ea30..74bcdc0739 100644
--- a/paimon-python/pypaimon/table/source/full_text_scan.py
+++ b/paimon-python/pypaimon/execution/python/read/search/full_text_scan.py
@@ -15,46 +15,21 @@
# specific language governing permissions and limitations
# under the License.
+
+from pypaimon.table.source.search_types import (FullTextScanPlan, FullTextScan)
"""Full-text scan to scan index files."""
-from abc import ABC, abstractmethod
from collections import defaultdict
-from typing import List
-from pypaimon.globalindex.data_evolution_global_index_coverage import DataEvolutionGlobalIndexCoverage
-from pypaimon.globalindex.full_text.native_full_text_global_index_reader import (
- FULL_TEXT_IDENTIFIER,
-)
+from pypaimon.execution.python.globalindex.data_evolution_global_index_coverage import DataEvolutionGlobalIndexCoverage
+from pypaimon.globalindex.full_text_options import FULL_TEXT_IDENTIFIER
from pypaimon.table.source.full_text_search_split import (
- FullTextSearchSplit,
IndexFullTextSearchSplit,
RawFullTextSearchSplit,
)
from pypaimon.utils.range import Range
-class FullTextScanPlan:
- """Plan of full-text scan."""
-
- def __init__(self, splits: List[FullTextSearchSplit], snapshot=None):
- self._splits = splits
- self._snapshot = snapshot
-
- def snapshot(self):
- return self._snapshot
-
- def splits(self) -> List[FullTextSearchSplit]:
- return self._splits
-
-
-class FullTextScan(ABC):
- """Full-text scan to scan index files."""
-
- @abstractmethod
- def scan(self) -> FullTextScanPlan:
- pass
-
-
class DataEvolutionFullTextScan(FullTextScan):
"""Implementation for FullTextScan."""
diff --git a/paimon-python/pypaimon/table/source/global_index_live_row_filter.py b/paimon-python/pypaimon/execution/python/read/search/global_index_live_row_filter.py
similarity index 100%
rename from paimon-python/pypaimon/table/source/global_index_live_row_filter.py
rename to paimon-python/pypaimon/execution/python/read/search/global_index_live_row_filter.py
diff --git a/paimon-python/pypaimon/table/source/global_index_row_filter.py b/paimon-python/pypaimon/execution/python/read/search/global_index_row_filter.py
similarity index 96%
rename from paimon-python/pypaimon/table/source/global_index_row_filter.py
rename to paimon-python/pypaimon/execution/python/read/search/global_index_row_filter.py
index 2e9820c3e4..b357859e87 100644
--- a/paimon-python/pypaimon/table/source/global_index_row_filter.py
+++ b/paimon-python/pypaimon/execution/python/read/search/global_index_row_filter.py
@@ -20,7 +20,7 @@
from pypaimon.common.like_optimization import try_optimize_like
from pypaimon.read.table_read import _ClosableArrowBatchReader
from pypaimon.table.special_fields import SpecialFields
-from pypaimon.table.source.global_index_live_row_filter import table_at_snapshot
+from pypaimon.execution.python.read.search.global_index_live_row_filter import table_at_snapshot
from pypaimon.utils.roaring_bitmap import RoaringBitmap64
diff --git a/paimon-python/pypaimon/execution/python/read/search/hybrid_search.py b/paimon-python/pypaimon/execution/python/read/search/hybrid_search.py
new file mode 100644
index 0000000000..b9af5c8b3a
--- /dev/null
+++ b/paimon-python/pypaimon/execution/python/read/search/hybrid_search.py
@@ -0,0 +1,184 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Python hybrid route execution and rank fusion."""
+
+import heapq
+from copy import copy
+from concurrent.futures import FIRST_EXCEPTION, ThreadPoolExecutor, wait
+from typing import List
+
+from pypaimon.catalog.table_query_auth import reject_search_under_query_auth
+from pypaimon.globalindex.global_index_result import GlobalIndexResult
+from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult, ScoredGlobalIndexResult
+from pypaimon.table.source.hybrid_search_builder import (
+ HybridSearchRouteBuilder, HybridSearchRouteResult,
+ WEIGHTED_SCORE_RANKER, MRR_RANKER, _MAX_ROUTE_WORKERS, _RRF_K,
+)
+
+
+def execute_local(self) -> ScoredGlobalIndexResult:
+ """Execute hybrid index search locally."""
+ route_builders = self.route_builders()
+ if len(route_builders) <= 1:
+ return self.rank([
+ self.to_route_result(
+ route_builder, route_builder.execute_local())
+ for route_builder in route_builders
+ ])
+
+ workers = min(len(route_builders), _MAX_ROUTE_WORKERS)
+ with ThreadPoolExecutor(
+ max_workers=workers,
+ thread_name_prefix="paimon-hybrid-search") as executor:
+ futures = [
+ executor.submit(route_builder.execute_local)
+ for route_builder in route_builders
+ ]
+ done, pending = wait(futures, return_when=FIRST_EXCEPTION)
+ failed = next(
+ (future for future in futures
+ if future in done and future.exception() is not None),
+ None,
+ )
+ if failed is not None:
+ for future in pending:
+ future.cancel()
+ failed.result()
+
+ route_results = [
+ self.to_route_result(route_builder, future.result())
+ for route_builder, future in zip(route_builders, futures)
+ ]
+ return self.rank(route_results)
+
+
+def route_builders(self) -> List[HybridSearchRouteBuilder]:
+ reject_search_under_query_auth(self._table)
+ self._validate_search()
+ from pypaimon.snapshot.time_travel_util import TimeTravelUtil
+ execution = copy(self)
+ execution._table = self._table._copy_with_snapshot(TimeTravelUtil.resolve_snapshot(self._table))
+ builders = []
+ for route in self._routes:
+ if route.is_vector():
+ builders.append(
+ HybridSearchRouteBuilder(
+ route, execution._new_vector_search_builder(route)))
+ else:
+ builders.append(
+ HybridSearchRouteBuilder(
+ route, execution._new_full_text_search_builder(route)))
+ return builders
+
+
+def to_route_result(
+ self,
+ route_builder: HybridSearchRouteBuilder,
+ result: GlobalIndexResult) -> HybridSearchRouteResult:
+ if isinstance(result, ScoredGlobalIndexResult):
+ scored = result
+ elif result.results().is_empty():
+ scored = ScoredGlobalIndexResult.create_empty()
+ else:
+ raise ValueError(
+ "Hybrid search requires scored index results, but got: %s"
+ % result.__class__.__name__)
+ return HybridSearchRouteResult(route_builder.route, scored)
+
+
+def rank(
+ self,
+ route_results: List[HybridSearchRouteResult]) -> ScoredGlobalIndexResult:
+ self._validate_search()
+ non_empty = [
+ route_result for route_result in route_results
+ if not route_result.result.results().is_empty()
+ ]
+ if self._ranker == WEIGHTED_SCORE_RANKER:
+ return self._weighted_score(non_empty)
+ if self._ranker == MRR_RANKER:
+ return self._mrr(non_empty)
+ return self._rrf(non_empty)
+
+
+def _rrf(self, route_results):
+ scores = {}
+ for route_result in route_results:
+ for rank, row_id in enumerate(_ranked_row_ids(route_result.result)):
+ contribution = route_result.route.weight / (_RRF_K + rank + 1.0)
+ scores[row_id] = scores.get(row_id, 0.0) + contribution
+ return _top_k(scores, self._limit)
+
+
+def _mrr(self, route_results):
+ scores = {}
+ for route_result in route_results:
+ for rank, row_id in enumerate(_ranked_row_ids(route_result.result)):
+ contribution = route_result.route.weight / (rank + 1.0)
+ scores[row_id] = scores.get(row_id, 0.0) + contribution
+ return _top_k(scores, self._limit)
+
+
+def _weighted_score(self, route_results):
+ scores = {}
+ for route_result in route_results:
+ result = route_result.result
+ weight = route_result.route.weight
+ score_getter = result.score_getter()
+
+ # Route score scales are heterogeneous (e.g. bounded vector similarity
+ # vs unbounded BM25), so raw scores are not comparable across routes.
+ # Min-max normalize each route into [0, 1] before weighting, so that
+ # weights -- not a route's numeric magnitude -- decide its influence on
+ # the fused score. This mirrors the Java HybridSearchRanker.
+ route_scores = {
+ row_id: (score_getter(row_id) or 0.0)
+ for row_id in result.results()
+ }
+ if not route_scores:
+ continue
+ min_score = min(route_scores.values())
+ score_range = max(route_scores.values()) - min_score
+ for row_id, raw in route_scores.items():
+ # No spread within the route (single hit or all ties) carries no
+ # relative signal, so every hit maps to 1.0 rather than zeroed out.
+ normalized = (
+ (raw - min_score) / score_range if score_range > 0.0 else 1.0)
+ scores[row_id] = scores.get(row_id, 0.0) + weight * normalized
+ return _top_k(scores, self._limit)
+
+
+def _ranked_row_ids(result):
+ score_getter = result.score_getter()
+ return sorted(
+ result.results(),
+ key=lambda row_id: (-(score_getter(row_id) or 0.0), row_id))
+
+
+def _top_k(scores, limit):
+ if not scores:
+ return ScoredGlobalIndexResult.create_empty()
+ heap = []
+ for row_id, score in scores.items():
+ item = (score, -row_id)
+ if len(heap) < limit:
+ heapq.heappush(heap, item)
+ elif item > heap[0]:
+ heapq.heapreplace(heap, item)
+ top_scores = {-neg_row_id: score for score, neg_row_id in heap}
+ return DictBasedScoredIndexResult(top_scores)
diff --git a/paimon-python/pypaimon/table/source/primary_key_full_text_read.py b/paimon-python/pypaimon/execution/python/read/search/primary_key_full_text_read.py
similarity index 95%
rename from paimon-python/pypaimon/table/source/primary_key_full_text_read.py
rename to paimon-python/pypaimon/execution/python/read/search/primary_key_full_text_read.py
index 2386cdf2d2..5538cd237b 100644
--- a/paimon-python/pypaimon/table/source/primary_key_full_text_read.py
+++ b/paimon-python/pypaimon/execution/python/read/search/primary_key_full_text_read.py
@@ -18,13 +18,13 @@
from concurrent.futures import wait
from pypaimon.index.pk.primary_key_index_source_meta import PrimaryKeyIndexSourceMeta
-from pypaimon.table.source.full_text_read import (
+from pypaimon.execution.python.read.search.full_text_read import (
DataEvolutionFullTextRead, _search_raw_full_text)
-from pypaimon.table.source.primary_key_full_text_scan import PrimaryKeyFullTextScanPlan
+from pypaimon.execution.python.read.search.primary_key_full_text_scan import PrimaryKeyFullTextScanPlan
from pypaimon.table.source.primary_key_scored_result import (
PrimaryKeyScoredResult, PrimaryKeySearchPosition, _partition_bytes)
-from pypaimon.table.source.primary_key_vector_read import _localize
-from pypaimon.table.source.primary_key_vector_read import (
+from pypaimon.execution.python.read.search.primary_key_vector_read import _localize
+from pypaimon.execution.python.read.search.primary_key_vector_read import (
_allowed, _deleted_positions, _live_positions)
from pypaimon.read.split import DataSplit
from pypaimon.globalindex.indexed_split import IndexedSplit
diff --git a/paimon-python/pypaimon/table/source/primary_key_full_text_scan.py b/paimon-python/pypaimon/execution/python/read/search/primary_key_full_text_scan.py
similarity index 92%
rename from paimon-python/pypaimon/table/source/primary_key_full_text_scan.py
rename to paimon-python/pypaimon/execution/python/read/search/primary_key_full_text_scan.py
index 4a7d7c515e..290b5e3bee 100644
--- a/paimon-python/pypaimon/table/source/primary_key_full_text_scan.py
+++ b/paimon-python/pypaimon/execution/python/read/search/primary_key_full_text_scan.py
@@ -15,7 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-from dataclasses import dataclass
+
+from pypaimon.table.source.search_types import (PrimaryKeyFullTextSearchSplit, PrimaryKeyFullTextScanPlan)
from pypaimon.globalindex.indexed_split import IndexedSplit
from pypaimon.index.index_file_handler import IndexFileHandler
@@ -26,16 +27,8 @@ from pypaimon.index.pk.primary_key_index_source_policy import (
from pypaimon.read.query_auth_split import QueryAuthSplit
from pypaimon.read.split import DataSplit
from pypaimon.snapshot.time_travel_util import TimeTravelUtil
-from pypaimon.table.source.global_index_live_row_filter import table_at_snapshot
-from pypaimon.table.source.full_text_scan import FullTextScan, FullTextScanPlan
-
-
-@dataclass(frozen=True)
-class PrimaryKeyFullTextSearchSplit:
- data_split: DataSplit
- payloads: tuple
- uncovered_data_files: tuple
- row_ranges_by_file: dict
+from pypaimon.execution.python.read.search.global_index_live_row_filter import table_at_snapshot
+from pypaimon.execution.python.read.search.full_text_scan import FullTextScan
class PrimaryKeyFullTextScan(FullTextScan):
@@ -170,9 +163,3 @@ def _current_payloads(active_files, active_payloads):
source_meta = PrimaryKeyIndexSourceMeta.from_index_file(payload)
covered.update(source.file_name for source in source_meta.source_files)
return current, covered
-
-
-class PrimaryKeyFullTextScanPlan(FullTextScanPlan):
- def __init__(self, snapshot_id, splits):
- super().__init__(splits)
- self.snapshot_id = snapshot_id
diff --git a/paimon-python/pypaimon/table/source/primary_key_sorted_index_scan.py b/paimon-python/pypaimon/execution/python/read/search/primary_key_sorted_index_scan.py
similarity index 98%
rename from paimon-python/pypaimon/table/source/primary_key_sorted_index_scan.py
rename to paimon-python/pypaimon/execution/python/read/search/primary_key_sorted_index_scan.py
index 865a52eebf..290394f64b 100644
--- a/paimon-python/pypaimon/table/source/primary_key_sorted_index_scan.py
+++ b/paimon-python/pypaimon/execution/python/read/search/primary_key_sorted_index_scan.py
@@ -17,11 +17,11 @@
import logging
from dataclasses import dataclass
-from pypaimon.globalindex.global_index_evaluator import GlobalIndexEvaluator
+from pypaimon.execution.python.globalindex.global_index_evaluator import GlobalIndexEvaluator
from pypaimon.globalindex.global_index_meta import GlobalIndexIOMeta
from pypaimon.globalindex.global_index_reader import GlobalIndexReader, _map_future
from pypaimon.globalindex.global_index_result import GlobalIndexResult
-from pypaimon.globalindex.data_evolution_global_index_scanner import _create_inner_readers
+from pypaimon.execution.python.globalindex.data_evolution_global_index_scanner import _create_inner_readers
from pypaimon.common.options.core_options import CoreOptions
from pypaimon.index.pk.primary_key_index_source_file import PrimaryKeyIndexSourceFile
from pypaimon.index.pk.primary_key_index_source_policy import should_read
diff --git a/paimon-python/pypaimon/table/source/primary_key_vector_read.py b/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_read.py
similarity index 98%
rename from paimon-python/pypaimon/table/source/primary_key_vector_read.py
rename to paimon-python/pypaimon/execution/python/read/search/primary_key_vector_read.py
index e379aaa802..c21808d36b 100644
--- a/paimon-python/pypaimon/table/source/primary_key_vector_read.py
+++ b/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_read.py
@@ -22,9 +22,9 @@ from pypaimon.common.options.core_options import GlobalIndexSearchMode
from pypaimon.index.pk.primary_key_index_source_meta import PrimaryKeyIndexSourceMeta
from pypaimon.table.source.primary_key_scored_result import (
PrimaryKeyScoredResult, PrimaryKeySearchPosition, _partition_bytes)
-from pypaimon.table.source.primary_key_vector_scan import PrimaryKeyVectorScanPlan
-from pypaimon.table.source.vector_search_read import DataEvolutionVectorRead
-from pypaimon.table.source.vector_search_read import (
+from pypaimon.execution.python.read.search.primary_key_vector_scan import PrimaryKeyVectorScanPlan
+from pypaimon.execution.python.read.search.vector_search_read import DataEvolutionVectorRead
+from pypaimon.execution.python.read.search.vector_search_read import (
_iter_arrow_scores)
from pypaimon.read.split import DataSplit
from pypaimon.globalindex.indexed_split import IndexedSplit
diff --git a/paimon-python/pypaimon/table/source/primary_key_vector_scan.py b/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py
similarity index 95%
rename from paimon-python/pypaimon/table/source/primary_key_vector_scan.py
rename to paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py
index 0ccdf9bf3f..11f44ee60d 100644
--- a/paimon-python/pypaimon/table/source/primary_key_vector_scan.py
+++ b/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py
@@ -15,7 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-from dataclasses import dataclass
+
+from pypaimon.table.source.search_types import (PrimaryKeyVectorSearchSplit, PrimaryKeyVectorScanPlan)
from pypaimon.common.options.core_options import CoreOptions
from pypaimon.common.options.options import Options
@@ -28,20 +29,12 @@ from pypaimon.read.split import DataSplit
from pypaimon.globalindex.indexed_split import IndexedSplit
from pypaimon.deletionvectors.deletion_vector import DeletionVector
from pypaimon.snapshot.time_travel_util import TimeTravelUtil
-from pypaimon.table.source.global_index_live_row_filter import table_at_snapshot
-from pypaimon.table.source.vector_search_scan import VectorSearchScan, VectorSearchScanPlan
+from pypaimon.execution.python.read.search.global_index_live_row_filter import table_at_snapshot
+from pypaimon.execution.python.read.search.vector_search_scan import VectorSearchScan
from pypaimon.table.row.generic_row import GenericRow
from pypaimon.utils.range import Range
-@dataclass(frozen=True)
-class PrimaryKeyVectorSearchSplit:
- data_split: DataSplit
- payloads: tuple
- uncovered_data_files: tuple
- row_ranges_by_file: dict
-
-
class PrimaryKeyVectorScan(VectorSearchScan):
"""Plan source-backed vector payloads with their current PK data files."""
@@ -97,12 +90,6 @@ class PrimaryKeyVectorScan(VectorSearchScan):
snapshot.id, _bucket_splits(source_splits, entries))
-class PrimaryKeyVectorScanPlan(VectorSearchScanPlan):
- def __init__(self, snapshot_id, splits):
- super().__init__(splits)
- self.snapshot_id = snapshot_id
-
-
def _matches(entry, field_id, index_type, partition_filter):
meta = entry.index_file.global_index_meta
return (entry.kind == 0 and entry.index_file.index_type == index_type
diff --git a/paimon-python/pypaimon/table/source/vector_search_read.py b/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py
similarity index 96%
rename from paimon-python/pypaimon/table/source/vector_search_read.py
rename to paimon-python/pypaimon/execution/python/read/search/vector_search_read.py
index 607aff7998..8464b09888 100644
--- a/paimon-python/pypaimon/table/source/vector_search_read.py
+++ b/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py
@@ -15,10 +15,11 @@
# specific language governing permissions and limitations
# under the License.
+
+from pypaimon.table.source.search_types import (VectorSearchRead, BatchVectorSearchRead)
"""Vector search read to read index files."""
import logging
-from abc import ABC, abstractmethod
from collections import deque
from concurrent.futures import ThreadPoolExecutor
from contextlib import closing
@@ -29,7 +30,7 @@ from pypaimon.common.options.options import Options
from pypaimon.globalindex.batch_vector_search import BatchVectorSearch
from pypaimon.globalindex.global_index_meta import GlobalIndexIOMeta
from pypaimon.globalindex.global_index_result import GlobalIndexResult
-from pypaimon.globalindex.offset_global_index_reader import OffsetGlobalIndexReader
+from pypaimon.execution.python.globalindex.offset_global_index_reader import OffsetGlobalIndexReader
from pypaimon.globalindex.vector_search import VectorSearch
from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult
from pypaimon.table.special_fields import SpecialFields
@@ -37,43 +38,11 @@ from pypaimon.table.source.vector_search_split import (
IndexVectorSearchSplit,
RawVectorSearchSplit,
)
-from pypaimon.table.source import global_index_live_row_filter
+from pypaimon.execution.python.read.search import global_index_live_row_filter
from pypaimon.utils.range import Range
from pypaimon.utils.roaring_bitmap import RoaringBitmap64
-class VectorSearchRead(ABC):
- """Vector search read to read index files."""
-
- def read_plan(self, plan):
- # type: (VectorSearchScanPlan) -> GlobalIndexResult
- return self._read(plan.splits(), plan.snapshot())
-
- def read(self, splits):
- # type: (List[VectorSearchSplit]) -> GlobalIndexResult
- return self._read(splits, None)
-
- @abstractmethod
- def _read(self, splits, snapshot):
- pass
-
-
-class BatchVectorSearchRead(ABC):
- """Batch vector search read to read index files."""
-
- def read_batch_plan(self, plan):
- # type: (VectorSearchScanPlan) -> List[GlobalIndexResult]
- return self._read_batch(plan.splits(), plan.snapshot())
-
- def read_batch(self, splits):
- # type: (List[VectorSearchSplit]) -> List[GlobalIndexResult]
- return self._read_batch(splits, None)
-
- @abstractmethod
- def _read_batch(self, splits, snapshot):
- pass
-
-
class AbstractVectorSearchReadImpl:
"""Base implementation for vector search reads."""
@@ -209,7 +178,9 @@ class AbstractVectorSearchReadImpl:
if not scalar_files:
return None
- from pypaimon.globalindex.data_evolution_global_index_scanner import DataEvolutionGlobalIndexScanner
+ from pypaimon.execution.python.globalindex.data_evolution_global_index_scanner import (
+ DataEvolutionGlobalIndexScanner,
+ )
scanner = DataEvolutionGlobalIndexScanner.create(
self._table,
index_files=scalar_files,
@@ -224,7 +195,7 @@ class AbstractVectorSearchReadImpl:
scanner.close()
def _refine_scalar_result(self, splits, result, snapshot=None):
- from pypaimon.table.source.global_index_row_filter import is_exact
+ from pypaimon.execution.python.read.search.global_index_row_filter import is_exact
if is_exact(self._filter, result):
return result.results()
@@ -244,7 +215,7 @@ class AbstractVectorSearchReadImpl:
return self._matching_candidate_rows(candidates, snapshot)
def _matching_candidate_rows(self, candidates, snapshot):
- from pypaimon.table.source.global_index_row_filter import matching_rows
+ from pypaimon.execution.python.read.search.global_index_row_filter import matching_rows
return matching_rows(self._table, self._filter, candidates, self._partition_filter, snapshot)
@@ -279,7 +250,9 @@ class AbstractVectorSearchReadImpl:
if not scalar_files:
return None
- from pypaimon.globalindex.data_evolution_global_index_scanner import DataEvolutionGlobalIndexScanner
+ from pypaimon.execution.python.globalindex.data_evolution_global_index_scanner import (
+ DataEvolutionGlobalIndexScanner,
+ )
scanner = DataEvolutionGlobalIndexScanner.create(
self._table,
index_files=scalar_files,
@@ -854,14 +827,12 @@ def _merge_index_scores(merged_scores, result):
def _create_vector_reader(index_type, file_io, index_path, index_io_meta_list, options=None):
"""Create a global index reader for vector search."""
- from pypaimon.globalindex.lumina.lumina_vector_global_index_reader import (
+ from pypaimon.execution.python.globalindex.lumina.lumina_vector_global_index_reader import (
LUMINA_IDENTIFIERS,
LuminaVectorGlobalIndexReader,
)
- from pypaimon.globalindex.vindex.vindex_vector_global_index_reader import (
- VINDEX_IDENTIFIERS,
- VindexVectorGlobalIndexReader,
- )
+ from pypaimon.globalindex.vindex_options import VINDEX_IDENTIFIERS
+ from pypaimon.globalindex.vindex_reader import VindexVectorGlobalIndexReader
if index_type in LUMINA_IDENTIFIERS:
return LuminaVectorGlobalIndexReader(
file_io, index_path, index_io_meta_list, options
@@ -1021,8 +992,8 @@ def _configured_vector_metric(options, vector_column, index_type=None):
def _raw_search_metric(table, vector_column, options, index_type=None):
- from pypaimon.globalindex.lumina.lumina_vector_global_index_reader import LUMINA_IDENTIFIERS
- from pypaimon.globalindex.vindex.vindex_vector_global_index_reader import VINDEX_IDENTIFIERS
+ from pypaimon.execution.python.globalindex.lumina.lumina_vector_global_index_reader import LUMINA_IDENTIFIERS
+ from pypaimon.globalindex.vindex_options import VINDEX_IDENTIFIERS
table_map = _table_options_map(table)
for source in (options, table_map):
diff --git a/paimon-python/pypaimon/table/source/vector_search_scan.py b/paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py
similarity index 92%
rename from paimon-python/pypaimon/table/source/vector_search_scan.py
rename to paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py
index 4203fe6235..e70d825d4e 100644
--- a/paimon-python/pypaimon/table/source/vector_search_scan.py
+++ b/paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py
@@ -15,15 +15,16 @@
# specific language governing permissions and limitations
# under the License.
+
+from pypaimon.table.source.search_types import (VectorSearchScanPlan, VectorSearchScan)
"""Vector search scan to scan index files."""
-from abc import ABC, abstractmethod
from collections import defaultdict
from pypaimon.common.options.core_options import CoreOptions, GlobalIndexSearchMode
from pypaimon.common.options.options import Options
-from pypaimon.globalindex.data_evolution_global_index_coverage import DataEvolutionGlobalIndexCoverage
-from pypaimon.globalindex.data_evolution_global_index_scanner import (
+from pypaimon.execution.python.globalindex.data_evolution_global_index_coverage import DataEvolutionGlobalIndexCoverage
+from pypaimon.execution.python.globalindex.data_evolution_global_index_scanner import (
is_supported_scalar_index,
)
from pypaimon.table.source.vector_search_split import (
@@ -34,31 +35,6 @@ from pypaimon.table.source.vector_search_split import VectorSearchSplit # noqa
from pypaimon.utils.range import Range
-class VectorSearchScanPlan:
- """Plan of vector search scan."""
-
- def __init__(self, splits, snapshot=None):
- # type: (list, object) -> None
- self._splits = splits
- self._snapshot = snapshot
-
- def splits(self):
- # type: () -> list
- return self._splits
-
- def snapshot(self):
- return self._snapshot
-
-
-class VectorSearchScan(ABC):
- """Vector search scan to scan index files."""
-
- @abstractmethod
- def scan(self):
- # type: () -> VectorSearchScanPlan
- pass
-
-
class DataEvolutionVectorScan(VectorSearchScan):
"""Implementation for VectorSearchScan."""
diff --git a/paimon-python/pypaimon/read/split_read.py b/paimon-python/pypaimon/execution/python/read/split_read.py
similarity index 95%
rename from paimon-python/pypaimon/read/split_read.py
rename to paimon-python/pypaimon/execution/python/read/split_read.py
index 19ecce4356..c7167b11fd 100644
--- a/paimon-python/pypaimon/read/split_read.py
+++ b/paimon-python/pypaimon/execution/python/read/split_read.py
@@ -20,7 +20,7 @@ from abc import ABC, abstractmethod
from functools import partial
from typing import Callable, Dict, List, Optional, Tuple
-from pypaimon.common.merge_engine_dispatch import build_merge_function
+from pypaimon.execution.python.merge_engine_dispatch import build_merge_function
from pypaimon.common.options.core_options import CoreOptions, MergeEngine
from pypaimon.casting.schema_evolution import read_arrow_schema
from pypaimon.common.predicate import Predicate
@@ -36,6 +36,7 @@ from pypaimon.data.map_shared_shredding import (
)
from pypaimon.globalindex import Range
from pypaimon.manifest.schema.data_file_meta import DataFileMeta
+from pypaimon.read.blob_read_support import deferred_blob_field_names
from pypaimon.read.interval_partition import IntervalPartition, SortedRun
from pypaimon.read.partition_info import PartitionInfo
from pypaimon.read.push_down_utils import (
@@ -46,47 +47,47 @@ from pypaimon.read.push_down_utils import (
rewrite_predicate_indices,
trim_predicate_by_fields,
)
-from pypaimon.read.reader.concat_batch_reader import (
+from pypaimon.execution.python.read.reader.concat_batch_reader import (
BlobFallbackBatchReader, ConcatBatchReader,
MergeAllBatchReader, DataEvolutionMergeReader)
-from pypaimon.read.reader.concat_record_reader import ConcatRecordReader
+from pypaimon.execution.python.read.reader.concat_record_reader import ConcatRecordReader
-from pypaimon.read.reader.auth_masking_reader import AuthFilterReader
-from pypaimon.read.reader.data_file_batch_reader import DataFileBatchReader
-from pypaimon.read.reader.deferred_blob_resolve_reader import \
+from pypaimon.execution.python.read.reader.auth_masking_reader import AuthFilterReader
+from pypaimon.execution.python.read.reader.data_file_batch_reader import DataFileBatchReader
+from pypaimon.execution.python.read.reader.deferred_blob_resolve_reader import \
DeferredBlobResolveReader
-from pypaimon.read.reader.drop_delete_reader import DropDeleteRecordReader
-from pypaimon.read.reader.empty_record_reader import EmptyFileRecordReader
-from pypaimon.read.reader.field_bunch import BlobBunch, DataBunch, FieldBunch, VectorBunch
+from pypaimon.execution.python.read.reader.drop_delete_reader import DropDeleteRecordReader
+from pypaimon.execution.python.read.reader.empty_record_reader import EmptyFileRecordReader
+from pypaimon.execution.python.read.reader.field_bunch import BlobBunch, DataBunch, FieldBunch, VectorBunch
from pypaimon.read.reader.field_indices import (
blob_field_indices, descriptor_field_indices_for_table,
descriptor_field_names_for_table, vector_field_indices)
-from pypaimon.read.reader.filter_record_reader import FilterRecordReader
-from pypaimon.read.reader.format_avro_reader import FormatAvroReader
-from pypaimon.read.reader.blob_descriptor_convert_reader import (
+from pypaimon.execution.python.read.reader.filter_record_reader import FilterRecordReader
+from pypaimon.execution.python.read.reader.format_avro_reader import FormatAvroReader
+from pypaimon.execution.python.read.reader.blob_descriptor_convert_reader import (
BlobDataBatchReader, BlobInlineConvertReader)
-from pypaimon.read.reader.blob_view_read_support import (
+from pypaimon.execution.python.read.reader.blob_view_read_support import (
needs_blob_inline_convert, wrap_record_reader_with_blob_inline_convert)
-from pypaimon.read.reader.filter_record_batch_reader import FilterRecordBatchReader
-from pypaimon.read.reader.limited_record_reader import LimitedRecordBatchReader, LimitedRecordReader
-from pypaimon.read.reader.row_range_filter_record_reader import RowIdFilterRecordBatchReader
-from pypaimon.read.reader.format_blob_reader import FormatBlobReader
-from pypaimon.read.reader.format_lance_reader import FormatLanceReader
-from pypaimon.read.reader.format_pyarrow_reader import FormatPyArrowReader
-from pypaimon.read.reader.format_row_reader import FormatRowReader
-from pypaimon.read.reader.format_mosaic_reader import FormatMosaicReader
-from pypaimon.read.reader.format_vortex_reader import FormatVortexReader
+from pypaimon.execution.python.read.reader.filter_record_batch_reader import FilterRecordBatchReader
+from pypaimon.execution.python.read.reader.limited_record_reader import LimitedRecordBatchReader, LimitedRecordReader
+from pypaimon.execution.python.read.reader.row_range_filter_record_reader import RowIdFilterRecordBatchReader
+from pypaimon.execution.python.read.reader.format_blob_reader import FormatBlobReader
+from pypaimon.execution.python.read.reader.format_lance_reader import FormatLanceReader
+from pypaimon.execution.python.read.reader.format_pyarrow_reader import FormatPyArrowReader
+from pypaimon.execution.python.read.reader.format_row_reader import FormatRowReader
+from pypaimon.execution.python.read.reader.format_mosaic_reader import FormatMosaicReader
+from pypaimon.execution.python.read.reader.format_vortex_reader import FormatVortexReader
from pypaimon.read.reader.iface.record_batch_reader import (RecordBatchReader,
RowPositionReader, EmptyRecordBatchReader)
from pypaimon.read.reader.iface.record_reader import RecordReader
-from pypaimon.read.reader.key_value_unwrap_reader import \
+from pypaimon.execution.python.read.reader.key_value_unwrap_reader import \
KeyValueUnwrapRecordReader
-from pypaimon.read.reader.key_value_wrap_reader import KeyValueWrapReader
-from pypaimon.read.reader.shard_batch_reader import ShardBatchReader
-from pypaimon.read.reader.aggregation_merge_function import (
+from pypaimon.execution.python.read.reader.key_value_wrap_reader import KeyValueWrapReader
+from pypaimon.execution.python.read.reader.shard_batch_reader import ShardBatchReader
+from pypaimon.execution.python.read.reader.aggregation_merge_function import (
AggregateMergeFunction, build_field_aggregators)
-from pypaimon.read.reader.sort_merge_reader import (SortMergeReaderWithMinHeap,
- builtin_seq_comparator)
+from pypaimon.execution.python.read.reader.sort_merge_reader import (
+ SortMergeReaderWithMinHeap, builtin_seq_comparator)
from pypaimon.read.split import Split
from pypaimon.read.sliced_split import SlicedSplit
from pypaimon.schema.data_types import DataField, MapType, PyarrowFieldParser
@@ -100,35 +101,6 @@ KEY_FIELD_ID_START = SpecialFields.SYSTEM_FIELD_ID_START
NULL_FIELD_INDEX = -1
-def deferred_blob_field_names(table, read_fields: List[DataField],
- predicate: Optional[Predicate],
- limit: Optional[int],
- has_post_filter: bool = False) -> set:
- # Only append tables have dedicated Blob files to defer. Primary-key
- # managed packs are resolved inside Rust's read pipeline.
- if table.is_primary_key_table:
- return set()
- # An auth filter also selects rows; defer past it too, like a predicate/limit.
- if ((predicate is None and limit is None and not has_post_filter)
- or CoreOptions.blob_as_descriptor(table.options)):
- return set()
-
- non_payload_fields = (
- CoreOptions.blob_descriptor_fields(table.options)
- | CoreOptions.blob_view_fields(table.options)
- | CoreOptions.video_frame_fields(table.options)
- )
- predicate_fields = (
- predicate_field_names(predicate) if predicate is not None else set()
- )
- return {
- read_fields[index].name
- for index in blob_field_indices(read_fields)
- if read_fields[index].name not in non_payload_fields
- and read_fields[index].name not in predicate_fields
- }
-
-
ROW_SIDECAR_FORMAT = CoreOptions.FILE_FORMAT_ROW
_COMPRESS_EXTENSIONS = frozenset(['gz', 'bz2', 'deflate', 'snappy', 'lz4', 'zst'])
@@ -1112,7 +1084,7 @@ class RawFileSplitRead(SplitRead):
schema_fields=self.read_fields,
)
if self.outer_extract_name_paths:
- from pypaimon.read.reader.nested_leaf_batch_reader import \
+ from pypaimon.execution.python.read.reader.nested_leaf_batch_reader import \
NestedLeafBatchReader
reader = NestedLeafBatchReader(
reader, self.outer_extract_name_paths,
@@ -1221,7 +1193,7 @@ class MergeFileSplitRead(SplitRead):
"""Pick the MergeFunction for the table's ``merge-engine`` option.
Delegates to the shared dispatch in
- ``pypaimon.common.merge_engine_dispatch`` so the read path and
+ ``pypaimon.execution.python.merge_engine_dispatch`` so the read path and
the in-memory merge buffer on the write path cannot drift.
``AGGREGATE`` is special-cased here because building the per-
field aggregators needs the full ``DataField`` objects, the
@@ -1311,7 +1283,7 @@ class MergeFileSplitRead(SplitRead):
reader = prescan_read.create_reader()
if isinstance(reader, RecordBatchReader):
return reader
- from pypaimon.read.reader.auth_masking_reader import RecordReaderToBatchAdapter
+ from pypaimon.execution.python.read.reader.auth_masking_reader import RecordReaderToBatchAdapter
schema = PyarrowFieldParser.from_paimon_schema(prescan_fields)
return RecordReaderToBatchAdapter(reader, schema)
@@ -1347,7 +1319,7 @@ class MergeFileSplitRead(SplitRead):
reader = wrap_record_reader_with_blob_inline_convert(
reader, self, value_fields)
if self.outer_extract_name_paths:
- from pypaimon.read.reader.outer_projection_record_reader import \
+ from pypaimon.execution.python.read.reader.outer_projection_record_reader import \
OuterProjectionRecordReader
inner_value_fields = self.read_fields[-self.value_arity:]
inner_descriptor_indices = getattr(reader, 'descriptor_field_indices', None)
@@ -1511,7 +1483,7 @@ class DataEvolutionSplitRead(SplitRead):
raise ValueError(
"outer_flat_read_type is required when outer_extract_name_paths "
"is set")
- from pypaimon.read.reader.nested_leaf_batch_reader import \
+ from pypaimon.execution.python.read.reader.nested_leaf_batch_reader import \
NestedLeafBatchReader
reader = NestedLeafBatchReader(
reader, self.outer_extract_name_paths, self.outer_flat_read_type,
diff --git a/paimon-python/pypaimon/read/streaming_table_scan.py b/paimon-python/pypaimon/execution/python/read/streaming_table_scan.py
similarity index 82%
copy from paimon-python/pypaimon/read/streaming_table_scan.py
copy to paimon-python/pypaimon/execution/python/read/streaming_table_scan.py
index 34ed9ba489..1698131d7a 100644
--- a/paimon-python/pypaimon/read/streaming_table_scan.py
+++ b/paimon-python/pypaimon/execution/python/read/streaming_table_scan.py
@@ -15,14 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-"""
-AsyncStreamingTableScan for continuous streaming reads from Paimon tables.
-
-This module provides async-based streaming reads that continuously poll for
-new snapshots and yield Plans as new data arrives. It is the Python equivalent
-of Java's DataTableStreamScan.
-"""
-
import asyncio
import logging
import os
@@ -34,19 +26,19 @@ from pypaimon.common.predicate import Predicate
from pypaimon.consumer.consumer import Consumer
from pypaimon.manifest.manifest_file_manager import ManifestFileManager
from pypaimon.manifest.manifest_list_manager import ManifestListManager
-from pypaimon.read.native_plan import _raise_if_native_fork_safety_error
+from pypaimon.execution.dispatch import propagate_fork_safety_error as _raise_if_native_fork_safety_error
from pypaimon.read.plan import Plan
from pypaimon.read.query_auth_split import wrap_plan_with_auth
-from pypaimon.read.scanner.append_table_split_generator import \
+from pypaimon.execution.python.read.scanner.append_table_split_generator import \
AppendTableSplitGenerator
-from pypaimon.read.scanner.changelog_follow_up_scanner import \
+from pypaimon.execution.python.read.scanner.changelog_follow_up_scanner import \
ChangelogFollowUpScanner
-from pypaimon.read.scanner.delta_follow_up_scanner import DeltaFollowUpScanner
-from pypaimon.read.scanner.file_scanner import FileScanner
-from pypaimon.read.scanner.follow_up_scanner import FollowUpScanner
-from pypaimon.read.scanner.incremental_diff_scanner import \
+from pypaimon.execution.python.read.scanner.delta_follow_up_scanner import DeltaFollowUpScanner
+from pypaimon.execution.python.read.scanner.file_scanner import FileScanner
+from pypaimon.execution.python.read.scanner.follow_up_scanner import FollowUpScanner
+from pypaimon.execution.python.read.scanner.incremental_diff_scanner import \
IncrementalDiffScanner
-from pypaimon.read.scanner.primary_key_table_split_generator import \
+from pypaimon.execution.python.read.scanner.primary_key_table_split_generator import \
PrimaryKeyTableSplitGenerator
from pypaimon.snapshot.snapshot import Snapshot
from pypaimon.table.bucket_mode import BucketMode
@@ -279,7 +271,7 @@ class AsyncStreamingTableScan:
return wrap_plan_with_auth(self.__auth_query(), plan)
def __auth_query(self):
- from pypaimon.read.table_scan import authorize
+ from pypaimon.read.auth_support import authorize
return authorize(self.table, self._query_auth_fn, self._read_type, self.predicate)
def _start_prefetch(self, snapshot_id: int) -> None:
@@ -354,7 +346,7 @@ class AsyncStreamingTableScan:
)
starting_scanner.only_read_real_buckets = self._only_read_real_buckets()
if auth_result is not None:
- from pypaimon.read.table_scan import prune_scanner_by_auth
+ from pypaimon.execution.python.read.auth_support import prune_scanner_by_auth
prune_scanner_by_auth(self.table, starting_scanner, auth_result)
plan = starting_scanner.scan()
if self._bucket_filter is not None:
@@ -460,88 +452,3 @@ class AsyncStreamingTableScan:
else:
plan = IncrementalDiffScanner(self.table).scan(start_snapshot, end_snapshot)
return wrap_plan_with_auth(auth_result, plan)
-
-
-class StreamTableScan:
- """Python iterator adapter for Rust's stateful StreamTableScan.
-
- Rust owns snapshot selection, split planning, checkpoints and consumer
- persistence. Python only polls, converts splits and acknowledges a yielded
- plan when the caller resumes the iterator after processing it.
- """
-
- def __init__(self, table, predicate=None, read_type=None, poll_interval_ms=1000,
- bucket_filter=None, consumer_id=None):
- from pypaimon.read.native_plan import native_stream_scan
- self.table = table
- self.poll_interval = poll_interval_ms / 1000.0
- self._scan = native_stream_scan(
- table, predicate=predicate, read_type=read_type,
- bucket_filter=bucket_filter, consumer_id=consumer_id)
- self._pending_consumer_snapshot = None
-
- @property
- def next_snapshot_id(self):
- return self.checkpoint()
-
- @next_snapshot_id.setter
- def next_snapshot_id(self, next_snapshot_id):
- self.restore(next_snapshot_id)
-
- def checkpoint(self):
- return self._scan.checkpoint()
-
- def watermark(self):
- return self._scan.watermark()
-
- def restore(self, next_snapshot_id=None):
- self._scan.restore(next_snapshot_id)
- self._pending_consumer_snapshot = None
-
- def notify_checkpoint_complete(self, next_snapshot):
- self._scan.notify_checkpoint_complete(next_snapshot)
-
- def plan(self):
- from pypaimon.read.native_plan import _from_native_plan
- # Decode before returning or acknowledging progress. A failed conversion
- # must leave the selected frame available for retry.
- checkpoint = self.checkpoint()
- native_plan = self._scan.plan()
- if native_plan is None:
- return None
- try:
- return _from_native_plan(self.table, native_plan)
- except Exception:
- self._scan.restore(checkpoint)
- raise
-
- def _flush_pending_consumer(self):
- if self._pending_consumer_snapshot is not None:
- self.notify_checkpoint_complete(self._pending_consumer_snapshot)
- self._pending_consumer_snapshot = None
-
- async def stream(self):
- while True:
- self._flush_pending_consumer()
- checkpoint = self.checkpoint()
- plan = self.plan()
- if plan is None:
- next_snapshot = self.checkpoint()
- if next_snapshot is not None and next_snapshot != checkpoint:
- # Skipped frames have no rows awaiting caller processing.
- self._pending_consumer_snapshot = next_snapshot
- self._flush_pending_consumer()
- await asyncio.sleep(self.poll_interval)
- continue
- self._pending_consumer_snapshot = self.checkpoint()
- yield plan
-
- def stream_sync(self):
- loop = asyncio.new_event_loop()
- iterator = self.stream()
- try:
- while True:
- yield loop.run_until_complete(iterator.__anext__())
- finally:
- loop.run_until_complete(iterator.aclose())
- loop.close()
diff --git a/paimon-python/pypaimon/execution/python/read/system_table_read.py b/paimon-python/pypaimon/execution/python/read/system_table_read.py
new file mode 100644
index 0000000000..0f9b75e491
--- /dev/null
+++ b/paimon-python/pypaimon/execution/python/read/system_table_read.py
@@ -0,0 +1,235 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Read pipeline for system tables.
+
+Mirrors the user-facing surface of :class:`TableRead` (``to_arrow``,
+``to_pandas``, ``to_iterator``, ``to_record_batch_iterator``,
+``to_duckdb``) so callers reach for the same API as a regular data
+table. The implementation operates purely on the in-memory PyArrow
+tables produced by :class:`SystemSplit` — no manifest reads, no data
+files.
+"""
+
+import re
+from typing import Any, Iterator, List, Optional
+
+import pyarrow
+
+from pypaimon.common.predicate import Like, Predicate
+from pypaimon.read.split import Split
+from pypaimon.schema.data_types import (
+ AtomicType, DataField, PyarrowFieldParser)
+from pypaimon.table.row.generic_row import GenericRow
+from pypaimon.table.system.system_table_scan import SystemSplit
+
+
+_PREDICATE_NOT_SUPPORTED = "This predicate is not supported on system tables"
+_STRING_PREDICATES = ("startsWith", "endsWith", "contains", "like")
+
+# Value comparisons (equal / greater-than / is-in / ...) have no PyArrow
+# compute kernel for nested columns such as ``$files.write_cols`` or
+# ``$table_indexes.dv_ranges``; only null checks work there.
+_COMPARISON_NEEDS_SCALAR = ("isNull", "isNotNull")
+
+
+def to_arrow(self, splits: List[Split]) -> pyarrow.Table:
+ return _materialise(self, splits)
+
+
+def to_iterator(self, splits: List[Split]) -> Iterator[dict]:
+ arrow_table = self.to_arrow(splits)
+ column_names = arrow_table.schema.names
+ for batch in arrow_table.to_batches():
+ py_columns = [batch.column(i).to_pylist()
+ for i in range(batch.num_columns)]
+ for row_idx in range(batch.num_rows):
+ yield {column_names[c]: py_columns[c][row_idx]
+ for c in range(batch.num_columns)}
+
+
+def to_record_batch_iterator(
+ self, splits: List[Split]
+) -> Iterator[pyarrow.RecordBatch]:
+ for batch in self.to_arrow(splits).to_batches():
+ yield batch
+
+
+def to_duckdb(
+ self,
+ splits: List[Split],
+ table_name: str,
+ connection: Optional[Any] = None,
+):
+ import duckdb
+
+ con = connection or duckdb.connect(database=":memory:")
+ con.register(table_name, self.to_arrow(splits))
+ return con
+
+
+def _materialise(self, splits: List[Split]) -> pyarrow.Table:
+ from pypaimon.read.push_down_utils import (
+ predicate_field_names, predicate_supports_arrow_filter, rewrite_predicate_indices)
+
+ predicate = self.predicate
+ if predicate is not None and not isinstance(predicate, Predicate):
+ # Preserve the public error contract for non-Predicate filter
+ # objects (e.g. ``with_filter(object())``): predicate_supports_
+ # arrow_filter dereferences ``.method`` and would otherwise leak
+ # an internal AttributeError.
+ raise NotImplementedError(_PREDICATE_NOT_SUPPORTED)
+ has_string_filter = _validate_filter(self, predicate) if predicate is not None else False
+ if predicate is not None and not has_string_filter and not predicate_supports_arrow_filter(
+ predicate
+ ):
+ raise NotImplementedError(_PREDICATE_NOT_SUPPORTED)
+
+ if not splits:
+ return _empty_table(self)
+
+ row_predicate = None
+ arrow_predicate = None
+ if has_string_filter:
+ predicate_fields = predicate_field_names(predicate)
+ filter_fields = [f for f in self.system_table.row_type().fields if f.name in predicate_fields]
+ row_predicate = rewrite_predicate_indices(predicate, filter_fields)
+ elif predicate is not None:
+ arrow_predicate = predicate.to_arrow()
+
+ projected_names = [f.name for f in self.read_type]
+ slices: List[pyarrow.Table] = []
+ for split in splits:
+ if not isinstance(split, SystemSplit):
+ raise TypeError(
+ "SystemTableRead expects SystemSplit but received "
+ + type(split).__name__
+ )
+ arrow_table = split.arrow_table()
+ # Filter on the full schema before projection so a predicate may
+ # reference columns that are not part of the projection.
+ if row_predicate is not None:
+ arrow_table = _filter_rows(self, arrow_table, row_predicate, filter_fields)
+ elif arrow_predicate is not None:
+ arrow_table = arrow_table.filter(arrow_predicate)
+ if projected_names:
+ arrow_table = arrow_table.select(projected_names)
+ slices.append(arrow_table)
+
+ combined = pyarrow.concat_tables(slices)
+ if self.limit is not None and combined.num_rows > self.limit:
+ combined = combined.slice(0, self.limit)
+ return combined.cast(PyarrowFieldParser.from_paimon_schema(self.read_type))
+
+
+def _filter_rows(
+ self, arrow_table: pyarrow.Table, predicate: Predicate, fields: List[DataField]
+) -> pyarrow.Table:
+ # String to_arrow() conversions may silently fall back to True or None.
+ # Evaluate the whole tree locally, including AND / OR.
+ columns = [arrow_table.column(f.name).to_pylist() for f in fields]
+ mask = pyarrow.array([
+ _test_row(self, predicate, GenericRow(list(values), fields))
+ for values in zip(*columns)
+ ], type=pyarrow.bool_())
+ return arrow_table.filter(mask)
+
+
+def _test_row(self, predicate: Predicate, row: GenericRow) -> bool:
+ if predicate.method in ("and", "or"):
+ results = (_test_row(self, child, row) for child in predicate.literals)
+ return all(results) if predicate.method == "and" else any(results)
+ if predicate.method in (
+ "equal", "notEqual", "lessThan", "lessOrEqual", "greaterThan", "greaterOrEqual"
+ ) and predicate.literals[0] is None:
+ # Arrow comparisons with NULL never select a row.
+ return False
+ if predicate.method in ("between", "notBetween") and any(v is None for v in predicate.literals):
+ value = row.get_field(predicate.index)
+ if predicate.method == "between" or value is None:
+ return False
+ # NOT BETWEEN is an OR: a non-null bound can still match.
+ lower, upper = predicate.literals
+ return ((lower is not None and value < lower)
+ or (upper is not None and value > upper))
+ if predicate.method == "like":
+ value = row.get_field(predicate.index)
+ # SQL '_' matches a newline too, as in Java's Like predicate.
+ return value is not None and re.fullmatch(
+ Like._sql_like_to_regex(predicate.literals[0]), value, flags=re.DOTALL
+ ) is not None
+ return predicate.test(row)
+
+
+def _validate_filter(self, predicate: Predicate) -> bool:
+ """Validate operands and report whether exact row evaluation is needed.
+
+ System tables expose list/map/row columns (e.g. ``$files.write_cols``,
+ ``$table_indexes.dv_ranges``). PyArrow has no equality / comparison /
+ is-in kernel for those types, so ``arrow_table.filter`` would raise an
+ internal ``ArrowNotImplementedError``. Surface the documented
+ ``NotImplementedError`` instead. Null checks are left alone because the
+ ``is_null`` / ``is_valid`` kernels do accept nested input.
+ """
+ field_types = {
+ f.name: f.type for f in self.system_table.row_type().fields
+ }
+ has_string_filter = False
+ pending: List[Predicate] = [predicate]
+ while pending:
+ current = pending.pop()
+ if current.method in ("and", "or"):
+ pending.extend(current.literals or [])
+ continue
+ if current.method not in Predicate.testers:
+ raise ValueError("Unsupported predicate method: " + current.method)
+ if current.method in _COMPARISON_NEEDS_SCALAR:
+ continue
+ field_type = field_types.get(current.field)
+ if field_type is not None and not isinstance(field_type, AtomicType):
+ raise NotImplementedError(
+ "predicate pushdown of "
+ + repr(current.method)
+ + " on the non-scalar system-table column "
+ + repr(current.field)
+ + " (type "
+ + type(field_type).__name__
+ + ") is not supported; filter the resulting PyArrow / "
+ + "pandas table on the client side"
+ )
+ if current.method in _STRING_PREDICATES:
+ if field_type is None:
+ raise ValueError("Unknown system-table column: " + repr(current.field))
+ if not pyarrow.types.is_string(PyarrowFieldParser.from_paimon_type(field_type)):
+ raise NotImplementedError(
+ "String predicates require a string system-table column: " + repr(current.field))
+ if (not current.literals or len(current.literals) != 1
+ or not isinstance(current.literals[0], str)):
+ raise ValueError("String predicates require a single string literal")
+ if current.method == "like":
+ # Validate even when input is empty or another branch matches.
+ Like._sql_like_to_regex(current.literals[0])
+ has_string_filter = True
+ return has_string_filter
+
+
+def _empty_table(self) -> pyarrow.Table:
+ target_schema = PyarrowFieldParser.from_paimon_schema(self.read_type)
+ return pyarrow.Table.from_arrays(
+ [pyarrow.array([], type=f.type) for f in target_schema],
+ schema=target_schema,
+ )
diff --git a/paimon-python/pypaimon/globalindex/btree/memory_slice_input.py b/paimon-python/pypaimon/execution/python/read/system_table_scan.py
similarity index 73%
copy from paimon-python/pypaimon/globalindex/btree/memory_slice_input.py
copy to paimon-python/pypaimon/execution/python/read/system_table_scan.py
index 4cf97a10e7..f4aaa52d7b 100644
--- a/paimon-python/pypaimon/globalindex/btree/memory_slice_input.py
+++ b/paimon-python/pypaimon/execution/python/read/system_table_scan.py
@@ -15,8 +15,11 @@
# specific language governing permissions and limitations
# under the License.
-"""Compatibility re-export for global index byte-slice input."""
+"""Materialize a Python system-table provider into its shared split contract."""
-from pypaimon.globalindex.memory_slice_input import MemorySliceInput
+from pypaimon.read.plan import Plan
+from pypaimon.table.system.system_table_scan import SystemSplit
-__all__ = ['MemorySliceInput']
+
+def plan(scan) -> Plan:
+ return Plan(_splits=[SystemSplit(scan.system_table._build_arrow_table())])
diff --git a/paimon-python/pypaimon/execution/python/read/table_read.py b/paimon-python/pypaimon/execution/python/read/table_read.py
new file mode 100644
index 0000000000..bf964ab019
--- /dev/null
+++ b/paimon-python/pypaimon/execution/python/read/table_read.py
@@ -0,0 +1,785 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Iterator, List, Optional
+
+import pyarrow
+
+from pypaimon.casting.schema_evolution import read_arrow_schema
+
+from pypaimon.common.predicate_json_parser import (
+ _collect_all_field_refs_from_transform, extract_referenced_fields)
+from pypaimon.data.map_shared_shredding import map_selected_keys_field
+from pypaimon.read.push_down_utils import (
+ exclude_predicate_with_fields, predicate_field_names, rewrite_predicate_indices)
+from pypaimon.read.query_auth_split import QueryAuthSplit
+from pypaimon.execution.python.read.reader.auth_masking_reader import (
+ AuthFilterReader, AuthMaskingReader, ColumnProjectReader,
+ RecordReaderToBatchAdapter, BatchToRecordReaderAdapter,
+ apply_auth_masking, compile_auth_masking_rules)
+from pypaimon.read.reader.iface.record_batch_reader import RecordBatchReader
+from pypaimon.execution.python.read.reader.limited_record_reader import LimitedRecordBatchReader
+from pypaimon.execution.python.read.reader.filter_record_batch_reader import FilterRecordBatchReader
+from pypaimon.read.split import Split
+from pypaimon.read.read_type import adapter_output_paths
+from pypaimon.execution.python.read.split_read import (
+ DataEvolutionSplitRead, MergeFileSplitRead, RawFileSplitRead, SplitRead)
+from pypaimon.schema.data_types import DataField, MapType, is_map_blob_type
+from pypaimon.table.row.offset_row import OffsetRow
+
+from pypaimon.read.constants import ROW_KIND_COLUMN, _DEFAULT_PREDICATE
+
+logger = logging.getLogger(__name__)
+
+
+class _RemainingRows:
+ """Thread-safe remaining-rows counter for parallel reads.
+
+ Row quota is pre-debited under a single lock so that any rows that
+ threads commit to emit are guaranteed not to overshoot the limit, even
+ if individual readers keep decoding one extra batch after the quota is
+ exhausted.
+
+ When ``limit`` is None the counter is unbounded and ``try_consume``
+ always returns the requested row count.
+ """
+
+ def __init__(self, limit: Optional[int]):
+ self._lock = threading.Lock()
+ self._remaining = limit # None == unlimited
+
+ def try_consume(self, requested: int) -> int:
+ if self._remaining is None:
+ return requested
+ if requested <= 0:
+ return 0
+ with self._lock:
+ if self._remaining <= 0:
+ return 0
+ allowed = min(requested, self._remaining)
+ self._remaining -= allowed
+ return allowed
+
+ def exhausted(self) -> bool:
+ if self._remaining is None:
+ return False
+ with self._lock:
+ return self._remaining <= 0
+
+
+class PythonTableReadMixin:
+ def _python_row_iterator(self, splits, limit):
+ self._check_python_merge_supported()
+ count = 0
+ for split in splits:
+ if limit is not None and count >= limit:
+ return
+ remaining = None if limit is None else limit - count
+ reader = self._TableRead__create_reader_for_split(split, limit=remaining)
+ try:
+ if self.output_projection is not None or not self._adapter_read_type:
+ from pypaimon.execution.python.read.reader.outer_projection_record_reader import (
+ OuterProjectionRecordReader)
+ _, paths = self._reader_adapter()
+ outputs = adapter_output_paths(self.output_projection, self._adapter_read_type, paths)
+ reader = OuterProjectionRecordReader(
+ reader, [field.name for field in self._adapter_read_type],
+ [path for _, path in outputs], file_io=getattr(reader, 'file_io', None),
+ blob_field_indices=getattr(reader, 'blob_field_indices', None),
+ descriptor_field_indices=getattr(reader, 'descriptor_field_indices', None),
+ vector_field_indices=getattr(reader, 'vector_field_indices', None))
+ for batch in iter(reader.read_batch, None):
+ for row in iter(batch.next, None):
+ yield row
+ count += 1
+ if limit is not None and count >= limit:
+ return
+ finally:
+ reader.close()
+
+ def _arrow_batch_generator(self, splits: List[Split], schema: pyarrow.Schema,
+ blob_parallelism: int = 1) -> Iterator[pyarrow.RecordBatch]:
+ self._begin_auth_read(splits)
+ chunk_size = 65536
+ # ``remaining`` tracks how many rows we are still allowed to emit
+ # across all splits. ``None`` means unlimited.
+ remaining = self.limit
+
+ for split in splits:
+ if remaining is not None and remaining <= 0:
+ break
+ reader = self._TableRead__create_reader_for_split(
+ split, blob_parallelism, limit=remaining)
+ try:
+ if isinstance(reader, RecordBatchReader):
+ for batch in iter(reader.read_arrow_batch, None):
+ if batch.num_rows == 0:
+ continue
+ if remaining is not None and batch.num_rows > remaining:
+ batch = batch.slice(0, remaining)
+ batch = self._serialize_blob_views_as_descriptors(
+ batch, reader)
+ batch = self._project_batch_to_output(batch)
+ if self.include_row_kind:
+ if "_row_kind" not in batch.schema.names:
+ batch = self._add_row_kind_column_to_batch(batch, "+I")
+ yield batch
+ if remaining is not None:
+ remaining -= batch.num_rows
+ if remaining <= 0:
+ break
+ else:
+ row_tuple_chunk = []
+ row_kind_chunk = []
+ while True:
+ row_iterator = reader.read_batch()
+ if row_iterator is None:
+ break
+ stop = False
+ for row in iter(row_iterator.next, None):
+ if not isinstance(row, OffsetRow):
+ raise TypeError(f"Expected OffsetRow, but got {type(row).__name__}")
+ row_tuple_chunk.append(
+ self._serialize_blob_view_row_tuple(row, reader))
+ if self.include_row_kind:
+ row_kind_chunk.append(row.get_row_kind().to_string())
+
+ if remaining is not None:
+ remaining -= 1
+ if remaining <= 0:
+ stop = True
+ break
+
+ if len(row_tuple_chunk) >= chunk_size:
+ yield from self._convert_rows_to_output_batches(
+ row_tuple_chunk, row_kind_chunk, schema
+ )
+ row_tuple_chunk = []
+ row_kind_chunk = []
+ if stop:
+ break
+
+ if row_tuple_chunk:
+ yield from self._convert_rows_to_output_batches(
+ row_tuple_chunk, row_kind_chunk, schema
+ )
+ finally:
+ reader.close()
+
+ def _to_arrow_parallel(
+ self,
+ splits: List[Split],
+ schema: pyarrow.Schema,
+ effective: int,
+ blob_parallelism: int = 1,
+ ) -> pyarrow.Table:
+ """Read ``splits`` concurrently and assemble the result in input order.
+
+ Each split is read in its own worker thread; row quota for ``limit``
+ is shared through :class:`_RemainingRows` so the combined output
+ never exceeds ``self.limit`` rows. Per-split batches are collected
+ by submission index, so the merged table preserves the order of the
+ input ``splits`` list.
+ """
+ self._begin_auth_read(splits)
+ remaining_state = _RemainingRows(self.limit)
+ results: List[Optional[List[pyarrow.RecordBatch]]] = [None] * len(splits)
+ workers = min(effective, len(splits))
+ blob_parallelism = self._cap_blob_parallelism(workers, blob_parallelism)
+ with ThreadPoolExecutor(
+ max_workers=workers,
+ thread_name_prefix="pypaimon-read",
+ ) as executor:
+ futures = {
+ executor.submit(
+ self._read_one_split_to_batches,
+ split,
+ schema,
+ remaining_state,
+ blob_parallelism,
+ ): idx
+ for idx, split in enumerate(splits)
+ }
+ for fut in as_completed(futures):
+ results[futures[fut]] = fut.result()
+
+ table_list: List[pyarrow.RecordBatch] = []
+ for split_batches in results:
+ if split_batches is None:
+ continue
+ for batch in split_batches:
+ if batch.num_rows == 0:
+ continue
+ table_list.append(self._try_to_pad_batch_by_schema(
+ batch, schema, allow_type_cast=True))
+
+ if not table_list:
+ return pyarrow.Table.from_arrays(
+ [pyarrow.array([], type=field.type) for field in schema],
+ schema=schema,
+ )
+ return pyarrow.Table.from_batches(table_list)
+
+ def _read_one_split_to_batches(
+ self,
+ split: Split,
+ schema: pyarrow.Schema,
+ remaining_state: _RemainingRows,
+ blob_parallelism: int = 1,
+ ) -> List[pyarrow.RecordBatch]:
+ """Read a single split into arrow batches under soft-stop control.
+
+ Row quota is debited against the shared ``remaining_state``; once a
+ request returns 0, the worker stops emitting further batches. The
+ reader is always closed via ``finally``.
+ """
+ chunk_size = 65536
+ out: List[pyarrow.RecordBatch] = []
+ reader = self._TableRead__create_reader_for_split(split, blob_parallelism)
+ try:
+ if isinstance(reader, RecordBatchReader):
+ for batch in iter(reader.read_arrow_batch, None):
+ if batch.num_rows == 0:
+ continue
+ allowed = remaining_state.try_consume(batch.num_rows)
+ if allowed == 0:
+ break
+ if allowed < batch.num_rows:
+ batch = batch.slice(0, allowed)
+ batch = self._serialize_blob_views_as_descriptors(
+ batch, reader)
+ batch = self._project_batch_to_output(batch)
+ if self.include_row_kind:
+ if "_row_kind" not in batch.schema.names:
+ batch = self._add_row_kind_column_to_batch(batch, "+I")
+ out.append(batch)
+ if remaining_state.exhausted():
+ break
+ else:
+ row_tuple_chunk: List[tuple] = []
+ row_kind_chunk: List[str] = []
+ stop = False
+ while not stop:
+ row_iterator = reader.read_batch()
+ if row_iterator is None:
+ break
+ for row in iter(row_iterator.next, None):
+ if not isinstance(row, OffsetRow):
+ raise TypeError(
+ f"Expected OffsetRow, but got {type(row).__name__}")
+ if remaining_state.try_consume(1) == 0:
+ stop = True
+ break
+ row_tuple_chunk.append(
+ self._serialize_blob_view_row_tuple(row, reader))
+ if self.include_row_kind:
+ row_kind_chunk.append(row.get_row_kind().to_string())
+
+ if len(row_tuple_chunk) >= chunk_size:
+ out.extend(self._convert_rows_to_output_batches(
+ row_tuple_chunk, row_kind_chunk, schema))
+ row_tuple_chunk = []
+ row_kind_chunk = []
+ if row_tuple_chunk:
+ out.extend(self._convert_rows_to_output_batches(
+ row_tuple_chunk, row_kind_chunk, schema))
+ finally:
+ reader.close()
+ return out
+
+ def _convert_rows_to_output_batches(
+ self,
+ row_tuples: List[tuple],
+ row_kinds: List[str],
+ schema: pyarrow.Schema,
+ ) -> Iterator[pyarrow.RecordBatch]:
+ if self.output_projection is None:
+ yield from self._convert_rows_to_arrow_batches_with_row_kind(
+ row_tuples, row_kinds, schema)
+ return
+ physical_schema = read_arrow_schema(self._adapter_read_type)
+ if self.include_row_kind:
+ physical_schema = self._add_row_kind_to_schema(physical_schema)
+ for batch in self._convert_rows_to_arrow_batches_with_row_kind(
+ row_tuples, row_kinds, physical_schema):
+ yield self._try_to_pad_batch_by_schema(
+ self._project_batch_to_output(batch), schema, allow_type_cast=True)
+
+ def _convert_rows_to_arrow_batches_with_row_kind(
+ self,
+ row_tuples: List[tuple],
+ row_kinds: List[str],
+ schema: pyarrow.Schema
+ ) -> Iterator[pyarrow.RecordBatch]:
+ """Convert rows to one or more Arrow batches, optionally including row kind column.
+
+ Yields more than one batch only when a column overflows pyarrow's 2GB
+ per-column limit (see ``_emit_overflow_safe_batches``); otherwise a single
+ batch is produced as before.
+ """
+ if not self.include_row_kind or not row_kinds:
+ # No row kind - use original schema (without _row_kind column)
+ data_schema = schema
+ columns_data = zip(*row_tuples)
+ pydict = {name: list(column) for name, column in zip(data_schema.names, columns_data)}
+ else:
+ # Include row kind as first column
+ # Schema already has _row_kind as first field
+ data_field_names = [f.name for f in schema if f.name != ROW_KIND_COLUMN]
+ columns_data = zip(*row_tuples)
+ pydict = {ROW_KIND_COLUMN: row_kinds}
+ for name, column in zip(data_field_names, columns_data):
+ pydict[name] = list(column)
+ yield from self._emit_overflow_safe_batches(pydict, len(row_tuples), schema)
+
+ def _create_split_read(self, split: Split, blob_parallelism: int = 1,
+ read_type=None, limit: Optional[int] = None,
+ push_down_limit: bool = True,
+ post_merge_filter=None,
+ eager_blob_fields=None,
+ post_filter_after_inline: bool = False,
+ predicate=_DEFAULT_PREDICATE,
+ defer_blob_resolution: bool = False) -> SplitRead:
+ sr = self._build_split_read(
+ split,
+ read_type,
+ limit,
+ push_down_limit,
+ post_merge_filter,
+ eager_blob_fields,
+ post_filter_after_inline,
+ predicate,
+ defer_blob_resolution,
+ )
+ sr._blob_parallelism = blob_parallelism
+ sr._parquet_row_group_cache = self._parquet_row_group_cache
+ return sr
+
+ def _build_split_read(self, split: Split, read_type=None,
+ limit: Optional[int] = None,
+ push_down_limit: bool = True,
+ post_merge_filter=None,
+ eager_blob_fields=None,
+ post_filter_after_inline: bool = False,
+ predicate=_DEFAULT_PREDICATE,
+ defer_blob_resolution: bool = False) -> SplitRead:
+ predicate = self.predicate if predicate is _DEFAULT_PREDICATE else predicate
+ effective_limit = (
+ self.limit if limit is None else limit
+ ) if push_down_limit else None
+ effective_read_type = read_type if read_type is not None else self._adapter_read_type
+ scan_read_type = self._with_predicate_extra_fields(read_type) if read_type is not None else self._scan_read_type
+ # the columns authorization widens the read with are plain top-level ones, so each
+ # gets a one-segment path; without them the outer extraction sees fewer paths than fields
+ nested_name_paths = self.nested_name_paths
+ if nested_name_paths and len(nested_name_paths) < len(effective_read_type):
+ nested_name_paths = list(nested_name_paths) + [
+ [f.name] for f in effective_read_type[len(nested_name_paths):]]
+ if self.table.is_primary_key_table and (
+ getattr(split, 'is_streaming', False) or not split.raw_convertible):
+ inner_read_type = scan_read_type
+ outer_extract_name_paths: Optional[List[List[str]]] = None
+ if nested_name_paths and any(
+ len(p) > 1 for p in nested_name_paths):
+ # Inner: full ROW for the merge function. Outer: extract
+ # the requested sub-paths back to the user's flat schema.
+ inner_read_type = self._with_predicate_extra_fields(
+ self._widen_to_top_level_for_merge(nested_name_paths))
+ outer_extract_name_paths = nested_name_paths
+
+ # When the user's projection drops a ``sequence.field``, the merge
+ # heap can't compare it. Inject the missing sequence field(s) into
+ # the value row so the comparator resolves, then project them back
+ # out after merging (mirrors Java MergeFileSplitRead.withReadType +
+ # projectOuter). Reuses the OuterProjectionRecordReader machinery.
+ seq_fields = self.table.options.sequence_field()
+ if seq_fields:
+ present = {f.name for f in inner_read_type}
+ missing = [name for name in seq_fields if name not in present]
+ if missing:
+ table_fields_by_name = {f.name: f for f in self.table.fields}
+ extra = []
+ for name in missing:
+ field = table_fields_by_name.get(name)
+ if field is None:
+ raise ValueError(
+ "sequence.field %r not found in table schema"
+ % (name,))
+ extra.append(field)
+ inner_read_type = list(inner_read_type) + extra
+ if outer_extract_name_paths is None:
+ # Drop the injected seq columns: project back to the
+ # user's requested (flat) columns in order.
+ outer_extract_name_paths = [
+ [f.name] for f in effective_read_type]
+ if read_type is None and outer_extract_name_paths is None and self._needs_output_projection():
+ outer_extract_name_paths = self._output_extract_name_paths()
+ return MergeFileSplitRead(
+ table=self.table,
+ predicate=predicate,
+ read_type=inner_read_type,
+ split=split,
+ row_tracking_enabled=False,
+ outer_extract_name_paths=outer_extract_name_paths,
+ outer_flat_read_type=(
+ effective_read_type if outer_extract_name_paths else None),
+ limit=effective_limit,
+ )
+ elif self.table.options.data_evolution_enabled():
+ if nested_name_paths and any(
+ len(p) > 1 for p in nested_name_paths):
+ if not self._only_map_key_nested_paths():
+ raise NotImplementedError(
+ "ROW nested-field projection on data-evolution tables "
+ "is not yet supported")
+ scan_read_type = self._with_predicate_extra_fields(
+ self._widen_to_top_level_for_merge(nested_name_paths))
+ outer_extract_name_paths = nested_name_paths
+ else:
+ outer_extract_name_paths = None
+ if (outer_extract_name_paths is None
+ and read_type is None
+ and self._needs_output_projection()):
+ outer_extract_name_paths = self._output_extract_name_paths()
+ return DataEvolutionSplitRead(
+ table=self.table,
+ predicate=predicate,
+ read_type=scan_read_type,
+ split=split,
+ row_tracking_enabled=True,
+ nested_name_paths=(
+ None if outer_extract_name_paths
+ else nested_name_paths),
+ outer_extract_name_paths=outer_extract_name_paths,
+ outer_flat_read_type=(
+ self._adapter_read_type if outer_extract_name_paths else None),
+ limit=effective_limit,
+ post_merge_filter=post_merge_filter,
+ eager_blob_fields=eager_blob_fields,
+ post_filter_after_inline=post_filter_after_inline,
+ defer_blob_resolution=defer_blob_resolution,
+ )
+ else:
+ inner_read_type = scan_read_type
+ outer_extract_name_paths: Optional[List[List[str]]] = None
+ if nested_name_paths and any(
+ len(p) > 1 for p in nested_name_paths):
+ # Mirror the merge path: read the full top-level columns so
+ # the per-file field-id normalization applies (a leaf path is
+ # only valid against the latest schema, not each file's own
+ # names/types), then extract the requested sub-paths back to
+ # the user's flat schema.
+ inner_read_type = self._with_predicate_extra_fields(
+ self._widen_to_top_level_for_merge(nested_name_paths))
+ outer_extract_name_paths = nested_name_paths
+ if read_type is None and outer_extract_name_paths is None and self._needs_output_projection():
+ outer_extract_name_paths = self._output_extract_name_paths()
+ return RawFileSplitRead(
+ table=self.table,
+ predicate=predicate,
+ read_type=inner_read_type,
+ split=split,
+ row_tracking_enabled=self.table.options.row_tracking_enabled(),
+ outer_extract_name_paths=outer_extract_name_paths,
+ outer_flat_read_type=(
+ effective_read_type if outer_extract_name_paths else None),
+ limit=effective_limit,
+ )
+
+ def _serialize_blob_views_as_descriptors(self, batch: pyarrow.RecordBatch, reader) -> pyarrow.RecordBatch:
+ if not self._blob_as_descriptor or not self._blob_view_output_indices:
+ return batch
+ lookup = getattr(reader, 'blob_view_lookup', None)
+ if lookup is None:
+ return batch
+ for index in self._blob_view_output_indices:
+ field_name = self._adapter_read_type[index].name
+ if field_name not in batch.schema.names:
+ continue
+ converted = [
+ lookup.serialize_view_field_value(value)
+ for value in batch.column(field_name).to_pylist()
+ ]
+ column_idx = batch.schema.names.index(field_name)
+ batch = batch.set_column(
+ column_idx,
+ pyarrow.field(field_name, pyarrow.large_binary(), nullable=True),
+ pyarrow.array(converted, type=pyarrow.large_binary()),
+ )
+ return batch
+
+ def _serialize_blob_view_row_tuple(self, row: OffsetRow, reader) -> tuple:
+ values = row.row_tuple[row.offset: row.offset + row.arity]
+ if not self._blob_as_descriptor or not self._blob_view_output_indices:
+ return values
+ lookup = getattr(reader, 'blob_view_lookup', None)
+ if lookup is None:
+ return values
+ converted = list(values)
+ for index in self._blob_view_output_indices:
+ converted[index] = lookup.serialize_view_field_value(converted[index])
+ return tuple(converted)
+
+ def _widen_to_top_level_for_merge(self, name_paths=None) -> List[DataField]:
+ """Unique top-level fields from ``name_paths``, in path order."""
+ paths = self.nested_name_paths if name_paths is None else name_paths
+ table_fields_by_name = {f.name: f for f in self._table_read_fields()}
+ paths_by_top = {}
+ for path in paths or []:
+ paths_by_top.setdefault(path[0], []).append(path)
+ seen = set()
+ widened: List[DataField] = []
+ for path in paths or []:
+ top_name = path[0]
+ if top_name in seen:
+ continue
+ seen.add(top_name)
+ field = table_fields_by_name.get(top_name)
+ if field is None:
+ raise ValueError(
+ "Nested projection top-level field %r not found in "
+ "table schema" % (top_name,))
+ paths = paths_by_top[top_name]
+ if (isinstance(field.type, MapType)
+ and all(len(path) > 1 for path in paths)
+ and not is_map_blob_type(field.type)
+ and not self._map_has_aggregator(top_name)):
+ keys = []
+ for path in paths:
+ if path[1] not in keys:
+ keys.append(path[1])
+ try:
+ field = map_selected_keys_field(field, keys)
+ except ValueError:
+ # Keys that cannot be encoded by the Java/Spark metadata
+ # contract still work through the complete-MAP fallback.
+ pass
+ widened.append(field)
+ return widened
+
+ def _only_map_key_nested_paths(self) -> bool:
+ table_fields = {field.name: field for field in self.table.fields}
+ for path in self.nested_name_paths or []:
+ if len(path) == 1:
+ continue
+ field = table_fields.get(path[0])
+ if len(path) != 2 or field is None \
+ or not isinstance(field.type, MapType):
+ return False
+ return True
+
+ def _has_map_key_projection(self) -> bool:
+ table_fields = {field.name: field for field in self.table.fields}
+ for path in self.nested_name_paths or []:
+ field = table_fields.get(path[0])
+ if len(path) == 2 and field is not None \
+ and isinstance(field.type, MapType):
+ return True
+ return False
+
+ def _map_has_aggregator(self, field_name: str) -> bool:
+ options = self.table.options.options.data
+ return (
+ "fields.{}.aggregate-function".format(field_name) in options
+ or "fields.default-aggregate-function" in options
+ )
+
+ def _check_python_merge_supported(self):
+ if getattr(self, '_python_merge_validation_pending', False):
+ from pypaimon.execution.python.read.merge_engine_support import check_supported
+ check_supported(self.table)
+ self._python_merge_validation_pending = False
+
+ def _TableRead__create_reader_for_split(
+ self, split, blob_parallelism=1, limit: Optional[int] = None):
+ # Raw-convertible splits skip MergeFileSplitRead, so validate before
+ # dispatch, including when native setup failed or bindings are absent.
+ self._check_python_merge_supported()
+ auth_result = None
+ if isinstance(split, QueryAuthSplit):
+ auth_result = split.auth_result
+ split = split.split
+
+ if auth_result is not None:
+ return self._TableRead__authed_reader(
+ split, auth_result, blob_parallelism, limit)
+ if limit is None:
+ return self._create_split_read(
+ split, blob_parallelism=blob_parallelism).create_reader()
+ return self._create_split_read(
+ split,
+ blob_parallelism=blob_parallelism,
+ limit=limit,
+ ).create_reader()
+
+ def _TableRead__authed_reader(
+ self, split, auth_result, blob_parallelism=1, limit: Optional[int] = None):
+ if self._has_map_key_projection():
+ raise NotImplementedError(
+ "MAP-key projection with query authorization is not supported")
+ table_fields = self.table.fields
+ read_fields = self._adapter_read_type
+ masks = auth_result.column_masking or {}
+ masked_fields = set(masks)
+ # Split reads may filter loosely (merged rows, nested leaves), so filter exactly before LIMIT.
+ post_filter = self.predicate
+ filter_read_fields = self._with_predicate_extra_fields(read_fields) if post_filter is not None else read_fields
+ from pypaimon.read.auth_support import latest_auth_fields, validate_auth_rules
+ # a split can carry rules without having been planned by a validating scan
+ snapshot = getattr(self, "_auth_schema", None)
+ if snapshot is None:
+ snapshot = self._auth_schema = latest_auth_fields(self.table)
+ latest_fields = validate_auth_rules(self.table, auth_result, snapshot)
+ # bind to the latest schema: a generated alias may since have become a real column
+ auth_result.validate_read_type(
+ latest_fields, filter_read_fields, self.nested_name_paths, table_fields)
+
+ extra_fields = auth_result.get_extra_fields(filter_read_fields, table_fields)
+ effective_read_type = filter_read_fields + extra_fields
+
+ filter_fn = auth_result.extract_row_filter()
+ effective_limit = self.limit if limit is None else limit
+ auth_fields = (
+ self._auth_filter_field_names(auth_result, effective_read_type)
+ if filter_fn is not None else set()
+ )
+ eager_blob_fields = set(auth_fields)
+ if post_filter is not None:
+ # Only inputs of the filter and its masks are read eagerly.
+ filter_names = predicate_field_names(post_filter)
+ filter_masks = compile_auth_masking_rules(
+ {name: rule for name, rule in masks.items() if name in filter_names},
+ effective_read_type)
+ eager_blob_fields.update(filter_names - masked_fields)
+ for transform in filter_masks.values():
+ eager_blob_fields.update(_collect_all_field_refs_from_transform(transform))
+ inline_blob_fields = (
+ self.table.options.blob_descriptor_fields()
+ | self.table.options.blob_view_fields()
+ )
+ embed_filter = (
+ filter_fn is not None
+ and self.table.options.data_evolution_enabled()
+ )
+ # If the auth filter references an inline BLOB, run it after inline resolution (in
+ # the split read) so it sees resolved payloads while scalar BLOBs still defer.
+ post_filter_after_inline = embed_filter and bool(auth_fields & inline_blob_fields)
+ split_read = self._create_split_read(
+ split,
+ blob_parallelism=blob_parallelism,
+ read_type=effective_read_type,
+ limit=limit,
+ push_down_limit=post_filter is None and (filter_fn is None or embed_filter),
+ post_merge_filter=filter_fn if embed_filter else None,
+ eager_blob_fields=eager_blob_fields,
+ post_filter_after_inline=post_filter_after_inline,
+ # push down conjuncts without masked columns
+ predicate=exclude_predicate_with_fields(post_filter, masked_fields),
+ defer_blob_resolution=post_filter is not None,
+ )
+ reader = split_read.create_reader()
+
+ needs_convert_back = False
+ if not isinstance(reader, RecordBatchReader):
+ schema = read_arrow_schema(effective_read_type)
+ reader = RecordReaderToBatchAdapter(reader, schema, include_row_kind=self.include_row_kind)
+ if getattr(reader, 'blob_field_indices', None) is None:
+ from pypaimon.read.reader.field_indices import (
+ blob_field_indices, descriptor_field_indices_for_table,
+ vector_field_indices)
+ reader.file_io = self.table.file_io
+ reader.blob_field_indices = blob_field_indices(effective_read_type)
+ reader.descriptor_field_indices = descriptor_field_indices_for_table(
+ self.table, effective_read_type)
+ reader.vector_field_indices = vector_field_indices(effective_read_type)
+ needs_convert_back = True
+
+ if filter_fn and not embed_filter:
+ reader = AuthFilterReader(reader, filter_fn)
+
+ if post_filter is not None:
+ # Filter on masked values but keep raw rows; masks apply after LIMIT.
+ reader = FilterRecordBatchReader(
+ reader, rewrite_predicate_indices(post_filter, effective_read_type),
+ field_names=[field.name for field in effective_read_type],
+ schema_fields=effective_read_type,
+ filter_view=lambda batch: apply_auth_masking(batch, filter_masks))
+ if effective_limit is not None and (post_filter is not None or (filter_fn and not embed_filter)):
+ reader = LimitedRecordBatchReader(reader, effective_limit)
+
+ if post_filter is not None and isinstance(split_read, DataEvolutionSplitRead):
+ # Fetch only payloads output unmasked or read by a mask.
+ needed = {f.name for f in read_fields} - masked_fields
+ for transform in compile_auth_masking_rules(masks, effective_read_type).values():
+ needed.update(_collect_all_field_refs_from_transform(transform))
+ reader = split_read._resolve_deferred_blobs(
+ reader, {f.name for f in effective_read_type} - needed)
+
+ if auth_result.column_masking:
+ reader = AuthMaskingReader(reader, auth_result.column_masking, effective_read_type)
+
+ if len(effective_read_type) > len(read_fields):
+ original_columns = [f.name for f in read_fields]
+ reader = ColumnProjectReader(reader, original_columns)
+
+ if needs_convert_back:
+ reader = BatchToRecordReaderAdapter(reader)
+
+ return reader
+
+ @staticmethod
+ def _auth_filter_field_names(auth_result, read_fields) -> set:
+ filters = getattr(auth_result, "filter", None)
+ if not filters:
+ return {field.name for field in read_fields}
+ names = set()
+ for filter_json in filters:
+ names.update(extract_referenced_fields(filter_json))
+ return names
+
+ def _python_to_arrow(self, splits, schema, parallelism, effective_bp):
+ self._check_python_merge_supported()
+ effective = self._effective_parallelism(parallelism, len(splits))
+ if self._should_run_parallel(splits, effective):
+ return self._to_arrow_parallel(splits, schema, effective, effective_bp)
+
+ batch_iterator = self._arrow_batch_generator(
+ splits, schema, effective_bp)
+ batch_reader = pyarrow.ipc.RecordBatchReader.from_batches(
+ schema, batch_iterator)
+
+ table_list = []
+ for batch in iter(batch_reader.read_next_batch, None):
+ if batch.num_rows == 0:
+ continue
+ table_list.append(self._try_to_pad_batch_by_schema(
+ batch, schema, allow_type_cast=True))
+
+ return self._batches_to_arrow(
+ table_list, schema, allow_type_cast=True)
+
+ def _python_batch_iterator(self, splits, schema, parallelism, effective_bp):
+ self._check_python_merge_supported()
+ effective = self._effective_parallelism(parallelism, len(splits))
+ if self._should_run_parallel(splits, effective):
+ effective_bp = self._cap_blob_parallelism(
+ min(effective, len(splits)), effective_bp)
+ return self._arrow_batch_generator(
+ splits, schema, effective_bp)
diff --git a/paimon-python/pypaimon/execution/python/read/table_scan.py b/paimon-python/pypaimon/execution/python/read/table_scan.py
new file mode 100644
index 0000000000..dcdfe5d907
--- /dev/null
+++ b/paimon-python/pypaimon/execution/python/read/table_scan.py
@@ -0,0 +1,129 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Python file scanner construction for the public TableScan."""
+
+from pypaimon.common.options.core_options import CoreOptions, ChangelogProducer
+from pypaimon.manifest.manifest_list_manager import ManifestListManager
+from pypaimon.execution.python.read.auth_support import prune_scanner_by_auth
+from pypaimon.read.query_auth_split import wrap_plan_with_auth
+from pypaimon.execution.python.read.scanner.file_scanner import FileScanner
+
+
+class PythonTableScanMixin:
+ def _create_file_scanner(self) -> FileScanner:
+ options = self.table.options.options
+ snapshot_manager = self.table.snapshot_manager()
+ manifest_list_manager = ManifestListManager(self.table)
+
+ from pypaimon.snapshot.time_travel_util import TimeTravelUtil, SCAN_KEYS
+ has_time_travel = any(options.contains_key(key) for key in SCAN_KEYS)
+ has_incremental = options.contains(CoreOptions.INCREMENTAL_BETWEEN_TIMESTAMP)
+
+ if has_incremental:
+ if self._incremental_snapshot_range is None:
+ return FileScanner(
+ self.table,
+ lambda: ([], None),
+ partition_predicate=self.partition_predicate,
+ )
+ start_id, end_id = self._incremental_snapshot_range
+ # Java's AUTO mode uses physical changelogs when a producer is
+ # configured, and APPEND deltas otherwise.
+ changelog = self.table.options.changelog_producer() != ChangelogProducer.NONE
+
+ def incremental_manifest():
+ snapshots_in_range = []
+ end_snapshot = snapshot_manager.get_snapshot_by_id(end_id) if end_id >= 1 else None
+ for snapshot_id in range(start_id + 1, end_id + 1):
+ snapshot = snapshot_manager.get_snapshot_by_id(snapshot_id)
+ end_snapshot = snapshot
+ if (snapshot.commit_kind != "OVERWRITE" and snapshot.changelog_manifest_list
+ if changelog else snapshot.commit_kind == "APPEND"):
+ snapshots_in_range.append(snapshot)
+
+ manifests = []
+
+ for snapshot in snapshots_in_range:
+ manifest_files = (manifest_list_manager.read_changelog(snapshot) if changelog else
+ manifest_list_manager.read_delta(snapshot))
+ manifests.extend(manifest_files)
+ return manifests, end_snapshot
+
+ return FileScanner(
+ self.table,
+ incremental_manifest,
+ self.predicate,
+ self.limit,
+ partition_predicate=self.partition_predicate,
+ skip_level0=False,
+ is_streaming=True,
+ )
+
+ if has_time_travel:
+ def time_travel_manifest_scanner():
+ snapshot = TimeTravelUtil.resolve_snapshot(self.table)
+ if snapshot is None:
+ raise ValueError(
+ "Could not resolve time travel snapshot from scan options."
+ )
+ return manifest_list_manager.read_all(snapshot), snapshot
+
+ return FileScanner(
+ self.table,
+ time_travel_manifest_scanner,
+ self.predicate,
+ self.limit,
+ partition_predicate=self.partition_predicate,
+ skip_level0=self.table.options.batch_scan_skip_level0(),
+ )
+
+ def all_manifests():
+ snapshot = TimeTravelUtil.resolve_snapshot(self.table)
+ return manifest_list_manager.read_all(snapshot), snapshot
+
+ return FileScanner(
+ self.table,
+ all_manifests,
+ self.predicate,
+ self.limit,
+ partition_predicate=self.partition_predicate,
+ skip_level0=self.table.options.batch_scan_skip_level0(),
+ )
+
+ def _python_scanner(self, auth_result):
+ scanner = self.file_scanner
+ if self._query_auth_fn is not None:
+ prune_scanner_by_auth(self.table, scanner, auth_result)
+ return scanner
+
+ def _python_plan(self, auth_result):
+ return wrap_plan_with_auth(auth_result, self._python_scanner(auth_result).scan())
+
+ def _python_plan_all_files(self):
+ scanner = self._python_scanner(None)
+ # Writer restore/overwrite must see L0 even when batch reads hide it.
+ skip_level0 = scanner.skip_level0
+ try:
+ scanner.skip_level0 = False
+ return scanner.scan()
+ finally:
+ scanner.skip_level0 = skip_level0
+
+ def _python_scan_with_stats(self, auth_result):
+ plan, stats = self._python_scanner(auth_result).scan_with_stats()
+ return wrap_plan_with_auth(auth_result, plan), stats
diff --git a/paimon-python/pypaimon/read/scanner/__init__.py b/paimon-python/pypaimon/execution/python/write/__init__.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/__init__.py
copy to paimon-python/pypaimon/execution/python/write/__init__.py
diff --git a/paimon-python/pypaimon/write/blob_format_writer.py b/paimon-python/pypaimon/execution/python/write/blob_format_writer.py
similarity index 100%
rename from paimon-python/pypaimon/write/blob_format_writer.py
rename to paimon-python/pypaimon/execution/python/write/blob_format_writer.py
diff --git a/paimon-python/pypaimon/read/reader/deduplicate_merge_function.py b/paimon-python/pypaimon/execution/python/write/bucket_metadata.py
similarity index 50%
rename from paimon-python/pypaimon/read/reader/deduplicate_merge_function.py
rename to paimon-python/pypaimon/execution/python/write/bucket_metadata.py
index 5b669aae55..10fd803930 100644
--- a/paimon-python/pypaimon/read/reader/deduplicate_merge_function.py
+++ b/paimon-python/pypaimon/execution/python/write/bucket_metadata.py
@@ -16,35 +16,15 @@
# limitations under the License.
################################################################################
-"""Default merge function for primary-key tables.
+"""Python live-file metadata lookup used by persistent input assignment."""
-Mirrors Java ``DeduplicateMergeFunction`` -- for a run of KVs sharing
-the same primary key, keep only the one with the highest sequence
-number (by virtue of ``add`` being called in sequence-number order).
-"""
-from typing import Optional
+def dynamic_bucket_data_buckets(table, partition, snapshot):
+ from pypaimon.index.dynamic_bucket import bucket_read_builder
-from pypaimon.table.row.key_value import KeyValue
-
-
-class DeduplicateMergeFunction:
- """Keep only the latest KV per primary key.
-
- Used by both the read path (``SortMergeReaderWithMinHeap``) and the
- write path (``KeyValueDataWriter`` in-memory merge buffer) -- the
- latter is what enforces the LSM "PK unique within a file"
- invariant on flush.
- """
-
- def __init__(self):
- self.latest_kv: Optional[KeyValue] = None
-
- def reset(self) -> None:
- self.latest_kv = None
-
- def add(self, kv: KeyValue) -> None:
- self.latest_kv = kv
-
- def get_result(self) -> Optional[KeyValue]:
- return self.latest_kv
+ read_builder = bucket_read_builder(table, partition, snapshot)
+ # This internal manifest lookup returns no rows and must include every
+ # live bucket, independently of a SELECT policy or level-zero read option.
+ scanner = read_builder.new_scan().file_scanner
+ scanner.skip_level0 = False
+ return {entry.bucket for entry in scanner.plan_files()}
diff --git a/paimon-python/pypaimon/read/scanner/__init__.py b/paimon-python/pypaimon/execution/python/write/commit/__init__.py
similarity index 100%
copy from paimon-python/pypaimon/read/scanner/__init__.py
copy to paimon-python/pypaimon/execution/python/write/commit/__init__.py
diff --git a/paimon-python/pypaimon/write/commit/commit_rollback.py b/paimon-python/pypaimon/execution/python/write/commit/commit_rollback.py
similarity index 100%
rename from paimon-python/pypaimon/write/commit/commit_rollback.py
rename to paimon-python/pypaimon/execution/python/write/commit/commit_rollback.py
diff --git a/paimon-python/pypaimon/write/commit/commit_scanner.py b/paimon-python/pypaimon/execution/python/write/commit/commit_scanner.py
similarity index 99%
rename from paimon-python/pypaimon/write/commit/commit_scanner.py
rename to paimon-python/pypaimon/execution/python/write/commit/commit_scanner.py
index b08bafcb81..154134af67 100644
--- a/paimon-python/pypaimon/write/commit/commit_scanner.py
+++ b/paimon-python/pypaimon/execution/python/write/commit/commit_scanner.py
@@ -25,7 +25,7 @@ from pypaimon.manifest.index_manifest_file import IndexManifestFile
from pypaimon.manifest.manifest_file_manager import ManifestFileManager
from pypaimon.manifest.manifest_list_manager import ManifestListManager
from pypaimon.manifest.schema.manifest_entry import ManifestEntry
-from pypaimon.read.scanner.file_scanner import FileScanner
+from pypaimon.execution.python.read.scanner.file_scanner import (FileScanner)
from pypaimon.snapshot.snapshot import Snapshot
diff --git a/paimon-python/pypaimon/write/commit/conflict_detection.py b/paimon-python/pypaimon/execution/python/write/commit/conflict_detection.py
similarity index 99%
rename from paimon-python/pypaimon/write/commit/conflict_detection.py
rename to paimon-python/pypaimon/execution/python/write/commit/conflict_detection.py
index e97d5a27b6..4c18e16868 100644
--- a/paimon-python/pypaimon/write/commit/conflict_detection.py
+++ b/paimon-python/pypaimon/execution/python/write/commit/conflict_detection.py
@@ -28,7 +28,7 @@ from pypaimon.manifest.schema.file_entry import FileEntry
from pypaimon.table.special_fields import SpecialFields
from pypaimon.utils.range import Range
from pypaimon.utils.range_helper import RangeHelper
-from pypaimon.write.commit.commit_scanner import CommitScanner
+from pypaimon.execution.python.write.commit.commit_scanner import CommitScanner
class RowIdColumnConflictChecker:
@@ -287,7 +287,7 @@ class ConflictDetection:
if conflict is not None:
return conflict
- from pypaimon.write.commit.global_index_source_check import check_global_index_sources
+ from pypaimon.execution.python.write.commit.global_index_source_check import check_global_index_sources
conflict = check_global_index_sources(
self, latest_snapshot, merged_entries, delta_entries, delta_index_entries)
if conflict is not None:
diff --git a/paimon-python/pypaimon/write/commit/fixed_bucket_commit_check.py b/paimon-python/pypaimon/execution/python/write/commit/fixed_bucket_commit_check.py
similarity index 100%
rename from paimon-python/pypaimon/write/commit/fixed_bucket_commit_check.py
rename to paimon-python/pypaimon/execution/python/write/commit/fixed_bucket_commit_check.py
diff --git a/paimon-python/pypaimon/write/commit/global_index_source_check.py b/paimon-python/pypaimon/execution/python/write/commit/global_index_source_check.py
similarity index 97%
rename from paimon-python/pypaimon/write/commit/global_index_source_check.py
rename to paimon-python/pypaimon/execution/python/write/commit/global_index_source_check.py
index c4d9bb52ea..207f12d55a 100644
--- a/paimon-python/pypaimon/write/commit/global_index_source_check.py
+++ b/paimon-python/pypaimon/execution/python/write/commit/global_index_source_check.py
@@ -58,7 +58,7 @@ def check_global_index_sources(detection, latest_snapshot, merged_entries, delta
def _check_column_changes(detection, sources, changes, snapshot_id=None):
- from pypaimon.write.commit.conflict_detection import RowIdColumnConflictChecker
+ from pypaimon.execution.python.write.commit.conflict_detection import RowIdColumnConflictChecker
field_checker = RowIdColumnConflictChecker([], detection.table.schema_manager)
for change in changes:
diff --git a/paimon-python/pypaimon/write/commit/overwrite_changes_provider.py b/paimon-python/pypaimon/execution/python/write/commit/overwrite_changes_provider.py
similarity index 99%
rename from paimon-python/pypaimon/write/commit/overwrite_changes_provider.py
rename to paimon-python/pypaimon/execution/python/write/commit/overwrite_changes_provider.py
index acd88833f5..cdbe140e53 100644
--- a/paimon-python/pypaimon/write/commit/overwrite_changes_provider.py
+++ b/paimon-python/pypaimon/execution/python/write/commit/overwrite_changes_provider.py
@@ -23,7 +23,7 @@ from pypaimon.manifest.index_manifest_file import IndexManifestFile
from pypaimon.manifest.manifest_file_manager import ManifestFileManager
from pypaimon.manifest.schema.file_entry import FileEntry
from pypaimon.manifest.schema.manifest_entry import ManifestEntry
-from pypaimon.read.scanner.file_scanner import FileScanner
+from pypaimon.execution.python.read.scanner.file_scanner import (FileScanner)
from pypaimon.snapshot.snapshot import Snapshot
from pypaimon.table.row.generic_row import GenericRow
diff --git a/paimon-python/pypaimon/write/commit/row_id_conflict_rewriter.py b/paimon-python/pypaimon/execution/python/write/commit/row_id_conflict_rewriter.py
similarity index 99%
rename from paimon-python/pypaimon/write/commit/row_id_conflict_rewriter.py
rename to paimon-python/pypaimon/execution/python/write/commit/row_id_conflict_rewriter.py
index fa0efbb8d2..d2a445c1a8 100644
--- a/paimon-python/pypaimon/write/commit/row_id_conflict_rewriter.py
+++ b/paimon-python/pypaimon/execution/python/write/commit/row_id_conflict_rewriter.py
@@ -27,7 +27,7 @@ from pypaimon.table.row.generic_row import GenericRow
from pypaimon.table.special_fields import SpecialFields
from pypaimon.utils.range import Range
... 29211 lines suppressed ...