You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/10/21 05:57:46 UTC
(paimon) branch master updated: [Python] Decouple public APIs from execution backends (#10707)
This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 062b22e0ad [Python] Decouple public APIs from execution backends (#10707)
062b22e0ad is described below
commit 062b22e0ad182021248bcf131cdca8352c864216
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Wed Oct 21 13:57:40 2026 +0800
[Python] Decouple public APIs from execution backends (#10707)
---
paimon-python/dev/test_packaging.py | 9 +
paimon-python/pypaimon/execution/README.md | 46 +-
paimon-python/pypaimon/execution/dispatch.py | 26 +-
paimon-python/pypaimon/execution/index_dispatch.py | 40 ++
.../execution/native/read/native_search_plan.py | 103 +--
.../pypaimon/execution/native/read/table_read.py | 36 +-
paimon-python/pypaimon/execution/native/runtime.py | 15 +-
.../execution/native/write/native_update.py | 20 +-
.../execution/native/write/native_write.py | 140 +---
.../python/globalindex/create_global_index.py | 103 +--
.../python/globalindex/ray_build.py} | 0
.../python/read/reader/concat_batch_reader.py | 61 +-
.../python/read/search/global_index_row_filter.py | 2 +-
.../execution/python/read/search/hybrid_search.py | 12 +-
.../python/read/search/vector_search_read.py | 9 +-
.../pypaimon/execution/python/read/table_read.py | 45 +-
.../pypaimon/execution/python/read/table_scan.py | 6 +-
.../pypaimon/execution/python/write/merge_into.py | 2 +-
.../python/write/postpone_batch_table_write.py | 19 +-
.../execution/python/write/table_update.py | 53 +-
.../execution/python/write/table_upsert_by_key.py | 93 +--
paimon-python/pypaimon/execution/read_dispatch.py | 279 ++++++++
paimon-python/pypaimon/execution/read_explain.py | 247 +++++++
paimon-python/pypaimon/execution/read_support.py | 647 ++++++++++++++++++
.../pypaimon/execution/search_dispatch.py | 154 +++++
.../pypaimon/execution/update_dispatch.py | 229 +++++++
paimon-python/pypaimon/execution/write_dispatch.py | 239 +++++++
.../pypaimon/filesystem/hdfs_native_file_io.py | 3 +-
paimon-python/pypaimon/filesystem/local_file_io.py | 3 +-
.../pypaimon/filesystem/pyarrow_file_io.py | 3 +-
.../pypaimon/globalindex/create_global_index.py | 99 +--
.../pypaimon/globalindex/drop_global_index.py | 14 +-
paimon-python/pypaimon/multimodal/temporal.py | 2 +-
paimon-python/pypaimon/ray/batch_vector_search.py | 4 +-
paimon-python/pypaimon/ray/full_text_search.py | 2 +-
paimon-python/pypaimon/ray/vector_search.py | 4 +-
.../pypaimon/read/datasource/torch_dataset.py | 2 +-
paimon-python/pypaimon/read/read_builder.py | 216 +-----
paimon-python/pypaimon/read/stream_read_builder.py | 40 +-
.../pypaimon/read/streaming_table_scan.py | 28 +-
paimon-python/pypaimon/read/table_read.py | 721 +++++++++------------
paimon-python/pypaimon/read/table_scan.py | 106 +--
paimon-python/pypaimon/schema/arrow_schema.py | 106 ++-
paimon-python/pypaimon/table/file_store_table.py | 18 +-
.../table/source/batch_vector_search_builder.py | 13 +-
.../table/source/full_text_search_builder.py | 16 +-
.../pypaimon/table/source/hybrid_search_builder.py | 32 +-
.../pypaimon/table/source/vector_search_builder.py | 16 +-
.../pypaimon/table/system/system_table_scan.py | 8 +-
.../pypaimon/tests/arrow_write_nullability_test.py | 171 +++++
.../pypaimon/tests/batch_vector_raw_scan_test.py | 2 +-
.../tests/daft/daft_pk_distributed_write_test.py | 2 +-
.../tests/data_file_path_directory_test.py | 27 +-
.../tests/execution_backend_isolation_test.py | 84 +++
.../pypaimon/tests/execution_dispatch_test.py | 25 +-
.../tests/execution_table_read_boundary_test.py | 5 +-
.../pypaimon/tests/global_index_build_test.py | 3 +-
.../tests/global_index_execution_boundary_test.py | 4 +-
.../pypaimon/tests/hybrid_search_ranker_test.py | 16 +-
.../pypaimon/tests/lazy_search_execution_test.py | 24 +-
.../pypaimon/tests/multimodal_table_test.py | 4 +-
.../tests/native_blob_collection_write_test.py | 2 +-
.../pypaimon/tests/native_blob_consumer_test.py | 2 +-
.../pypaimon/tests/native_blob_row_write_test.py | 2 +-
.../pypaimon/tests/native_blob_update_test.py | 2 +-
.../pypaimon/tests/native_blob_uri_reader_test.py | 2 +-
.../pypaimon/tests/native_blob_write_test.py | 2 +-
.../pypaimon/tests/native_branch_write_test.py | 2 +-
.../tests/native_cross_partition_write_test.py | 2 +-
.../pypaimon/tests/native_data_directory_test.py | 2 +-
.../tests/native_data_evolution_commit_test.py | 2 +-
.../tests/native_dv_merge_engine_write_test.py | 2 +-
.../pypaimon/tests/native_dynamic_write_test.py | 2 +-
.../pypaimon/tests/native_external_write_test.py | 2 +-
.../tests/native_filtered_write_validation_test.py | 2 +-
.../native_filtered_writer_configuration_test.py | 2 +-
.../tests/native_generic_index_build_test.py | 8 +-
.../tests/native_historical_write_commit_test.py | 2 +-
.../pypaimon/tests/native_loaded_schema_io_test.py | 2 +-
.../tests/native_managed_blob_write_test.py | 2 +-
.../tests/native_map_shredding_write_test.py | 2 +-
.../pypaimon/tests/native_merge_into_test.py | 2 +-
.../pypaimon/tests/native_mod_bucket_write_test.py | 2 +-
.../tests/native_nullable_primary_key_test.py | 2 +-
.../tests/native_overwrite_restore_test.py | 2 +-
.../tests/native_parallel_index_build_test.py | 3 +-
.../pypaimon/tests/native_partition_types_test.py | 2 +-
.../pypaimon/tests/native_pk_blob_limit_test.py | 2 +-
paimon-python/pypaimon/tests/native_plan_test.py | 11 -
.../tests/native_postpone_fixed_write_test.py | 2 +-
.../pypaimon/tests/native_postpone_write_test.py | 2 +-
.../tests/native_precomputed_bucket_test.py | 2 +-
.../pypaimon/tests/native_read_type_test.py | 3 +-
.../pypaimon/tests/native_row_sidecar_test.py | 3 +-
.../pypaimon/tests/native_schema_cast_test.py | 2 +-
.../pypaimon/tests/native_search_plan_test.py | 2 +-
.../pypaimon/tests/native_sequence_test.py | 2 +-
.../tests/native_sorted_index_build_test.py | 12 +-
.../tests/native_stream_query_auth_test.py | 2 +-
.../tests/native_timestamp_bucket_write_test.py | 2 +-
.../pypaimon/tests/native_variant_update_test.py | 2 +-
.../tests/native_vector_index_build_test.py | 3 +-
.../tests/native_write_capabilities_test.py | 2 +-
.../tests/native_write_fork_safety_test.py | 14 +-
.../tests/native_write_inactive_options_test.py | 6 +-
.../tests/native_write_path_completion_test.py | 6 +-
.../pypaimon/tests/native_write_routing_test.py | 2 +-
paimon-python/pypaimon/tests/native_write_test.py | 6 +-
.../pypaimon/tests/native_write_type_test.py | 2 +-
.../tests/pure_writer_creation_time_test.py | 2 +-
.../tests/py36/arrow_write_nullability_test.py | 63 ++
.../pypaimon/tests/ray_vector_index_build_test.py | 6 +-
.../pypaimon/tests/read_execution_boundary_test.py | 268 ++++++++
.../pypaimon/tests/reader_parallel_test.py | 3 +-
.../pypaimon/tests/shard_table_updator_test.py | 2 +-
.../pypaimon/tests/shared_execution_import_test.py | 114 ++++
.../pypaimon/tests/stream_read_builder_test.py | 2 +-
.../tests/streaming_scan_import_boundary_test.py | 101 +++
.../tests/system/system_string_filter_test.py | 3 +
.../tests/test_read_builder_nested_projection.py | 2 +-
paimon-python/pypaimon/tests/torch_read_test.py | 2 +
paimon-python/pypaimon/tests/variant_test.py | 1 -
.../pypaimon/tests/vindex_batch_write_test.py | 4 +-
.../pypaimon/tests/write/backend_boundary_test.py | 115 ++++
.../tests/write/execution_lifecycle_test.py | 12 +-
.../pypaimon/tests/write/object_row_kind_test.py | 2 +-
.../write/postpone_execution_boundary_test.py | 56 +-
.../write/retained_delete_nullability_test.py | 2 +-
.../tests/write/row_kind_nullability_test.py | 4 +-
.../pypaimon/tests/write/row_kind_write_test.py | 2 +-
.../pypaimon/tests/write/table_write_test.py | 2 +-
.../pypaimon/write/postpone_batch_table_write.py | 69 +-
paimon-python/pypaimon/write/postpone_bucket.py | 9 +-
paimon-python/pypaimon/write/table_delete.py | 56 +-
paimon-python/pypaimon/write/table_update.py | 294 +++++----
.../pypaimon/write/table_update_by_row_id.py | 113 +++-
.../pypaimon/write/table_upsert_by_key.py | 129 ++--
paimon-python/pypaimon/write/table_write.py | 107 ++-
paimon-python/pypaimon/write/update_validation.py | 93 +++
139 files changed, 4412 insertions(+), 1833 deletions(-)
diff --git a/paimon-python/dev/test_packaging.py b/paimon-python/dev/test_packaging.py
index 7130077c1e..c9b9cfca35 100644
--- a/paimon-python/dev/test_packaging.py
+++ b/paimon-python/dev/test_packaging.py
@@ -124,6 +124,13 @@ class PackagingTest(unittest.TestCase):
self.assertIn(name, names)
execution_files = (
"pypaimon/execution/dispatch.py",
+ "pypaimon/execution/read_dispatch.py",
+ "pypaimon/execution/read_support.py",
+ "pypaimon/execution/read_explain.py",
+ "pypaimon/execution/index_dispatch.py",
+ "pypaimon/execution/search_dispatch.py",
+ "pypaimon/execution/update_dispatch.py",
+ "pypaimon/execution/write_dispatch.py",
"pypaimon/execution/vindex_input.py",
"pypaimon/execution/python/read/table_read.py",
"pypaimon/execution/python/read/table_scan.py",
@@ -138,6 +145,7 @@ class PackagingTest(unittest.TestCase):
"pypaimon/execution/python/merge_engine_dispatch.py",
"pypaimon/execution/python/read/search/vector_search_read.py",
"pypaimon/execution/python/read/search/hybrid_search.py",
+ "pypaimon/execution/python/globalindex/ray_build.py",
"pypaimon/execution/python/globalindex/btree/btree_index_reader.py",
"pypaimon/execution/python/globalindex/btree/block_handle.py",
"pypaimon/execution/python/globalindex/block_compression.py",
@@ -167,6 +175,7 @@ class PackagingTest(unittest.TestCase):
for name in execution_files:
self.assertIn(name, names)
internal_aliases = (
+ "pypaimon/ray/vector_index_build.py",
"pypaimon/common/merge_engine_dispatch.py",
"pypaimon/read/native_plan.py",
"pypaimon/read/native_row_iterator.py",
diff --git a/paimon-python/pypaimon/execution/README.md b/paimon-python/pypaimon/execution/README.md
index 995d90eb75..1d7ff7f352 100644
--- a/paimon-python/pypaimon/execution/README.md
+++ b/paimon-python/pypaimon/execution/README.md
@@ -29,6 +29,13 @@ pypaimon/
write/ Public write/commit/update APIs and shared inputs
execution/
dispatch.py Preparation and safe backend selection
+ read_dispatch.py Batch/stream planning and read selection
+ read_support.py Shared Arrow output, resource and dataset adapters
+ read_explain.py Explain planning and counter aggregation
+ search_dispatch.py Search selection and lazy transported-plan reads
+ write_dispatch.py Writer selection, diagnostics and safe switching
+ update_dispatch.py Update, delete, upsert and MERGE coordination
+ index_dispatch.py Index build/drop selection
python/
read/ Python scanning, file readers and merge execution
Format/object file I/O and system-table filtering
@@ -36,6 +43,7 @@ pypaimon/
write/ Python writers, committers and update execution
Format-table writing and explicit cleanup
globalindex/ Python index readers, writers, build and evaluation
+ ray_build.py Driver coordination and Python shard workers
merge_engine_dispatch.py Python merge-function selection for read and write
native/
runtime.py Lazy runtime discovery and process safety
@@ -50,7 +58,9 @@ pypaimon/
Both implementations consume shared requests, predicates, schemas, plans, splits,
commit messages and input adapters. These contracts stay outside either backend.
Neither implementation imports the other. Public APIs retain configuration and
-select an executor when needed. Python scanners, readers, writers and committers
+validation and delegate execution without inheriting either backend. Explicit
+delegates retain the public object as the request context so subclass and instance
+overrides continue to run. Python scanners, readers, writers and committers
are created only when Python execution is selected. Direct access to a Python
scanner, writer router or committer through an existing public property explicitly
selects that implementation. Attribute inspection and configuration getters do
@@ -59,6 +69,9 @@ shared metadata remain in `globalindex`; shared search plans and splits remain i
`table/source`. Python index file codecs, compression and reader/writer helpers
live in `execution/python/globalindex`. Vector-index stream ownership and I/O
callbacks are shared by both backends through `execution/vindex_input.py`.
+Public streaming scan names resolve to the original classes on demand, without
+loading the other backend. Python 3.6 retains eager exports because it does not
+support module-level lazy attribute lookup.
Public Format, Object and System table classes keep their existing paths and
delegate file enumeration, decoding and filtering to Python read execution.
Native system-table planning uses `execution/native/read/system_table.py`.
@@ -73,6 +86,11 @@ not require or initialize it.
Python schema evolution uses Python cast rules and builds Arrow read schemas
directly, including runtime child nullability. It does not select a cast engine
based on whether the Native extension is installed.
+The shared Arrow write validator also checks nested nullability directly in
+Python. It never probes the Native extension; visible NULLs in required children
+are rejected, while values hidden by NULL parents retain their existing meaning.
+Table metadata and FileIO classes load Python row-key extractors and BLOB writers
+only when their corresponding Python operation is invoked.
Ray and Daft input grouping share bucket arithmetic, binary-row size estimation
and fixed/dynamic hash routing in `write/bucket_planning.py` and
@@ -84,6 +102,9 @@ requires a complete HASH index for existing data buckets. Python coordinators
use their Python metadata loader. Selection finishes before assigning input
keys. Python's Arrow grouping capability probe runs only when its grouping
implementation is used.
+The public postpone planner uses this shared input planning directly. Supplied
+bucket metadata avoids loading either backend; its Python metadata loader is
+imported only when existing bucket layouts are requested.
## Selection and failure boundaries
@@ -93,6 +114,12 @@ a Native committer. Transportable `Plan`, `Split` and `CommitMessage` contracts
connect these stages.
`dispatch.py` owns preparation fallbacks for writers, committers and updates.
+The writer coordinator in `write_dispatch.py` holds the selected executor and
+any safe preparation fallback. Native writers only adapt inputs/results and
+invoke Rust; they neither construct nor retain a Python writer. Update and index
+coordination likewise lives outside both backends. Python index builders own
+their construction methods; the public index builder no longer exposes private
+Python writer hooks, and Ray workers receive the Python executor directly.
MERGE selects Python only for an explicit unsupported result, because preparing
it may consume user input. Native read/scan adapters keep their existing
capability checks and preparation boundaries. An empty successful result is distinct from an
@@ -104,12 +131,22 @@ publication calls run outside preparation fallback handlers. A failure may occur
after a write or publication succeeded, so execution is never replayed through
another implementation and prepared files are preserved.
-Native search keeps a lazy Python factory for transported plans or Ray split
-workers. Creating a Native scan or reader, executing its Native plan, and
+The shared search coordinator keeps a lazy Python factory for transported plans
+or Ray split workers. Native search adapters contain only Rust handles and plan
+metadata; they do not construct or retain a Python executor. Creating a Native
+scan or reader, executing its Native plan, and
serializing it do not construct that Python executor. Search execution errors
propagate without a second search through Python. Explicit split reads and Ray
split workers select their Python factory when invoked, not when their methods
or optional attributes are inspected.
+Readers capture query vectors and options at construction for both the live
+Native plan and a later transported-plan read.
+
+Disabling Native options does not import Native adapters. Fork safety checks in
+shared error handling inspect only an already loaded SDK, so Python streaming
+prefetch failures cannot load Native execution. Independent stage options still
+apply: a Python index builder may use a Native reader selected through the public
+read API without importing that reader's implementation itself.
## Tests
@@ -120,3 +157,6 @@ fork errors, public updater types and prevention of write replay.
The lazy read/search and execution lifecycle tests also reject Python executor
construction on Native paths. Global index boundary tests verify those imports
in fresh processes and preserve the exported public index contracts.
+Strict import guards also exercise complete Python write/update/commit/index
+workflows and Native read/scan/stream preparation, rejecting imports from the
+other backend rather than only checking constructor mocks.
diff --git a/paimon-python/pypaimon/execution/dispatch.py b/paimon-python/pypaimon/execution/dispatch.py
index 7386fd087d..bf79eaf25e 100644
--- a/paimon-python/pypaimon/execution/dispatch.py
+++ b/paimon-python/pypaimon/execution/dispatch.py
@@ -23,13 +23,23 @@ The independent plan/read/write/commit options keep their existing meaning.
"""
import logging
+import sys
logger = logging.getLogger(__name__)
def propagate_fork_safety_error(error):
- from pypaimon.execution.native.runtime import _raise_if_native_fork_safety_error
- _raise_if_native_fork_safety_error(error)
+ """Recognize an already-loaded SDK error without loading another backend."""
+ runtime = sys.modules.get('pypaimon_rust')
+ error_type = getattr(runtime, 'ForkSafetyError', None)
+ if error_type is None:
+ return
+ current, seen = error, set()
+ while current is not None and id(current) not in seen:
+ seen.add(id(current))
+ if isinstance(current, error_type):
+ raise current
+ current = current.__cause__ or current.__context__
def try_prepare_native(factory, *args, **kwargs):
@@ -65,7 +75,8 @@ def create_write(table, commit_user, static_partition=None, stream=False,
restore_snapshot_id=restore_snapshot_id, fixed_bucket=fixed_bucket, bucket_plan=bucket_plan)
native = try_prepare_native(prepare)
if native is not None:
- return native
+ from pypaimon.execution.write_dispatch import NativeTableWrite
+ return NativeTableWrite.from_native(native)
return create_python_write(
table, commit_user, static_partition, stream, restore_snapshot_id, fixed_bucket, bucket_plan)
@@ -119,6 +130,8 @@ def prepare_native_commit(owner, messages, *, batch=False):
def prepare_native_write_operation(operation, table, commit_user, *args, **kwargs):
+ if not table.options.native_write_enabled():
+ return None
names = {
'update': 'create_native_update',
'update_by_row_id': 'create_native_update_by_row_id',
@@ -142,13 +155,6 @@ def prepare_native_write_operation(operation, table, commit_user, *args, **kwarg
return try_prepare_native(prepare)
-def try_native_predicate_row_ids(scan_table, predicate, splits):
- def prepare():
- from pypaimon.execution.native.write.native_update import native_predicate_row_ids
- return native_predicate_row_ids(scan_table, predicate, splits)
- return try_prepare_native(prepare)
-
-
def prepare_native_merge_into(*args, **kwargs):
# MERGE preparation can consume user input. Preserve its existing boundary:
# only an explicit unsupported result selects Python; failures propagate.
diff --git a/paimon-python/pypaimon/execution/index_dispatch.py b/paimon-python/pypaimon/execution/index_dispatch.py
new file mode 100644
index 0000000000..2484e91cc2
--- /dev/null
+++ b/paimon-python/pypaimon/execution/index_dispatch.py
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Select an index executor before building files or deletion messages."""
+
+
+def build_global_index(request, partition_filter, *, execution="local", concurrency=None, ray_remote_args=None):
+ if execution == "local" and request._table.options.native_write_enabled():
+ from pypaimon.execution.native.write.native_index_build import build_native_global_index
+ messages = build_native_global_index(request, partition_filter)
+ if messages is not None:
+ return messages
+ from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
+ return PythonGlobalIndexBuilder(request).build(
+ partition_filter, execution=execution, concurrency=concurrency,
+ ray_remote_args=ray_remote_args)
+
+
+def build_global_index_drop(request, partition_filter):
+ if request._table.options.native_write_enabled():
+ from pypaimon.execution.native.write.native_index_drop import build_native_global_index_drop
+ messages = build_native_global_index_drop(request, partition_filter)
+ if messages is not None:
+ return messages
+ from pypaimon.execution.python.globalindex.drop_global_index import build_global_index_drop as execute
+ return execute(request, partition_filter)
diff --git a/paimon-python/pypaimon/execution/native/read/native_search_plan.py b/paimon-python/pypaimon/execution/native/read/native_search_plan.py
index ddb3eec8a3..e55f812243 100644
--- a/paimon-python/pypaimon/execution/native/read/native_search_plan.py
+++ b/paimon-python/pypaimon/execution/native/read/native_search_plan.py
@@ -17,7 +17,6 @@
"""Native search plans with ordinary, transportable Python source metadata."""
-from copy import copy
import hashlib
import json
import pickle
@@ -127,14 +126,14 @@ def _result(table, value, primary_key):
if primary_key else DictBasedScoredIndexResult(value.row_ids()))
-class _NativeScan:
- def __init__(self, table, scan, full_text, primary_key, python_factory, source_modes):
+class NativeSearchScan:
+ """Convert Rust scan metadata without selecting another executor."""
+
+ def __init__(self, table, scan, full_text, primary_key, source_modes):
self._table = table
self._scan = scan
self._full_text = full_text
self._primary_key = primary_key
- self._python_factory = python_factory
- self._python_executor = None
self._source_modes = source_modes
def __getstate__(self):
@@ -142,74 +141,45 @@ class _NativeScan:
state['_scan'] = None
return state
+ def available(self):
+ return self._scan is not None
+
def scan(self):
- if self._scan is None:
- if self._python_executor is None:
- self._python_executor = self._python_factory()
- return self._python_executor.scan()
return _plan_from_native(self._table, self._scan.scan(), self._full_text, self._primary_key, self._source_modes)
-class _NativeRead:
- def __init__(self, table, native, python_factory, primary_key, source_modes):
+class NativeSearchRead:
+ """Read a compatible live Rust plan; transport fallback belongs to dispatch."""
+
+ def __init__(self, table, native, primary_key, source_modes):
self._table = table
self._native = native
- self._python_factory = python_factory
- self._python_executor = None
self._primary_key = primary_key
self._source_modes = source_modes
self._table_options = dict(table.options.options.to_map())
- def _python_reader(self):
- if self._python_executor is None:
- self._python_executor = self._python_factory()
- return self._python_executor
-
def __getstate__(self):
state = self.__dict__.copy()
state['_native'] = None
return state
def read_plan(self, plan):
- native = self._native_plan(plan)
- if native is not None and self._native is not None:
- # Once execution starts, propagate errors without a second search.
- return _result(self._table, self._native.read_plan(native), self._primary_key)
- reader = copy(self._python_reader())
- if plan.snapshot() is not None:
- reader._table = reader._table._copy_with_snapshot(plan.snapshot())
- return reader.read_plan(plan)
+ return _result(self._table, self._native.read_plan(plan), self._primary_key)
def read_batch_plan(self, plan):
- native = self._native_plan(plan)
- if native is not None and self._native is not None:
- return [_result(self._table, result, False) for result in self._native.read_batch_plan(native)]
- reader = copy(self._python_reader())
- if plan.snapshot() is not None:
- reader._table = reader._table._copy_with_snapshot(plan.snapshot())
- return reader.read_batch_plan(plan)
-
- def _native_plan(self, plan):
+ return [_result(self._table, result, False)
+ for result in self._native.read_batch_plan(plan)]
+
+ def resolve_plan(self, plan):
# Search modes choose source work at scan time. A new reader may use
# another query or limit, but must still consume these captured splits.
- if (isinstance(plan, _NativePlan) and plan._source_modes == self._source_modes
+ if (self._native is not None and isinstance(plan, _NativePlan)
+ and plan._source_modes == self._source_modes
and plan._table_options == self._table_options):
return plan._unchanged_native_plan()
return None
-class _NativeSplitRead(_NativeRead):
- def read(self, splits):
- # Explicit split reads select Python only when invoked. Attribute probes
- # and method inspection must not construct a fallback executor.
- return self._python_reader().read(splits)
-
-
-class _NativeBatchRead(_NativeRead):
- def read_batch(self, splits):
- return self._python_reader().read_batch(splits)
-
-
def _source_modes(builder, full_text=False):
options = dict(builder._table.options.options.to_map())
options.update(getattr(builder, '_options', {}))
@@ -218,47 +188,40 @@ def _source_modes(builder, full_text=False):
(core.vector_index_search_mode(), core.scalar_index_search_mode()))
-def python_search_reader(reader):
- """Keep the existing Ray split workers independent of a live native reader."""
- return reader._python_reader() if isinstance(reader, _NativeRead) else reader
-
-
-def native_vector_scan(builder, python_factory, batch=False):
+def native_vector_scan(builder, batch=False):
from pypaimon.execution.native.read.native_vector_search import _native_vector_builder
native = _native_vector_builder(builder, batch)
- return python_factory() if native is None else _NativeScan(
- builder._table, native.new_vector_search_scan(), False, builder._is_primary_key_vector_search(), python_factory,
- _source_modes(builder))
+ return None if native is None else NativeSearchScan(
+ builder._table, native.new_vector_search_scan(), False,
+ builder._is_primary_key_vector_search(), _source_modes(builder))
-def native_vector_read(builder, python_factory, batch=False):
+def native_vector_read(builder, batch=False):
from pypaimon.execution.native.read.native_vector_search import _native_vector_builder
native = _native_vector_builder(builder, batch)
if native is None:
- return python_factory()
+ return None
if batch:
native.with_query_vectors(builder._query_vectors)
reader = native.new_batch_vector_search_read()
else:
native.with_query_vector(builder._query_vector)
reader = native.new_vector_search_read()
- read_class = _NativeBatchRead if batch else _NativeSplitRead
- return read_class(
- builder._table, reader, python_factory, builder._is_primary_key_vector_search(), _source_modes(builder))
+ return NativeSearchRead(
+ builder._table, reader, builder._is_primary_key_vector_search(), _source_modes(builder))
-def native_full_text_scan(builder, python_factory):
+def native_full_text_scan(builder):
from pypaimon.execution.native.read.native_full_text_search import _native_full_text_builder
native = _native_full_text_builder(builder, require_limit=False)
- return python_factory() if native is None else _NativeScan(
+ return None if native is None else NativeSearchScan(
builder._table, native.new_full_text_scan(), True,
- builder._primary_key_full_text_definition() is not None, python_factory, _source_modes(builder, True))
+ builder._primary_key_full_text_definition() is not None, _source_modes(builder, True))
-def native_full_text_read(builder, python_factory):
+def native_full_text_read(builder):
from pypaimon.execution.native.read.native_full_text_search import _native_full_text_builder
native = _native_full_text_builder(builder)
- return python_factory() if native is None else _NativeSplitRead(
- builder._table, native.new_full_text_read(), python_factory,
- builder._primary_key_full_text_definition() is not None,
- _source_modes(builder, True))
+ return None if native is None else NativeSearchRead(
+ builder._table, native.new_full_text_read(),
+ builder._primary_key_full_text_definition() is not None, _source_modes(builder, True))
diff --git a/paimon-python/pypaimon/execution/native/read/table_read.py b/paimon-python/pypaimon/execution/native/read/table_read.py
index b5590e18d4..652f7f277b 100644
--- a/paimon-python/pypaimon/execution/native/read/table_read.py
+++ b/paimon-python/pypaimon/execution/native/read/table_read.py
@@ -23,6 +23,7 @@ from typing import List, Optional
import pyarrow
from pypaimon.casting.schema_evolution import read_arrow_schema
+from pypaimon.execution.read_support import close_read_source
from pypaimon.common.options.core_options import CoreOptions
from pypaimon.common.predicate import Predicate
from pypaimon.execution.native import runtime
@@ -31,7 +32,7 @@ from pypaimon.execution.native.read import builder as native_builder
from pypaimon.read.read_type import OutputProjection, adapter_output_paths, extract_array, output_schema
from pypaimon.read.split import Split
from pypaimon.read.variant_read_type import has_variant_extractions
-from pypaimon.schema.data_types import DataField, is_blob_file_field
+from pypaimon.schema.data_types import DataField
logger = logging.getLogger(__name__)
_NATIVE_READ_FILE_FORMATS = frozenset({
@@ -182,6 +183,7 @@ class NativeTableReadMixin:
return True
def _convert_native_batches(self, batches, schema):
+ failure = None
try:
for batch in batches:
if batch.num_rows == 0:
@@ -189,10 +191,11 @@ class NativeTableReadMixin:
batch = self._prepare_native_batch(batch, schema)
batch = self._project_batch_to_output(batch)
yield self._try_to_pad_batch_by_schema(batch, schema)
+ except BaseException as error:
+ failure = error
+ raise
finally:
- close = getattr(batches, 'close', None)
- if close is not None:
- close()
+ close_read_source(batches, failure)
def _prepare_native_batch(self, batch, schema):
if self.include_row_kind and 'rowkind' in batch.schema.names:
@@ -232,23 +235,6 @@ class NativeTableReadMixin:
from pypaimon.execution.native.table import _native_blob_view_supported
return _native_blob_view_supported(self.table, (field.name for field in self._scan_read_type))
- def _native_resolved_blob_fields(self) -> set:
- """BLOB fields Rust resolves before emitting a batch."""
- options = self.table.options
- if options.blob_as_descriptor():
- return set()
- if self.table.is_primary_key_table:
- return {field.name for field in self._scan_read_type if is_blob_file_field(field)}
- inline_fields = (
- options.blob_descriptor_fields()
- | options.blob_view_fields()
- )
- read_names = {
- field.name
- for field in getattr(self, '_scan_read_type', self._adapter_read_type)
- }
- return inline_fields & read_names
-
def _try_native_rows(self, splits):
from copy import copy
from pypaimon.execution.native.read.row_iterator import native_rows
@@ -265,11 +251,13 @@ class NativeTableReadMixin:
return None
def rows():
+ failure = None
try:
yield from native_rows(batches, self.table, fields)
+ except BaseException as error:
+ failure = error
+ raise
finally:
- close = getattr(batches, 'close', None)
- if close is not None:
- close()
+ close_read_source(batches, failure)
return rows()
diff --git a/paimon-python/pypaimon/execution/native/runtime.py b/paimon-python/pypaimon/execution/native/runtime.py
index 6e4d0460ff..f4b469d190 100644
--- a/paimon-python/pypaimon/execution/native/runtime.py
+++ b/paimon-python/pypaimon/execution/native/runtime.py
@@ -21,18 +21,9 @@ from importlib import import_module
def _raise_if_native_fork_safety_error(error):
- """Propagate native fork errors instead of using inherited SDK state."""
- if not native_runtime_available():
- return
- from pypaimon_rust import ForkSafetyError
-
- current = error
- seen = set()
- while current is not None and id(current) not in seen:
- seen.add(id(current))
- if isinstance(current, ForkSafetyError):
- raise current
- current = current.__cause__ or current.__context__
+ """Propagate native fork errors without probing or initializing the SDK."""
+ from pypaimon.execution.dispatch import propagate_fork_safety_error
+ propagate_fork_safety_error(error)
def native_runtime_available() -> bool:
diff --git a/paimon-python/pypaimon/execution/native/write/native_update.py b/paimon-python/pypaimon/execution/native/write/native_update.py
index 0d21285428..9a55461c62 100644
--- a/paimon-python/pypaimon/execution/native/write/native_update.py
+++ b/paimon-python/pypaimon/execution/native/write/native_update.py
@@ -133,7 +133,7 @@ class NativeShardTableUpdator:
self.read = read
def arrow_reader(self):
- from pypaimon.read.table_read import _ClosableArrowBatchReader
+ from pypaimon.execution.read_support import _ClosableArrowBatchReader
schema = self.read._output_arrow_schema()
batches = self.read._convert_native_batches(self.writer.arrow_reader(), schema)
return _ClosableArrowBatchReader(pa.ipc.RecordBatchReader.from_batches(schema, batches), batches)
@@ -405,24 +405,6 @@ def create_native_predicate_update(table, commit_user, predicate, columns=None):
return NativePredicateTableUpdate(table, writer, native_predicate)
-def native_predicate_row_ids(scan_table, predicate, splits):
- """Match a batch delete predicate in Rust and return its row IDs."""
- from pypaimon.execution.native.read.table_read import _prepare_native_read
- from pypaimon.execution.native.read.split_adapter import native_split_from_python
- if not native_runtime_available():
- return None
- reader = _prepare_native_read(
- scan_table, predicate=predicate, projection=['_ROW_ID']
- )
- row_ids = []
- ranges_use_row_ids = scan_table.options.row_tracking_enabled() or scan_table.options.data_evolution_enabled()
- for split in splits:
- for batch in reader([native_split_from_python(
- split, ranges_use_row_ids=ranges_use_row_ids)]):
- row_ids.extend(batch.column('_ROW_ID').to_pylist())
- return row_ids
-
-
def create_native_predicate_delete(table, commit_user, predicate):
"""Delegate matching and partition metadata deletion to the core updater."""
from pypaimon.execution.native.read.builder import _predicate_to_native
diff --git a/paimon-python/pypaimon/execution/native/write/native_write.py b/paimon-python/pypaimon/execution/native/write/native_write.py
index 11a3d86321..801f76e8b6 100644
--- a/paimon-python/pypaimon/execution/native/write/native_write.py
+++ b/paimon-python/pypaimon/execution/native/write/native_write.py
@@ -71,12 +71,7 @@ def create_native_write(table, commit_user, static_partition=None, stream=False,
class NativeTableWrite:
- """Use Rust for Arrow batches while retaining PyPaimon's commit-message API.
-
- Advanced Python writer methods switch to the Python writer before the first
- native data write. Once Rust has written data, switching would split one
- logical write across two writers, so it is rejected.
- """
+ """Adapt Python inputs and commit messages to one Rust writer."""
def __init__(self, table, commit_user, static_partition, stream, native_writer,
restore_snapshot_id=None):
@@ -85,7 +80,6 @@ class NativeTableWrite:
self.static_partition = static_partition
self.stream = stream
self._native_writer = native_writer
- self._python_writer = None
self._written = False
self._batch_committed = False
self._table_schema = PyarrowFieldParser.from_paimon_schema(table.table_schema.fields)
@@ -101,98 +95,20 @@ class NativeTableWrite:
self._blob_rows = NativeBlobRows()
self._native_writer.with_blob_uri_reader_factory(self._blob_rows)
- def _switch_to_python(self):
- from pypaimon.execution.dispatch import switch_writer_to_python
- return switch_writer_to_python(self)
-
- def _new_python_writer(self):
- from pypaimon.execution.dispatch import create_python_write
- writer = create_python_write(
- self.table, self.commit_user, self.static_partition, self.stream,
- restore_snapshot_id=self._restore_snapshot_id)
- return self._configure_python_writer(writer)
-
- def _configure_python_writer(self, writer):
- if not self.stream:
- # Empty batch preparation can leave _written false. Explicit
- # diagnostics must not reset the one-time preparation guard.
- writer.batch_committed = self._batch_committed
- if self._write_cols is not None:
- writer.with_write_type(self._write_cols)
- if self._blob_consumer is not None:
- writer.with_blob_consumer(self._blob_consumer)
- if self._blob_uri_reader_factory is not None:
- writer.with_blob_uri_reader_factory(self._blob_uri_reader_factory)
- return writer
-
- @property
- def file_store_write(self):
- """Explicitly request the Python writer's diagnostic implementation."""
- return self._switch_to_python().file_store_write
-
- @file_store_write.setter
- def file_store_write(self, writer):
- self._switch_to_python().file_store_write = writer
-
- @property
- def row_key_extractor(self):
- """Explicitly request the Python writer's routing implementation."""
- return self._switch_to_python().row_key_extractor
-
- @row_key_extractor.setter
- def row_key_extractor(self, extractor):
- self._switch_to_python().row_key_extractor = extractor
-
- @property
- def table_pyarrow_schema(self):
- return (self._python_writer.table_pyarrow_schema
- if self._python_writer is not None else self._table_schema)
-
- @table_pyarrow_schema.setter
- def table_pyarrow_schema(self, schema):
- self._switch_to_python().table_pyarrow_schema = schema
-
- @property
- def restore_snapshot_id(self):
- return (self._python_writer.restore_snapshot_id
- if self._python_writer is not None else self._restore_snapshot_id)
-
- @restore_snapshot_id.setter
- def restore_snapshot_id(self, snapshot_id):
- self._switch_to_python().restore_snapshot_id = snapshot_id
-
- @property
- def batch_committed(self):
- if self._python_writer is not None:
- return self._python_writer.batch_committed
- if self.stream:
- raise AttributeError('batch_committed')
- return self._batch_committed
-
- @batch_committed.setter
- def batch_committed(self, value):
- self._switch_to_python().batch_committed = value
-
def with_write_type(self, write_cols):
- if self._python_writer is not None:
- return self._python_writer.with_write_type(write_cols)
names = list(write_cols)
if self.table.is_primary_key_table:
if len(names) == len(self._table_schema.names) and set(names) == set(self._table_schema.names):
# Python treats a complete PK write type as the unchanged table
# schema. It does not configure a PK column-subset writer.
return self
- # Core's write type is an append-table operation. Keep the existing
- # Python writer contract for PK setters before any data is staged.
- return self._switch_to_python().with_write_type(names)
+ raise ValueError('Column-subset primary-key writes require the Python executor')
self._native_writer.with_write_type(names)
self._schema = pa.schema([self._table_schema.field(name) for name in names])
self._write_cols = names
return self
def with_blob_consumer(self, blob_consumer):
- if self._python_writer is not None:
- return self._python_writer.with_blob_consumer(blob_consumer)
if blob_consumer is None:
self._configure_native_blob('with_blob_consumer', None)
self._blob_consumer = None
@@ -210,8 +126,6 @@ class NativeTableWrite:
return self
def with_blob_uri_reader_factory(self, uri_reader_factory):
- if self._python_writer is not None:
- return self._python_writer.with_blob_uri_reader_factory(uri_reader_factory)
if self._blob_rows is not None:
if uri_reader_factory is not None and not callable(getattr(uri_reader_factory, 'create', None)):
raise TypeError('URI reader factory must provide callable create')
@@ -238,8 +152,6 @@ class NativeTableWrite:
raise
def write_arrow(self, data, bucket=None):
- if self._python_writer is not None:
- return self._python_writer.write_arrow(data, bucket)
data = self._prepare_native_arrow_data(data)
if isinstance(data, pa.RecordBatch):
return self.write_arrow_batch(data, bucket)
@@ -247,8 +159,6 @@ class NativeTableWrite:
self.write_arrow_batch(batch, bucket)
def write_arrow_batch(self, data, bucket=None):
- if self._python_writer is not None:
- return self._python_writer.write_arrow_batch(data, bucket)
data = self._prepare_native_arrow_data(data)
self._write_native_batch(data, bucket)
@@ -276,34 +186,16 @@ class NativeTableWrite:
return data
def write_pandas(self, dataframe):
- if self._python_writer is not None:
- return self._python_writer.write_pandas(dataframe)
from pypaimon.write.input_adapter import _pandas_write_batch
self.write_arrow_batch(_pandas_write_batch(self.table, dataframe, self._schema, self._table_schema))
- def write_ray(self, dataset, overwrite=False, concurrency=None, ray_remote_args=None,
- hash_fixed_precluster='auto', static_partition=None):
- if self._python_writer is not None:
- return self._python_writer.write_ray(
- dataset, overwrite, concurrency, ray_remote_args, hash_fixed_precluster, static_partition)
- from pypaimon.write.input_adapter import _write_ray
- _write_ray(self, dataset, overwrite, concurrency, ray_remote_args,
- hash_fixed_precluster, static_partition)
-
- def _distributed_write_options(self, *, overwrite=False):
- return {}
-
def roll_before_group_if_needed(self, row_count):
- if self._python_writer is not None:
- return self._python_writer.roll_before_group_if_needed(row_count)
if isinstance(row_count, bool) or not isinstance(row_count, int) or row_count <= 0:
raise ValueError('Group row count must be a positive integer.')
# Python's group boundary only affects packed video writers. Native
# selection excludes video, so ordinary Parquet/BLOB writes do not roll.
def write_row(self, row):
- if self._python_writer is not None:
- return self._python_writer.write_row(row)
values = row_to_named_values(row, self.table.table_schema.fields)
names = self._schema.names
require_columns(values, names, 'write_row')
@@ -341,10 +233,6 @@ class NativeTableWrite:
self._blob_rows.clear()
def prepare_commit(self, commit_identifier=None):
- if self._python_writer is not None:
- if self.stream:
- return self._python_writer.prepare_commit(commit_identifier)
- return self._python_writer.prepare_commit()
if self.stream:
if commit_identifier is None:
raise TypeError('StreamTableWrite.prepare_commit requires an identifier')
@@ -361,22 +249,16 @@ class NativeTableWrite:
return from_native_commit_messages(self.table, messages)
def close(self):
- if self._python_writer is not None:
- self._python_writer.close()
- elif self._native_writer is not None:
+ if self._native_writer is not None:
self._native_writer.close()
self._native_writer = None
if self._blob_rows is not None:
self._blob_rows.clear()
def abort(self):
- if self._python_writer is not None:
- self._python_writer.abort()
- else:
- # Closing Rust cleans only files still owned by its writer.
- # Never delete prepared CommitMessage files: publication can
- # succeed even when its response raises an exception.
- self.close()
+ # Rust cleans only files still owned by its writer. Prepared commit
+ # messages must survive failures whose publication outcome is unknown.
+ self.close()
class NativePostponeFixedBucketTableWrite(NativeTableWrite):
@@ -392,13 +274,3 @@ class NativePostponeFixedBucketTableWrite(NativeTableWrite):
# Automatic planning stages input until all partition sizes are known.
self._defer_blob_input = (bucket_plan is None
and table.options.postpone_default_bucket_num() is None)
-
- def _new_python_writer(self):
- from pypaimon.execution.dispatch import create_python_write
- return self._configure_python_writer(create_python_write(
- self.table, self.commit_user, self.static_partition,
- fixed_bucket=True, bucket_plan=self._bucket_plan))
-
- def _distributed_write_options(self, *, overwrite=False):
- from pypaimon.write.input_adapter import postpone_distributed_write_options
- return postpone_distributed_write_options(self.table, self._bucket_plan, overwrite)
diff --git a/paimon-python/pypaimon/execution/python/globalindex/create_global_index.py b/paimon-python/pypaimon/execution/python/globalindex/create_global_index.py
index c98c9898eb..8fc6ad2025 100644
--- a/paimon-python/pypaimon/execution/python/globalindex/create_global_index.py
+++ b/paimon-python/pypaimon/execution/python/globalindex/create_global_index.py
@@ -61,60 +61,67 @@ _GENERIC_INDEX_IDENTIFIERS = tuple(VINDEX_IDENTIFIERS) + (FULL_TEXT_IDENTIFIER,)
_SORTED_INDEX_RECORDS_PER_FILE_FLOATING = 1.2
-def build_global_index(self, partition_filter, *, execution="local", concurrency=None, ray_remote_args=None):
- read_builder = self._table.new_read_builder()
- if partition_filter is not None:
- read_builder = read_builder.with_partition_filter(partition_filter)
-
- scan = read_builder.new_scan()
- plan = scan.plan_all_files()
- splits = plan.splits()
- if not splits:
- return []
-
- index_field = self._table.field_dict[self._index_columns[0]]
- snapshot = self._snapshot_for_plan(plan)
- unindexed_ranges = _unindexed_row_ranges(
- self._table,
- snapshot,
- partition_filter,
- index_field.id,
- self._index_type,
- )
- if not unindexed_ranges:
- return []
+class PythonGlobalIndexBuilder:
+ """Algorithms invoked only by the Python index build path."""
- if self._index_type in _GENERIC_INDEX_IDENTIFIERS:
- splits = _filter_non_indexable_splits(
- self._table, splits, self._index_columns)
+ def __init__(self, request):
+ self._table = request._table
+ self._index_columns = list(request._index_columns)
+ self._index_type = request._index_type
+ self._options = request._options
+ self._core_options = request._core_options
+ self._user_options = dict(request._user_options)
+
+ def build(self, partition_filter, *, execution="local", concurrency=None, ray_remote_args=None):
+ read_builder = self._table.new_read_builder()
+ if partition_filter is not None:
+ read_builder = read_builder.with_partition_filter(partition_filter)
+
+ scan = read_builder.new_scan()
+ plan = scan.plan_all_files()
+ splits = plan.splits()
if not splits:
return []
- read_type = [index_field, SpecialFields.ROW_ID]
- from pypaimon.read.table_read import TableRead
-
- table_read = TableRead(
- table=self._table,
- predicate=None,
- read_type=read_type,
- )
- index_path_factory = self._table.path_factory().global_index_path_factory()
- index_path = index_path_factory.global_index_root_path()
+ index_field = self._table.field_dict[self._index_columns[0]]
+ snapshot = self._snapshot_for_plan(plan)
+ unindexed_ranges = _unindexed_row_ranges(
+ self._table,
+ snapshot,
+ partition_filter,
+ index_field.id,
+ self._index_type,
+ )
+ if not unindexed_ranges:
+ return []
- if self._index_type in _SORTED_INDEX_IDENTIFIERS:
- return self._build_sorted_index(
- splits, unindexed_ranges, index_field, table_read, index_path)
- if execution == "ray":
- from pypaimon.ray.vector_index_build import build_vector_index
- return build_vector_index(
- self, splits, unindexed_ranges, index_field, table_read,
- index_path, concurrency, ray_remote_args)
- return self._build_generic_index(
- splits, unindexed_ranges, index_field, table_read, index_path)
+ if self._index_type in _GENERIC_INDEX_IDENTIFIERS:
+ splits = _filter_non_indexable_splits(
+ self._table, splits, self._index_columns)
+ if not splits:
+ return []
+ read_type = [index_field, SpecialFields.ROW_ID]
+ from pypaimon.read.table_read import TableRead
-class PythonGlobalIndexBuilder:
- """Algorithms invoked only by the Python index build path."""
+ table_read = TableRead(
+ table=self._table,
+ predicate=None,
+ read_type=read_type,
+ )
+ index_path_factory = self._table.path_factory().global_index_path_factory()
+ index_path = index_path_factory.global_index_root_path()
+
+ if self._index_type in _SORTED_INDEX_IDENTIFIERS:
+ return self._build_sorted_index(
+ splits, unindexed_ranges, index_field, table_read, index_path)
+ if execution == "ray":
+ from pypaimon.execution.python.globalindex.ray_build import build_vector_index
+ return build_vector_index(
+ self, splits, unindexed_ranges, index_field, table_read,
+ index_path, concurrency, ray_remote_args)
+ return self._build_generic_index(
+ splits, unindexed_ranges, index_field, table_read, index_path)
def _snapshot_for_plan(self, plan):
snapshot_id = getattr(plan, "snapshot_id", None)
@@ -292,7 +299,7 @@ class PythonGlobalIndexBuilder:
def _build_generic_shard(
self, index_split, index_range, index_field, table_read, index_path: str, file_name=None
) -> Optional[CommitMessage]:
- from pypaimon.read.table_read import _ClosableArrowBatchReader
+ from pypaimon.execution.read_support import _ClosableArrowBatchReader
writer = None
try:
diff --git a/paimon-python/pypaimon/ray/vector_index_build.py b/paimon-python/pypaimon/execution/python/globalindex/ray_build.py
similarity index 100%
rename from paimon-python/pypaimon/ray/vector_index_build.py
rename to paimon-python/pypaimon/execution/python/globalindex/ray_build.py
diff --git a/paimon-python/pypaimon/execution/python/read/reader/concat_batch_reader.py b/paimon-python/pypaimon/execution/python/read/reader/concat_batch_reader.py
index 30aa021004..7bf51bdb6e 100644
--- a/paimon-python/pypaimon/execution/python/read/reader/concat_batch_reader.py
+++ b/paimon-python/pypaimon/execution/python/read/reader/concat_batch_reader.py
@@ -16,7 +16,6 @@
# under the License.
import collections
-import struct
from typing import Callable, Dict, List, Optional, Tuple
import pyarrow as pa
@@ -29,65 +28,7 @@ from pypaimon.schema.data_types import DataField, PyarrowFieldParser
from pypaimon.table.row.blob import Blob
from pypaimon.utils.range import Range
-_MAX_ARROW_OFFSET = (1 << 31) - 1
-
-
-def _offset_bounds(array, width=4):
- offsets = array.buffers()[1]
- if offsets is None:
- return 0, 0
- value_format = "=i" if width == 4 else "=q"
- start = struct.unpack_from(
- value_format, offsets, array.offset * width)[0]
- end = struct.unpack_from(
- value_format, offsets, (array.offset + len(array)) * width)[0]
- return start, end
-
-
-def _collect_offset_usage(array, path, usage):
- data_type = array.type
- if pa.types.is_string(data_type) or pa.types.is_binary(data_type):
- start, end = _offset_bounds(array)
- usage[path] = end - start
- return
-
- if pa.types.is_list(data_type) or pa.types.is_map(data_type):
- start, end = _offset_bounds(array)
- usage[path] = end - start
- if pa.types.is_map(data_type):
- _collect_offset_usage(
- array.keys.slice(start, end - start), path + ("key",), usage)
- _collect_offset_usage(
- array.items.slice(start, end - start), path + ("item",), usage)
- else:
- _collect_offset_usage(
- array.values.slice(start, end - start), path + ("item",), usage)
- return
-
- if pa.types.is_large_list(data_type):
- start, end = _offset_bounds(array, 8)
- _collect_offset_usage(
- array.values.slice(start, end - start), path + ("item",), usage)
- return
-
- if pa.types.is_fixed_size_list(data_type):
- size = data_type.list_size
- start = array.offset * size
- _collect_offset_usage(
- array.values.slice(start, len(array) * size), path + ("item",), usage)
- return
-
- if pa.types.is_struct(data_type):
- for index in range(data_type.num_fields):
- _collect_offset_usage(
- array.field(index), path + ("field", index), usage)
-
-
-def _batch_offset_usage(batch):
- usage = {}
- for index in range(batch.num_columns):
- _collect_offset_usage(batch.column(index), (index,), usage)
- return usage
+from pypaimon.execution.read_support import _MAX_ARROW_OFFSET, _batch_offset_usage
class _BlobFileState:
diff --git a/paimon-python/pypaimon/execution/python/read/search/global_index_row_filter.py b/paimon-python/pypaimon/execution/python/read/search/global_index_row_filter.py
index b357859e87..eb28f646a6 100644
--- a/paimon-python/pypaimon/execution/python/read/search/global_index_row_filter.py
+++ b/paimon-python/pypaimon/execution/python/read/search/global_index_row_filter.py
@@ -18,7 +18,7 @@
"""Exact row filtering for data-evolution search candidates."""
from pypaimon.common.like_optimization import try_optimize_like
-from pypaimon.read.table_read import _ClosableArrowBatchReader
+from pypaimon.execution.read_support import _ClosableArrowBatchReader
from pypaimon.table.special_fields import SpecialFields
from pypaimon.execution.python.read.search.global_index_live_row_filter import table_at_snapshot
from pypaimon.utils.roaring_bitmap import RoaringBitmap64
diff --git a/paimon-python/pypaimon/execution/python/read/search/hybrid_search.py b/paimon-python/pypaimon/execution/python/read/search/hybrid_search.py
index b9af5c8b3a..55ead01bd6 100644
--- a/paimon-python/pypaimon/execution/python/read/search/hybrid_search.py
+++ b/paimon-python/pypaimon/execution/python/read/search/hybrid_search.py
@@ -27,10 +27,14 @@ from pypaimon.globalindex.global_index_result import GlobalIndexResult
from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult, ScoredGlobalIndexResult
from pypaimon.table.source.hybrid_search_builder import (
HybridSearchRouteBuilder, HybridSearchRouteResult,
- WEIGHTED_SCORE_RANKER, MRR_RANKER, _MAX_ROUTE_WORKERS, _RRF_K,
+ WEIGHTED_SCORE_RANKER, MRR_RANKER,
)
+_MAX_ROUTE_WORKERS = 4
+_RRF_K = 60.0
+
+
def execute_local(self) -> ScoredGlobalIndexResult:
"""Execute hybrid index search locally."""
route_builders = self.route_builders()
@@ -110,10 +114,10 @@ def rank(
if not route_result.result.results().is_empty()
]
if self._ranker == WEIGHTED_SCORE_RANKER:
- return self._weighted_score(non_empty)
+ return _weighted_score(self, non_empty)
if self._ranker == MRR_RANKER:
- return self._mrr(non_empty)
- return self._rrf(non_empty)
+ return _mrr(self, non_empty)
+ return _rrf(self, non_empty)
def _rrf(self, route_results):
diff --git a/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py b/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py
index 8464b09888..20e71a3d7a 100644
--- a/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py
+++ b/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py
@@ -623,7 +623,7 @@ class AbstractVectorSearchReadImpl:
return [_scored_result(heap) for heap in heaps]
def _score_refine_splits(self, table_read, splits, queries_by_row, query_vectors, metric):
- from pypaimon.read.table_read import _ClosableArrowBatchReader
+ from pypaimon.execution.read_support import _ClosableArrowBatchReader
heaps = [[] for _ in query_vectors]
reader, batches = table_read._new_arrow_batch_reader(splits)
@@ -792,7 +792,7 @@ class BatchVectorSearchReadImpl(AbstractVectorSearchReadImpl,
return [_scored_result(heap) for heap in heaps]
def _score_raw_splits(self, table_read, splits, metric):
- from pypaimon.read.table_read import _ClosableArrowBatchReader
+ from pypaimon.execution.read_support import _ClosableArrowBatchReader
reader, batches = table_read._new_arrow_batch_reader(splits)
# Close the underlying iterator as well if scoring fails mid-batch.
@@ -832,13 +832,14 @@ def _create_vector_reader(index_type, file_io, index_path, index_io_meta_list, o
LuminaVectorGlobalIndexReader,
)
from pypaimon.globalindex.vindex_options import VINDEX_IDENTIFIERS
- from pypaimon.globalindex.vindex_reader import VindexVectorGlobalIndexReader
+ from pypaimon.execution.python.globalindex.vindex.vindex_vector_global_index_reader import (
+ PythonVindexVectorGlobalIndexReader)
if index_type in LUMINA_IDENTIFIERS:
return LuminaVectorGlobalIndexReader(
file_io, index_path, index_io_meta_list, options
)
if index_type in VINDEX_IDENTIFIERS:
- return VindexVectorGlobalIndexReader(
+ return PythonVindexVectorGlobalIndexReader(
file_io, index_path, index_io_meta_list, options
)
raise ValueError("Unsupported vector index type: '%s'" % index_type)
diff --git a/paimon-python/pypaimon/execution/python/read/table_read.py b/paimon-python/pypaimon/execution/python/read/table_read.py
index bf964ab019..92528cea49 100644
--- a/paimon-python/pypaimon/execution/python/read/table_read.py
+++ b/paimon-python/pypaimon/execution/python/read/table_read.py
@@ -18,7 +18,7 @@
import logging
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Iterator, List, Optional
+from typing import Dict, Iterator, List, Optional
import pyarrow
@@ -45,6 +45,7 @@ from pypaimon.schema.data_types import DataField, MapType, is_map_blob_type
from pypaimon.table.row.offset_row import OffsetRow
from pypaimon.read.constants import ROW_KIND_COLUMN, _DEFAULT_PREDICATE
+from pypaimon.utils.arrow_utils import zero_column_batch
logger = logging.getLogger(__name__)
@@ -85,6 +86,46 @@ class _RemainingRows:
class PythonTableReadMixin:
+ @staticmethod
+ def _emit_overflow_safe_batches(
+ pydict: Dict[str, list],
+ row_count: int,
+ schema: pyarrow.Schema,
+ ) -> Iterator[pyarrow.RecordBatch]:
+ """Yield RecordBatches from a ``{column: list}`` dict, keeping every column
+ within pyarrow's per-column size limit.
+
+ A STRING/BYTES column maps to ``pyarrow.string()``/``binary()`` which use
+ 32-bit offsets (max 2GB per column). A chunk of large values can overflow
+ that, in which case ``pyarrow.array()`` returns a ``ChunkedArray`` that a
+ single ``RecordBatch`` cannot hold. When that happens we split the rows in
+ half and recurse so every emitted batch keeps each column under the limit.
+ """
+ if not schema:
+ yield zero_column_batch(row_count, schema.metadata)
+ return
+ arrays = []
+ for field in schema:
+ arr = pyarrow.array(pydict[field.name], type=field.type)
+ if isinstance(arr, pyarrow.ChunkedArray):
+ # A column overflowed the 2GB limit and was auto-chunked; split.
+ break
+ arrays.append(arr)
+ else:
+ yield pyarrow.RecordBatch.from_arrays(arrays, schema=schema)
+ return
+
+ if row_count <= 1:
+ raise ValueError(
+ "A single row exceeds the 2GB per-column limit of "
+ "pyarrow.string()/binary(); cannot build a RecordBatch for this row."
+ )
+ mid = row_count // 2
+ left = {name: column[:mid] for name, column in pydict.items()}
+ right = {name: column[mid:] for name, column in pydict.items()}
+ yield from PythonTableReadMixin._emit_overflow_safe_batches(left, mid, schema)
+ yield from PythonTableReadMixin._emit_overflow_safe_batches(right, row_count - mid, schema)
+
def _python_row_iterator(self, splits, limit):
self._check_python_merge_supported()
count = 0
@@ -347,7 +388,7 @@ class PythonTableReadMixin:
pydict = {ROW_KIND_COLUMN: row_kinds}
for name, column in zip(data_field_names, columns_data):
pydict[name] = list(column)
- yield from self._emit_overflow_safe_batches(pydict, len(row_tuples), schema)
+ yield from PythonTableReadMixin._emit_overflow_safe_batches(pydict, len(row_tuples), schema)
def _create_split_read(self, split: Split, blob_parallelism: int = 1,
read_type=None, limit: Optional[int] = None,
diff --git a/paimon-python/pypaimon/execution/python/read/table_scan.py b/paimon-python/pypaimon/execution/python/read/table_scan.py
index dcdfe5d907..ffea17dd5e 100644
--- a/paimon-python/pypaimon/execution/python/read/table_scan.py
+++ b/paimon-python/pypaimon/execution/python/read/table_scan.py
@@ -112,10 +112,10 @@ class PythonTableScanMixin:
return scanner
def _python_plan(self, auth_result):
- return wrap_plan_with_auth(auth_result, self._python_scanner(auth_result).scan())
+ return wrap_plan_with_auth(auth_result, PythonTableScanMixin._python_scanner(self, auth_result).scan())
def _python_plan_all_files(self):
- scanner = self._python_scanner(None)
+ scanner = PythonTableScanMixin._python_scanner(self, None)
# Writer restore/overwrite must see L0 even when batch reads hide it.
skip_level0 = scanner.skip_level0
try:
@@ -125,5 +125,5 @@ class PythonTableScanMixin:
scanner.skip_level0 = skip_level0
def _python_scan_with_stats(self, auth_result):
- plan, stats = self._python_scanner(auth_result).scan_with_stats()
+ plan, stats = PythonTableScanMixin._python_scanner(self, auth_result).scan_with_stats()
return wrap_plan_with_auth(auth_result, plan), stats
diff --git a/paimon-python/pypaimon/execution/python/write/merge_into.py b/paimon-python/pypaimon/execution/python/write/merge_into.py
index c68de13037..fb004fde9d 100644
--- a/paimon-python/pypaimon/execution/python/write/merge_into.py
+++ b/paimon-python/pypaimon/execution/python/write/merge_into.py
@@ -74,7 +74,7 @@ def merge_into(
"""
# Empty-target INSERT skips the target scan, but still needs the same
# unrestricted whole-table grant before any user callback or staging.
- from pypaimon.write.table_update import _authorize_update
+ from pypaimon.write.update_validation import _authorize_update
_authorize_update(target_table)
if any(clause.condition is not None
for clauses in (when_matched, when_not_matched) for clause in clauses):
diff --git a/paimon-python/pypaimon/execution/python/write/postpone_batch_table_write.py b/paimon-python/pypaimon/execution/python/write/postpone_batch_table_write.py
index 6c6f2bb60c..9dea73a7e7 100644
--- a/paimon-python/pypaimon/execution/python/write/postpone_batch_table_write.py
+++ b/paimon-python/pypaimon/execution/python/write/postpone_batch_table_write.py
@@ -22,6 +22,7 @@ from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER
from pypaimon.write.commit_message import CommitMessage
from pypaimon.execution.python.write.file_store_write import PostponeFixedBucketFileStoreWrite
from pypaimon.write.postpone_bucket import PostponeBucketPlanner
+from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
from pypaimon.execution.python.write.row_key_extractor import PostponeFixedBucketRowKeyExtractor
from pypaimon.write.row_utils import (
require_columns,
@@ -54,7 +55,7 @@ class PythonPostponeFixedBucketWriteMixin:
)
self._plan_provided = bucket_plan is not None
self._pending_inputs = []
- super().__init__(table, commit_user, static_partition)
+ PythonTableWriteMixin.__init__(self, table, commit_user, static_partition)
def _create_file_store_write(self, commit_user):
return PostponeFixedBucketFileStoreWrite(self.table, commit_user)
@@ -95,7 +96,7 @@ class PythonPostponeFixedBucketWriteMixin:
raise ValueError('Precomputed bucket writes require HASH_FIXED or HASH_DYNAMIC tables')
table = self._prepare_arrow_data(table)
if not self._buffer_input(table):
- return super().write_arrow(table)
+ return PythonTableWriteMixin.write_arrow(self, table)
self._pending_inputs.extend(
("batch", batch) for batch in table.to_batches())
@@ -104,12 +105,12 @@ class PythonPostponeFixedBucketWriteMixin:
raise ValueError('Precomputed bucket writes require HASH_FIXED or HASH_DYNAMIC tables')
data = self._prepare_arrow_data(data)
if not self._buffer_input(data):
- return super().write_arrow_batch(data)
+ return PythonTableWriteMixin.write_arrow_batch(self, data)
self._pending_inputs.append(("batch", data))
def write_row(self, row):
if self._plan_provided:
- return super().write_row(row)
+ return PythonTableWriteMixin.write_row(self, row)
values_by_name = row_to_named_values(
row, self.table.table_schema.fields)
@@ -127,7 +128,7 @@ class PythonPostponeFixedBucketWriteMixin:
partition = tuple(
values_by_name[key] for key in self.table.partition_keys)
if self._bucket_plan.contains(partition):
- return super().write_row(row)
+ return PythonTableWriteMixin.write_row(self, row)
arrow_row = row_values_to_arrow_table(
values_by_name, self.table.table_schema.fields, column_names)
@@ -160,9 +161,9 @@ class PythonPostponeFixedBucketWriteMixin:
self._pending_inputs = []
for input_type, value in inputs:
if input_type == "batch":
- super().write_arrow_batch(value)
+ PythonTableWriteMixin.write_arrow_batch(self, value)
else:
- super().write_row(value[0])
+ PythonTableWriteMixin.write_row(self, value[0])
def prepare_commit(self) -> List[CommitMessage]:
if self.batch_committed:
@@ -179,8 +180,8 @@ class PythonPostponeFixedBucketWriteMixin:
def close(self):
self._pending_inputs = []
- super().close()
+ PythonTableWriteMixin.close(self)
def abort(self):
self._pending_inputs = []
- super().abort()
+ PythonTableWriteMixin.abort(self)
diff --git a/paimon-python/pypaimon/execution/python/write/table_update.py b/paimon-python/pypaimon/execution/python/write/table_update.py
index 5d1fd06c7b..3bde203300 100644
--- a/paimon-python/pypaimon/execution/python/write/table_update.py
+++ b/paimon-python/pypaimon/execution/python/write/table_update.py
@@ -36,7 +36,6 @@ from pypaimon.manifest.manifest_list_manager import ManifestListManager
from pypaimon.manifest.schema.data_file_meta import DataFileMeta
from pypaimon.execution.python.read.scanner.file_scanner import (FileScanner)
from pypaimon.execution.python.read.scanner.data_evolution_split_generator import (DataEvolutionSplitGenerator)
-from pypaimon.read.query_auth_split import QueryAuthSplit
from pypaimon.read.split import DataSplit
from pypaimon.schema.data_types import PyarrowFieldParser
from pypaimon.snapshot.time_travel_util import TimeTravelUtil
@@ -123,14 +122,6 @@ class PythonTableUpdateMixin:
scan = read_builder.new_scan()
splits = scan.plan_all_files().splits()
- if (splits
- and self.table.options.native_write_enabled()
- and not any(isinstance(split, QueryAuthSplit)
- for split in splits)):
- from pypaimon.execution.dispatch import try_native_predicate_row_ids
- row_ids = try_native_predicate_row_ids(scan_table, predicate, splits)
- if row_ids is not None:
- return row_ids
matched = read_builder.new_read().to_arrow(splits)
if matched.num_rows == 0:
return []
@@ -213,47 +204,9 @@ class PythonTableUpdateMixin:
has_callable: bool,
has_array: bool,
):
- if not self.table.options.data_evolution_enabled():
- raise ValueError(
- "update_by_predicate requires "
- "'data-evolution.enabled' = 'true'."
- )
- if not self.table.options.row_tracking_enabled():
- raise ValueError(
- "update_by_predicate requires "
- "'row-tracking.enabled' = 'true'."
- )
- if not assignments:
- raise ValueError("assignments must not be empty.")
- if read_columns and not has_callable:
- raise ValueError(
- "read_columns requires a callable assignment."
- )
- if has_callable:
- if has_array:
- raise ValueError(
- "Callable assignments cannot be combined with Arrow "
- "array assignments."
- )
- if not read_columns:
- raise ValueError(
- "Callable assignments require read_columns."
- )
- for col in read_columns:
- if col not in self.table.field_names:
- raise ValueError(
- f"Read column {col} is not in table schema."
- )
+ from pypaimon.write.update_validation import _validate_predicate_update
- partition_keys = set(self.table.partition_keys)
- for col in assignments:
- if col not in self.table.field_names:
- raise ValueError(f"Column {col} is not in table schema.")
- if col in partition_keys:
- raise ValueError(
- "update_by_predicate does not support updating "
- f"partition column '{col}'."
- )
+ return _validate_predicate_update(self, assignments, read_columns, has_callable, has_array)
def _build_predicate_update_table(
self,
@@ -465,6 +418,7 @@ class PythonTableUpdateMixin:
return self._delete_by_row_id(row_ids, commit_identifier)
def _new_python_shard_updator(self, shard_num, total_shard_count):
+ from pypaimon.write.table_update import ShardTableUpdator
return ShardTableUpdator(self.table, self.projection, self.update_cols,
self.commit_user, shard_num, total_shard_count)
@@ -550,6 +504,7 @@ class ShardTableUpdator:
self.update_by_arrow_batch(rest)
def _init_writer(self):
+ from pypaimon.write.table_update import SingleWriter
if self.writer is None:
if self.write_pos >= len(self.row_ranges):
raise RuntimeError(
diff --git a/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py b/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py
index 1e6226c608..46bfe3de1e 100644
--- a/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py
+++ b/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py
@@ -22,7 +22,6 @@ import pyarrow as pa
from pypaimon.read.table_read import TableRead
from pypaimon.table.row.blob import Blob
-from pypaimon.table.row.internal_row import InternalRow
from pypaimon.table.special_fields import SpecialFields
from pypaimon.write.commit_message import CommitMessage
from pypaimon.write.row_utils import require_columns
@@ -45,9 +44,9 @@ class PythonTableUpsertMixin:
@staticmethod
def _normalize_rows(rows) -> List:
- if isinstance(rows, InternalRow):
- return [rows]
- return list(rows)
+ from pypaimon.write.update_validation import _normalize_rows
+
+ return _normalize_rows(rows)
def _group_rows_by_partition(
self,
@@ -278,96 +277,18 @@ class PythonTableUpsertMixin:
def _validate_inputs(self, data: pa.Table, upsert_keys: List[str],
update_cols: Optional[List[str]]):
- """Validate inputs before processing."""
- if not self.table.options.data_evolution_enabled():
- raise ValueError(
- "upsert_by_arrow_with_key requires 'data-evolution.enabled' = 'true'."
- )
-
- if not self.table.options.row_tracking_enabled():
- raise ValueError(
- "upsert_by_arrow_with_key requires 'row-tracking.enabled' = 'true'."
- )
-
- if not upsert_keys:
- raise ValueError("upsert_keys must not be empty.")
-
- for key in upsert_keys:
- if key not in self.table.field_names:
- raise ValueError(
- f"upsert_key '{key}' is not in table schema fields: {self.table.field_names}"
- )
- if key not in data.column_names:
- raise ValueError(
- f"upsert_key '{key}' is not in input data columns: {data.column_names}"
- )
-
- # For partitioned tables, input data must contain partition columns
- partition_keys = self.table.partition_keys
- if partition_keys:
- missing_in_data = [pk for pk in partition_keys if pk not in data.column_names]
- if missing_in_data:
- raise ValueError(
- f"For partitioned tables, input data must contain all partition key "
- f"columns. Missing: {missing_in_data}"
- )
-
- if update_cols is not None:
- for col in update_cols:
- if col not in self.table.field_names:
- raise ValueError(
- f"Column '{col}' in update_cols is not in table schema fields: "
- f"{self.table.field_names}"
- )
+ from pypaimon.write.update_validation import _validate_inputs
- if data.num_rows == 0:
- raise ValueError("Input data is empty.")
+ return _validate_inputs(self, data, upsert_keys, update_cols)
def _validate_row_inputs(
self,
values_by_name: Dict[str, Any],
upsert_keys: List[str],
update_cols: Optional[List[str]]):
- if not self.table.options.data_evolution_enabled():
- raise ValueError(
- "upsert_by_key requires 'data-evolution.enabled' = 'true'."
- )
-
- if not self.table.options.row_tracking_enabled():
- raise ValueError(
- "upsert_by_key requires 'row-tracking.enabled' = 'true'."
- )
-
- if not upsert_keys:
- raise ValueError("upsert_keys must not be empty.")
-
- for key in upsert_keys:
- if key not in self.table.field_names:
- raise ValueError(
- f"upsert_key '{key}' is not in table schema fields: {self.table.field_names}"
- )
-
- unknown_fields = [
- field_name
- for field_name in values_by_name
- if field_name not in self.table.field_names
- ]
- if unknown_fields:
- raise ValueError(
- f"upsert_by_key got row field(s) {unknown_fields} "
- f"that are not in table schema fields: {self.table.field_names}"
- )
-
- require_columns(values_by_name, upsert_keys, "upsert_by_key")
- require_columns(values_by_name, self.table.partition_keys, "upsert_by_key")
+ from pypaimon.write.update_validation import _validate_row_inputs
- if update_cols is not None:
- for col in update_cols:
- if col not in self.table.field_names:
- raise ValueError(
- f"Column '{col}' in update_cols is not in table schema fields: "
- f"{self.table.field_names}"
- )
+ return _validate_row_inputs(self, values_by_name, upsert_keys, update_cols)
def _build_key_to_row_ids_map(
self,
diff --git a/paimon-python/pypaimon/execution/read_dispatch.py b/paimon-python/pypaimon/execution/read_dispatch.py
new file mode 100644
index 0000000000..29f9fc8357
--- /dev/null
+++ b/paimon-python/pypaimon/execution/read_dispatch.py
@@ -0,0 +1,279 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Preparation selection for reading and planning; execution is never replayed."""
+
+import logging
+from typing import Iterator, List, Optional, Tuple
+
+import pyarrow
+
+from pypaimon.catalog.catalog_exception import TableNoPermissionException
+from pypaimon.read.plan import Plan
+from pypaimon.read.split import Split
+from pypaimon.read.scan_stats import ScanStats
+
+
+def to_iterator(self, splits: List[Split]) -> Iterator:
+ """Lazily read InternalRows, using native reads when available.
+
+ Rows retain their kind separately from projected fields. Like the
+ Python split reader, the iterator reuses its row view and visits splits
+ in order. Close the iterator when stopping before exhaustion.
+ """
+ if self.output_projection is not None and self.output_projection.named:
+ raise RuntimeError(
+ "Projection expressions are not supported by to_iterator(); "
+ "use to_arrow() or to_arrow_batch_reader()")
+ self._begin_auth_read(splits)
+ limit = self.limit
+
+ def _record_generator():
+ if limit is not None and limit <= 0:
+ return
+ rows = self._try_native_rows(splits)
+ if rows is not None:
+ try:
+ yield from rows
+ finally:
+ rows.close()
+ return
+
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ yield from PythonTableReadMixin._python_row_iterator(self, splits, limit)
+
+ return _record_generator()
+
+
+def _new_arrow_batch_reader(
+ self,
+ splits: List[Split],
+ blob_parallelism: Optional[int] = None,
+ parallelism: Optional[int] = None):
+ schema = self._output_arrow_schema(splits)
+ if self.include_row_kind:
+ schema = self._add_row_kind_to_schema(schema)
+ effective = self._resolve_parallelism(parallelism, len(splits))
+ effective_bp = self._resolve_blob_parallelism(blob_parallelism)
+ native_batches = self._try_native_batches(
+ splits,
+ schema,
+ parallelism=effective,
+ blob_parallelism=effective_bp,
+ ordered=False,
+ )
+ if native_batches is not None:
+ batch_iterator = iter(native_batches)
+ else:
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ batch_iterator = PythonTableReadMixin._python_batch_iterator(
+ self, splits, schema, parallelism, effective_bp)
+ reader_type = pyarrow.ipc.RecordBatchReader
+ reader = reader_type.from_batches(schema, batch_iterator)
+ return reader, batch_iterator
+
+
+def to_arrow(
+ self,
+ splits: List[Split],
+ parallelism: Optional[int] = None,
+ blob_parallelism: Optional[int] = None,
+) -> Optional[pyarrow.Table]:
+ """Read ``splits`` into a single arrow ``Table``.
+
+ Args:
+ splits: scan-plan splits returned from a ``TableScan``.
+ parallelism: optional runtime override of the
+ ``read.parallelism`` table option. ``None`` (default) falls
+ back to the table option; when that is also unset the read
+ auto-scales to ``min(number of splits, CPU count)``. ``1``
+ keeps reads serial; ``>= 2`` caps the thread pool that reads
+ byte-balanced contiguous split groups concurrently and
+ assembles the final table in input order. An unfiltered row
+ limit caps this further to avoid speculative work. Must be
+ ``>= 1``. Note that with ``>= 2`` (or auto)
+ and a ``limit`` set, the returned rows are an arbitrary
+ subset of the requested size, since which splits fill the row
+ quota first is non-deterministic. Reads with BLOB payloads run
+ serially when a limit may discard rows, so discarded payloads
+ are not fetched by speculative split workers.
+ blob_parallelism: maximum concurrent blob range reads within each
+ split reader. ``None`` or ``1`` (default) reads blobs serially;
+ ``>= 2`` enables concurrent ranged reads. On the
+ parallel path, peak blob threads (``parallelism`` *
+ ``blob_parallelism``) are capped at
+ ``_MAX_TOTAL_BLOB_WORKERS``; per-split ``blob_parallelism`` is
+ shrunk to stay within it.
+ """
+ effective_bp = self._resolve_blob_parallelism(blob_parallelism)
+ effective = self._resolve_parallelism(parallelism, len(splits))
+ schema = self._output_arrow_schema(splits)
+ if self.include_row_kind:
+ schema = self._add_row_kind_to_schema(schema)
+
+ native_batches = self._try_native_batches(
+ splits,
+ schema,
+ parallelism=effective,
+ blob_parallelism=effective_bp,
+ ordered=True,
+ )
+ if native_batches is not None:
+ return self._batches_to_arrow(native_batches, schema)
+
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ return PythonTableReadMixin._python_to_arrow(self, splits, schema, parallelism, effective_bp)
+
+
+def plan_scan(self) -> Plan:
+ native_auth = (self.table.options.query_auth_enabled
+ and self._native_requested() and self._native_plan_supported())
+ if native_auth:
+ # Rust asks the REST server and carries its complete response in
+ # QueryAuthSplit. Both readers then enforce that captured policy.
+ native = self._try_native_plan()
+ if native is not None:
+ return native
+ auth_result = self._TableScan__auth_query()
+ if self._query_auth_fn is not None and self._file_scanner is not None:
+ from pypaimon.execution.python.read.auth_support import prune_scanner_by_auth
+ prune_scanner_by_auth(self.table, self._file_scanner, auth_result)
+ # The native planner bypasses the auth-aware file scanner. Resolve auth
+ # before selecting a planning backend.
+ if (not native_auth and auth_result is None and self._native_requested()
+ and self._native_plan_supported()):
+ native = self._try_native_plan()
+ if native is not None:
+ return native
+ from pypaimon.execution.python.read.table_scan import PythonTableScanMixin
+ return PythonTableScanMixin._python_plan(self, auth_result)
+
+
+def plan_all_files(self) -> Plan:
+ """Plan matching files including L0 and all data-evolution column files.
+
+ Existing filters and limits still apply. Query authorization must be
+ unrestricted, as required by writer restore, overwrite and indexing.
+ """
+ if self._native_requested():
+ from pypaimon.execution.native.table import _native_query_auth_supported
+ native_supported = _native_query_auth_supported(self.table) and self._native_plan_supported()
+ else:
+ native_supported = False
+ if native_supported:
+ plan = self._try_native_plan(all_files=True)
+ if plan is not None:
+ return plan
+ if self._TableScan__auth_query(all_columns=True) is not None:
+ raise TableNoPermissionException(self.table.identifier)
+ from pypaimon.execution.python.read.table_scan import PythonTableScanMixin
+ return PythonTableScanMixin._python_plan_all_files(self)
+
+
+def scan_with_stats(self) -> Tuple[Plan, Optional[ScanStats]]:
+ """Run :meth:`plan` while recording manifest / pruning counters.
+
+ Only used by :meth:`ReadBuilder.explain`; the regular read path
+ keeps going through :meth:`plan`. Native scans always collect lightweight
+ counters in the same core planning pass, without opening data files.
+ """
+ native_auth = (self.table.options.query_auth_enabled
+ and self._native_requested() and self._native_plan_supported())
+ if native_auth:
+ native = self._try_native_plan()
+ if native is not None:
+ return native, native.stats()
+ auth_result = self._TableScan__auth_query()
+ if self._query_auth_fn is not None and self._file_scanner is not None:
+ from pypaimon.execution.python.read.auth_support import prune_scanner_by_auth
+ prune_scanner_by_auth(self.table, self._file_scanner, auth_result)
+ if (not native_auth and auth_result is None and self._native_requested()
+ and self._native_plan_supported()):
+ native = self._try_native_plan()
+ if native is not None:
+ return native, native.stats()
+ from pypaimon.execution.python.read.table_scan import PythonTableScanMixin
+ return PythonTableScanMixin._python_scan_with_stats(self, auth_result)
+
+
+def new_streaming_scan(self):
+ """Create a stateful streaming scan using the configured backend."""
+ from pypaimon.execution.dispatch import propagate_fork_safety_error
+
+ projection = self._projection_builder()
+ projection._validate_map_key_filter()
+ if self.table.options.native_plan_enabled():
+ from pypaimon.execution.native.table import _native_query_auth_supported
+ native_supported = _native_query_auth_supported(self.table)
+ else:
+ native_supported = False
+ if native_supported:
+ try:
+ from pypaimon.execution.native.read.streaming_table_scan import StreamTableScan
+ return StreamTableScan(
+ self.table, predicate=self._predicate, read_type=projection.read_type(),
+ poll_interval_ms=self._poll_interval_ms, bucket_filter=self._bucket_filter,
+ consumer_id=self._consumer_id)
+ except Exception as error:
+ propagate_fork_safety_error(error)
+ logging.warning("Cannot create Native stream scan, using Python: %s", error)
+ from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan
+ scan = AsyncStreamingTableScan(
+ table=self.table,
+ predicate=self._predicate,
+ poll_interval_ms=self._poll_interval_ms,
+ bucket_filter=self._bucket_filter,
+ consumer_id=self._consumer_id
+ )
+ scan._read_type = projection.read_type()
+ return scan
+
+
+def native_plan_requested(scan) -> bool:
+ """Resolve Native planning preference without loading a disabled backend."""
+ if scan.table.options.native_plan_enabled():
+ return True
+ if not scan.table.options.native_read_enabled():
+ return False
+ from pypaimon.execution.native.runtime import native_runtime_available
+ return native_runtime_available()
+
+
+def native_read_fallback(read, reason, error=None, log=False):
+ """Validate features that cannot fall back before importing either executor."""
+ from pypaimon.read.variant_read_type import has_variant_extractions
+ if has_variant_extractions(read.read_type):
+ raise RuntimeError(
+ "Variant extraction requires a compatible native reader: %s" % reason) from error
+ if log:
+ logging.getLogger('pypaimon.execution.native.read.table_read').warning(
+ 'Native read failed, falling back to the Python reader: %s', reason)
+ return None
+
+
+def plan_system_table(scan) -> Plan:
+ """Choose a metadata provider before importing its read implementation."""
+ table = scan.system_table
+ options = getattr(getattr(table, 'base_table', None), 'options', None)
+ if options is not None and options.native_read_enabled():
+ from pypaimon.execution.native.read.system_table import try_read_native_system_table
+ arrow_table = try_read_native_system_table(table)
+ if arrow_table is not None:
+ from pypaimon.table.system.system_table_scan import SystemSplit
+ return Plan(_splits=[SystemSplit(arrow_table)])
+ from pypaimon.execution.python.read.system_table_scan import plan
+ return plan(scan)
diff --git a/paimon-python/pypaimon/execution/read_explain.py b/paimon-python/pypaimon/execution/read_explain.py
new file mode 100644
index 0000000000..1f34f979f3
--- /dev/null
+++ b/paimon-python/pypaimon/execution/read_explain.py
@@ -0,0 +1,247 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Build public explain results from one selected backend's plan and counters."""
+
+from typing import List, Optional
+
+from pypaimon.read.explain import ExplainResult, ExplainSplitInfo, PruningStat
+from pypaimon.read.explain_render import render_predicate
+from pypaimon.read.query_auth_split import QueryAuthSplit
+from pypaimon.read.scan_stats import ScanStats
+from pypaimon.read.split import Split
+from pypaimon.read.table_scan import TableScan
+
+
+def explain(builder, verbose: bool = False) -> ExplainResult:
+ """Produce a structured scan plan for this builder.
+
+ Runs one planning pass (manifest list + manifest reads, no data
+ files) and returns an :class:`ExplainResult` summarising the
+ target snapshot, the pushed-down predicate / projection / limit,
+ the partition / bucket / file-stats pruning funnel, and split-
+ level execution signals (raw-convertible ratio, deletion-vector
+ ratio, level histogram, files-per-split and split-size
+ distribution). With ``verbose=True``, every split is listed.
+
+ Cost: ``explain()`` reads manifest list + manifests but never
+ opens data files. To produce accurate before/after counters it
+ suppresses early bucket and sidecar pruning and counts candidates
+ before applying LIMIT, so it can be measurably
+ heavier than a regular ``new_scan().plan()`` on tables where the
+ early filter usually prunes aggressively (e.g. very wide
+ HASH_FIXED tables with a tight predicate).
+ """
+ scan = builder.new_scan()
+ plan, stats = scan.scan_with_stats()
+ return _build_explain_result(
+ table=builder.table,
+ scan=scan,
+ plan=plan,
+ stats=stats,
+ predicate=builder._predicate,
+ projection=([alias for alias, _ in builder._output_projection.columns]
+ if builder._output_projection else None),
+ limit=builder._limit,
+ verbose=verbose,
+ )
+
+
+def _build_explain_result(table, scan: TableScan, plan, stats: Optional[ScanStats],
+ predicate, projection, limit, verbose: bool) -> ExplainResult:
+ """Translate one (Plan, ScanStats) pair into an ExplainResult."""
+ splits: List[Split] = plan.splits()
+
+ table_schema = table.table_schema
+ bucket_mode_str = _safe_bucket_mode(table)
+
+ native_planned = stats is None or stats.native_planned
+ if stats is None or native_planned:
+ partition_pruning = bucket_pruning = file_skipping = None
+ else:
+ partition_pruning = _partition_pruning(stats, scan)
+ bucket_pruning = _bucket_pruning(stats, scan)
+ file_skipping = _file_skipping(stats, scan)
+
+ files_per_split = [len(getattr(s, 'files', []) or []) for s in splits]
+ sizes = [int(getattr(s, 'file_size', 0) or 0) for s in splits]
+
+ rows_total = sum(int(getattr(s, 'row_count', 0) or 0) for s in splits)
+ merged_per_split = [s.merged_row_count() for s in splits]
+ if splits and all(v is not None for v in merged_per_split):
+ merged_total: Optional[int] = sum(merged_per_split)
+ else:
+ merged_total = None
+
+ file_count = sum(files_per_split)
+ total_size = sum(sizes)
+ level_hist: dict = {}
+ deletion_file_total = 0
+ splits_raw_convertible = 0
+ splits_with_dv = 0
+ splits_all_above_l0 = 0
+ split_infos: List[ExplainSplitInfo] = []
+
+ plan_has_auth = any(isinstance(s, QueryAuthSplit) for s in splits)
+
+ for split in splits:
+ files = getattr(split, 'files', []) or []
+ per_split_levels: dict = {}
+ for f in files:
+ lv = getattr(f, 'level', 0) or 0
+ level_hist[lv] = level_hist.get(lv, 0) + 1
+ per_split_levels[lv] = per_split_levels.get(lv, 0) + 1
+ dvs = getattr(split, 'data_deletion_files', None) or []
+ dv_count_here = sum(1 for d in dvs if d is not None)
+ deletion_file_total += dv_count_here
+ has_dv = dv_count_here > 0
+ raw = bool(getattr(split, 'raw_convertible', False))
+ if raw:
+ splits_raw_convertible += 1
+ if has_dv:
+ splits_with_dv += 1
+ if files and all((getattr(f, 'level', 0) or 0) > 0 for f in files):
+ splits_all_above_l0 += 1
+
+ if verbose:
+ split_infos.append(ExplainSplitInfo(
+ partition=_format_partition(split, table),
+ bucket=int(getattr(split, 'bucket', -1)),
+ file_count=len(files),
+ row_count=int(getattr(split, 'row_count', 0) or 0),
+ merged_row_count=split.merged_row_count(),
+ file_size=int(getattr(split, 'file_size', 0) or 0),
+ raw_convertible=raw,
+ has_deletion_vectors=has_dv,
+ level_histogram=per_split_levels,
+ deletion_file_count=dv_count_here,
+ file_paths=list(getattr(split, 'file_paths', []) or []),
+ data_files=list(files),
+ ))
+
+ fps_min, fps_max, fps_avg = _min_max_avg(files_per_split)
+ sz_min, sz_max, sz_avg = _min_max_avg(sizes)
+ sz_p50 = _percentile(sizes, 50)
+ sz_p95 = _percentile(sizes, 95)
+
+ return ExplainResult(
+ table_identifier=str(table.identifier.get_full_name()),
+ is_primary_key_table=bool(table.is_primary_key_table),
+ bucket_mode=bucket_mode_str,
+ deletion_vectors_enabled=bool(table.options.deletion_vectors_enabled()),
+ data_evolution_enabled=bool(table.options.data_evolution_enabled()),
+ snapshot_id=plan.snapshot_id,
+ schema_id=table_schema.id if plan.snapshot_id is not None else None,
+ predicate=render_predicate(predicate) if predicate is not None else None,
+ projection=list(projection) if projection else None,
+ limit=limit,
+ partition_pruning=partition_pruning,
+ bucket_pruning=bucket_pruning,
+ file_skipping=file_skipping,
+ file_count=file_count,
+ total_file_size=total_size,
+ estimated_row_count=rows_total,
+ estimated_merged_row_count=merged_total,
+ deletion_file_count=deletion_file_total,
+ level_histogram=level_hist,
+ split_count=len(splits),
+ splits_raw_convertible=splits_raw_convertible,
+ splits_with_deletion_vectors=splits_with_dv,
+ splits_all_above_l0=splits_all_above_l0,
+ files_per_split_min=fps_min,
+ files_per_split_max=fps_max,
+ files_per_split_avg=fps_avg,
+ split_size_min=sz_min,
+ split_size_max=sz_max,
+ split_size_avg=sz_avg,
+ split_size_p50=sz_p50,
+ split_size_p95=sz_p95,
+ has_auth=plan_has_auth,
+ native_planned=native_planned,
+ planning_counters=stats.planning_counters if stats is not None else None,
+ splits=split_infos if verbose else None,
+ )
+
+
+def _partition_pruning(stats: ScanStats, scan: TableScan) -> Optional[PruningStat]:
+ if scan.predicate is None:
+ return None
+ table_partition_keys = scan.table.partition_keys or []
+ if not table_partition_keys:
+ return None
+ # ``entries_potential_total`` is the count from manifest-file metadata
+ # (manifest-level pruning has not been applied yet). The "after" side
+ # is everything that survived both manifest-stats and per-entry
+ # partition filters.
+ return PruningStat(
+ before=stats.entries_potential_total,
+ after=stats.entries_after_partition,
+ )
+
+
+def _bucket_pruning(stats: ScanStats, scan: TableScan) -> Optional[PruningStat]:
+ # Visible whenever the scan applies any bucket-level filtering — the
+ # HASH_FIXED predicate-driven selector OR the POSTPONE_BUCKET
+ # synthetic-bucket skip. Tables with neither (e.g. BUCKET_UNAWARE
+ # append) leave this counter as ``None``.
+ fs = scan.file_scanner
+ if fs._bucket_selector is None and not fs.only_read_real_buckets:
+ return None
+ return PruningStat(before=stats.entries_after_partition, after=stats.entries_after_bucket)
+
+
+def _file_skipping(stats: ScanStats, scan: TableScan) -> Optional[PruningStat]:
+ # Captures the funnel between bucket-stage survivors and the entries
+ # that actually feed the split generator. The drop here includes both
+ # predicate-driven file-stats pruning AND structural skips that fire
+ # in ``_filter_manifest_entry`` once a file is fully decoded (most
+ # notably the "do not read level-0 file" rule for DV-enabled PK
+ # tables, which is an LSM-shape decision rather than a predicate
+ # test).
+ if scan.predicate is None:
+ return None
+ return PruningStat(before=stats.entries_after_bucket, after=stats.entries_after_stats)
+
+
+def _safe_bucket_mode(table) -> str:
+ try:
+ return table.bucket_mode().name
+ except Exception:
+ return "UNKNOWN"
+
+
+def _format_partition(split, table) -> dict:
+ keys = list(table.partition_keys or [])
+ partition = getattr(split, 'partition', None)
+ if partition is None or not keys:
+ return {}
+ values = getattr(partition, 'values', None) or []
+ return {k: v for k, v in zip(keys, values)}
+
+
+def _min_max_avg(values):
+ if not values:
+ return 0, 0, 0.0
+ return min(values), max(values), sum(values) / float(len(values))
+
+
+def _percentile(values, pct: int) -> int:
+ if not values:
+ return 0
+ ordered = sorted(values)
+ idx = int(round((pct / 100.0) * (len(ordered) - 1)))
+ return int(ordered[idx])
diff --git a/paimon-python/pypaimon/execution/read_support.py b/paimon-python/pypaimon/execution/read_support.py
new file mode 100644
index 0000000000..ccf3396ad6
--- /dev/null
+++ b/paimon-python/pypaimon/execution/read_support.py
@@ -0,0 +1,647 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Backend-independent Arrow output, resource ownership and dataset adapters."""
+
+import os
+import struct
+from typing import Any, Callable, Dict, List, Optional
+
+import pyarrow
+
+from pypaimon.casting.schema_evolution import read_arrow_schema
+from pypaimon.read.constants import ROW_KIND_COLUMN
+from pypaimon.read.query_auth_split import QueryAuthSplit
+from pypaimon.read.read_type import OutputProjection, adapter_output_paths, extract_array, output_schema
+from pypaimon.read.split import Split
+from pypaimon.schema.data_types import is_blob_file_field
+from pypaimon.utils.arrow_utils import zero_column_batch
+
+_RECORD_BATCH_READER_FROM_STREAM = getattr(pyarrow.ipc.RecordBatchReader, 'from_stream', None)
+
+
+class _ClosableArrowBatchReader:
+
+ def __init__(self, reader, batch_iterator):
+ self._reader = reader
+ self._batch_iterator = batch_iterator
+
+ def __getattr__(self, name):
+ return getattr(self._reader, name)
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return self._reader.read_next_batch()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.close()
+
+ def close(self):
+ try:
+ close = getattr(self._batch_iterator, "close", None)
+ if close is not None:
+ close()
+ finally:
+ close = getattr(self._reader, "close", None)
+ if close is not None:
+ close()
+
+
+def to_arrow_batch_reader(
+ self,
+ splits: List[Split],
+ blob_parallelism: Optional[int] = None,
+ parallelism: Optional[int] = None) -> pyarrow.ipc.RecordBatchReader:
+ """Lazily read batches using bounded native split concurrency.
+
+ ``parallelism`` follows :meth:`to_arrow`: an explicit value overrides
+ ``read.parallelism``, otherwise the reader auto-scales to the available
+ splits and CPUs. Native workers buffer at most one batch each; batches
+ from different split groups are emitted as they become ready because
+ table reads do not guarantee row order. Python fallback reads remain
+ serial.
+ """
+ # PyArrow does not close a suspended Python batch iterator. Keep its
+ # cleanup even when LIMIT or BLOB pruning serializes native reads.
+ effective = self._resolve_parallelism(parallelism, len(splits))
+ reader, batch_iterator = self._new_arrow_batch_reader(
+ splits, blob_parallelism, parallelism)
+ if self._requires_batch_iterator_cleanup(splits, effective):
+ return _ClosableArrowBatchReader(reader, batch_iterator)
+ return reader
+
+
+def _to_managed_arrow_batch_reader(
+ self,
+ splits: List[Split],
+ blob_parallelism: Optional[int] = None,
+ parallelism: Optional[int] = None):
+ """Return a closeable batch reader supporting context management.
+
+ PyArrow readers do not close the Python iterator passed to
+ ``from_batches``. Retain it explicitly so native reads release their
+ resources when a caller ends the read early, including serial reads.
+ """
+ # Keep the explicit iterator cleanup chain even when LIMIT caps the
+ # native worker count inside _new_arrow_batch_reader to one.
+ effective = self._resolve_parallelism(parallelism, len(splits))
+ reader, batch_iterator = self._new_arrow_batch_reader(
+ splits, blob_parallelism, parallelism)
+ if self._requires_batch_iterator_cleanup(splits, effective):
+ return _ClosableArrowBatchReader(reader, batch_iterator)
+ if (_RECORD_BATCH_READER_FROM_STREAM is not None
+ and hasattr(reader, "close")):
+ return _RECORD_BATCH_READER_FROM_STREAM(reader)
+ return _ClosableArrowBatchReader(reader, batch_iterator)
+
+
+def _add_row_kind_to_schema(schema: pyarrow.Schema, column_name=ROW_KIND_COLUMN) -> pyarrow.Schema:
+ """Prepend the row-kind transport column, normally named _row_kind."""
+ row_kind_field = pyarrow.field(column_name, pyarrow.string())
+ return pyarrow.schema([row_kind_field] + list(schema))
+
+
+def _try_to_pad_batch_by_schema(
+ batch: pyarrow.RecordBatch,
+ target_schema,
+ allow_type_cast: bool = False):
+ """Align a batch with the requested schema.
+
+ Native batches stay strict so an outdated Rust BLOB representation
+ cannot be silently converted. Python query-auth masking is allowed
+ to cast internal dependency columns because masking intentionally may
+ change their logical type.
+ """
+ if batch.schema.equals(target_schema, check_metadata=True):
+ return batch
+
+ columns = []
+ num_rows = batch.num_rows
+ same_order = batch.schema.names == target_schema.names
+
+ for index, field in enumerate(target_schema):
+ if field.name in batch.schema.names:
+ # Result projections may repeat a column. A name lookup is
+ # ambiguous even when the batch already has the correct order.
+ col = batch.column(index if same_order else field.name)
+ if col.type != field.type:
+ if not allow_type_cast:
+ raise TypeError(
+ "Batch field '%s' has type %s, expected %s" % (
+ field.name, col.type, field.type))
+ col = col.cast(field.type)
+ else:
+ col = pyarrow.nulls(num_rows, type=field.type)
+ columns.append(col)
+
+ return pyarrow.RecordBatch.from_arrays(columns, schema=target_schema)
+
+
+def close_read_source(source, primary_error=None):
+ """Close a stream without replacing a failure from reading or adapting it."""
+ try:
+ close = getattr(source, 'close', None)
+ if close is not None:
+ close()
+ except BaseException:
+ # GeneratorExit is ordinary early cancellation, rather than a read
+ # failure: an explicit close must still surface its own cleanup error.
+ if primary_error is None or isinstance(primary_error, GeneratorExit):
+ raise
+
+
+def _batches_to_arrow(batches, schema, allow_type_cast: bool = False):
+ """Materialize batches and close their source, preserving read failures."""
+ failure = None
+ try:
+ collected = [
+ _try_to_pad_batch_by_schema(
+ batch, schema, allow_type_cast=allow_type_cast)
+ for batch in batches
+ if batch.num_rows > 0
+ ]
+ if not collected:
+ return pyarrow.Table.from_arrays(
+ [pyarrow.array([], type=field.type) for field in schema],
+ schema=schema)
+ return pyarrow.Table.from_batches(collected)
+ except BaseException as error:
+ failure = error
+ raise
+ finally:
+ close_read_source(batches, failure)
+
+
+def _resolve_parallelism(self, runtime: Optional[int], num_splits: int) -> int:
+ """Pick the effective parallelism and reject illegal values.
+
+ Priority: explicit ``parallelism`` argument > ``read.parallelism``
+ table option > auto. When neither the argument nor the option is set
+ the read auto-scales to ``min(num_splits, CPU count)``. A value >= 1
+ caps the thread pool; ``1`` forces serial reads. The validation
+ message names whichever source produced the offending value.
+ """
+ if runtime is not None:
+ value, source = runtime, "parallelism"
+ elif self._read_parallelism is not None:
+ value, source = self._read_parallelism, "read.parallelism"
+ else:
+ # os.cpu_count() may return None on exotic platforms; fall back
+ # to 1. min() with num_splits avoids more workers than work.
+ return max(1, min(num_splits, os.cpu_count() or 1))
+ if value < 1:
+ raise ValueError(f"{source} must be >= 1, got {value}")
+ return value
+
+
+def _effective_parallelism(
+ self, runtime: Optional[int], num_splits: int) -> int:
+ """Resolve parallelism and bound speculative unfiltered LIMIT reads."""
+ effective = self._resolve_parallelism(runtime, num_splits)
+ if self.limit is None or (self.limit > 0 and self.predicate is not None):
+ return effective
+ return min(effective, max(1, self.limit))
+
+
+def _resolve_blob_parallelism(runtime: Optional[int]) -> int:
+ if runtime is None:
+ return 1
+ if runtime < 1:
+ raise ValueError(f"blob_parallelism must be >= 1, got {runtime}")
+ return runtime
+
+
+def _cap_blob_parallelism(cls, workers: int, blob_parallelism: int) -> int:
+ """Shrink per-split blob_parallelism so workers*B <= cap (peak reads)."""
+ if blob_parallelism <= 1 or workers * blob_parallelism <= cls._MAX_TOTAL_BLOB_WORKERS:
+ return blob_parallelism
+ return max(1, cls._MAX_TOTAL_BLOB_WORKERS // workers)
+
+
+def _should_run_parallel(
+ self,
+ splits: List[Split],
+ effective: int,
+) -> bool:
+ """Decide whether to take the parallel read path.
+
+ ``effective == 1`` falls back to the serial path (no thread pool
+ overhead, no behavior change). A single split is never
+ parallelized since there is nothing to fan out across.
+ """
+ deferred_limit_may_prune = self._deferred_blob_limit_may_prune(splits)
+ return (effective >= 2 and len(splits) >= 2
+ and not deferred_limit_may_prune)
+
+
+def _deferred_blob_limit_may_prune(self, splits: List[Split]) -> bool:
+ return (self.limit is not None
+ and (self._deferred_blob_fields
+ or self._native_resolved_blob_fields())
+ and not self._limit_covers_all_splits(splits))
+
+
+def _requires_batch_iterator_cleanup(self, splits: List[Split], effective: int) -> bool:
+ # A serial Core stream owns resources too. PyArrow's reader does not
+ # close the Python iterator supplied to from_batches on an early close.
+ return (self.table.options.native_read_enabled()
+ or self._should_run_parallel(splits, effective)
+ or self._deferred_blob_limit_may_prune(splits))
+
+
+def _limit_covers_all_splits(self, splits: List[Split]) -> bool:
+ """Return whether split metadata proves that LIMIT cannot drop rows."""
+ total_rows = 0
+ for split in splits:
+ merged_row_count = split.merged_row_count()
+ if merged_row_count is None:
+ return False
+ total_rows += merged_row_count
+ if total_rows > self.limit:
+ return False
+ return True
+
+
+def _add_row_kind_column_to_batch(
+ self,
+ batch: pyarrow.RecordBatch,
+ default_row_kind: str = "+I"
+) -> pyarrow.RecordBatch:
+ """Add a _row_kind column to an existing batch."""
+ row_kind_array = pyarrow.array([default_row_kind] * batch.num_rows, type=pyarrow.string())
+ new_schema = self._add_row_kind_to_schema(batch.schema)
+ columns = [row_kind_array] + [batch.column(i) for i in range(batch.num_columns)]
+ return pyarrow.RecordBatch.from_arrays(columns, schema=new_schema)
+
+
+def to_duckdb(self, splits: List[Split], table_name: str,
+ connection: Optional["DuckDBPyConnection"] = None,
+ parallelism: Optional[int] = None) -> "DuckDBPyConnection":
+ """Materialize ``splits`` into an in-memory table registered with DuckDB.
+
+ See :meth:`to_arrow` for the semantics of ``parallelism``.
+ """
+ import duckdb
+
+ con = connection or duckdb.connect(database=":memory:")
+ con.register(table_name, self.to_arrow(splits, parallelism=parallelism))
+ return con
+
+
+def to_ray(
+ self,
+ splits: List[Split],
+ *,
+ ray_remote_args: Optional[Dict[str, Any]] = None,
+ concurrency: Optional[int] = None,
+ override_num_blocks: Optional[int] = None,
+ **read_args,
+) -> "ray.data.dataset.Dataset":
+ """Convert Paimon table data to Ray Dataset.
+ Args:
+ splits: List of splits to read from the Paimon table.
+ ray_remote_args: Optional kwargs passed to :func:`ray.remote` in read tasks.
+ For example, ``{"num_cpus": 2, "max_retries": 3}``.
+ concurrency: Optional max number of Ray tasks to run concurrently.
+ By default, dynamically decided based on available resources.
+ override_num_blocks: Optional override for the number of output blocks.
+ You needn't manually set this in most cases.
+ **read_args: Additional kwargs passed to the datasource.
+ For example, ``per_task_row_limit`` (Ray 2.52.0+).
+
+ See `Ray Data API <https://docs.ray.io/en/latest/data/api/doc/ray.data.read_datasource.html>`_
+ for details.
+ """
+ import ray
+
+ if not splits:
+ schema = self._output_arrow_schema()
+ if self.include_row_kind:
+ schema = self._add_row_kind_to_schema(schema)
+ empty_table = pyarrow.Table.from_arrays(
+ [pyarrow.array([], type=field.type) for field in schema],
+ schema=schema
+ )
+ return ray.data.from_arrow(empty_table)
+
+ if override_num_blocks is not None and override_num_blocks < 1:
+ raise ValueError(f"override_num_blocks must be at least 1, got {override_num_blocks}")
+
+ from pypaimon.read.datasource.ray_datasource import RayDatasource
+ from pypaimon.read.datasource.split_provider import PreResolvedSplitProvider
+
+ datasource = RayDatasource(
+ PreResolvedSplitProvider(
+ table=self.table,
+ splits=splits,
+ read_type=self.read_type,
+ predicate=self.predicate,
+ limit=self.limit,
+ output_projection=self.output_projection,
+ include_row_kind=self.include_row_kind,
+ )
+ )
+ ds = ray.data.read_datasource(
+ datasource,
+ ray_remote_args=ray_remote_args,
+ concurrency=concurrency,
+ override_num_blocks=override_num_blocks,
+ **read_args
+ )
+ # Each Ray worker applies the per-task limit independently, so N
+ # workers can collectively yield up to N * limit rows. Cap the
+ # final dataset to the user-visible limit on top.
+ if self.limit is not None:
+ ds = ds.limit(self.limit)
+ return ds
+
+
+def to_torch(
+ self,
+ splits: List[Split],
+ streaming: bool = False,
+ prefetch_concurrency: int = 1,
+ *,
+ batch_format: str = "row",
+ batch_size: Optional[int] = None,
+ to_tensor_fn: Optional[Callable] = None,
+ shuffle: bool = False,
+ seed: int = 0,
+ buffer_size: int = 1000,
+ max_buffer_input_splits: int = 10,
+ auto_detect_rank: bool = False,
+ sharding_rank: Optional[int] = None,
+ sharding_world_size: Optional[int] = None,
+) -> "torch.utils.data.Dataset":
+ """Wrap Paimon table data in a PyTorch Dataset.
+
+ Args:
+ splits: Splits to read.
+ streaming: Whether to return an iterable dataset. Non-streaming
+ eligible data-evolution reads fetch map-style batches lazily
+ by row ID.
+ prefetch_concurrency: Reader threads per DataLoader worker in row
+ format.
+ batch_format: ``"row"``, ``"pyarrow"``, or ``"torch"``. Batch
+ formats require streaming.
+ batch_size: Rows per batch; ``None`` preserves reader batches when
+ unshuffled, or emits shuffle blocks of at most buffer_size rows.
+ to_tensor_fn: Optional RecordBatch converter for Torch batches.
+ shuffle: Whether to shuffle rows in a bounded buffer before output.
+ auto_detect_rank: Whether streaming reads shard by DDP rank.
+ sharding_rank: Explicit rank in the intended DDP process group.
+ sharding_world_size: Explicit size of that process group.
+ """
+ valid_batch_formats = {"row", "pyarrow", "torch"}
+ if batch_format not in valid_batch_formats:
+ raise ValueError(
+ "batch_format must be one of %s, got %r"
+ % (sorted(valid_batch_formats), batch_format)
+ )
+ if (
+ auto_detect_rank
+ or sharding_rank is not None
+ or sharding_world_size is not None
+ ) and not streaming:
+ raise ValueError("distributed sharding requires streaming=True")
+ if batch_size is not None and (
+ isinstance(batch_size, bool)
+ or not isinstance(batch_size, int)
+ or batch_size <= 0
+ ):
+ raise ValueError("batch_size must be a positive int or None")
+ if batch_format == "row":
+ if self.output_projection is not None and self.output_projection.named:
+ raise RuntimeError(
+ "Projection expressions are not supported by Torch row format; "
+ "use streaming=True with batch_format='pyarrow' or 'torch'")
+ if batch_size is not None:
+ raise ValueError(
+ "batch_size requires batch_format='pyarrow' or 'torch'"
+ )
+ if to_tensor_fn is not None:
+ raise ValueError("to_tensor_fn requires batch_format='torch'")
+ else:
+ if not streaming:
+ raise ValueError(
+ "batch_format=%r requires streaming=True" % batch_format
+ )
+ if batch_format == "pyarrow" and to_tensor_fn is not None:
+ raise ValueError("to_tensor_fn requires batch_format='torch'")
+ if to_tensor_fn is not None and not callable(to_tensor_fn):
+ raise ValueError("to_tensor_fn must be callable")
+ if (
+ isinstance(prefetch_concurrency, bool)
+ or not isinstance(prefetch_concurrency, int)
+ or prefetch_concurrency != 1
+ ):
+ raise ValueError(
+ "batch formats require prefetch_concurrency=1"
+ )
+
+ from pypaimon.read.datasource.torch_dataset import (
+ TorchBatchIterDataset,
+ TorchShuffledBatchIterDataset,
+ )
+ dataset_type = TorchShuffledBatchIterDataset if shuffle else TorchBatchIterDataset
+ shuffle_options = (dict(seed=seed, buffer_size=buffer_size,
+ max_buffer_input_splits=max_buffer_input_splits)
+ if shuffle else {})
+ return dataset_type(
+ self,
+ splits,
+ batch_format=batch_format,
+ batch_size=batch_size,
+ to_tensor_fn=to_tensor_fn,
+ auto_detect_rank=auto_detect_rank,
+ sharding_rank=sharding_rank,
+ sharding_world_size=sharding_world_size,
+ **shuffle_options,
+ )
+
+ if shuffle:
+ if not streaming:
+ raise ValueError("shuffle=True only supports streaming=True")
+ if prefetch_concurrency > 1:
+ raise ValueError("shuffle=True does not support prefetch_concurrency > 1")
+ from pypaimon.read.datasource.torch_dataset import TorchShuffledIterDataset
+ dataset = TorchShuffledIterDataset(
+ self,
+ splits,
+ seed=seed,
+ buffer_size=buffer_size,
+ max_buffer_input_splits=max_buffer_input_splits,
+ auto_detect_rank=auto_detect_rank,
+ sharding_rank=sharding_rank,
+ sharding_world_size=sharding_world_size,
+ )
+ return dataset
+
+ if streaming:
+ from pypaimon.read.datasource.torch_dataset import TorchIterDataset
+ dataset = TorchIterDataset(
+ self,
+ splits,
+ prefetch_concurrency,
+ auto_detect_rank=auto_detect_rank,
+ sharding_rank=sharding_rank,
+ sharding_world_size=sharding_world_size,
+ )
+ return dataset
+ else:
+ from pypaimon.read.datasource.torch_dataset import TorchDataset
+ dataset = TorchDataset(self, splits)
+ return dataset
+
+
+def _project_batch_to_output(self, batch: pyarrow.RecordBatch) -> pyarrow.RecordBatch:
+ if self.output_projection is not None:
+ _, paths = self._reader_adapter()
+ outputs = adapter_output_paths(self.output_projection, self._adapter_read_type, paths)
+ arrays = [extract_array(batch, path) for _, path in outputs]
+ # Project the already masked schema, retaining captured nullability.
+ adapted_projection = OutputProjection([(alias, path) for alias, path in outputs])
+ actual = output_schema(batch.schema, adapted_projection)
+ declared = output_schema(read_arrow_schema(self.read_type), self.output_projection)
+ fields = [field.with_nullable(field.nullable or actual[i].nullable) for i, field in enumerate(declared)]
+ if self.include_row_kind and self._row_kind_column in batch.schema.names:
+ arrays.insert(0, batch.column(self._row_kind_column))
+ fields.insert(0, batch.schema.field(self._row_kind_column))
+ if not arrays:
+ return zero_column_batch(batch.num_rows, batch.schema.metadata)
+ return pyarrow.RecordBatch.from_arrays(
+ arrays, schema=pyarrow.schema(fields, metadata=batch.schema.metadata))
+ if not self._needs_output_projection():
+ return batch
+ output_names = list(self._output_column_names)
+ if self.include_row_kind and self._row_kind_column in batch.schema.names:
+ output_names.insert(0, self._row_kind_column)
+ if batch.schema.names == output_names:
+ return batch
+ name_to_pos = {name: i for i, name in enumerate(batch.schema.names)}
+ arrays = [batch.column(name_to_pos[name]) for name in output_names]
+ fields = [batch.schema.field(name_to_pos[name]) for name in output_names]
+ if not arrays:
+ return zero_column_batch(batch.num_rows, batch.schema.metadata)
+ return pyarrow.RecordBatch.from_arrays(
+ arrays, schema=pyarrow.schema(fields))
+
+
+def _begin_auth_read(self, splits):
+ """Refresh the schema the rules are validated against, once for this read.
+
+ Splits are read concurrently and temporal alignment gives each one its own rule object,
+ so a per-split refresh would ask the catalog once per worker; a per-reader one would let
+ a later read reuse a schema fetched for an earlier one.
+ """
+ from pypaimon.read.auth_support import latest_auth_fields
+
+ self._auth_schema = None
+ for split in splits:
+ if isinstance(split, QueryAuthSplit) and split.auth_result is not None:
+ self._auth_schema = latest_auth_fields(self.table)
+ return
+
+
+def convert_rows_to_arrow_batch(row_tuples: List[tuple], schema: pyarrow.Schema) -> pyarrow.RecordBatch:
+ columns_data = zip(*row_tuples)
+ pydict = {name: list(column) for name, column in zip(schema.names, columns_data)}
+ return pyarrow.RecordBatch.from_pydict(pydict, schema=schema)
+
+
+def _native_resolved_blob_fields(self) -> set:
+ """BLOB fields Rust resolves before emitting a batch."""
+ options = self.table.options
+ if options.blob_as_descriptor():
+ return set()
+ if self.table.is_primary_key_table:
+ return {field.name for field in self._scan_read_type if is_blob_file_field(field)}
+ inline_fields = (
+ options.blob_descriptor_fields()
+ | options.blob_view_fields()
+ )
+ read_names = {
+ field.name
+ for field in getattr(self, '_scan_read_type', self._adapter_read_type)
+ }
+ return inline_fields & read_names
+
+
+_MAX_ARROW_OFFSET = (1 << 31) - 1
+
+
+def _offset_bounds(array, width=4):
+ offsets = array.buffers()[1]
+ if offsets is None:
+ return 0, 0
+ value_format = "=i" if width == 4 else "=q"
+ start = struct.unpack_from(
+ value_format, offsets, array.offset * width)[0]
+ end = struct.unpack_from(
+ value_format, offsets, (array.offset + len(array)) * width)[0]
+ return start, end
+
+
+def _collect_offset_usage(array, path, usage):
+ data_type = array.type
+ if pyarrow.types.is_string(data_type) or pyarrow.types.is_binary(data_type):
+ start, end = _offset_bounds(array)
+ usage[path] = end - start
+ return
+
+ if pyarrow.types.is_list(data_type) or pyarrow.types.is_map(data_type):
+ start, end = _offset_bounds(array)
+ usage[path] = end - start
+ if pyarrow.types.is_map(data_type):
+ _collect_offset_usage(
+ array.keys.slice(start, end - start), path + ("key",), usage)
+ _collect_offset_usage(
+ array.items.slice(start, end - start), path + ("item",), usage)
+ else:
+ _collect_offset_usage(
+ array.values.slice(start, end - start), path + ("item",), usage)
+ return
+
+ if pyarrow.types.is_large_list(data_type):
+ start, end = _offset_bounds(array, 8)
+ _collect_offset_usage(
+ array.values.slice(start, end - start), path + ("item",), usage)
+ return
+
+ if pyarrow.types.is_fixed_size_list(data_type):
+ size = data_type.list_size
+ start = array.offset * size
+ _collect_offset_usage(
+ array.values.slice(start, len(array) * size), path + ("item",), usage)
+ return
+
+ if pyarrow.types.is_struct(data_type):
+ for index in range(data_type.num_fields):
+ _collect_offset_usage(
+ array.field(index), path + ("field", index), usage)
+
+
+def _batch_offset_usage(batch):
+ usage = {}
+ for index in range(batch.num_columns):
+ _collect_offset_usage(batch.column(index), (index,), usage)
+ return usage
diff --git a/paimon-python/pypaimon/execution/search_dispatch.py b/paimon-python/pypaimon/execution/search_dispatch.py
new file mode 100644
index 0000000000..6589a3fe8e
--- /dev/null
+++ b/paimon-python/pypaimon/execution/search_dispatch.py
@@ -0,0 +1,154 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Coordinate search execution and transport without coupling the backends.
+
+A live compatible plan executes in Rust. Transported plans and explicit split
+reads select Python lazily. An execution error never triggers another search.
+"""
+
+from copy import copy
+
+
+class _SearchScan:
+ def __init__(self, native, python_factory):
+ self._native_scan = native
+ self._python_factory = python_factory
+ self._python_executor = None
+
+ def scan(self):
+ if self._native_scan.available():
+ return self._native_scan.scan()
+ if self._python_executor is None:
+ self._python_executor = self._python_factory()
+ return self._python_executor.scan()
+
+
+class _SearchRead:
+ def __init__(self, native, python_factory):
+ self._native_reader = native
+ self._python_factory = python_factory
+ self._python_executor = None
+
+ def _python_reader(self):
+ if self._python_executor is None:
+ self._python_executor = self._python_factory()
+ return self._python_executor
+
+ def _reader_for_plan(self, plan):
+ reader = copy(self._python_reader())
+ if plan.snapshot() is not None:
+ reader._table = reader._table._copy_with_snapshot(plan.snapshot())
+ return reader
+
+ def read_plan(self, plan):
+ native = self._native_reader.resolve_plan(plan)
+ if native is not None:
+ return self._native_reader.read_plan(native)
+ return self._reader_for_plan(plan).read_plan(plan)
+
+ def read_batch_plan(self, plan):
+ native = self._native_reader.resolve_plan(plan)
+ if native is not None:
+ return self._native_reader.read_batch_plan(native)
+ return self._reader_for_plan(plan).read_batch_plan(plan)
+
+
+class _SplitSearchRead(_SearchRead):
+ def read(self, splits):
+ return self._python_reader().read(splits)
+
+
+class _BatchSearchRead(_SearchRead):
+ def read_batch(self, splits):
+ return self._python_reader().read_batch(splits)
+
+
+def python_search_reader(reader):
+ """Select Python explicitly for Ray split workers or plan comparisons."""
+ return reader._python_reader() if isinstance(reader, _SearchRead) else reader
+
+
+def _native_requested(builder):
+ enabled = getattr(getattr(builder._table, 'options', None), 'native_read_enabled', None)
+ return callable(enabled) and enabled()
+
+
+def create_vector_scan(builder, python_factory, batch=False):
+ if not _native_requested(builder):
+ return python_factory()
+ from pypaimon.execution.native.read.native_search_plan import native_vector_scan
+ native = native_vector_scan(builder, batch)
+ return python_factory() if native is None else _SearchScan(native, python_factory)
+
+
+def create_vector_read(builder, python_factory, batch=False):
+ if not _native_requested(builder):
+ return python_factory()
+ from pypaimon.execution.native.read.native_search_plan import native_vector_read
+ native = native_vector_read(builder, batch)
+ if native is None:
+ return python_factory()
+ reader_type = _BatchSearchRead if batch else _SplitSearchRead
+ return reader_type(native, python_factory)
+
+
+def create_full_text_scan(builder, python_factory):
+ if not _native_requested(builder):
+ return python_factory()
+ from pypaimon.execution.native.read.native_search_plan import native_full_text_scan
+ native = native_full_text_scan(builder)
+ return python_factory() if native is None else _SearchScan(native, python_factory)
+
+
+def create_full_text_read(builder, python_factory):
+ if not _native_requested(builder):
+ return python_factory()
+ from pypaimon.execution.native.read.native_search_plan import native_full_text_read
+ native = native_full_text_read(builder)
+ return python_factory() if native is None else _SplitSearchRead(native, python_factory)
+
+
+def execute_vector_search(builder, queries, batch=False):
+ if _native_requested(builder):
+ from pypaimon.execution.native.read.native_vector_search import try_native_vector_search
+ result = try_native_vector_search(builder, queries, batch)
+ if result is not None:
+ return result
+ reader = (builder.new_batch_vector_search_read() if batch
+ else builder.new_vector_search_read())
+ plan = builder.new_vector_search_scan().scan()
+ return reader.read_batch_plan(plan) if batch else reader.read_plan(plan)
+
+
+def execute_full_text_search(builder):
+ if _native_requested(builder):
+ from pypaimon.execution.native.read.native_full_text_search import try_native_full_text_search
+ result = try_native_full_text_search(builder)
+ if result is not None:
+ return result
+ return builder.new_full_text_read().read_plan(builder.new_full_text_scan().scan())
+
+
+def execute_hybrid_search(builder):
+ if _native_requested(builder):
+ from pypaimon.execution.native.read.native_hybrid_search import try_native_hybrid_search
+ result = try_native_hybrid_search(builder)
+ if result is not None:
+ return result
+ from pypaimon.execution.python.read.search.hybrid_search import execute_local
+ return execute_local(builder)
diff --git a/paimon-python/pypaimon/execution/update_dispatch.py b/paimon-python/pypaimon/execution/update_dispatch.py
new file mode 100644
index 0000000000..56fe72cbed
--- /dev/null
+++ b/paimon-python/pypaimon/execution/update_dispatch.py
@@ -0,0 +1,229 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Select update execution before mutations, preserving public owner hooks."""
+
+from typing import Any, Iterable, List, Mapping, Optional, Sequence
+
+import pyarrow as pa
+
+from pypaimon.common.predicate import Predicate
+from pypaimon.execution.dispatch import prepare_native_write_operation
+from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER
+from pypaimon.table.special_fields import SpecialFields
+from pypaimon.write.commit_message import CommitMessage
+from pypaimon.write.row_utils import row_to_named_values
+from pypaimon.write.table_update_by_row_id import TableUpdateByRowId
+from pypaimon.write.table_upsert_by_key import TableUpsertByKey
+from pypaimon.write.update_validation import _authorize_update
+
+
+class TableUpdateOperations:
+
+ def _new_row_id_updater(self, commit_identifier: int, _precomputed_files_info=None) -> TableUpdateByRowId:
+ _authorize_update(self.table)
+ if _precomputed_files_info is None:
+ native = prepare_native_write_operation('update_by_row_id', self.table, self.commit_user, commit_identifier)
+ if native is not None:
+ return native
+ return TableUpdateByRowId(self.table, self.commit_user, commit_identifier,
+ _precomputed_files_info=_precomputed_files_info)
+
+ def new_shard_updator(self, shard_num: int, total_shard_count: int):
+ from pypaimon.write.table_update import StreamTableUpdate
+ """Create a shard updater for scan and rewrite style updates."""
+ if total_shard_count <= 0 or shard_num < 0 or shard_num >= total_shard_count:
+ raise ValueError('Shard index must be smaller than a positive shard count')
+ if self.update_cols == []:
+ raise ValueError('Shard update columns must not be empty')
+ native = prepare_native_write_operation(
+ 'shard_update',
+ self.table,
+ self.commit_user,
+ self.projection,
+ self.update_cols,
+ shard_num,
+ total_shard_count,
+ stream=isinstance(
+ self,
+ StreamTableUpdate))
+ if native is not None:
+ return native
+ return self._new_python_shard_updator(shard_num, total_shard_count)
+
+ def _update_by_arrow_with_row_id(self, table: pa.Table, commit_identifier: int) -> List[CommitMessage]:
+ cols = self.update_cols if self.update_cols is not None else [
+ c for c in table.column_names if c != SpecialFields.ROW_ID.name]
+ if table.num_rows and cols and (SpecialFields.ROW_ID.name in table.column_names):
+ native = prepare_native_write_operation('update', self.table, self.commit_user, cols)
+ if native is not None:
+ return native.update_by_arrow_with_row_id(table)
+ return self._new_row_id_updater(commit_identifier).update_columns(table, cols)
+
+ def _update_by_arrow_batches_with_row_id(
+ self, tables: Iterable[pa.Table], commit_identifier: int) -> List[CommitMessage]:
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._update_by_arrow_batches_with_row_id(self, tables, commit_identifier)
+
+ def _upsert_by_arrow_with_key(
+ self, table: pa.Table, upsert_keys: List[str], commit_identifier: int) -> List[CommitMessage]:
+ """Shared implementation for ``upsert_by_arrow_with_key``.
+
+ For each row in the input Arrow table:
+
+ * If one or more rows with the same composite ``upsert_keys`` value
+ already exist → update all of them in-place.
+ * Otherwise → append as a new row.
+
+ The public method lives on the concrete subclasses so each can
+ expose the signature appropriate to its mode (batch vs stream).
+
+ Args:
+ table: Input Arrow table containing rows to upsert.
+ upsert_keys: One or more column names forming the composite match key.
+ commit_identifier: Identifier to tag the produced commit messages with.
+
+ Returns:
+ List of :class:`CommitMessage` objects to be committed.
+ """
+ return TableUpsertByKey(self.table, self.commit_user, commit_identifier).upsert(
+ table, upsert_keys, self.update_cols)
+
+ def _upsert_by_key(self, rows, upsert_keys: List[str], commit_identifier: int) -> List[CommitMessage]:
+ return TableUpsertByKey(self.table, self.commit_user, commit_identifier).upsert_rows(
+ rows, upsert_keys, self.update_cols)
+
+ def _merge_into(self, source: Any, on, when_matched: Sequence, when_not_matched: Sequence,
+ commit_identifier: int) -> List[CommitMessage]:
+ from pypaimon.table.data_evolution_merge_into import merge_into
+ return merge_into(self.table, source, on=on, when_matched=when_matched, when_not_matched=when_not_matched,
+ commit_user=self.commit_user, commit_identifier=commit_identifier)
+
+ def _update_by_predicate(
+ self, predicate: Optional[Predicate], assignments: Mapping[str, Any],
+ commit_identifier: int, read_columns: Optional[Sequence[str]] = None) -> List[CommitMessage]:
+ """Shared implementation for SQL-like ``UPDATE ... WHERE ...``.
+
+ ``predicate`` identifies the target rows. Assignment values may be
+ literals or callables receiving one matched logical file group as an
+ Arrow table.
+ """
+ has_callable = any((callable(value) for value in assignments.values()))
+ has_array = any((isinstance(value, (pa.Array, pa.ChunkedArray)) for value in assignments.values()))
+ read_columns = tuple(read_columns or ())
+ self._validate_predicate_update(assignments, read_columns, has_callable, has_array)
+ if self.table.options.native_write_enabled():
+ native = prepare_native_write_operation(
+ 'predicate_update',
+ self.table,
+ self.commit_user,
+ predicate,
+ list(assignments))
+ if native is not None:
+ return native.update(assignments, read_columns)
+ return self._execute_predicate_update(
+ predicate, assignments, commit_identifier, read_columns, has_callable, has_array)
+
+ def _delete_by_predicate(self, predicate: Optional[Predicate], commit_identifier: int) -> List[CommitMessage]:
+ native = prepare_native_write_operation('predicate_delete', self.table, self.commit_user, predicate)
+ if native is not None:
+ return native.delete()
+ return self._execute_predicate_delete(predicate, commit_identifier)
+
+ def _delete_by_row_id(self, row_ids: Sequence[int], commit_identifier: int) -> List[CommitMessage]:
+ if len(row_ids):
+ native = prepare_native_write_operation('delete', self.table, self.commit_user)
+ if native is not None:
+ return native.delete_by_row_id(row_ids)
+ return self._delete_rows_with_python(row_ids)
+
+ def _matched_delete_row_ids(self, predicate: Optional[Predicate]) -> List[int]:
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._matched_delete_row_ids(self, predicate)
+
+ def update_by_arrow_batches_with_row_id(self, tables: Iterable[pa.Table]) -> List[CommitMessage]:
+ """Apply row-id updates from batches using one target-file index.
+
+ For each updated column, batches must target disjoint ``first_row_id``
+ file groups. Conflicting overlap is rejected and all files staged by
+ earlier batches are aborted.
+ """
+ native = prepare_native_write_operation('update', self.table, self.commit_user, self.update_cols)
+ if native is not None:
+ return native.update_by_arrow_batches_with_row_id(tables)
+ return self._update_by_arrow_batches_with_row_id(tables, BATCH_COMMIT_IDENTIFIER)
+
+
+class UpsertOperations:
+
+ def upsert(self, data: pa.Table, upsert_keys: List[str],
+ update_cols: Optional[List[str]] = None) -> List[CommitMessage]:
+ """
+ Upsert rows into an append-only table by the specified key columns.
+
+ Execution is driven partition-by-partition:
+ 1. Group input rows by their partition values.
+ 2. For each partition, scan only that partition to build the
+ key -> _ROW_ID map.
+ 3. Split the partition's input rows into matched (update) and
+ unmatched (append).
+ 4. Perform updates and appends.
+
+ This keeps memory usage proportional to a single partition's key set
+ and avoids scanning the entire table.
+
+ Args:
+ data: Input Arrow table containing rows to upsert.
+ Must contain all upsert_keys columns.
+ upsert_keys: One or more column names used together as a composite
+ match key.
+ update_cols: Columns to update for matched rows.
+ If None, all columns in the table schema are updated.
+
+ Returns:
+ List of CommitMessages to be committed.
+ """
+ self._validate_inputs(data, upsert_keys, update_cols)
+ if update_cols is None or len(update_cols) == len(self.table.field_names):
+ effective_update_cols = None
+ else:
+ effective_update_cols = update_cols
+ columns = list(effective_update_cols) if effective_update_cols else list(self.table.field_names)
+ native = self._create_native_upsert(data, upsert_keys, columns)
+ if native is not None:
+ return native.upsert()
+ return self._upsert_arrow(data, upsert_keys, effective_update_cols)
+
+ def upsert_rows(self, rows, upsert_keys: List[str], update_cols: Optional[List[str]] = None) -> List[CommitMessage]:
+ row_list = self._normalize_rows(rows)
+ if not row_list:
+ raise ValueError('rows must not be empty.')
+ row_items = [(row, row_to_named_values(row, self.table.table_schema.fields)) for row in row_list]
+ for _, values_by_name in row_items:
+ self._validate_row_inputs(values_by_name, upsert_keys, update_cols)
+ if update_cols is None or len(update_cols) == len(self.table.field_names):
+ effective_update_cols = None
+ else:
+ effective_update_cols = update_cols
+ columns = list(effective_update_cols) if effective_update_cols is not None else list(self.table.field_names)
+ native = self._create_native_upsert([values for _, values in row_items], upsert_keys, columns)
+ if native is not None:
+ return native.upsert()
+ return self._upsert_rows(row_items, upsert_keys, effective_update_cols)
+
+ def _create_native_upsert(self, data, upsert_keys, columns):
+ return prepare_native_write_operation('upsert', self.table, self.commit_user, data, upsert_keys, columns)
diff --git a/paimon-python/pypaimon/execution/write_dispatch.py b/paimon-python/pypaimon/execution/write_dispatch.py
new file mode 100644
index 0000000000..76ee599ff7
--- /dev/null
+++ b/paimon-python/pypaimon/execution/write_dispatch.py
@@ -0,0 +1,239 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Writer backend selection and lifecycle, outside either execution backend."""
+
+import pyarrow as pa
+
+
+class NativeTableWrite:
+ """Keep one selected writer and allow explicit pre-mutation diagnostics."""
+
+ def __init__(self, table, commit_user, static_partition, stream, native_writer,
+ restore_snapshot_id=None):
+ from pypaimon.execution.native.write.native_write import NativeTableWrite as NativeWrite
+
+ self._execution = NativeWrite(table, commit_user, static_partition, stream,
+ native_writer, restore_snapshot_id)
+ self._python_writer = None
+
+ @classmethod
+ def from_native(cls, native):
+ from pypaimon.execution.native.write.native_write import (
+ NativePostponeFixedBucketTableWrite as NativePostponeWrite,
+ )
+
+ selected = (NativePostponeFixedBucketTableWrite
+ if isinstance(native, NativePostponeWrite) else cls)
+ result = object.__new__(selected)
+ result._execution = native
+ result._python_writer = None
+ return result
+
+ def __getattr__(self, name):
+ # Metadata or unknown-attribute inspection must never select a backend.
+ execution = object.__getattribute__(self, '_execution')
+ return getattr(execution, name)
+
+ @property
+ def _native_writer(self):
+ return self._execution._native_writer
+
+ @_native_writer.setter
+ def _native_writer(self, writer):
+ self._execution._native_writer = writer
+
+ @property
+ def _written(self):
+ return self._execution._written
+
+ def _new_python_writer(self):
+ from pypaimon.execution.dispatch import create_python_write
+
+ writer = create_python_write(
+ self.table, self.commit_user, self.static_partition, self.stream,
+ restore_snapshot_id=self._restore_snapshot_id)
+ return self._configure_python_writer(writer)
+
+ def _configure_python_writer(self, writer):
+ if not self.stream:
+ writer.batch_committed = self._batch_committed
+ if self._write_cols is not None:
+ writer.with_write_type(self._write_cols)
+ if self._blob_consumer is not None:
+ writer.with_blob_consumer(self._blob_consumer)
+ if self._blob_uri_reader_factory is not None:
+ writer.with_blob_uri_reader_factory(self._blob_uri_reader_factory)
+ return writer
+
+ def _switch_to_python(self):
+ from pypaimon.execution.dispatch import switch_writer_to_python
+
+ return switch_writer_to_python(self)
+
+ @property
+ def file_store_write(self):
+ return self._switch_to_python().file_store_write
+
+ @file_store_write.setter
+ def file_store_write(self, writer):
+ self._switch_to_python().file_store_write = writer
+
+ @property
+ def row_key_extractor(self):
+ return self._switch_to_python().row_key_extractor
+
+ @row_key_extractor.setter
+ def row_key_extractor(self, extractor):
+ self._switch_to_python().row_key_extractor = extractor
+
+ @property
+ def table_pyarrow_schema(self):
+ return (self._python_writer.table_pyarrow_schema
+ if self._python_writer is not None else self._table_schema)
+
+ @table_pyarrow_schema.setter
+ def table_pyarrow_schema(self, schema):
+ self._switch_to_python().table_pyarrow_schema = schema
+
+ @property
+ def restore_snapshot_id(self):
+ return (self._python_writer.restore_snapshot_id
+ if self._python_writer is not None else self._restore_snapshot_id)
+
+ @restore_snapshot_id.setter
+ def restore_snapshot_id(self, snapshot_id):
+ self._switch_to_python().restore_snapshot_id = snapshot_id
+
+ @property
+ def batch_committed(self):
+ if self._python_writer is not None:
+ return self._python_writer.batch_committed
+ if self.stream:
+ raise AttributeError('batch_committed')
+ return self._batch_committed
+
+ @batch_committed.setter
+ def batch_committed(self, value):
+ self._switch_to_python().batch_committed = value
+
+ def with_write_type(self, write_cols):
+ if self._python_writer is not None:
+ return self._python_writer.with_write_type(write_cols)
+ names = list(write_cols)
+ full_names = self._table_schema.names
+ if (self.table.is_primary_key_table
+ and not (len(names) == len(full_names) and set(names) == set(full_names))):
+ return self._switch_to_python().with_write_type(names)
+ self._execution.with_write_type(names)
+ return self
+
+ def with_blob_consumer(self, blob_consumer):
+ if self._python_writer is not None:
+ return self._python_writer.with_blob_consumer(blob_consumer)
+ self._execution.with_blob_consumer(blob_consumer)
+ return self
+
+ def with_blob_uri_reader_factory(self, uri_reader_factory):
+ if self._python_writer is not None:
+ return self._python_writer.with_blob_uri_reader_factory(uri_reader_factory)
+ self._execution.with_blob_uri_reader_factory(uri_reader_factory)
+ return self
+
+ def write_arrow(self, data, bucket=None):
+ if self._python_writer is not None:
+ return self._python_writer.write_arrow(data, bucket)
+ data = self._execution._prepare_native_arrow_data(data)
+ if isinstance(data, pa.RecordBatch):
+ return self.write_arrow_batch(data, bucket)
+ for batch in data.to_batches():
+ self.write_arrow_batch(batch, bucket)
+
+ def write_arrow_batch(self, data, bucket=None):
+ execution = self._python_writer if self._python_writer is not None else self._execution
+ return execution.write_arrow_batch(data, bucket)
+
+ def write_pandas(self, dataframe):
+ if self._python_writer is not None:
+ return self._python_writer.write_pandas(dataframe)
+ from pypaimon.write.input_adapter import _pandas_write_batch
+
+ return self.write_arrow_batch(_pandas_write_batch(
+ self.table, dataframe, self._schema, self._table_schema))
+
+ def write_row(self, row):
+ execution = self._python_writer if self._python_writer is not None else self._execution
+ return execution.write_row(row)
+
+ def write_ray(self, dataset, overwrite=False, concurrency=None, ray_remote_args=None,
+ hash_fixed_precluster='auto', static_partition=None):
+ if self._python_writer is not None:
+ return self._python_writer.write_ray(
+ dataset, overwrite, concurrency, ray_remote_args, hash_fixed_precluster, static_partition)
+ from pypaimon.write.input_adapter import _write_ray
+
+ _write_ray(self, dataset, overwrite, concurrency, ray_remote_args,
+ hash_fixed_precluster, static_partition)
+
+ def _distributed_write_options(self, *, overwrite=False):
+ return {}
+
+ def roll_before_group_if_needed(self, row_count):
+ execution = self._python_writer if self._python_writer is not None else self._execution
+ return execution.roll_before_group_if_needed(row_count)
+
+ def prepare_commit(self, commit_identifier=None):
+ if self._python_writer is not None:
+ if self.stream:
+ return self._python_writer.prepare_commit(commit_identifier)
+ return self._python_writer.prepare_commit()
+ return self._execution.prepare_commit(commit_identifier)
+
+ def close(self):
+ if self._python_writer is not None:
+ self._python_writer.close()
+ return self._execution.close()
+
+ def abort(self):
+ if self._python_writer is not None:
+ return self._python_writer.abort()
+ return self._execution.abort()
+
+
+class NativePostponeFixedBucketTableWrite(NativeTableWrite):
+ """Share postpone planning without coupling either backend to the other."""
+
+ def __init__(self, table, commit_user, static_partition, bucket_plan, native_writer):
+ from pypaimon.execution.native.write.native_write import (
+ NativePostponeFixedBucketTableWrite as NativePostponeWrite,
+ )
+
+ self._execution = NativePostponeWrite(
+ table, commit_user, static_partition, bucket_plan, native_writer)
+ self._python_writer = None
+
+ def _new_python_writer(self):
+ from pypaimon.execution.dispatch import create_python_write
+
+ return self._configure_python_writer(create_python_write(
+ self.table, self.commit_user, self.static_partition,
+ fixed_bucket=True, bucket_plan=self._bucket_plan))
+
+ def _distributed_write_options(self, *, overwrite=False):
+ from pypaimon.write.input_adapter import postpone_distributed_write_options
+
+ return postpone_distributed_write_options(self.table, self._bucket_plan, overwrite)
diff --git a/paimon-python/pypaimon/filesystem/hdfs_native_file_io.py b/paimon-python/pypaimon/filesystem/hdfs_native_file_io.py
index ef2e55cf3e..9e9e14ac8a 100644
--- a/paimon-python/pypaimon/filesystem/hdfs_native_file_io.py
+++ b/paimon-python/pypaimon/filesystem/hdfs_native_file_io.py
@@ -34,7 +34,6 @@ from pypaimon.common.options.config import HdfsOptions, SecurityOptions
from pypaimon.common.uri_reader import UriReaderFactory
from pypaimon.filesystem import _kerberos
from pypaimon.schema.data_types import AtomicType, DataField, PyarrowFieldParser
-from pypaimon.execution.python.write.blob_format_writer import BlobFormatWriter
class _HdfsFileInfo:
@@ -634,6 +633,8 @@ class HdfsNativeFileIO(FileIO):
raise RuntimeError(f"Failed to write row file {path}: {e}") from e
def write_blob(self, path: str, data: pyarrow.Table, **kwargs):
+ from pypaimon.execution.python.write.blob_format_writer import BlobFormatWriter
+
try:
if data.num_columns != 1:
raise RuntimeError(
diff --git a/paimon-python/pypaimon/filesystem/local_file_io.py b/paimon-python/pypaimon/filesystem/local_file_io.py
index 091b7b2eaa..2e53eb3123 100644
--- a/paimon-python/pypaimon/filesystem/local_file_io.py
+++ b/paimon-python/pypaimon/filesystem/local_file_io.py
@@ -32,7 +32,6 @@ from pypaimon.common.options import Options
from pypaimon.common.uri_reader import UriReaderFactory
from pypaimon.filesystem.local import PaimonLocalFileSystem
from pypaimon.schema.data_types import DataField, AtomicType, PyarrowFieldParser
-from pypaimon.execution.python.write.blob_format_writer import BlobFormatWriter
def _normalize_naive_datetimes(value):
@@ -480,6 +479,8 @@ class LocalFileIO(FileIO):
raise RuntimeError(f"Failed to write row file {path}: {e}") from e
def write_blob(self, path: str, data: pyarrow.Table, **kwargs):
+ from pypaimon.execution.python.write.blob_format_writer import BlobFormatWriter
+
try:
if data.num_columns != 1:
raise RuntimeError(f"Blob format only supports a single column, got {data.num_columns} columns")
diff --git a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py
index efe851700a..3d9bc985ee 100644
--- a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py
+++ b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py
@@ -45,7 +45,6 @@ from pypaimon.filesystem import oss_user_agent
from pypaimon.filesystem.jindo_file_system_handler import JindoFileSystemHandler, JINDO_AVAILABLE
from pypaimon.schema.data_types import (AtomicType, DataField,
PyarrowFieldParser)
-from pypaimon.execution.python.write.blob_format_writer import BlobFormatWriter
def _pyarrow_lt_7():
@@ -1027,6 +1026,8 @@ class PyArrowFileIO(FileIO):
raise RuntimeError(f"Failed to write row file {path}: {e}") from e
def write_blob(self, path: str, data: pyarrow.Table, **kwargs):
+ from pypaimon.execution.python.write.blob_format_writer import BlobFormatWriter
+
try:
if data.num_columns != 1:
raise RuntimeError(f"Blob format only supports a single column, got {data.num_columns} columns")
diff --git a/paimon-python/pypaimon/globalindex/create_global_index.py b/paimon-python/pypaimon/globalindex/create_global_index.py
index b7fd42d9af..463403c806 100644
--- a/paimon-python/pypaimon/globalindex/create_global_index.py
+++ b/paimon-python/pypaimon/globalindex/create_global_index.py
@@ -19,12 +19,6 @@
from typing import Dict, List, Optional, Sequence, Union
-import pyarrow as pa
-
-from pypaimon.table.row.generic_row import GenericRow
-from pypaimon.manifest.index_manifest_entry import IndexManifestEntry
-from pypaimon.utils.range import Range
-
from pypaimon.common.options.core_options import CoreOptions
from pypaimon.common.options.options import Options
from pypaimon.common.predicate import Predicate
@@ -137,61 +131,15 @@ class GlobalIndexBuilder:
if execution == "local" and (concurrency is not None or ray_remote_args is not None):
raise ValueError("Ray options require execution='ray'.")
if execution == "ray":
- from pypaimon.ray.vector_index_build import validate_build_options
+ from pypaimon.execution.python.globalindex.ray_build import validate_build_options
concurrency, ray_remote_args = validate_build_options(
self, concurrency, ray_remote_args)
partition_filter = self._resolve_partition_filter()
- if execution == "local":
- from pypaimon.execution.native.write.native_index_build import build_native_global_index
- messages = build_native_global_index(self, partition_filter)
- if messages is not None:
- return messages
- from pypaimon.execution.python.globalindex.create_global_index import build_global_index
+ from pypaimon.execution.index_dispatch import build_global_index
return build_global_index(
self, partition_filter, execution=execution, concurrency=concurrency,
ray_remote_args=ray_remote_args)
- def _snapshot_for_plan(self, plan):
- from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
- return PythonGlobalIndexBuilder._snapshot_for_plan(self, plan)
-
- def _build_sorted_index(
- self, splits, unindexed_ranges, index_field, table_read, index_path: str
- ) -> List[CommitMessage]:
- from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
- return PythonGlobalIndexBuilder._build_sorted_index(
- self, splits, unindexed_ranges, index_field, table_read, index_path)
-
- def _create_sorted_index_writer(self, index_path: str, key_serializer):
- from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
- return PythonGlobalIndexBuilder._create_sorted_index_writer(self, index_path, key_serializer)
-
- def _build_generic_index(
- self, splits, unindexed_ranges, index_field, table_read, index_path: str
- ) -> List[CommitMessage]:
- from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
- return PythonGlobalIndexBuilder._build_generic_index(
- self, splits, unindexed_ranges, index_field, table_read, index_path)
-
- def _build_generic_shard(
- self, index_split, index_range, index_field, table_read, index_path: str, file_name=None
- ) -> Optional[CommitMessage]:
- from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
- return PythonGlobalIndexBuilder._build_generic_shard(
- self, index_split, index_range, index_field, table_read, index_path, file_name)
-
- def _delete_writer_output(self, writer, index_path: str) -> None:
- from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
- return PythonGlobalIndexBuilder._delete_writer_output(self, writer, index_path)
-
- def _delete_uncommitted_indexes(self, messages) -> None:
- from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
- return PythonGlobalIndexBuilder._delete_uncommitted_indexes(self, messages)
-
- def _create_generic_index_writer(self, index_path: str, index_field):
- from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
- return PythonGlobalIndexBuilder._create_generic_index_writer(self, index_path, index_field)
-
def _validate_unaware_bucket_table(self) -> None:
# Global index files live in bucket 0 and are keyed by the table-wide
# _ROW_ID, so every index type needs an unaware-bucket table. Check the
@@ -256,46 +204,3 @@ def _merged_options(table, options: Optional[Dict[str, object]]) -> Options:
if options:
merged.update(options)
return Options(merged)
-
-
-def _extract_sorted_rows(
- table: pa.Table,
- index_column: str,
- row_id_column: str,
- key_serializer,
- row_range: Optional[Range] = None,
-):
- from pypaimon.execution.python.globalindex.create_global_index import _extract_sorted_rows as execute
- return execute(table, index_column, row_id_column, key_serializer, row_range)
-
-
-def _write_vector_batch(writer, batch, index_column, row_range):
- from pypaimon.execution.python.globalindex.create_global_index import _write_vector_batch as execute
- return execute(writer, batch, index_column, row_range)
-
-
-def _extract_index_rows(
- table: Union[pa.Table, pa.RecordBatch],
- index_column: str,
- row_id_column: str,
- row_range: Optional[Range] = None,
-):
- from pypaimon.execution.python.globalindex.create_global_index import _extract_index_rows as execute
- return execute(table, index_column, row_id_column, row_range)
-
-
-def _chunks(rows, size):
- from pypaimon.execution.python.globalindex.create_global_index import _chunks as execute
- return execute(rows, size)
-
-
-def _to_index_manifest_entries(
- table,
- partition: GenericRow,
- row_range: Range,
- index_field_id: int,
- index_type: str,
- result_entries,
-) -> List[IndexManifestEntry]:
- from pypaimon.execution.python.globalindex.create_global_index import _to_index_manifest_entries as execute
- return execute(table, partition, row_range, index_field_id, index_type, result_entries)
diff --git a/paimon-python/pypaimon/globalindex/drop_global_index.py b/paimon-python/pypaimon/globalindex/drop_global_index.py
index e7714b88ed..449d5c8348 100644
--- a/paimon-python/pypaimon/globalindex/drop_global_index.py
+++ b/paimon-python/pypaimon/globalindex/drop_global_index.py
@@ -88,16 +88,9 @@ class GlobalIndexDropper:
def build(self) -> List[CommitMessage]:
partition_filter = self._resolve_partition_filter()
- messages = self._native_build(partition_filter)
- if messages is not None:
- return messages
- from pypaimon.execution.python.globalindex.drop_global_index import build_global_index_drop
+ from pypaimon.execution.index_dispatch import build_global_index_drop
return build_global_index_drop(self, partition_filter)
- def _native_build(self, partition_filter):
- from pypaimon.execution.native.write.native_index_drop import build_native_global_index_drop
- return build_native_global_index_drop(self, partition_filter)
-
def _resolve_partition_filter(self) -> Optional[Predicate]:
if self._partition_filter is not None:
return self._partition_filter
@@ -133,8 +126,3 @@ def _normalize_index_columns(index_column: Union[str, Sequence[str]]) -> List[st
if isinstance(index_column, str):
return [c.strip() for c in index_column.split(",") if c.strip()]
return [str(c).strip() for c in index_column if str(c).strip()]
-
-
-def _indexed_field_ids(global_meta) -> List[int]:
- from pypaimon.execution.python.globalindex.drop_global_index import _indexed_field_ids as execute
- return execute(global_meta)
diff --git a/paimon-python/pypaimon/multimodal/temporal.py b/paimon-python/pypaimon/multimodal/temporal.py
index d3257e04a5..8b3b90eed5 100644
--- a/paimon-python/pypaimon/multimodal/temporal.py
+++ b/paimon-python/pypaimon/multimodal/temporal.py
@@ -36,7 +36,7 @@ from pypaimon.globalindex.indexed_split import IndexedSplit
from pypaimon.multimodal.query import ScanQuery
from pypaimon.read.query_auth_split import QueryAuthSplit, resolve_auth_result
from pypaimon.execution.python.read.reader.format_pyarrow_reader import (_DecodedRowGroupCache)
-from pypaimon.read.table_read import _ClosableArrowBatchReader
+from pypaimon.execution.read_support import _ClosableArrowBatchReader
from pypaimon.schema.data_types import PyarrowFieldParser
from pypaimon.snapshot.time_travel_util import TimeTravelUtil
from pypaimon.table.special_fields import SpecialFields
diff --git a/paimon-python/pypaimon/ray/batch_vector_search.py b/paimon-python/pypaimon/ray/batch_vector_search.py
index c7c8479b63..cedaaf4a12 100644
--- a/paimon-python/pypaimon/ray/batch_vector_search.py
+++ b/paimon-python/pypaimon/ray/batch_vector_search.py
@@ -31,7 +31,7 @@ from pypaimon.execution.python.read.search.vector_search_read import (
def _execute_batch_vector_search(builder, *, concurrency=None, ray_remote_args=None):
concurrency, remote_args = _execution_options(concurrency, ray_remote_args)
- from pypaimon.execution.native.read.native_search_plan import python_search_reader
+ from pypaimon.execution.search_dispatch import python_search_reader
reader = python_search_reader(builder.new_batch_vector_search_read())
if (type(reader) is not BatchVectorSearchReadImpl
or not reader._table.options.data_evolution_enabled()):
@@ -130,7 +130,7 @@ def _search_batch_index_split(context, split, scalar_filter=None):
def _search_batch_raw_split(context, split):
- from pypaimon.read.table_read import _ClosableArrowBatchReader
+ from pypaimon.execution.read_support import _ClosableArrowBatchReader
table_read, column, queries, limit, metric = context
scorer = BatchVectorSearchReadImpl(table_read.table, limit, column, queries)
diff --git a/paimon-python/pypaimon/ray/full_text_search.py b/paimon-python/pypaimon/ray/full_text_search.py
index 861b6d2b34..0ca078fca8 100644
--- a/paimon-python/pypaimon/ray/full_text_search.py
+++ b/paimon-python/pypaimon/ray/full_text_search.py
@@ -34,7 +34,7 @@ from pypaimon.utils.roaring_bitmap import RoaringBitmap64
def _execute_full_text_search(builder, *, concurrency=None, ray_remote_args=None):
concurrency, remote_args = _execution_options(concurrency, ray_remote_args)
- from pypaimon.execution.native.read.native_search_plan import python_search_reader
+ from pypaimon.execution.search_dispatch import python_search_reader
reader = python_search_reader(builder.new_full_text_read())
if (type(reader) is not DataEvolutionFullTextRead
or not reader._table.options.data_evolution_enabled()):
diff --git a/paimon-python/pypaimon/ray/vector_search.py b/paimon-python/pypaimon/ray/vector_search.py
index 33dc7a815e..dd117c58c4 100644
--- a/paimon-python/pypaimon/ray/vector_search.py
+++ b/paimon-python/pypaimon/ray/vector_search.py
@@ -40,7 +40,7 @@ from pypaimon.utils.roaring_bitmap import RoaringBitmap64
def _execute_vector_search(builder, *, concurrency=None, ray_remote_args=None):
"""Execute the builder of an already snapshot-pinned ``VectorQuery``."""
concurrency, remote_args = _execution_options(concurrency, ray_remote_args)
- from pypaimon.execution.native.read.native_search_plan import python_search_reader
+ from pypaimon.execution.search_dispatch import python_search_reader
reader = python_search_reader(builder.new_vector_search_read())
if (type(reader) is not DataEvolutionVectorRead
or not reader._table.options.data_evolution_enabled()):
@@ -199,7 +199,7 @@ def _shared_scalar_filters(splits, predicate, remote_args, worker=None):
def _search_raw_split(context, split):
- from pypaimon.read.table_read import _ClosableArrowBatchReader
+ from pypaimon.execution.read_support import _ClosableArrowBatchReader
table_read, column, query, limit, metric = context
scorer = DataEvolutionVectorRead(table_read.table, limit, column, query)
diff --git a/paimon-python/pypaimon/read/datasource/torch_dataset.py b/paimon-python/pypaimon/read/datasource/torch_dataset.py
index d1c05a95d3..7f76ccce0c 100644
--- a/paimon-python/pypaimon/read/datasource/torch_dataset.py
+++ b/paimon-python/pypaimon/read/datasource/torch_dataset.py
@@ -35,7 +35,7 @@ from torch.utils.data import Dataset, IterableDataset
from pypaimon.globalindex.indexed_split import IndexedSplit
from pypaimon.read.query_auth_split import QueryAuthSplit
-from pypaimon.execution.python.read.reader.concat_batch_reader import (_MAX_ARROW_OFFSET, _batch_offset_usage)
+from pypaimon.execution.read_support import _MAX_ARROW_OFFSET, _batch_offset_usage
from pypaimon.read.split import DataSplit, Split
from pypaimon.read.table_read import TableRead
from pypaimon.table.special_fields import SpecialFields
diff --git a/paimon-python/pypaimon/read/read_builder.py b/paimon-python/pypaimon/read/read_builder.py
index e4d3297d71..2e99f553ba 100644
--- a/paimon-python/pypaimon/read/read_builder.py
+++ b/paimon-python/pypaimon/read/read_builder.py
@@ -16,18 +16,14 @@
# under the License.
import ast
-from typing import Dict, List, Optional, Sequence, Union
+from typing import Dict, List, Optional, Sequence, Union, TYPE_CHECKING
import pyarrow
from pypaimon.common.predicate import Predicate
from pypaimon.common.predicate_builder import PredicateBuilder
-from pypaimon.read.explain import ExplainResult, ExplainSplitInfo, PruningStat
-from pypaimon.read.explain_render import render_predicate
+from pypaimon.read.explain import ExplainResult
from pypaimon.read.push_down_utils import predicate_field_names, predicate_matches_physical_names
-from pypaimon.read.query_auth_split import QueryAuthSplit
-from pypaimon.read.scan_stats import ScanStats
-from pypaimon.read.split import Split
from pypaimon.read.table_read import ROW_KIND_COLUMN, TableRead
from pypaimon.read.table_scan import TableScan
from pypaimon.read.read_type import OutputProjection, output_fields, project_read_type, reader_adapter
@@ -37,6 +33,9 @@ from pypaimon.table.special_fields import SpecialFields
from pypaimon.utils.projection import MapKey, Projection, is_row_type
+if TYPE_CHECKING:
+ from pypaimon.table.file_store_table import FileStoreTable
+
ProjectionPath = Sequence[Union[int, MapKey]]
@@ -110,9 +109,7 @@ class ReadBuilder:
"""Implementation of ReadBuilder for native Python reading."""
def __init__(self, table):
- from pypaimon.table.file_store_table import FileStoreTable
-
- self.table: FileStoreTable = table
+ self.table: "FileStoreTable" = table
self._predicate: Optional[Predicate] = None
self._read_type: Optional[List[DataField]] = None
self._output_projection: Optional[OutputProjection] = None
@@ -243,19 +240,8 @@ class ReadBuilder:
early filter usually prunes aggressively (e.g. very wide
HASH_FIXED tables with a tight predicate).
"""
- scan = self.new_scan()
- plan, stats = scan.scan_with_stats()
- return _build_explain_result(
- table=self.table,
- scan=scan,
- plan=plan,
- stats=stats,
- predicate=self._predicate,
- projection=([alias for alias, _ in self._output_projection.columns]
- if self._output_projection else None),
- limit=self._limit,
- verbose=verbose,
- )
+ from pypaimon.execution.read_explain import explain
+ return explain(self, verbose)
def read_type(self) -> List[DataField]:
"""Return the structured reader request, preserving source field IDs.
@@ -483,189 +469,3 @@ def _is_string_key_map(data_type) -> bool:
and isinstance(data_type.key, AtomicType)
and data_type.key.type.upper() == 'STRING'
)
-
-
-def _build_explain_result(table, scan: TableScan, plan, stats: Optional[ScanStats],
- predicate, projection, limit, verbose: bool) -> ExplainResult:
- """Translate one (Plan, ScanStats) pair into an ExplainResult."""
- splits: List[Split] = plan.splits()
-
- table_schema = table.table_schema
- bucket_mode_str = _safe_bucket_mode(table)
-
- native_planned = stats is None or stats.native_planned
- if stats is None or native_planned:
- partition_pruning = bucket_pruning = file_skipping = None
- else:
- partition_pruning = _partition_pruning(stats, scan)
- bucket_pruning = _bucket_pruning(stats, scan)
- file_skipping = _file_skipping(stats, scan)
-
- files_per_split = [len(getattr(s, 'files', []) or []) for s in splits]
- sizes = [int(getattr(s, 'file_size', 0) or 0) for s in splits]
-
- rows_total = sum(int(getattr(s, 'row_count', 0) or 0) for s in splits)
- merged_per_split = [s.merged_row_count() for s in splits]
- if splits and all(v is not None for v in merged_per_split):
- merged_total: Optional[int] = sum(merged_per_split)
- else:
- merged_total = None
-
- file_count = sum(files_per_split)
- total_size = sum(sizes)
- level_hist: dict = {}
- deletion_file_total = 0
- splits_raw_convertible = 0
- splits_with_dv = 0
- splits_all_above_l0 = 0
- split_infos: List[ExplainSplitInfo] = []
-
- plan_has_auth = any(isinstance(s, QueryAuthSplit) for s in splits)
-
- for split in splits:
- files = getattr(split, 'files', []) or []
- per_split_levels: dict = {}
- for f in files:
- lv = getattr(f, 'level', 0) or 0
- level_hist[lv] = level_hist.get(lv, 0) + 1
- per_split_levels[lv] = per_split_levels.get(lv, 0) + 1
- dvs = getattr(split, 'data_deletion_files', None) or []
- dv_count_here = sum(1 for d in dvs if d is not None)
- deletion_file_total += dv_count_here
- has_dv = dv_count_here > 0
- raw = bool(getattr(split, 'raw_convertible', False))
- if raw:
- splits_raw_convertible += 1
- if has_dv:
- splits_with_dv += 1
- if files and all((getattr(f, 'level', 0) or 0) > 0 for f in files):
- splits_all_above_l0 += 1
-
- if verbose:
- split_infos.append(ExplainSplitInfo(
- partition=_format_partition(split, table),
- bucket=int(getattr(split, 'bucket', -1)),
- file_count=len(files),
- row_count=int(getattr(split, 'row_count', 0) or 0),
- merged_row_count=split.merged_row_count(),
- file_size=int(getattr(split, 'file_size', 0) or 0),
- raw_convertible=raw,
- has_deletion_vectors=has_dv,
- level_histogram=per_split_levels,
- deletion_file_count=dv_count_here,
- file_paths=list(getattr(split, 'file_paths', []) or []),
- data_files=list(files),
- ))
-
- fps_min, fps_max, fps_avg = _min_max_avg(files_per_split)
- sz_min, sz_max, sz_avg = _min_max_avg(sizes)
- sz_p50 = _percentile(sizes, 50)
- sz_p95 = _percentile(sizes, 95)
-
- return ExplainResult(
- table_identifier=str(table.identifier.get_full_name()),
- is_primary_key_table=bool(table.is_primary_key_table),
- bucket_mode=bucket_mode_str,
- deletion_vectors_enabled=bool(table.options.deletion_vectors_enabled()),
- data_evolution_enabled=bool(table.options.data_evolution_enabled()),
- snapshot_id=plan.snapshot_id,
- schema_id=table_schema.id if plan.snapshot_id is not None else None,
- predicate=render_predicate(predicate) if predicate is not None else None,
- projection=list(projection) if projection else None,
- limit=limit,
- partition_pruning=partition_pruning,
- bucket_pruning=bucket_pruning,
- file_skipping=file_skipping,
- file_count=file_count,
- total_file_size=total_size,
- estimated_row_count=rows_total,
- estimated_merged_row_count=merged_total,
- deletion_file_count=deletion_file_total,
- level_histogram=level_hist,
- split_count=len(splits),
- splits_raw_convertible=splits_raw_convertible,
- splits_with_deletion_vectors=splits_with_dv,
- splits_all_above_l0=splits_all_above_l0,
- files_per_split_min=fps_min,
- files_per_split_max=fps_max,
- files_per_split_avg=fps_avg,
- split_size_min=sz_min,
- split_size_max=sz_max,
- split_size_avg=sz_avg,
- split_size_p50=sz_p50,
- split_size_p95=sz_p95,
- has_auth=plan_has_auth,
- native_planned=native_planned,
- planning_counters=stats.planning_counters if stats is not None else None,
- splits=split_infos if verbose else None,
- )
-
-
-def _partition_pruning(stats: ScanStats, scan: TableScan) -> Optional[PruningStat]:
- if scan.predicate is None:
- return None
- table_partition_keys = scan.table.partition_keys or []
- if not table_partition_keys:
- return None
- # ``entries_potential_total`` is the count from manifest-file metadata
- # (manifest-level pruning has not been applied yet). The "after" side
- # is everything that survived both manifest-stats and per-entry
- # partition filters.
- return PruningStat(
- before=stats.entries_potential_total,
- after=stats.entries_after_partition,
- )
-
-
-def _bucket_pruning(stats: ScanStats, scan: TableScan) -> Optional[PruningStat]:
- # Visible whenever the scan applies any bucket-level filtering — the
- # HASH_FIXED predicate-driven selector OR the POSTPONE_BUCKET
- # synthetic-bucket skip. Tables with neither (e.g. BUCKET_UNAWARE
- # append) leave this counter as ``None``.
- fs = scan.file_scanner
- if fs._bucket_selector is None and not fs.only_read_real_buckets:
- return None
- return PruningStat(before=stats.entries_after_partition, after=stats.entries_after_bucket)
-
-
-def _file_skipping(stats: ScanStats, scan: TableScan) -> Optional[PruningStat]:
- # Captures the funnel between bucket-stage survivors and the entries
- # that actually feed the split generator. The drop here includes both
- # predicate-driven file-stats pruning AND structural skips that fire
- # in ``_filter_manifest_entry`` once a file is fully decoded (most
- # notably the "do not read level-0 file" rule for DV-enabled PK
- # tables, which is an LSM-shape decision rather than a predicate
- # test).
- if scan.predicate is None:
- return None
- return PruningStat(before=stats.entries_after_bucket, after=stats.entries_after_stats)
-
-
-def _safe_bucket_mode(table) -> str:
- try:
- return table.bucket_mode().name
- except Exception:
- return "UNKNOWN"
-
-
-def _format_partition(split, table) -> dict:
- keys = list(table.partition_keys or [])
- partition = getattr(split, 'partition', None)
- if partition is None or not keys:
- return {}
- values = getattr(partition, 'values', None) or []
- return {k: v for k, v in zip(keys, values)}
-
-
-def _min_max_avg(values):
- if not values:
- return 0, 0, 0.0
- return min(values), max(values), sum(values) / float(len(values))
-
-
-def _percentile(values, pct: int) -> int:
- if not values:
- return 0
- ordered = sorted(values)
- idx = int(round((pct / 100.0) * (len(ordered) - 1)))
- return int(ordered[idx])
diff --git a/paimon-python/pypaimon/read/stream_read_builder.py b/paimon-python/pypaimon/read/stream_read_builder.py
index eac056a4a3..ff73f1382e 100644
--- a/paimon-python/pypaimon/read/stream_read_builder.py
+++ b/paimon-python/pypaimon/read/stream_read_builder.py
@@ -22,19 +22,19 @@ This module provides a builder for configuring streaming reads from Paimon
tables, similar to ReadBuilder but for continuous streaming use cases.
"""
-import logging
-
-from typing import Callable, Dict, List, Optional, Set, Union
+from typing import Callable, Dict, List, Optional, Set, Union, TYPE_CHECKING
from pypaimon.common.predicate import Predicate
from pypaimon.common.predicate_builder import PredicateBuilder
-from pypaimon.execution.dispatch import propagate_fork_safety_error as _raise_if_native_fork_safety_error
from pypaimon.read.read_builder import ReadBuilder
-from pypaimon.read.streaming_table_scan import AsyncStreamingTableScan, StreamTableScan
from pypaimon.read.table_read import TableRead
from pypaimon.schema.data_types import DataField
+if TYPE_CHECKING:
+ from pypaimon.table.file_store_table import FileStoreTable
+
+
class StreamReadBuilder:
"""
Builder for streaming reads from Paimon tables.
@@ -53,9 +53,7 @@ class StreamReadBuilder:
def __init__(self, table):
"""Initialize the StreamReadBuilder."""
- from pypaimon.table.file_store_table import FileStoreTable
-
- self.table: FileStoreTable = table
+ self.table: "FileStoreTable" = table
self._predicate: Optional[Predicate] = None
self._read_builder = ReadBuilder(table)
self._poll_interval_ms: int = 1000
@@ -125,30 +123,10 @@ class StreamReadBuilder:
bucket_set: Set[int] = set(bucket_ids)
return self.with_bucket_filter(lambda bucket: bucket in bucket_set)
- def new_streaming_scan(self) -> Union[AsyncStreamingTableScan, StreamTableScan]:
+ def new_streaming_scan(self) -> Union["AsyncStreamingTableScan", "StreamTableScan"]:
"""Create a stateful streaming scan using the configured backend."""
- projection = self._projection_builder()
- projection._validate_map_key_filter()
- from pypaimon.execution.native.table import _native_query_auth_supported
- if (self.table.options.native_plan_enabled()
- and _native_query_auth_supported(self.table)):
- try:
- return StreamTableScan(
- self.table, predicate=self._predicate, read_type=projection.read_type(),
- poll_interval_ms=self._poll_interval_ms, bucket_filter=self._bucket_filter,
- consumer_id=self._consumer_id)
- except Exception as error:
- _raise_if_native_fork_safety_error(error)
- logging.warning("Cannot create Native stream scan, using Python: %s", error)
- scan = AsyncStreamingTableScan(
- table=self.table,
- predicate=self._predicate,
- poll_interval_ms=self._poll_interval_ms,
- bucket_filter=self._bucket_filter,
- consumer_id=self._consumer_id
- )
- scan._read_type = projection.read_type()
- return scan
+ from pypaimon.execution.read_dispatch import new_streaming_scan
+ return new_streaming_scan(self)
def new_read(self) -> TableRead:
"""Create a new TableRead with this builder's settings."""
diff --git a/paimon-python/pypaimon/read/streaming_table_scan.py b/paimon-python/pypaimon/read/streaming_table_scan.py
index 2f3a26c4b2..bf64f9a897 100644
--- a/paimon-python/pypaimon/read/streaming_table_scan.py
+++ b/paimon-python/pypaimon/read/streaming_table_scan.py
@@ -17,7 +17,31 @@
"""Public streaming scan classes backed by Python or Native execution."""
-from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan
-from pypaimon.execution.native.read.streaming_table_scan import StreamTableScan
+import importlib
+import sys
+
+if sys.version_info[:2] < (3, 7):
+ # Python 3.6 does not resolve module attributes through __getattr__.
+ from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan
+ from pypaimon.execution.native.read.streaming_table_scan import StreamTableScan
__all__ = ['AsyncStreamingTableScan', 'StreamTableScan']
+
+_MODULE_BY_EXPORT = {
+ 'AsyncStreamingTableScan': 'pypaimon.execution.python.read.streaming_table_scan',
+ 'StreamTableScan': 'pypaimon.execution.native.read.streaming_table_scan',
+}
+
+
+def __getattr__(name):
+ module_name = _MODULE_BY_EXPORT.get(name)
+ if module_name is None:
+ raise AttributeError(
+ "module 'pypaimon.read.streaming_table_scan' has no attribute {!r}".format(name))
+ value = getattr(importlib.import_module(module_name), name)
+ globals()[name] = value
+ return value
+
+
+def __dir__():
+ return sorted(set(globals()) | set(__all__))
diff --git a/paimon-python/pypaimon/read/table_read.py b/paimon-python/pypaimon/read/table_read.py
index 4936ea801d..58685b5c0b 100644
--- a/paimon-python/pypaimon/read/table_read.py
+++ b/paimon-python/pypaimon/read/table_read.py
@@ -15,9 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-import logging
-import os
-from typing import Any, Callable, Dict, Iterator, List, Optional
+from typing import Any, Callable, Dict, Iterator, List, Optional, TYPE_CHECKING
import pandas
import pyarrow
@@ -28,53 +26,18 @@ from pypaimon.common.predicate import Predicate
from pypaimon.read.push_down_utils import predicate_field_names
from pypaimon.read.query_auth_split import QueryAuthSplit
from pypaimon.read.split import Split
-from pypaimon.read.read_type import (
- OutputProjection, adapter_output_paths, extract_array, output_schema, reader_adapter)
+from pypaimon.read.read_type import OutputProjection, output_schema, reader_adapter
from pypaimon.read.blob_read_support import deferred_blob_field_names
from pypaimon.schema.data_types import DataField
-from pypaimon.utils.arrow_utils import zero_column_batch
-from pypaimon.read.constants import ROW_KIND_COLUMN
-from pypaimon.execution.python.read.table_read import PythonTableReadMixin, _RemainingRows
-from pypaimon.execution.native.read.table_read import NativeTableReadMixin
-logger = logging.getLogger(__name__)
-_RECORD_BATCH_READER_FROM_STREAM = getattr(
- pyarrow.ipc.RecordBatchReader, "from_stream", None)
+from pypaimon.read.constants import ROW_KIND_COLUMN, _DEFAULT_PREDICATE
-class _ClosableArrowBatchReader:
+if TYPE_CHECKING:
+ from pypaimon.table.file_store_table import FileStoreTable
- def __init__(self, reader, batch_iterator):
- self._reader = reader
- self._batch_iterator = batch_iterator
- def __getattr__(self, name):
- return getattr(self._reader, name)
-
- def __iter__(self):
- return self
-
- def __next__(self):
- return self._reader.read_next_batch()
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, traceback):
- self.close()
-
- def close(self):
- try:
- close = getattr(self._batch_iterator, "close", None)
- if close is not None:
- close()
- finally:
- close = getattr(self._reader, "close", None)
- if close is not None:
- close()
-
-
-class TableRead(PythonTableReadMixin, NativeTableReadMixin):
+class TableRead:
"""Public read facade selecting the configured execution backend."""
# Cap peak concurrent BLOB range reads across the whole parallel read:
@@ -92,12 +55,10 @@ class TableRead(PythonTableReadMixin, NativeTableReadMixin):
limit: Optional[int] = None,
):
from pypaimon.read.merge_engine_support import check_sequence_field_valid
- from pypaimon.table.file_store_table import FileStoreTable
-
if table.is_primary_key_table:
check_sequence_field_valid(table)
self._python_merge_validation_pending = True
- self.table: FileStoreTable = table
+ self.table: "FileStoreTable" = table
if not table.options.native_read_enabled():
self._check_python_merge_supported()
self.predicate = predicate
@@ -146,27 +107,8 @@ class TableRead(PythonTableReadMixin, NativeTableReadMixin):
Python split reader, the iterator reuses its row view and visits splits
in order. Close the iterator when stopping before exhaustion.
"""
- if self.output_projection is not None and self.output_projection.named:
- raise RuntimeError(
- "Projection expressions are not supported by to_iterator(); "
- "use to_arrow() or to_arrow_batch_reader()")
- self._begin_auth_read(splits)
- limit = self.limit
-
- def _record_generator():
- if limit is not None and limit <= 0:
- return
- rows = self._try_native_rows(splits)
- if rows is not None:
- try:
- yield from rows
- finally:
- rows.close()
- return
-
- yield from self._python_row_iterator(splits, limit)
-
- return _record_generator()
+ from pypaimon.execution.read_dispatch import to_iterator
+ return to_iterator(self, splits)
def to_arrow_batch_reader(
self,
@@ -182,14 +124,8 @@ class TableRead(PythonTableReadMixin, NativeTableReadMixin):
table reads do not guarantee row order. Python fallback reads remain
serial.
"""
- # PyArrow does not close a suspended Python batch iterator. Keep its
- # cleanup even when LIMIT or BLOB pruning serializes native reads.
- effective = self._resolve_parallelism(parallelism, len(splits))
- reader, batch_iterator = self._new_arrow_batch_reader(
- splits, blob_parallelism, parallelism)
- if self._requires_batch_iterator_cleanup(splits, effective):
- return _ClosableArrowBatchReader(reader, batch_iterator)
- return reader
+ from pypaimon.execution.read_support import to_arrow_batch_reader
+ return to_arrow_batch_reader(self, splits, blob_parallelism, parallelism)
def _to_managed_arrow_batch_reader(
self,
@@ -202,49 +138,22 @@ class TableRead(PythonTableReadMixin, NativeTableReadMixin):
``from_batches``. Retain it explicitly so native reads release their
resources when a caller ends the read early, including serial reads.
"""
- # Keep the explicit iterator cleanup chain even when LIMIT caps the
- # native worker count inside _new_arrow_batch_reader to one.
- effective = self._resolve_parallelism(parallelism, len(splits))
- reader, batch_iterator = self._new_arrow_batch_reader(
- splits, blob_parallelism, parallelism)
- if self._requires_batch_iterator_cleanup(splits, effective):
- return _ClosableArrowBatchReader(reader, batch_iterator)
- if (_RECORD_BATCH_READER_FROM_STREAM is not None
- and hasattr(reader, "close")):
- return _RECORD_BATCH_READER_FROM_STREAM(reader)
- return _ClosableArrowBatchReader(reader, batch_iterator)
+ from pypaimon.execution.read_support import _to_managed_arrow_batch_reader
+ return _to_managed_arrow_batch_reader(self, splits, blob_parallelism, parallelism)
def _new_arrow_batch_reader(
self,
splits: List[Split],
blob_parallelism: Optional[int] = None,
parallelism: Optional[int] = None):
- schema = self._output_arrow_schema(splits)
- if self.include_row_kind:
- schema = self._add_row_kind_to_schema(schema)
- effective = self._resolve_parallelism(parallelism, len(splits))
- effective_bp = self._resolve_blob_parallelism(blob_parallelism)
- native_batches = self._try_native_batches(
- splits,
- schema,
- parallelism=effective,
- blob_parallelism=effective_bp,
- ordered=False,
- )
- if native_batches is not None:
- batch_iterator = iter(native_batches)
- else:
- batch_iterator = self._python_batch_iterator(
- splits, schema, parallelism, effective_bp)
- reader_type = pyarrow.ipc.RecordBatchReader
- reader = reader_type.from_batches(schema, batch_iterator)
- return reader, batch_iterator
+ from pypaimon.execution.read_dispatch import _new_arrow_batch_reader
+ return _new_arrow_batch_reader(self, splits, blob_parallelism, parallelism)
@staticmethod
def _add_row_kind_to_schema(schema: pyarrow.Schema, column_name=ROW_KIND_COLUMN) -> pyarrow.Schema:
"""Prepend the row-kind transport column, normally named _row_kind."""
- row_kind_field = pyarrow.field(column_name, pyarrow.string())
- return pyarrow.schema([row_kind_field] + list(schema))
+ from pypaimon.execution.read_support import _add_row_kind_to_schema
+ return _add_row_kind_to_schema(schema, column_name)
@staticmethod
def _try_to_pad_batch_by_schema(
@@ -258,29 +167,8 @@ class TableRead(PythonTableReadMixin, NativeTableReadMixin):
to cast internal dependency columns because masking intentionally may
change their logical type.
"""
- if batch.schema.equals(target_schema, check_metadata=True):
- return batch
-
- columns = []
- num_rows = batch.num_rows
- same_order = batch.schema.names == target_schema.names
-
- for index, field in enumerate(target_schema):
- if field.name in batch.schema.names:
- # Result projections may repeat a column. A name lookup is
- # ambiguous even when the batch already has the correct order.
- col = batch.column(index if same_order else field.name)
- if col.type != field.type:
- if not allow_type_cast:
- raise TypeError(
- "Batch field '%s' has type %s, expected %s" % (
- field.name, col.type, field.type))
- col = col.cast(field.type)
- else:
- col = pyarrow.nulls(num_rows, type=field.type)
- columns.append(col)
-
- return pyarrow.RecordBatch.from_arrays(columns, schema=target_schema)
+ from pypaimon.execution.read_support import _try_to_pad_batch_by_schema
+ return _try_to_pad_batch_by_schema(batch, target_schema, allow_type_cast)
def to_arrow(
self,
@@ -314,37 +202,13 @@ class TableRead(PythonTableReadMixin, NativeTableReadMixin):
``_MAX_TOTAL_BLOB_WORKERS``; per-split ``blob_parallelism`` is
shrunk to stay within it.
"""
- effective_bp = self._resolve_blob_parallelism(blob_parallelism)
- effective = self._resolve_parallelism(parallelism, len(splits))
- schema = self._output_arrow_schema(splits)
- if self.include_row_kind:
- schema = self._add_row_kind_to_schema(schema)
-
- native_batches = self._try_native_batches(
- splits,
- schema,
- parallelism=effective,
- blob_parallelism=effective_bp,
- ordered=True,
- )
- if native_batches is not None:
- return self._batches_to_arrow(native_batches, schema)
-
- return self._python_to_arrow(splits, schema, parallelism, effective_bp)
+ from pypaimon.execution.read_dispatch import to_arrow
+ return to_arrow(self, splits, parallelism, blob_parallelism)
@staticmethod
def _batches_to_arrow(batches, schema, allow_type_cast: bool = False):
- batches = [
- TableRead._try_to_pad_batch_by_schema(
- batch, schema, allow_type_cast=allow_type_cast)
- for batch in batches
- if batch.num_rows > 0
- ]
- if not batches:
- return pyarrow.Table.from_arrays(
- [pyarrow.array([], type=field.type) for field in schema],
- schema=schema)
- return pyarrow.Table.from_batches(batches)
+ from pypaimon.execution.read_support import _batches_to_arrow
+ return _batches_to_arrow(batches, schema, allow_type_cast)
@property
def nested_name_paths(self):
@@ -442,40 +306,25 @@ class TableRead(PythonTableReadMixin, NativeTableReadMixin):
caps the thread pool; ``1`` forces serial reads. The validation
message names whichever source produced the offending value.
"""
- if runtime is not None:
- value, source = runtime, "parallelism"
- elif self._read_parallelism is not None:
- value, source = self._read_parallelism, "read.parallelism"
- else:
- # os.cpu_count() may return None on exotic platforms; fall back
- # to 1. min() with num_splits avoids more workers than work.
- return max(1, min(num_splits, os.cpu_count() or 1))
- if value < 1:
- raise ValueError(f"{source} must be >= 1, got {value}")
- return value
+ from pypaimon.execution.read_support import _resolve_parallelism
+ return _resolve_parallelism(self, runtime, num_splits)
def _effective_parallelism(
self, runtime: Optional[int], num_splits: int) -> int:
"""Resolve parallelism and bound speculative unfiltered LIMIT reads."""
- effective = self._resolve_parallelism(runtime, num_splits)
- if self.limit is None or (self.limit > 0 and self.predicate is not None):
- return effective
- return min(effective, max(1, self.limit))
+ from pypaimon.execution.read_support import _effective_parallelism
+ return _effective_parallelism(self, runtime, num_splits)
@staticmethod
def _resolve_blob_parallelism(runtime: Optional[int]) -> int:
- if runtime is None:
- return 1
- if runtime < 1:
- raise ValueError(f"blob_parallelism must be >= 1, got {runtime}")
- return runtime
+ from pypaimon.execution.read_support import _resolve_blob_parallelism
+ return _resolve_blob_parallelism(runtime)
@classmethod
def _cap_blob_parallelism(cls, workers: int, blob_parallelism: int) -> int:
"""Shrink per-split blob_parallelism so workers*B <= cap (peak reads)."""
- if blob_parallelism <= 1 or workers * blob_parallelism <= cls._MAX_TOTAL_BLOB_WORKERS:
- return blob_parallelism
- return max(1, cls._MAX_TOTAL_BLOB_WORKERS // workers)
+ from pypaimon.execution.read_support import _cap_blob_parallelism
+ return _cap_blob_parallelism(cls, workers, blob_parallelism)
def _should_run_parallel(
self,
@@ -488,74 +337,23 @@ class TableRead(PythonTableReadMixin, NativeTableReadMixin):
overhead, no behavior change). A single split is never
parallelized since there is nothing to fan out across.
"""
- deferred_limit_may_prune = self._deferred_blob_limit_may_prune(splits)
- return (effective >= 2 and len(splits) >= 2
- and not deferred_limit_may_prune)
+ from pypaimon.execution.read_support import _should_run_parallel
+ return _should_run_parallel(self, splits, effective)
def _deferred_blob_limit_may_prune(self, splits: List[Split]) -> bool:
- return (self.limit is not None
- and (self._deferred_blob_fields
- or self._native_resolved_blob_fields())
- and not self._limit_covers_all_splits(splits))
+ from pypaimon.execution.read_support import _deferred_blob_limit_may_prune
+ return _deferred_blob_limit_may_prune(self, splits)
def _requires_batch_iterator_cleanup(self, splits: List[Split], effective: int) -> bool:
# A serial Core stream owns resources too. PyArrow's reader does not
# close the Python iterator supplied to from_batches on an early close.
- return (self.table.options.native_read_enabled()
- or self._should_run_parallel(splits, effective)
- or self._deferred_blob_limit_may_prune(splits))
+ from pypaimon.execution.read_support import _requires_batch_iterator_cleanup
+ return _requires_batch_iterator_cleanup(self, splits, effective)
def _limit_covers_all_splits(self, splits: List[Split]) -> bool:
"""Return whether split metadata proves that LIMIT cannot drop rows."""
- total_rows = 0
- for split in splits:
- merged_row_count = split.merged_row_count()
- if merged_row_count is None:
- return False
- total_rows += merged_row_count
- if total_rows > self.limit:
- return False
- return True
-
- @staticmethod
- def _emit_overflow_safe_batches(
- pydict: Dict[str, list],
- row_count: int,
- schema: pyarrow.Schema,
- ) -> Iterator[pyarrow.RecordBatch]:
- """Yield RecordBatches from a ``{column: list}`` dict, keeping every column
- within pyarrow's per-column size limit.
-
- A STRING/BYTES column maps to ``pyarrow.string()``/``binary()`` which use
- 32-bit offsets (max 2GB per column). A chunk of large values can overflow
- that, in which case ``pyarrow.array()`` returns a ``ChunkedArray`` that a
- single ``RecordBatch`` cannot hold. When that happens we split the rows in
- half and recurse so every emitted batch keeps each column under the limit.
- """
- if not schema:
- yield zero_column_batch(row_count, schema.metadata)
- return
- arrays = []
- for field in schema:
- arr = pyarrow.array(pydict[field.name], type=field.type)
- if isinstance(arr, pyarrow.ChunkedArray):
- # A column overflowed the 2GB limit and was auto-chunked; split.
- break
- arrays.append(arr)
- else:
- yield pyarrow.RecordBatch.from_arrays(arrays, schema=schema)
- return
-
- if row_count <= 1:
- raise ValueError(
- "A single row exceeds the 2GB per-column limit of "
- "pyarrow.string()/binary(); cannot build a RecordBatch for this row."
- )
- mid = row_count // 2
- left = {name: column[:mid] for name, column in pydict.items()}
- right = {name: column[mid:] for name, column in pydict.items()}
- yield from TableRead._emit_overflow_safe_batches(left, mid, schema)
- yield from TableRead._emit_overflow_safe_batches(right, row_count - mid, schema)
+ from pypaimon.execution.read_support import _limit_covers_all_splits
+ return _limit_covers_all_splits(self, splits)
def _add_row_kind_column_to_batch(
self,
@@ -563,10 +361,8 @@ class TableRead(PythonTableReadMixin, NativeTableReadMixin):
default_row_kind: str = "+I"
) -> pyarrow.RecordBatch:
"""Add a _row_kind column to an existing batch."""
- row_kind_array = pyarrow.array([default_row_kind] * batch.num_rows, type=pyarrow.string())
- new_schema = self._add_row_kind_to_schema(batch.schema)
- columns = [row_kind_array] + [batch.column(i) for i in range(batch.num_columns)]
- return pyarrow.RecordBatch.from_arrays(columns, schema=new_schema)
+ from pypaimon.execution.read_support import _add_row_kind_column_to_batch
+ return _add_row_kind_column_to_batch(self, batch, default_row_kind)
def to_pandas(
self,
@@ -587,11 +383,8 @@ class TableRead(PythonTableReadMixin, NativeTableReadMixin):
See :meth:`to_arrow` for the semantics of ``parallelism``.
"""
- import duckdb
-
- con = connection or duckdb.connect(database=":memory:")
- con.register(table_name, self.to_arrow(splits, parallelism=parallelism))
- return con
+ from pypaimon.execution.read_support import to_duckdb
+ return to_duckdb(self, splits, table_name, connection, parallelism)
def to_ray(
self,
@@ -617,48 +410,15 @@ class TableRead(PythonTableReadMixin, NativeTableReadMixin):
See `Ray Data API <https://docs.ray.io/en/latest/data/api/doc/ray.data.read_datasource.html>`_
for details.
"""
- import ray
-
- if not splits:
- schema = self._output_arrow_schema()
- if self.include_row_kind:
- schema = self._add_row_kind_to_schema(schema)
- empty_table = pyarrow.Table.from_arrays(
- [pyarrow.array([], type=field.type) for field in schema],
- schema=schema
- )
- return ray.data.from_arrow(empty_table)
-
- if override_num_blocks is not None and override_num_blocks < 1:
- raise ValueError(f"override_num_blocks must be at least 1, got {override_num_blocks}")
-
- from pypaimon.read.datasource.ray_datasource import RayDatasource
- from pypaimon.read.datasource.split_provider import PreResolvedSplitProvider
-
- datasource = RayDatasource(
- PreResolvedSplitProvider(
- table=self.table,
- splits=splits,
- read_type=self.read_type,
- predicate=self.predicate,
- limit=self.limit,
- output_projection=self.output_projection,
- include_row_kind=self.include_row_kind,
- )
- )
- ds = ray.data.read_datasource(
- datasource,
+ from pypaimon.execution.read_support import to_ray
+ return to_ray(
+ self,
+ splits,
ray_remote_args=ray_remote_args,
concurrency=concurrency,
override_num_blocks=override_num_blocks,
- **read_args
+ **read_args,
)
- # Each Ray worker applies the per-task limit independently, so N
- # workers can collectively yield up to N * limit rows. Cap the
- # final dataset to the user-visible limit on top.
- if self.limit is not None:
- ds = ds.limit(self.limit)
- return ds
def to_torch(
self,
@@ -696,138 +456,27 @@ class TableRead(PythonTableReadMixin, NativeTableReadMixin):
sharding_rank: Explicit rank in the intended DDP process group.
sharding_world_size: Explicit size of that process group.
"""
- valid_batch_formats = {"row", "pyarrow", "torch"}
- if batch_format not in valid_batch_formats:
- raise ValueError(
- "batch_format must be one of %s, got %r"
- % (sorted(valid_batch_formats), batch_format)
- )
- if (
- auto_detect_rank
- or sharding_rank is not None
- or sharding_world_size is not None
- ) and not streaming:
- raise ValueError("distributed sharding requires streaming=True")
- if batch_size is not None and (
- isinstance(batch_size, bool)
- or not isinstance(batch_size, int)
- or batch_size <= 0
- ):
- raise ValueError("batch_size must be a positive int or None")
- if batch_format == "row":
- if self.output_projection is not None and self.output_projection.named:
- raise RuntimeError(
- "Projection expressions are not supported by Torch row format; "
- "use streaming=True with batch_format='pyarrow' or 'torch'")
- if batch_size is not None:
- raise ValueError(
- "batch_size requires batch_format='pyarrow' or 'torch'"
- )
- if to_tensor_fn is not None:
- raise ValueError("to_tensor_fn requires batch_format='torch'")
- else:
- if not streaming:
- raise ValueError(
- "batch_format=%r requires streaming=True" % batch_format
- )
- if batch_format == "pyarrow" and to_tensor_fn is not None:
- raise ValueError("to_tensor_fn requires batch_format='torch'")
- if to_tensor_fn is not None and not callable(to_tensor_fn):
- raise ValueError("to_tensor_fn must be callable")
- if (
- isinstance(prefetch_concurrency, bool)
- or not isinstance(prefetch_concurrency, int)
- or prefetch_concurrency != 1
- ):
- raise ValueError(
- "batch formats require prefetch_concurrency=1"
- )
-
- from pypaimon.read.datasource.torch_dataset import (
- TorchBatchIterDataset,
- TorchShuffledBatchIterDataset,
- )
- dataset_type = TorchShuffledBatchIterDataset if shuffle else TorchBatchIterDataset
- shuffle_options = (dict(seed=seed, buffer_size=buffer_size,
- max_buffer_input_splits=max_buffer_input_splits)
- if shuffle else {})
- return dataset_type(
- self,
- splits,
- batch_format=batch_format,
- batch_size=batch_size,
- to_tensor_fn=to_tensor_fn,
- auto_detect_rank=auto_detect_rank,
- sharding_rank=sharding_rank,
- sharding_world_size=sharding_world_size,
- **shuffle_options,
- )
-
- if shuffle:
- if not streaming:
- raise ValueError("shuffle=True only supports streaming=True")
- if prefetch_concurrency > 1:
- raise ValueError("shuffle=True does not support prefetch_concurrency > 1")
- from pypaimon.read.datasource.torch_dataset import TorchShuffledIterDataset
- dataset = TorchShuffledIterDataset(
- self,
- splits,
- seed=seed,
- buffer_size=buffer_size,
- max_buffer_input_splits=max_buffer_input_splits,
- auto_detect_rank=auto_detect_rank,
- sharding_rank=sharding_rank,
- sharding_world_size=sharding_world_size,
- )
- return dataset
-
- if streaming:
- from pypaimon.read.datasource.torch_dataset import TorchIterDataset
- dataset = TorchIterDataset(
- self,
- splits,
- prefetch_concurrency,
- auto_detect_rank=auto_detect_rank,
- sharding_rank=sharding_rank,
- sharding_world_size=sharding_world_size,
- )
- return dataset
- else:
- from pypaimon.read.datasource.torch_dataset import TorchDataset
- dataset = TorchDataset(self, splits)
- return dataset
+ from pypaimon.execution.read_support import to_torch
+ return to_torch(
+ self,
+ splits,
+ streaming,
+ prefetch_concurrency,
+ batch_format=batch_format,
+ batch_size=batch_size,
+ to_tensor_fn=to_tensor_fn,
+ shuffle=shuffle,
+ seed=seed,
+ buffer_size=buffer_size,
+ max_buffer_input_splits=max_buffer_input_splits,
+ auto_detect_rank=auto_detect_rank,
+ sharding_rank=sharding_rank,
+ sharding_world_size=sharding_world_size,
+ )
def _project_batch_to_output(self, batch: pyarrow.RecordBatch) -> pyarrow.RecordBatch:
- if self.output_projection is not None:
- _, paths = self._reader_adapter()
- outputs = adapter_output_paths(self.output_projection, self._adapter_read_type, paths)
- arrays = [extract_array(batch, path) for _, path in outputs]
- # Project the already masked schema, retaining captured nullability.
- adapted_projection = OutputProjection([(alias, path) for alias, path in outputs])
- actual = output_schema(batch.schema, adapted_projection)
- declared = output_schema(read_arrow_schema(self.read_type), self.output_projection)
- fields = [field.with_nullable(field.nullable or actual[i].nullable) for i, field in enumerate(declared)]
- if self.include_row_kind and self._row_kind_column in batch.schema.names:
- arrays.insert(0, batch.column(self._row_kind_column))
- fields.insert(0, batch.schema.field(self._row_kind_column))
- if not arrays:
- return zero_column_batch(batch.num_rows, batch.schema.metadata)
- return pyarrow.RecordBatch.from_arrays(
- arrays, schema=pyarrow.schema(fields, metadata=batch.schema.metadata))
- if not self._needs_output_projection():
- return batch
- output_names = list(self._output_column_names)
- if self.include_row_kind and self._row_kind_column in batch.schema.names:
- output_names.insert(0, self._row_kind_column)
- if batch.schema.names == output_names:
- return batch
- name_to_pos = {name: i for i, name in enumerate(batch.schema.names)}
- arrays = [batch.column(name_to_pos[name]) for name in output_names]
- fields = [batch.schema.field(name_to_pos[name]) for name in output_names]
- if not arrays:
- return zero_column_batch(batch.num_rows, batch.schema.metadata)
- return pyarrow.RecordBatch.from_arrays(
- arrays, schema=pyarrow.schema(fields))
+ from pypaimon.execution.read_support import _project_batch_to_output
+ return _project_batch_to_output(self, batch)
def _needs_output_projection(self) -> bool:
return bool(self._predicate_extra_fields or self.output_projection)
@@ -869,18 +518,232 @@ class TableRead(PythonTableReadMixin, NativeTableReadMixin):
so a per-split refresh would ask the catalog once per worker; a per-reader one would let
a later read reuse a schema fetched for an earlier one.
"""
- from pypaimon.read.auth_support import latest_auth_fields
-
- self._auth_schema = None
- for split in splits:
- if isinstance(split, QueryAuthSplit) and split.auth_result is not None:
- self._auth_schema = latest_auth_fields(self.table)
- return
+ from pypaimon.execution.read_support import _begin_auth_read
+ return _begin_auth_read(self, splits)
@staticmethod
def convert_rows_to_arrow_batch(row_tuples: List[tuple], schema: pyarrow.Schema) -> pyarrow.RecordBatch:
- columns_data = zip(*row_tuples)
- pydict = {name: list(column) for name, column in zip(schema.names, columns_data)}
- return pyarrow.RecordBatch.from_pydict(pydict, schema=schema)
+ from pypaimon.execution.read_support import convert_rows_to_arrow_batch
+ return convert_rows_to_arrow_batch(row_tuples, schema)
+
+ def _arrow_batch_generator(self, splits: List[Split], schema: pyarrow.Schema,
+ blob_parallelism: int = 1) -> Iterator[pyarrow.RecordBatch]:
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ yield from PythonTableReadMixin._arrow_batch_generator(self, splits, schema, blob_parallelism)
+
+ def _to_arrow_parallel(
+ self,
+ splits: List[Split],
+ schema: pyarrow.Schema,
+ effective: int,
+ blob_parallelism: int = 1,
+ ) -> pyarrow.Table:
+ """Read ``splits`` concurrently and assemble the result in input order.
+
+ Each split is read in its own worker thread; row quota for ``limit``
+ is shared through :class:`_RemainingRows` so the combined output
+ never exceeds ``self.limit`` rows. Per-split batches are collected
+ by submission index, so the merged table preserves the order of the
+ input ``splits`` list.
+ """
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ return PythonTableReadMixin._to_arrow_parallel(self, splits, schema, effective, blob_parallelism)
+
+ def _read_one_split_to_batches(
+ self,
+ split: Split,
+ schema: pyarrow.Schema,
+ remaining_state: "_RemainingRows",
+ blob_parallelism: int = 1,
+ ) -> List[pyarrow.RecordBatch]:
+ """Read a single split into arrow batches under soft-stop control.
+
+ Row quota is debited against the shared ``remaining_state``; once a
+ request returns 0, the worker stops emitting further batches. The
+ reader is always closed via ``finally``.
+ """
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ return PythonTableReadMixin._read_one_split_to_batches(self, split, schema, remaining_state, blob_parallelism)
+
+ def _convert_rows_to_output_batches(
+ self,
+ row_tuples: List[tuple],
+ row_kinds: List[str],
+ schema: pyarrow.Schema,
+ ) -> Iterator[pyarrow.RecordBatch]:
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ yield from PythonTableReadMixin._convert_rows_to_output_batches(self, row_tuples, row_kinds, schema)
+
+ def _convert_rows_to_arrow_batches_with_row_kind(
+ self,
+ row_tuples: List[tuple],
+ row_kinds: List[str],
+ schema: pyarrow.Schema
+ ) -> Iterator[pyarrow.RecordBatch]:
+ """Convert rows to one or more Arrow batches, optionally including row kind column.
+
+ Yields more than one batch only when a column overflows pyarrow's 2GB
+ per-column limit (see ``_emit_overflow_safe_batches``); otherwise a single
+ batch is produced as before.
+ """
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ yield from PythonTableReadMixin._convert_rows_to_arrow_batches_with_row_kind(
+ self,
+ row_tuples,
+ row_kinds,
+ schema,
+ )
+
+ def _create_split_read(self, split: Split, blob_parallelism: int = 1,
+ read_type=None, limit: Optional[int] = None,
+ push_down_limit: bool = True,
+ post_merge_filter=None,
+ eager_blob_fields=None,
+ post_filter_after_inline: bool = False,
+ predicate=_DEFAULT_PREDICATE,
+ defer_blob_resolution: bool = False) -> "SplitRead":
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ return PythonTableReadMixin._create_split_read(
+ self,
+ split,
+ blob_parallelism,
+ read_type,
+ limit,
+ push_down_limit,
+ post_merge_filter,
+ eager_blob_fields,
+ post_filter_after_inline,
+ predicate,
+ defer_blob_resolution,
+ )
+
+ def _build_split_read(self, split: Split, read_type=None,
+ limit: Optional[int] = None,
+ push_down_limit: bool = True,
+ post_merge_filter=None,
+ eager_blob_fields=None,
+ post_filter_after_inline: bool = False,
+ predicate=_DEFAULT_PREDICATE,
+ defer_blob_resolution: bool = False) -> "SplitRead":
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ return PythonTableReadMixin._build_split_read(
+ self,
+ split,
+ read_type,
+ limit,
+ push_down_limit,
+ post_merge_filter,
+ eager_blob_fields,
+ post_filter_after_inline,
+ predicate,
+ defer_blob_resolution,
+ )
+
+ def _serialize_blob_views_as_descriptors(self, batch: pyarrow.RecordBatch, reader) -> pyarrow.RecordBatch:
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ return PythonTableReadMixin._serialize_blob_views_as_descriptors(self, batch, reader)
+
+ def _serialize_blob_view_row_tuple(self, row: "OffsetRow", reader) -> tuple:
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ return PythonTableReadMixin._serialize_blob_view_row_tuple(self, row, reader)
+
+ def _widen_to_top_level_for_merge(self, name_paths=None) -> List[DataField]:
+ """Unique top-level fields from ``name_paths``, in path order."""
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ return PythonTableReadMixin._widen_to_top_level_for_merge(self, name_paths)
+
+ def _only_map_key_nested_paths(self) -> bool:
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ return PythonTableReadMixin._only_map_key_nested_paths(self)
+
+ def _has_map_key_projection(self) -> bool:
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ return PythonTableReadMixin._has_map_key_projection(self)
+
+ def _map_has_aggregator(self, field_name: str) -> bool:
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ return PythonTableReadMixin._map_has_aggregator(self, field_name)
+
+ def _check_python_merge_supported(self):
+ if getattr(self, '_python_merge_validation_pending', False):
+ from pypaimon.execution.python.read.merge_engine_support import check_supported
+ check_supported(self.table)
+ self._python_merge_validation_pending = False
+
+ def _TableRead__create_reader_for_split(
+ self, split, blob_parallelism=1, limit: Optional[int] = None):
+ # Raw-convertible splits skip MergeFileSplitRead, so validate before
+ # dispatch, including when native setup failed or bindings are absent.
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ return PythonTableReadMixin._TableRead__create_reader_for_split(self, split, blob_parallelism, limit)
+
+ def _TableRead__authed_reader(
+ self, split, auth_result, blob_parallelism=1, limit: Optional[int] = None):
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ return PythonTableReadMixin._TableRead__authed_reader(self, split, auth_result, blob_parallelism, limit)
+
+ @staticmethod
+ def _auth_filter_field_names(auth_result, read_fields) -> set:
+ from pypaimon.execution.python.read.table_read import PythonTableReadMixin
+ return PythonTableReadMixin._auth_filter_field_names(auth_result, read_fields)
+
+ def _native_fallback(
+ self,
+ reason: str,
+ error: Optional[Exception] = None,
+ log: bool = False,
+ ):
+ from pypaimon.execution.read_dispatch import native_read_fallback
+ return native_read_fallback(self, reason, error, log)
+
+ def _try_native_batches(
+ self,
+ splits: List[Split],
+ schema: pyarrow.Schema,
+ parallelism: Optional[int] = None,
+ blob_parallelism: Optional[int] = None,
+ ordered: bool = False):
+ """Return Rust-read batches, or ``None`` when this read must fall back."""
+ if not self.table.options.native_read_enabled():
+ return self._native_fallback('read.native.enabled is false')
+ from pypaimon.execution.native.read.table_read import NativeTableReadMixin
+ return NativeTableReadMixin._try_native_batches(self, splits, schema, parallelism, blob_parallelism, ordered)
+
+ def _native_read_kwargs(self, blob_parallelism=None):
+ from pypaimon.execution.native.read.table_read import NativeTableReadMixin
+ return NativeTableReadMixin._native_read_kwargs(self, blob_parallelism)
+
+ @staticmethod
+ def _native_split_files_supported(split):
+ from pypaimon.execution.native.read.table_read import NativeTableReadMixin
+ return NativeTableReadMixin._native_split_files_supported(split)
+
+ def _convert_native_batches(self, batches, schema):
+ from pypaimon.execution.native.read.table_read import NativeTableReadMixin
+ yield from NativeTableReadMixin._convert_native_batches(self, batches, schema)
+
+ def _prepare_native_batch(self, batch, schema):
+ from pypaimon.execution.native.read.table_read import NativeTableReadMixin
+ return NativeTableReadMixin._prepare_native_batch(self, batch, schema)
+
+ def _flatten_native_nested_batch(self, batch, schema):
+ from pypaimon.execution.native.read.table_read import NativeTableReadMixin
+ return NativeTableReadMixin._flatten_native_nested_batch(self, batch, schema)
+
+ def _native_blob_view_supported(self) -> bool:
+ from pypaimon.execution.native.read.table_read import NativeTableReadMixin
+ return NativeTableReadMixin._native_blob_view_supported(self)
+
+ def _native_resolved_blob_fields(self) -> set:
+ """BLOB fields Rust resolves before emitting a batch."""
+ from pypaimon.execution.read_support import _native_resolved_blob_fields
+ return _native_resolved_blob_fields(self)
+
+ def _try_native_rows(self, splits):
+ if not self.table.options.native_read_enabled():
+ return self._native_fallback('read.native.enabled is false')
+ from pypaimon.execution.native.read.table_read import NativeTableReadMixin
+ return NativeTableReadMixin._try_native_rows(self, splits)
+
-__all__ = ["TableRead", "ROW_KIND_COLUMN", "_RemainingRows"]
+__all__ = ["TableRead", "ROW_KIND_COLUMN"]
diff --git a/paimon-python/pypaimon/read/table_scan.py b/paimon-python/pypaimon/read/table_scan.py
index 81e03f934a..22faab5a05 100755
--- a/paimon-python/pypaimon/read/table_scan.py
+++ b/paimon-python/pypaimon/read/table_scan.py
@@ -16,13 +16,10 @@
# under the License.
from dataclasses import replace
-from typing import Optional, Tuple
+from typing import List, Optional, Tuple, TYPE_CHECKING
-from pypaimon.catalog.catalog_exception import TableNoPermissionException
from pypaimon.common.predicate import Predicate
from pypaimon.common.options.core_options import CoreOptions
-from pypaimon.execution.native.read.plan import NativeTableScanMixin
-from pypaimon.execution.python.read.table_scan import PythonTableScanMixin
from pypaimon.read.auth_support import (
authorize, latest_auth_fields, validate_auth_rules)
from pypaimon.read.plan import Plan
@@ -33,7 +30,11 @@ from pypaimon.read.scan_stats import ScanStats
from pypaimon.utils.range import Range
-class TableScan(PythonTableScanMixin, NativeTableScanMixin):
+if TYPE_CHECKING:
+ from pypaimon.table.file_store_table import FileStoreTable
+
+
+class TableScan:
"""Public scan facade selecting the configured execution backend."""
def __init__(
@@ -43,9 +44,7 @@ class TableScan(PythonTableScanMixin, NativeTableScanMixin):
limit: Optional[int],
partition_predicate: Optional[Predicate] = None,
):
- from pypaimon.table.file_store_table import FileStoreTable
-
- self.table: FileStoreTable = table
+ self.table: "FileStoreTable" = table
self.predicate = predicate
self.limit = limit
self.partition_predicate = partition_predicate
@@ -126,26 +125,8 @@ class TableScan(PythonTableScanMixin, NativeTableScanMixin):
)
def plan(self) -> Plan:
- native_auth = (self.table.options.query_auth_enabled
- and self._native_requested() and self._native_plan_supported())
- if native_auth:
- # Rust asks the REST server and carries its complete response in
- # QueryAuthSplit. Both readers then enforce that captured policy.
- native = self._try_native_plan()
- if native is not None:
- return native
- auth_result = self.__auth_query()
- if self._query_auth_fn is not None and self._file_scanner is not None:
- from pypaimon.execution.python.read.auth_support import prune_scanner_by_auth
- prune_scanner_by_auth(self.table, self._file_scanner, auth_result)
- # The native planner bypasses the auth-aware file scanner. Resolve auth
- # before selecting a planning backend.
- if (not native_auth and auth_result is None and self._native_requested()
- and self._native_plan_supported()):
- native = self._try_native_plan()
- if native is not None:
- return native
- return self._python_plan(auth_result)
+ from pypaimon.execution.read_dispatch import plan_scan
+ return plan_scan(self)
def plan_all_files(self) -> Plan:
"""Plan matching files including L0 and all data-evolution column files.
@@ -153,14 +134,8 @@ class TableScan(PythonTableScanMixin, NativeTableScanMixin):
Existing filters and limits still apply. Query authorization must be
unrestricted, as required by writer restore, overwrite and indexing.
"""
- from pypaimon.execution.native.table import _native_query_auth_supported
- if self._native_requested() and _native_query_auth_supported(self.table) and self._native_plan_supported():
- plan = self._try_native_plan(all_files=True)
- if plan is not None:
- return plan
- if self.__auth_query(all_columns=True) is not None:
- raise TableNoPermissionException(self.table.identifier)
- return self._python_plan_all_files()
+ from pypaimon.execution.read_dispatch import plan_all_files
+ return plan_all_files(self)
def __auth_query(self, all_columns=False):
return authorize(self.table, self._query_auth_fn, None if all_columns else self._read_type,
@@ -173,22 +148,8 @@ class TableScan(PythonTableScanMixin, NativeTableScanMixin):
keeps going through :meth:`plan`. Native scans always collect lightweight
counters in the same core planning pass, without opening data files.
"""
- native_auth = (self.table.options.query_auth_enabled
- and self._native_requested() and self._native_plan_supported())
- if native_auth:
- native = self._try_native_plan()
- if native is not None:
- return native, native.stats()
- auth_result = self.__auth_query()
- if self._query_auth_fn is not None and self._file_scanner is not None:
- from pypaimon.execution.python.read.auth_support import prune_scanner_by_auth
- prune_scanner_by_auth(self.table, self._file_scanner, auth_result)
- if (not native_auth and auth_result is None and self._native_requested()
- and self._native_plan_supported()):
- native = self._try_native_plan()
- if native is not None:
- return native, native.stats()
- return self._python_scan_with_stats(auth_result)
+ from pypaimon.execution.read_dispatch import scan_with_stats
+ return scan_with_stats(self)
def with_shard(self, idx_of_this_subtask, number_of_para_subtasks) -> 'TableScan':
if self._file_scanner is None:
@@ -228,6 +189,47 @@ class TableScan(PythonTableScanMixin, NativeTableScanMixin):
def _validate_scan_mode(self):
_validate_scan_mode(self.table)
+ def _create_file_scanner(self) -> "FileScanner":
+ from pypaimon.execution.python.read.table_scan import PythonTableScanMixin
+ return PythonTableScanMixin._create_file_scanner(self)
+
+ def _native_requested(self) -> bool:
+ from pypaimon.execution.read_dispatch import native_plan_requested
+ return native_plan_requested(self)
+
+ def _native_plan_supported(self) -> bool:
+ # Any probe failure (e.g. a remote schema/metadata read) must fall back, not fail the scan.
+ from pypaimon.execution.native.read.plan import NativeTableScanMixin
+ return NativeTableScanMixin._native_plan_supported(self)
+
+ def _native_plan_supported_impl(self) -> bool:
+ """Fall back to the Python scanner for scans native can't carry:
+ a primary-key table whose trimmed PK is empty (PK equals the partition
+ key; Rust rejects this schema), unsupported time travel selectors,
+ custom authorization, a missing pypaimon-rust, or a catalog / identifier Rust
+ cannot reconstruct. Keep this capability gate in sync when adding scan
+ features."""
+ from pypaimon.execution.native.read.plan import NativeTableScanMixin
+ return NativeTableScanMixin._native_plan_supported_impl(self)
+
+ def _native_global_index_result_supported(self, request: Optional[ScanRequest] = None) -> bool:
+ from pypaimon.execution.native.read.plan import NativeTableScanMixin
+ return NativeTableScanMixin._native_global_index_result_supported(self, request)
+
+ def _native_row_ranges(self) -> Optional[List[Tuple[int, int]]]:
+ from pypaimon.execution.native.read.plan import NativeTableScanMixin
+ return NativeTableScanMixin._native_row_ranges(self)
+
+ def _try_native_plan(self, all_files=False) -> Optional[Plan]:
+ """Plan via pypaimon_rust, then drop partitions the predicate rejects.
+
+ Predicate and limit are pushed into Rust planning and are still enforced
+ by the reader. Snapshot metadata is preserved even when pruning removes
+ every split.
+ """
+ from pypaimon.execution.native.read.plan import NativeTableScanMixin
+ return NativeTableScanMixin._try_native_plan(self, all_files)
+
__all__ = [
"TableScan", "authorize", "latest_auth_fields", "validate_auth_rules",
diff --git a/paimon-python/pypaimon/schema/arrow_schema.py b/paimon-python/pypaimon/schema/arrow_schema.py
index feae004b0c..4304d8c3fd 100644
--- a/paimon-python/pypaimon/schema/arrow_schema.py
+++ b/paimon-python/pypaimon/schema/arrow_schema.py
@@ -18,6 +18,7 @@
"""STRING compatibility and canonical Arrow input for core writers."""
import pyarrow as pa
+import pyarrow.compute as pc
def _normalize_string_type(data_type, ignore_child_nullability=False):
@@ -73,23 +74,27 @@ def arrow_schemas_compatible(source, target, check_top_level_nullability=True, a
def normalize_arrow_write_nullability(data, target):
- """Validate read-result NULLs in core before restoring writer child fields."""
- from pypaimon.execution.native.runtime import (native_runtime_available)
- if not native_runtime_available():
- return data
- from pypaimon_rust.data import normalize_array_for_write
+ """Restore declared child nullability after checking visible input values.
+
+ Runtime read schemas relax nested nullability. Validate only children of
+ non-NULL parents before restoring those declarations, independently of the
+ optional Native runtime. This does not convert physical Arrow types.
+ """
options = dict(check_top_level_nullability=False, allow_binary_compatibility=True)
if (arrow_schemas_compatible(data.schema, target, **options)
or not arrow_schemas_compatible(data.schema, target, check_nested_nullability=False, **options)):
return data
arrays, fields = [], []
for column, field, expected in zip(data.columns, data.schema, target):
- if column.type != expected.type:
- if isinstance(column, pa.ChunkedArray):
- column = pa.chunked_array(
- [normalize_array_for_write(chunk, expected.type) for chunk in column.chunks], type=expected.type)
- else:
- column = normalize_array_for_write(column, expected.type)
+ data_type = _write_child_nullability(column.type, expected.type)
+ if data_type != column.type:
+ chunks = column.chunks if isinstance(column, pa.ChunkedArray) else [column]
+ normalized = []
+ for chunk in chunks:
+ _check_write_nullability(chunk, expected.with_nullable(True), field.name)
+ normalized.append(_with_child_nullability(chunk, data_type))
+ column = (pa.chunked_array(normalized, type=data_type)
+ if isinstance(column, pa.ChunkedArray) else normalized[0])
arrays.append(column)
fields.append(field.with_type(column.type))
schema = pa.schema(fields, metadata=data.schema.metadata)
@@ -98,6 +103,85 @@ def normalize_arrow_write_nullability(data, target):
return pa.Table.from_arrays(arrays, schema=schema)
+def _write_child_nullability(source, target):
+ """Copy child declarations without changing layouts or scalar types."""
+ def child(actual, expected):
+ return expected.with_type(_write_child_nullability(actual.type, expected.type))
+
+ if pa.types.is_struct(source) and pa.types.is_struct(target):
+ return pa.struct([child(actual, expected) for actual, expected in zip(source, target)])
+ if ((pa.types.is_list(source) or pa.types.is_large_list(source) or pa.types.is_fixed_size_list(source))
+ and (pa.types.is_list(target) or pa.types.is_large_list(target) or pa.types.is_fixed_size_list(target))):
+ field = child(source.value_field, target.value_field)
+ if pa.types.is_large_list(source):
+ return pa.large_list(field)
+ if pa.types.is_fixed_size_list(source):
+ return pa.list_(field, source.list_size)
+ return pa.list_(field)
+ if pa.types.is_map(source) and pa.types.is_map(target):
+ sorted_keys = source == pa.map_(source.key_field, source.item_field, keys_sorted=True)
+ return pa.map_(child(source.key_field, target.key_field), child(source.item_field, target.item_field),
+ keys_sorted=sorted_keys)
+ return source
+
+
+def _check_write_nullability(array, field, path):
+ if not field.nullable and array.null_count:
+ raise ValueError("Input schema isn't consistent with table schema: non-nullable field %s contains NULL" % path)
+ if array.null_count == len(array):
+ return
+ data_type = field.type
+ if pa.types.is_struct(data_type):
+ valid = pc.is_valid(array) if array.null_count else None
+ for index, child in enumerate(data_type):
+ values = array.field(index)
+ if valid is not None:
+ values = pc.filter(values, valid)
+ _check_write_nullability(values, child, path + '.' + child.name)
+ elif (pa.types.is_list(data_type) or pa.types.is_large_list(data_type)
+ or pa.types.is_fixed_size_list(data_type)):
+ child = data_type.value_field
+ _check_write_nullability(pc.list_flatten(array), child, path + '.' + child.name)
+ elif pa.types.is_map(data_type):
+ # MAP and LIST<STRUCT<key, value>> share a layout. Retain the parent's
+ # validity and offset so flatten selects only visible entries, including
+ # with Arrow 6 where ListArray.from_arrays has no mask argument.
+ entries = pa.StructArray.from_arrays(
+ [array.keys, array.items],
+ fields=[array.type.key_field, array.type.item_field])
+ entries = pc.list_flatten(pa.Array.from_buffers(
+ pa.list_(entries.type), len(array), array.buffers()[:2], children=[entries],
+ offset=array.offset, null_count=array.null_count))
+ for index, child in enumerate((data_type.key_field, data_type.item_field)):
+ _check_write_nullability(entries.field(index), child, path + '.' + child.name)
+
+
+def _with_child_nullability(array, data_type):
+ """Retype nested wrappers while retaining payloads, slices and validity."""
+ if array.type == data_type:
+ return array
+ if pa.types.is_struct(data_type):
+ # StructArray.field() already applies the parent's slice. Construct an
+ # offset-zero wrapper so the offset is not applied to children twice.
+ children = [_with_child_nullability(array.field(index), child.type)
+ for index, child in enumerate(data_type)]
+ return pa.StructArray.from_arrays(children, fields=list(data_type), mask=pc.is_null(array))
+ if pa.types.is_map(data_type):
+ entries = pa.StructArray.from_arrays(
+ [_with_child_nullability(array.keys, data_type.key_type),
+ _with_child_nullability(array.items, data_type.item_type)],
+ fields=[data_type.key_field, data_type.item_field])
+ return pa.Array.from_buffers(data_type, len(array), array.buffers()[:2],
+ children=[entries], offset=array.offset, null_count=array.null_count)
+ if (pa.types.is_list(data_type) or pa.types.is_large_list(data_type)
+ or pa.types.is_fixed_size_list(data_type)):
+ children = [_with_child_nullability(array.values, data_type.value_type)]
+ buffers = array.buffers()[:1 if pa.types.is_fixed_size_list(data_type) else 2]
+ return pa.Array.from_buffers(data_type, len(array), buffers, children=children,
+ offset=array.offset, null_count=array.null_count)
+ return array
+
+
def normalize_arrow_strings(data):
"""Safely normalize large_string input before routing or buffering rows.
diff --git a/paimon-python/pypaimon/table/file_store_table.py b/paimon-python/pypaimon/table/file_store_table.py
index 19da5e4817..49324a28df 100644
--- a/paimon-python/pypaimon/table/file_store_table.py
+++ b/paimon-python/pypaimon/table/file_store_table.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from typing import List, Optional, Set
+from typing import List, Optional, Set, TYPE_CHECKING
from pypaimon.catalog.catalog_environment import CatalogEnvironment
from pypaimon.common.file_io import FileIO
@@ -28,15 +28,11 @@ from pypaimon.schema.schema_manager import SchemaManager
from pypaimon.schema.table_schema import TableSchema
from pypaimon.table.bucket_mode import BucketMode
from pypaimon.table.table import Table
-from pypaimon.execution.python.write.row_key_extractor import (
- DynamicBucketRowKeyExtractor,
- FixedBucketRowKeyExtractor,
- PostponeBucketRowKeyExtractor,
- RowKeyExtractor,
- UnawareBucketRowKeyExtractor,
-)
from pypaimon.write.write_builder import BatchWriteBuilder, StreamWriteBuilder
+if TYPE_CHECKING:
+ from pypaimon.execution.python.write.row_key_extractor import RowKeyExtractor
+
class FileStoreTable(Table):
def __init__(self, file_io: FileIO, identifier: Identifier, table_path: str,
@@ -492,7 +488,11 @@ class FileStoreTable(Table):
def create_row_key_extractor(
self, ignore_existing: bool = False
- ) -> RowKeyExtractor:
+ ) -> 'RowKeyExtractor':
+ from pypaimon.execution.python.write.row_key_extractor import (
+ DynamicBucketRowKeyExtractor, FixedBucketRowKeyExtractor,
+ PostponeBucketRowKeyExtractor, UnawareBucketRowKeyExtractor)
+
bucket_mode = self.bucket_mode()
if bucket_mode == BucketMode.HASH_FIXED:
return FixedBucketRowKeyExtractor(self.table_schema)
diff --git a/paimon-python/pypaimon/table/source/batch_vector_search_builder.py b/paimon-python/pypaimon/table/source/batch_vector_search_builder.py
index 6f7eecdac6..bef5a7e5b9 100644
--- a/paimon-python/pypaimon/table/source/batch_vector_search_builder.py
+++ b/paimon-python/pypaimon/table/source/batch_vector_search_builder.py
@@ -110,11 +110,8 @@ class BatchVectorSearchBuilderImpl(AbstractVectorSearchBuilderImpl,
return self
def execute_batch_local(self):
- from pypaimon.execution.native.read.native_vector_search import try_native_vector_search
- results = try_native_vector_search(self, self._query_vectors, batch=True)
- if results is not None:
- return results
- return super().execute_batch_local()
+ from pypaimon.execution.search_dispatch import execute_vector_search
+ return execute_vector_search(self, self._query_vectors, batch=True)
def new_batch_vector_search_read(self):
# type: () -> BatchVectorSearchRead
@@ -126,11 +123,11 @@ class BatchVectorSearchBuilderImpl(AbstractVectorSearchBuilderImpl,
if not self._query_vectors:
raise ValueError(
"Query vectors must be set via with_query_vectors()")
- from pypaimon.execution.native.read.native_search_plan import native_vector_read
+ from pypaimon.execution.search_dispatch import create_vector_read
execution = copy(self)
execution._options = dict(self._options)
- execution._query_vectors = list(self._query_vectors)
- return native_vector_read(self, execution._new_python_batch_vector_read, batch=True)
+ execution._query_vectors = [copy(vector) for vector in self._query_vectors]
+ return create_vector_read(execution, execution._new_python_batch_vector_read, batch=True)
def _new_python_batch_vector_read(self):
from pypaimon.execution.python.read.search.vector_search_read import BatchVectorSearchReadImpl
diff --git a/paimon-python/pypaimon/table/source/full_text_search_builder.py b/paimon-python/pypaimon/table/source/full_text_search_builder.py
index de01443e9b..dcb6679b7f 100644
--- a/paimon-python/pypaimon/table/source/full_text_search_builder.py
+++ b/paimon-python/pypaimon/table/source/full_text_search_builder.py
@@ -85,12 +85,8 @@ class FullTextSearchBuilderImpl(FullTextSearchBuilder):
return self
def execute_local(self) -> GlobalIndexResult:
- from pypaimon.execution.native.read.native_full_text_search import try_native_full_text_search
-
- result = try_native_full_text_search(self)
- if result is not None:
- return result
- return super().execute_local()
+ from pypaimon.execution.search_dispatch import execute_full_text_search
+ return execute_full_text_search(self)
def with_query(self, field_name: str, query: str) -> 'FullTextSearchBuilder':
self._field_name = field_name
@@ -147,10 +143,10 @@ class FullTextSearchBuilderImpl(FullTextSearchBuilder):
return predicate.new_index(name_to_idx[predicate.field])
def new_full_text_scan(self) -> FullTextScan:
- from pypaimon.execution.native.read.native_search_plan import native_full_text_scan
+ from pypaimon.execution.search_dispatch import create_full_text_scan
reject_search_under_query_auth(self._table)
self._primary_key_full_text_definition()
- return native_full_text_scan(self, copy(self)._new_python_full_text_scan)
+ return create_full_text_scan(self, copy(self)._new_python_full_text_scan)
def _new_python_full_text_scan(self):
definition = self._primary_key_full_text_definition()
@@ -167,7 +163,7 @@ class FullTextSearchBuilderImpl(FullTextSearchBuilder):
)
def new_full_text_read(self) -> FullTextRead:
- from pypaimon.execution.native.read.native_search_plan import native_full_text_read
+ from pypaimon.execution.search_dispatch import create_full_text_read
reject_search_under_query_auth(self._table)
if self._limit <= 0:
raise ValueError("Limit must be positive, set via with_limit()")
@@ -184,7 +180,7 @@ class FullTextSearchBuilderImpl(FullTextSearchBuilder):
"Primary-key full-text search only supports the FAST "
"full-text index search mode; FULL and DETAIL require "
"merge-aware logical-row fallback.")
- return native_full_text_read(self, copy(self)._new_python_full_text_read)
+ return create_full_text_read(self, copy(self)._new_python_full_text_read)
def _new_python_full_text_read(self):
definition = self._primary_key_full_text_definition()
diff --git a/paimon-python/pypaimon/table/source/hybrid_search_builder.py b/paimon-python/pypaimon/table/source/hybrid_search_builder.py
index c8e9b0eff0..552fb67037 100644
--- a/paimon-python/pypaimon/table/source/hybrid_search_builder.py
+++ b/paimon-python/pypaimon/table/source/hybrid_search_builder.py
@@ -31,8 +31,6 @@ from pypaimon.globalindex.vector_search_result import (
RRF_RANKER = "rrf"
WEIGHTED_SCORE_RANKER = "weighted_score"
MRR_RANKER = "mrr"
-_RRF_K = 60.0
-_MAX_ROUTE_WORKERS = 4
def _check_full_text_options(options: Dict[str, str]):
@@ -243,12 +241,8 @@ class HybridSearchBuilderImpl(HybridSearchBuilder):
self._partition_filter = None
def execute_local(self) -> ScoredGlobalIndexResult:
- from pypaimon.execution.native.read.native_hybrid_search import try_native_hybrid_search
-
- result = try_native_hybrid_search(self)
- if result is not None:
- return result
- return super().execute_local()
+ from pypaimon.execution.search_dispatch import execute_hybrid_search
+ return execute_hybrid_search(self)
def with_limit(self, limit: int) -> 'HybridSearchBuilder':
self._limit = limit
@@ -330,18 +324,6 @@ class HybridSearchBuilderImpl(HybridSearchBuilder):
builder.with_filter(self._filter)
return builder
- def _rrf(self, route_results):
- from pypaimon.execution.python.read.search.hybrid_search import _rrf
- return _rrf(self, route_results)
-
- def _mrr(self, route_results):
- from pypaimon.execution.python.read.search.hybrid_search import _mrr
- return _mrr(self, route_results)
-
- def _weighted_score(self, route_results):
- from pypaimon.execution.python.read.search.hybrid_search import _weighted_score
- return _weighted_score(self, route_results)
-
def _split_partition_filter(self, predicate):
partition_keys = list(self._table.partition_keys or [])
if not partition_keys:
@@ -393,13 +375,3 @@ class HybridSearchBuilderImpl(HybridSearchBuilder):
[cls._rebuild_leaf_indices_by_name(c, name_to_idx)
for c in (predicate.literals or [])])
return predicate.new_index(name_to_idx[predicate.field])
-
-
-def _ranked_row_ids(result):
- from pypaimon.execution.python.read.search.hybrid_search import _ranked_row_ids as execute
- return execute(result)
-
-
-def _top_k(scores, limit):
- from pypaimon.execution.python.read.search.hybrid_search import _top_k as execute
- return execute(scores, limit)
diff --git a/paimon-python/pypaimon/table/source/vector_search_builder.py b/paimon-python/pypaimon/table/source/vector_search_builder.py
index 65b46990fd..1869d3a847 100644
--- a/paimon-python/pypaimon/table/source/vector_search_builder.py
+++ b/paimon-python/pypaimon/table/source/vector_search_builder.py
@@ -220,10 +220,10 @@ class AbstractVectorSearchBuilderImpl:
reject_search_under_query_auth(self._table)
if self._vector_column is None:
raise ValueError("Vector column must be set via with_vector_column()")
- from pypaimon.execution.native.read.native_search_plan import native_vector_scan
+ from pypaimon.execution.search_dispatch import create_vector_scan
execution = copy(self)
execution._options = dict(self._options)
- return native_vector_scan(self, execution._new_python_vector_scan, self._native_batch_search)
+ return create_vector_scan(self, execution._new_python_vector_scan, self._native_batch_search)
def _new_python_vector_scan(self):
from pypaimon.execution.python.read.search.vector_search_scan import DataEvolutionVectorScan
@@ -259,11 +259,8 @@ class VectorSearchBuilderImpl(AbstractVectorSearchBuilderImpl, VectorSearchBuild
return self
def execute_local(self):
- from pypaimon.execution.native.read.native_vector_search import try_native_vector_search
- result = try_native_vector_search(self, self._query_vector)
- if result is not None:
- return result
- return super().execute_local()
+ from pypaimon.execution.search_dispatch import execute_vector_search
+ return execute_vector_search(self, self._query_vector)
def new_vector_search_read(self):
# type: () -> VectorSearchRead
@@ -274,10 +271,11 @@ class VectorSearchBuilderImpl(AbstractVectorSearchBuilderImpl, VectorSearchBuild
raise ValueError("Vector column must be set via with_vector_column()")
if self._query_vector is None:
raise ValueError("Query vector must be set via with_query_vector()")
- from pypaimon.execution.native.read.native_search_plan import native_vector_read
+ from pypaimon.execution.search_dispatch import create_vector_read
execution = copy(self)
execution._options = dict(self._options)
- return native_vector_read(self, execution._new_python_vector_read)
+ execution._query_vector = copy(self._query_vector)
+ return create_vector_read(execution, execution._new_python_vector_read)
def _new_python_vector_read(self):
from pypaimon.execution.python.read.search.vector_search_read import DataEvolutionVectorRead
diff --git a/paimon-python/pypaimon/table/system/system_table_scan.py b/paimon-python/pypaimon/table/system/system_table_scan.py
index 6892a5fb33..516d929de6 100644
--- a/paimon-python/pypaimon/table/system/system_table_scan.py
+++ b/paimon-python/pypaimon/table/system/system_table_scan.py
@@ -70,9 +70,5 @@ class SystemTableScan:
self.system_table = system_table
def plan(self) -> Plan:
- from pypaimon.execution.native.read.system_table import try_read_native_system_table
- arrow_table = try_read_native_system_table(self.system_table)
- if arrow_table is not None:
- return Plan(_splits=[SystemSplit(arrow_table)])
- from pypaimon.execution.python.read.system_table_scan import plan
- return plan(self)
+ from pypaimon.execution.read_dispatch import plan_system_table
+ return plan_system_table(self)
diff --git a/paimon-python/pypaimon/tests/arrow_write_nullability_test.py b/paimon-python/pypaimon/tests/arrow_write_nullability_test.py
new file mode 100644
index 0000000000..904ddd136f
--- /dev/null
+++ b/paimon-python/pypaimon/tests/arrow_write_nullability_test.py
@@ -0,0 +1,171 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Pure write schema validation is independent of Native installation."""
+
+import sys
+from types import ModuleType
+from unittest.mock import Mock, patch
+
+import pyarrow as pa
+import pytest
+
+from pypaimon.schema.arrow_schema import normalize_arrow_write_nullability
+
+
+def _source(array, batch=False):
+ schema = pa.schema([pa.field('nested', array.type, metadata={b'field': b'input'})],
+ metadata={b'schema': b'input'})
+ return (pa.RecordBatch if batch else pa.Table).from_arrays([array], schema=schema)
+
+
+def _target(data_type):
+ return pa.schema([pa.field('nested', data_type)])
+
+
+@pytest.mark.parametrize('installed', [False, True])
+@pytest.mark.parametrize('batch', [False, True])
+def test_visible_child_validation_never_loads_native(installed, batch):
+ sdk = ModuleType('pypaimon_rust.data')
+ sdk.normalize_array_for_write = Mock(side_effect=AssertionError('Native normalization called'))
+ modules = {'pypaimon.execution.native.runtime': None,
+ 'pypaimon_rust': ModuleType('pypaimon_rust') if installed else None,
+ 'pypaimon_rust.data': sdk if installed else None}
+ target = _target(pa.list_(pa.field('item', pa.int32(), nullable=False)))
+ data = _source(pa.array([[1, 2], None, []], type=pa.list_(pa.int32())), batch)
+ with patch.dict(sys.modules, modules):
+ result = normalize_arrow_write_nullability(data, target)
+ assert result.to_pydict() == data.to_pydict()
+ assert result.schema.field(0).type == target.field(0).type
+ with pytest.raises(ValueError, match='nested.item.*NULL'):
+ normalize_arrow_write_nullability(_source(pa.array([[1, None]], type=pa.list_(pa.int32())), batch), target)
+ sdk.normalize_array_for_write.assert_not_called()
+
+
+@pytest.mark.parametrize('kind', ['list', 'large-list', 'fixed-size-list', 'map', 'struct'])
+def test_null_parents_and_slices_hide_unselected_null_children(kind):
+ child = pa.field('item', pa.int32(), nullable=False)
+ if kind == 'struct':
+ data = pa.StructArray.from_arrays(
+ [pa.array([None, None, 7, None], type=pa.int32())],
+ names=['item'], mask=pa.array([False, True, False, False]))
+ data_type = pa.struct([child])
+ elif kind == 'map':
+ offsets = pa.array([0, 1, 2, 3, 4], type=pa.int32())
+ data = pa.MapArray.from_arrays(
+ offsets, pa.array(['outside', 'hidden', 'visible', 'outside']),
+ pa.array([None, None, 7, None], type=pa.int32()),
+ mask=pa.array([False, True, False, False]))
+ data_type = pa.map_(pa.string(), pa.field('value', pa.int32(), nullable=False))
+ else:
+ values = pa.array([None, None, 7, None], type=pa.int32())
+ mask = pa.array([False, True, False, False])
+ if kind == 'fixed-size-list':
+ data = pa.FixedSizeListArray.from_arrays(values, 1, mask=mask)
+ data_type = pa.list_(child, 1)
+ else:
+ large = kind == 'large-list'
+ cls = pa.LargeListArray if large else pa.ListArray
+ offsets = pa.array([0, 1, 2, 3, 4], type=pa.int64() if large else pa.int32())
+ data = cls.from_arrays(offsets, values, mask=mask)
+ data_type = pa.large_list(child) if large else pa.list_(child)
+ source = _source(data.slice(1, 2))
+ result = normalize_arrow_write_nullability(source, _target(data_type))
+ assert result.to_pydict() == source.to_pydict()
+ assert result.schema.field(0).type == data_type
+ assert result.schema.metadata == source.schema.metadata
+ assert result.schema.field(0).metadata == source.schema.field(0).metadata
+ result.validate(full=True)
+ with pytest.raises(ValueError, match='NULL'):
+ normalize_arrow_write_nullability(_source(data.slice(2, 2)), _target(data_type))
+
+
+def test_null_struct_masks_nested_non_nullable_containers():
+ lists = pa.ListArray.from_arrays(pa.array([0, 1, 2]), pa.array([None, 7], type=pa.int32()))
+ maps = pa.MapArray.from_arrays(
+ pa.array([0, 1, 2]), pa.array(['hidden', 'visible']),
+ pa.array([None, 8], type=pa.int32()))
+ source = _source(pa.StructArray.from_arrays(
+ [lists, maps], names=['items', 'attrs'], mask=pa.array([True, False])))
+ target = _target(pa.struct([
+ pa.field('items', pa.list_(pa.field('item', pa.int32(), nullable=False)), nullable=False),
+ pa.field('attrs', pa.map_(pa.string(), pa.field('value', pa.int32(), nullable=False)), nullable=False),
+ ]))
+ result = normalize_arrow_write_nullability(source, target)
+ assert result.to_pydict() == source.to_pydict()
+ assert result.schema.field(0).type == target.field(0).type
+
+
+def test_chunk_boundaries_and_payload_buffers_are_retained():
+ data_type = pa.list_(pa.field('item', pa.int32(), nullable=False))
+ chunks = [pa.array([[1, 2], [3]], type=pa.list_(pa.int32())),
+ pa.array([None, [4]], type=pa.list_(pa.int32()))]
+ data = pa.table({'nested': pa.chunked_array(chunks)})
+ result = normalize_arrow_write_nullability(data, _target(data_type))
+ assert [len(chunk) for chunk in result.column(0).chunks] == [2, 2]
+ assert result.to_pydict() == data.to_pydict()
+ for actual, original in zip(result.column(0).chunks, chunks):
+ assert actual.values.buffers()[1].address == original.values.buffers()[1].address
+
+
+def test_nullability_restoration_does_not_cast_other_layouts():
+ data = pa.table({'nested': pa.array([[1]], type=pa.list_(pa.int32())),
+ 'text': pa.array(['value'], type=pa.large_string()),
+ 'binary': pa.array([b'x'], type=pa.binary(1))})
+ target = pa.schema([('nested', pa.list_(pa.field('item', pa.int32(), nullable=False))),
+ ('text', pa.string()), ('binary', pa.binary())])
+ result = normalize_arrow_write_nullability(data, target)
+ assert result.schema.field('text').type == pa.large_string()
+ assert result.schema.field('binary').type == pa.binary(1)
+ incompatible = pa.schema([('nested', pa.list_(pa.field('item', pa.int64(), nullable=False))),
+ ('text', pa.string()), ('binary', pa.binary())])
+ assert normalize_arrow_write_nullability(data, incompatible) is data
+
+
+def test_empty_chunked_input_restores_only_declared_child_fields():
+ source = pa.table({'nested': pa.chunked_array([], type=pa.list_(pa.int32()))})
+ data_type = pa.list_(pa.field('item', pa.int32(), nullable=False))
+ result = normalize_arrow_write_nullability(source, _target(data_type))
+ assert result.num_rows == 0 and result.column(0).num_chunks == 0
+ assert result.schema.field(0).type == data_type
+
+
+def test_python_writer_accepts_visible_non_null_children_without_native(tmp_path):
+ from pypaimon import CatalogFactory, Schema
+
+ declared = pa.schema([('nested', pa.struct([pa.field('value', pa.int32(), nullable=False)]))])
+ catalog = CatalogFactory.create({'warehouse': str(tmp_path / 'warehouse')})
+ catalog.create_database('default', True)
+ catalog.create_table('default.nested', Schema.from_pyarrow_schema(declared, options={
+ 'read.native.enabled': 'false', 'scan.native-plan.enabled': 'false',
+ 'write.native.enabled': 'false', 'commit.native.enabled': 'false',
+ }), False)
+ table = catalog.get_table('default.nested')
+ source = _source(pa.StructArray.from_arrays(
+ [pa.array([None, 7], type=pa.int32())], names=['value'], mask=pa.array([True, False])))
+ builder = table.new_batch_write_builder()
+ writer, commit = builder.new_write(), builder.new_commit()
+ try:
+ with patch.dict(sys.modules, {'pypaimon.execution.native.runtime': None, 'pypaimon_rust': None}):
+ writer.write_arrow(source)
+ commit.commit(writer.prepare_commit())
+ finally:
+ writer.close()
+ commit.close()
+ reader = table.new_read_builder()
+ result = reader.new_read().to_arrow(reader.new_scan().plan().splits())
+ assert result.column('nested').to_pylist() == [None, {'value': 7}]
diff --git a/paimon-python/pypaimon/tests/batch_vector_raw_scan_test.py b/paimon-python/pypaimon/tests/batch_vector_raw_scan_test.py
index 6fde08c225..86d4182745 100644
--- a/paimon-python/pypaimon/tests/batch_vector_raw_scan_test.py
+++ b/paimon-python/pypaimon/tests/batch_vector_raw_scan_test.py
@@ -239,7 +239,7 @@ class BatchVectorRawScanTest(BatchModeMixin, DataEvolutionTestBase, unittest.Tes
state['closed'] += 1
with patch.object(TableRead, '_arrow_batch_generator', tracked), \
- patch('pypaimon.read.table_read.os.cpu_count', return_value=4):
+ patch('pypaimon.execution.read_support.os.cpu_count', return_value=4):
actual = (read_table.new_batch_vector_search_builder()
.with_vector_column('embedding').with_query_vectors(queries)
.with_option('metric', metric).with_limit(2).execute_batch_local())
diff --git a/paimon-python/pypaimon/tests/daft/daft_pk_distributed_write_test.py b/paimon-python/pypaimon/tests/daft/daft_pk_distributed_write_test.py
index 39588932d0..bd1f1858e6 100644
--- a/paimon-python/pypaimon/tests/daft/daft_pk_distributed_write_test.py
+++ b/paimon-python/pypaimon/tests/daft/daft_pk_distributed_write_test.py
@@ -318,7 +318,7 @@ def test_hash_dynamic_pk_restores_bucket_mapping_across_commits(catalog):
@pytest.mark.parametrize('overwrite', [False, True])
def test_grouped_pk_writes_use_native_without_python_fallback(
catalog, monkeypatch, bucket, partitioned, overwrite):
- from pypaimon.execution.native.write.native_write import NativeTableWrite
+ from pypaimon.execution.write_dispatch import NativeTableWrite
def reject_fallback(self):
raise AssertionError('Grouped Daft write fell back to Python')
diff --git a/paimon-python/pypaimon/tests/data_file_path_directory_test.py b/paimon-python/pypaimon/tests/data_file_path_directory_test.py
index 40ef1580a7..d678e4177b 100644
--- a/paimon-python/pypaimon/tests/data_file_path_directory_test.py
+++ b/paimon-python/pypaimon/tests/data_file_path_directory_test.py
@@ -109,9 +109,30 @@ class DataFilePathDirectoryTest(unittest.TestCase):
table = self._create_with_options(
"db.t_native_write", {"data-file.path-directory": "data",
"write.native.enabled": "true"})
- sentinel = object()
- with mock.patch("pypaimon.execution.native.write.native_write.create_native_write", return_value=sentinel):
- self.assertIs(table.new_batch_write_builder().new_write(), sentinel)
+ from pypaimon.execution.write_dispatch import NativeTableWrite
+
+ adapter = mock.Mock(spec=['write_arrow_batch', 'prepare_commit', 'close'])
+ messages = []
+ adapter.prepare_commit.return_value = messages
+ builder = table.new_batch_write_builder()
+ with mock.patch("pypaimon.execution.native.write.native_write.create_native_write",
+ return_value=adapter) as prepare:
+ writer = builder.new_write()
+ self.assertIsInstance(writer, NativeTableWrite)
+ prepare.assert_called_once_with(
+ table, builder.commit_user, None, False,
+ restore_snapshot_id=None, fixed_bucket=False, bucket_plan=None)
+ self.assertEqual(prepare.call_args[0][0].options.options.to_map()['data-file.path-directory'], 'data')
+ batch = pa.record_batch(
+ [pa.array([1], type=pa.int32()), ['a']], names=['id', 'v'])
+ try:
+ writer.write_arrow_batch(batch)
+ adapter.write_arrow_batch.assert_called_once_with(batch, None)
+ self.assertIs(writer.prepare_commit(), messages)
+ adapter.prepare_commit.assert_called_once_with(None)
+ finally:
+ writer.close()
+ adapter.close.assert_called_once_with()
def test_native_commit_dispatches_with_configured_directory(self):
table = self._create_with_options(
diff --git a/paimon-python/pypaimon/tests/execution_backend_isolation_test.py b/paimon-python/pypaimon/tests/execution_backend_isolation_test.py
new file mode 100644
index 0000000000..bbbbdef284
--- /dev/null
+++ b/paimon-python/pypaimon/tests/execution_backend_isolation_test.py
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Whole public Python workflows remain independent of Native execution."""
+
+from pathlib import Path
+import subprocess
+import sys
+import textwrap
+
+
+def test_python_read_update_and_index_roundtrip_never_loads_native(tmp_path):
+ source = r"""
+import importlib.abc
+import sys
+import pyarrow as pa
+
+class BlockNative(importlib.abc.MetaPathFinder):
+ def find_spec(self, fullname, path=None, target=None):
+ if fullname.startswith(('pypaimon.execution.native', 'pypaimon_rust')):
+ raise AssertionError('Python workflow loaded Native execution: ' + fullname)
+
+sys.meta_path.insert(0, BlockNative())
+from pypaimon import CatalogFactory, Schema
+
+catalog = CatalogFactory.create({'warehouse': sys.argv[1]})
+catalog.create_database('default', True)
+catalog.create_table('default.records', Schema.from_pyarrow_schema(
+ pa.schema([('id', pa.int64()), ('value', pa.string())]), options={
+ 'bucket': '-1', 'row-tracking.enabled': 'true', 'data-evolution.enabled': 'true',
+ 'scan.native-plan.enabled': 'false', 'read.native.enabled': 'false',
+ 'write.native.enabled': 'false', 'commit.native.enabled': 'false',
+ }), False)
+table = catalog.get_table('default.records')
+builder = table.new_batch_write_builder()
+writer, commit = builder.new_write(), builder.new_commit()
+try:
+ writer.write_arrow(pa.table({'id': [1, 2], 'value': ['old', 'kept']}))
+ commit.commit(writer.prepare_commit())
+finally:
+ writer.close()
+ commit.close()
+
+builder = table.new_batch_write_builder()
+commit = builder.new_commit()
+try:
+ messages = builder.new_update().update_by_arrow_with_row_id(
+ pa.table({'_ROW_ID': [0], 'value': ['changed']}))
+ commit.commit(messages)
+finally:
+ commit.close()
+
+read = table.new_read_builder()
+rows = read.new_read().to_arrow(read.new_scan().plan().splits()).to_pylist()
+assert sorted(rows, key=lambda row: row['id']) == [
+ {'id': 1, 'value': 'changed'}, {'id': 2, 'value': 'kept'}]
+assert table.create_global_index('value', 'btree') > 0
+filtered = table.new_read_builder()
+filtered.with_filter(filtered.new_predicate_builder().equal('value', 'changed'))
+assert filtered.new_read().to_arrow(filtered.new_scan().plan().splits()).to_pylist() == [
+ {'id': 1, 'value': 'changed'}]
+assert table.drop_global_index('value', 'btree', dry_run=True) > 0
+assert table.drop_global_index('value', 'btree') > 0
+assert not any(name.startswith(('pypaimon.execution.native', 'pypaimon_rust')) for name in sys.modules)
+"""
+ result = subprocess.run(
+ [sys.executable, '-c', textwrap.dedent(source), str(tmp_path / 'warehouse')],
+ cwd=str(Path(__file__).resolve().parents[2]),
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=45)
+ assert result.returncode == 0, result.stdout + result.stderr
diff --git a/paimon-python/pypaimon/tests/execution_dispatch_test.py b/paimon-python/pypaimon/tests/execution_dispatch_test.py
index 2fe3798da2..21c5a161c4 100644
--- a/paimon-python/pypaimon/tests/execution_dispatch_test.py
+++ b/paimon-python/pypaimon/tests/execution_dispatch_test.py
@@ -70,28 +70,6 @@ def test_native_write_error_cannot_trigger_python_replay():
python_write.assert_not_called()
-def test_native_predicate_row_ids_uses_the_read_adapter():
- import pyarrow as pa
- from pypaimon.execution.native.write.native_update import native_predicate_row_ids
-
- table = SimpleNamespace(options=SimpleNamespace(
- row_tracking_enabled=lambda: True, data_evolution_enabled=lambda: False))
- predicate = object()
- reader = Mock(side_effect=[
- [pa.record_batch([[2, 3]], names=['_ROW_ID'])],
- [pa.record_batch([[9]], names=['_ROW_ID'])],
- ])
- with patch('pypaimon.execution.native.write.native_update.native_runtime_available', return_value=True), \
- patch('pypaimon.execution.native.read.table_read._prepare_native_read', return_value=reader) as prepare, \
- patch('pypaimon.execution.native.read.split_adapter.native_split_from_python',
- side_effect=['native-first', 'native-second']) as convert:
- assert native_predicate_row_ids(table, predicate, ['first', 'second']) == [2, 3, 9]
- prepare.assert_called_once_with(table, predicate=predicate, projection=['_ROW_ID'])
- assert [call[0] for call in reader.call_args_list] == [(['native-first'],), (['native-second'],)]
- assert [call[1] for call in convert.call_args_list] == [
- {'ranges_use_row_ids': True}, {'ranges_use_row_ids': True}]
-
-
def test_native_row_id_updater_keeps_the_public_return_type():
from pypaimon.execution.native.write.native_update import NativeTableUpdateByRowId
from pypaimon.write.table_update_by_row_id import TableUpdateByRowId
@@ -99,7 +77,8 @@ def test_native_row_id_updater_keeps_the_public_return_type():
native = object.__new__(NativeTableUpdateByRowId)
with patch('pypaimon.execution.native.write.native_update.create_native_update_by_row_id',
return_value=native):
- result = dispatch.prepare_native_write_operation('update_by_row_id', object(), 'writer', 1)
+ table = SimpleNamespace(options=SimpleNamespace(native_write_enabled=lambda: True))
+ result = dispatch.prepare_native_write_operation('update_by_row_id', table, 'writer', 1)
assert result is native
assert isinstance(result, TableUpdateByRowId)
assert issubclass(NativeTableUpdateByRowId, TableUpdateByRowId)
diff --git a/paimon-python/pypaimon/tests/execution_table_read_boundary_test.py b/paimon-python/pypaimon/tests/execution_table_read_boundary_test.py
index 3311425efe..92cad4142f 100644
--- a/paimon-python/pypaimon/tests/execution_table_read_boundary_test.py
+++ b/paimon-python/pypaimon/tests/execution_table_read_boundary_test.py
@@ -59,7 +59,10 @@ def test_native_system_scan_never_materializes_the_python_provider():
from pypaimon.table.system.system_table_scan import SystemSplit, SystemTableScan
rows = pa.table({'id': [7, 11]})
- provider = SimpleNamespace(_build_arrow_table=Mock(side_effect=AssertionError('Python provider')))
+ provider = SimpleNamespace(
+ _build_arrow_table=Mock(side_effect=AssertionError('Python provider')),
+ base_table=SimpleNamespace(options=SimpleNamespace(native_read_enabled=lambda: True)),
+ )
with patch('pypaimon.execution.native.read.system_table.try_read_native_system_table', return_value=rows), \
patch.dict(sys.modules, {'pypaimon.execution.python.read.system_table_scan': None}):
plan = SystemTableScan(provider).plan()
diff --git a/paimon-python/pypaimon/tests/global_index_build_test.py b/paimon-python/pypaimon/tests/global_index_build_test.py
index 1c68fd4dcf..a49e3243f7 100644
--- a/paimon-python/pypaimon/tests/global_index_build_test.py
+++ b/paimon-python/pypaimon/tests/global_index_build_test.py
@@ -38,6 +38,7 @@ from pypaimon import Schema
from pypaimon.common.json_util import JSON
from pypaimon.execution.python.globalindex.bitmap.bitmap_index_writer import BITMAP_IDENTIFIER
from pypaimon.globalindex.create_global_index import GlobalIndexBuilder
+from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
from pypaimon.execution.python.globalindex.btree.btree_file_footer import BTreeFileFooter
from pypaimon.execution.python.globalindex.btree.btree_index_writer import BTREE_IDENTIFIER
from pypaimon.execution.python.globalindex.key_serializer import create_serializer
@@ -1441,7 +1442,7 @@ class GenericIndexStreamingTest(unittest.TestCase):
])
def setUp(self):
- self.builder = object.__new__(GlobalIndexBuilder)
+ self.builder = object.__new__(PythonGlobalIndexBuilder)
self.builder._table = Mock()
self.builder._core_options = Mock()
self.builder._core_options.global_index_row_count_per_shard.return_value = 10
diff --git a/paimon-python/pypaimon/tests/global_index_execution_boundary_test.py b/paimon-python/pypaimon/tests/global_index_execution_boundary_test.py
index 11603f5eb0..818279767d 100644
--- a/paimon-python/pypaimon/tests/global_index_execution_boundary_test.py
+++ b/paimon-python/pypaimon/tests/global_index_execution_boundary_test.py
@@ -36,7 +36,7 @@ from unittest.mock import Mock, patch
class RejectPythonIndexes(importlib.abc.MetaPathFinder):
def find_spec(self, fullname, path=None, target=None):
- if fullname.startswith('pypaimon.execution.python.globalindex'):
+ if fullname.startswith('pypaimon.execution.python'):
raise AssertionError('Python index execution imported: ' + fullname)
if fullname in ('paimon_vindex', 'paimon_ftindex', 'lumina'):
raise AssertionError('Python index SDK imported: ' + fullname)
@@ -148,7 +148,7 @@ with patch.object(vindex_reader, '_native_reader_type', return_value=RustReader)
reader.close()
reader.close()
assert stream.closed
-assert not any(name.startswith('pypaimon.execution.python.globalindex') for name in sys.modules)
+assert not any(name.startswith('pypaimon.execution.python') for name in sys.modules)
"""
result = subprocess.run(
[sys.executable, '-c', textwrap.dedent(script)],
diff --git a/paimon-python/pypaimon/tests/hybrid_search_ranker_test.py b/paimon-python/pypaimon/tests/hybrid_search_ranker_test.py
index 5dd054b180..5c34c60c39 100644
--- a/paimon-python/pypaimon/tests/hybrid_search_ranker_test.py
+++ b/paimon-python/pypaimon/tests/hybrid_search_ranker_test.py
@@ -25,7 +25,7 @@ import unittest
from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult
from pypaimon.table.source.hybrid_search_builder import (
HybridSearchBuilderImpl, HybridSearchRoute, HybridSearchRouteResult,
- MRR_RANKER)
+ MRR_RANKER, RRF_RANKER, WEIGHTED_SCORE_RANKER)
def _route_result(weight, id_to_scores):
@@ -39,6 +39,12 @@ def _builder(limit):
return builder
+def _rank(limit, ranker, results):
+ builder = _builder(limit).with_ranker(ranker)
+ builder._routes = [result.route for result in results]
+ return builder.rank(results)
+
+
class HybridSearchRankerTest(unittest.TestCase):
def test_weighted_score_normalizes_heterogeneous_route_scales(self):
@@ -46,7 +52,7 @@ class HybridSearchRankerTest(unittest.TestCase):
vector_route = _route_result(5.0, {1: 0.95, 2: 0.10})
text_route = _route_result(1.0, {1: 2.0, 2: 25.0})
- ranked = _builder(2)._weighted_score([vector_route, text_route])
+ ranked = _rank(2, WEIGHTED_SCORE_RANKER, [vector_route, text_route])
getter = ranked.score_getter()
# rowId 1: 5 * 1.0 (vector best) + 1 * 0.0 (text worst) = 5.0
@@ -59,14 +65,14 @@ class HybridSearchRankerTest(unittest.TestCase):
# min == max: must not divide by zero and must keep contributing.
single_hit = _route_result(2.0, {7: 42.0})
- ranked = _builder(1)._weighted_score([single_hit])
+ ranked = _rank(1, WEIGHTED_SCORE_RANKER, [single_hit])
self.assertAlmostEqual(ranked.score_getter()(7), 2.0, places=6)
def test_weighted_score_all_tied_route_maps_each_hit_to_full_weight(self):
tied = _route_result(2.0, {1: 5.0, 2: 5.0, 3: 5.0})
- ranked = _builder(3)._weighted_score([tied])
+ ranked = _rank(3, WEIGHTED_SCORE_RANKER, [tied])
getter = ranked.score_getter()
self.assertAlmostEqual(getter(1), 2.0, places=6)
@@ -77,7 +83,7 @@ class HybridSearchRankerTest(unittest.TestCase):
vector_route = _route_result(5.0, {1: 0.95, 2: 0.10})
text_route = _route_result(1.0, {1: 2.0, 2: 25.0})
- ranked = _builder(2)._rrf([vector_route, text_route])
+ ranked = _rank(2, RRF_RANKER, [vector_route, text_route])
getter = ranked.score_getter()
# Rank-based fusion respects the 5x vector weight: rowId 1 wins.
diff --git a/paimon-python/pypaimon/tests/lazy_search_execution_test.py b/paimon-python/pypaimon/tests/lazy_search_execution_test.py
index e1083038fa..ca3fc9dad6 100644
--- a/paimon-python/pypaimon/tests/lazy_search_execution_test.py
+++ b/paimon-python/pypaimon/tests/lazy_search_execution_test.py
@@ -27,7 +27,7 @@ import pyarrow as pa
import pytest
from pypaimon import CatalogFactory, Schema
-from pypaimon.execution.native.read import native_search_plan
+from pypaimon.execution import search_dispatch
from pypaimon.table.source.search_types import VectorSearchScanPlan
@@ -103,8 +103,8 @@ def test_native_search_prepares_scans_reads_and_serializes_without_python(search
results = read.read_batch_plan(plan) if family == 'batch' else [read.read_plan(plan)]
assert [list(result.results()) for result in results] == [[7]] * (2 if family == 'batch' else 1)
# Dropping live handles for transport must not create fallback readers.
- assert pickle.loads(pickle.dumps(read))._native is None
- assert pickle.loads(pickle.dumps(scan))._scan is None
+ assert pickle.loads(pickle.dumps(read))._native_reader._native is None
+ assert not pickle.loads(pickle.dumps(scan))._native_scan.available()
core_scan.scan.assert_called_once_with()
(core_read.read_batch_plan if family == 'batch' else core_read.read_plan).assert_called_once_with(plan._native_plan)
@@ -188,7 +188,7 @@ def test_native_reader_keeps_the_query_options_captured_at_construction(search_t
with patch(_native_factory('vector'), return_value=native):
reader = builder.new_vector_search_read()
builder.with_option('ivf-flat.metric', 'cosine')
- fallback = native_search_plan.python_search_reader(reader)
+ fallback = search_dispatch.python_search_reader(reader)
assert fallback._options == {'ivf-flat.metric': 'l2'}
@@ -199,7 +199,9 @@ def test_batch_reader_freezes_query_sequence_before_lazy_fallback(search_table):
with patch(_native_factory('batch'), return_value=native):
reader = builder.new_batch_vector_search_read()
queries.append([0.0, 1.0])
- fallback = native_search_plan.python_search_reader(pickle.loads(pickle.dumps(reader)))
+ queries[0][:] = [3.0, 4.0]
+ assert native.with_query_vectors.call_args[0][0] == [[1.0, 0.0]]
+ fallback = search_dispatch.python_search_reader(pickle.loads(pickle.dumps(reader)))
assert fallback._query_vectors == [[1.0, 0.0]]
@@ -217,3 +219,15 @@ def test_vector_split_uses_its_shared_contract_module():
from pypaimon.table.source.vector_search_split import VectorSearchSplit
assert pickle.loads(pickle.dumps(VectorSearchSplit)) is VectorSearchSplit
+
+
+def test_single_reader_freezes_query_values_before_transport_fallback(search_table):
+ query = [1.0, 0.0]
+ builder = _builder(search_table, 'vector').with_query_vector(query)
+ native, _, _ = _native_builder('vector')
+ with patch(_native_factory('vector'), return_value=native):
+ reader = builder.new_vector_search_read()
+ query[:] = [3.0, 4.0]
+ assert native.with_query_vector.call_args[0][0] == [1.0, 0.0]
+ fallback = search_dispatch.python_search_reader(pickle.loads(pickle.dumps(reader)))
+ assert fallback._query_vector == [1.0, 0.0]
diff --git a/paimon-python/pypaimon/tests/multimodal_table_test.py b/paimon-python/pypaimon/tests/multimodal_table_test.py
index 7154d27de7..a4e2153e6a 100644
--- a/paimon-python/pypaimon/tests/multimodal_table_test.py
+++ b/paimon-python/pypaimon/tests/multimodal_table_test.py
@@ -1551,7 +1551,7 @@ class MultimodalTableTest(unittest.TestCase):
with patch(
"pypaimon.read.table_read.TableRead._arrow_batch_generator",
new=batches), patch(
- "pypaimon.read.table_read."
+ "pypaimon.execution.read_support."
"_RECORD_BATCH_READER_FROM_STREAM", None):
reader = users.scan().select("id").to_arrow_batch_reader()
reader.read_next_batch()
@@ -1570,7 +1570,7 @@ class MultimodalTableTest(unittest.TestCase):
read_builder = users.raw_table.new_read_builder()
splits = read_builder.new_scan().plan().splits()
with patch(
- "pypaimon.read.table_read."
+ "pypaimon.execution.read_support."
"_RECORD_BATCH_READER_FROM_STREAM", None):
reader = read_builder.new_read().to_arrow_batch_reader(splits)
self.assertIsInstance(reader, pa.RecordBatchReader)
diff --git a/paimon-python/pypaimon/tests/native_blob_collection_write_test.py b/paimon-python/pypaimon/tests/native_blob_collection_write_test.py
index 71fabc512a..3309734d2e 100644
--- a/paimon-python/pypaimon/tests/native_blob_collection_write_test.py
+++ b/paimon-python/pypaimon/tests/native_blob_collection_write_test.py
@@ -23,7 +23,7 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.schema.data_types import AtomicType, ArrayType, DataField, MapType, PyarrowFieldParser
from pypaimon.table.row.blob import BlobDescriptor
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
from pypaimon.tests.native_blob_write_test import _read, _physical_files
pytestmark = pytest.mark.native_plan
diff --git a/paimon-python/pypaimon/tests/native_blob_consumer_test.py b/paimon-python/pypaimon/tests/native_blob_consumer_test.py
index b83af65136..ca038b388f 100644
--- a/paimon-python/pypaimon/tests/native_blob_consumer_test.py
+++ b/paimon-python/pypaimon/tests/native_blob_consumer_test.py
@@ -27,7 +27,7 @@ from pypaimon.schema.data_types import AtomicType, ArrayType, DataField, MapType
from pypaimon.table.row.blob import Blob, BlobDescriptor
from pypaimon.table.row.generic_row import GenericRow
from pypaimon.tests.native_blob_write_test import _table, _data, _read
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
diff --git a/paimon-python/pypaimon/tests/native_blob_row_write_test.py b/paimon-python/pypaimon/tests/native_blob_row_write_test.py
index 11aaffd374..867171175c 100644
--- a/paimon-python/pypaimon/tests/native_blob_row_write_test.py
+++ b/paimon-python/pypaimon/tests/native_blob_row_write_test.py
@@ -28,7 +28,7 @@ from pypaimon import CatalogFactory, Schema
from pypaimon.schema.data_types import AtomicType, ArrayType, DataField, MapType, PyarrowFieldParser
from pypaimon.table.row.blob import Blob, BlobDescriptor, BlobViewStruct
from pypaimon.table.row.generic_row import GenericRow
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
diff --git a/paimon-python/pypaimon/tests/native_blob_update_test.py b/paimon-python/pypaimon/tests/native_blob_update_test.py
index a02d3f65f1..65816ac1dc 100644
--- a/paimon-python/pypaimon/tests/native_blob_update_test.py
+++ b/paimon-python/pypaimon/tests/native_blob_update_test.py
@@ -32,7 +32,7 @@ from pypaimon.table.row.generic_row import GenericRow
from pypaimon.table.data_evolution_merge_into import WhenMatched, WhenNotMatched, source_col
from pypaimon.write.table_delete import TableDeleteByRowId
from pypaimon.execution.native.write.native_update import NativeTableUpdateByRowId
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
from pypaimon.write.table_update import TableUpdate
from pypaimon.write.table_update_by_row_id import TableUpdateByRowId
from pypaimon.write.table_upsert_by_key import TableUpsertByKey
diff --git a/paimon-python/pypaimon/tests/native_blob_uri_reader_test.py b/paimon-python/pypaimon/tests/native_blob_uri_reader_test.py
index c60ca3da20..6253c0fe69 100644
--- a/paimon-python/pypaimon/tests/native_blob_uri_reader_test.py
+++ b/paimon-python/pypaimon/tests/native_blob_uri_reader_test.py
@@ -25,7 +25,7 @@ import pytest
from pypaimon.schema.data_types import PyarrowFieldParser
from pypaimon.table.row.blob import BlobDescriptor
from pypaimon.tests.native_blob_write_test import _table, _read
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
diff --git a/paimon-python/pypaimon/tests/native_blob_write_test.py b/paimon-python/pypaimon/tests/native_blob_write_test.py
index efe11d9514..b283a7a881 100644
--- a/paimon-python/pypaimon/tests/native_blob_write_test.py
+++ b/paimon-python/pypaimon/tests/native_blob_write_test.py
@@ -29,7 +29,7 @@ from pypaimon.execution.native.read.plan import (native_plan)
from pypaimon.schema.data_types import AtomicType, DataField
from pypaimon.table.row.blob import Blob, BlobDescriptor
from pypaimon.table.row.generic_row import GenericRow
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
_SCHEMA = pa.schema([('id', pa.int32()), ('large', pa.large_binary()), ('small', pa.large_binary())])
diff --git a/paimon-python/pypaimon/tests/native_branch_write_test.py b/paimon-python/pypaimon/tests/native_branch_write_test.py
index b92ee886ac..a14b402b8a 100644
--- a/paimon-python/pypaimon/tests/native_branch_write_test.py
+++ b/paimon-python/pypaimon/tests/native_branch_write_test.py
@@ -24,7 +24,7 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.common.identifier import Identifier
from pypaimon.common.options.core_options import MergeEngine
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
_SCHEMA = pa.schema([('id', pa.int32()), ('value', pa.int32()), ('p', pa.string())])
diff --git a/paimon-python/pypaimon/tests/native_cross_partition_write_test.py b/paimon-python/pypaimon/tests/native_cross_partition_write_test.py
index 1928e68c34..4ed126e838 100644
--- a/paimon-python/pypaimon/tests/native_cross_partition_write_test.py
+++ b/paimon-python/pypaimon/tests/native_cross_partition_write_test.py
@@ -24,7 +24,7 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.table.row.generic_row import GenericRow
from pypaimon.table.row.row_kind import RowKind
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
_SCHEMA = pa.schema([('tenant', pa.string()), ('p', pa.string()),
diff --git a/paimon-python/pypaimon/tests/native_data_directory_test.py b/paimon-python/pypaimon/tests/native_data_directory_test.py
index 25e4fe71ea..fcd983bad4 100644
--- a/paimon-python/pypaimon/tests/native_data_directory_test.py
+++ b/paimon-python/pypaimon/tests/native_data_directory_test.py
@@ -28,7 +28,7 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.execution.native.read.plan import (native_plan)
from pypaimon.schema.data_types import AtomicType, DataField
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
from pypaimon.write.table_upsert_by_key import TableUpsertByKey
diff --git a/paimon-python/pypaimon/tests/native_data_evolution_commit_test.py b/paimon-python/pypaimon/tests/native_data_evolution_commit_test.py
index 47ee711fdc..482991cf62 100644
--- a/paimon-python/pypaimon/tests/native_data_evolution_commit_test.py
+++ b/paimon-python/pypaimon/tests/native_data_evolution_commit_test.py
@@ -29,7 +29,7 @@ from pypaimon.manifest.manifest_list_manager import ManifestListManager
from pypaimon.manifest.manifest_sidecar import read_sidecar, SUFFIX
from pypaimon.utils.range import Range
from pypaimon.execution.native.write.native_commit import native_messages_supported
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
diff --git a/paimon-python/pypaimon/tests/native_dv_merge_engine_write_test.py b/paimon-python/pypaimon/tests/native_dv_merge_engine_write_test.py
index 604eec327c..5763f74271 100644
--- a/paimon-python/pypaimon/tests/native_dv_merge_engine_write_test.py
+++ b/paimon-python/pypaimon/tests/native_dv_merge_engine_write_test.py
@@ -25,7 +25,7 @@ import pytest
from pypaimon import Schema
from pypaimon.table.bucket_mode import BucketMode
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
diff --git a/paimon-python/pypaimon/tests/native_dynamic_write_test.py b/paimon-python/pypaimon/tests/native_dynamic_write_test.py
index 9bbf685667..4d346474ba 100644
--- a/paimon-python/pypaimon/tests/native_dynamic_write_test.py
+++ b/paimon-python/pypaimon/tests/native_dynamic_write_test.py
@@ -24,7 +24,7 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.index.index_file_handler import IndexFileHandler
from pypaimon.execution.native.write.native_commit import from_native_commit_messages
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
SCHEMA = pa.schema([('id', pa.int64()), ('p', pa.string()), ('v', pa.string())])
diff --git a/paimon-python/pypaimon/tests/native_external_write_test.py b/paimon-python/pypaimon/tests/native_external_write_test.py
index a72c285bc5..4b5615068a 100644
--- a/paimon-python/pypaimon/tests/native_external_write_test.py
+++ b/paimon-python/pypaimon/tests/native_external_write_test.py
@@ -26,7 +26,7 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.execution.native.read.plan import (native_plan)
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
_SCHEMA = pa.schema([('id', pa.int32()), ('pt', pa.string()), ('value', pa.int32())])
diff --git a/paimon-python/pypaimon/tests/native_filtered_write_validation_test.py b/paimon-python/pypaimon/tests/native_filtered_write_validation_test.py
index ea827ea0e9..8ebf0cf6e0 100644
--- a/paimon-python/pypaimon/tests/native_filtered_write_validation_test.py
+++ b/paimon-python/pypaimon/tests/native_filtered_write_validation_test.py
@@ -22,7 +22,7 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.execution.native.runtime import (native_runtime_available)
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = [pytest.mark.python_write, pytest.mark.python_commit]
diff --git a/paimon-python/pypaimon/tests/native_filtered_writer_configuration_test.py b/paimon-python/pypaimon/tests/native_filtered_writer_configuration_test.py
index f2f63671e6..665cb3ba13 100644
--- a/paimon-python/pypaimon/tests/native_filtered_writer_configuration_test.py
+++ b/paimon-python/pypaimon/tests/native_filtered_writer_configuration_test.py
@@ -30,7 +30,7 @@ from pypaimon.table.row.blob import BlobDescriptor
from pypaimon.tests import native_plan_rest_test
from pypaimon.tests.native_blob_uri_reader_test import Factory
from pypaimon.tests.native_write_routing_test import _table
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = [pytest.mark.python_write, pytest.mark.python_commit]
diff --git a/paimon-python/pypaimon/tests/native_generic_index_build_test.py b/paimon-python/pypaimon/tests/native_generic_index_build_test.py
index 9d5b084f7c..54cf75f91d 100644
--- a/paimon-python/pypaimon/tests/native_generic_index_build_test.py
+++ b/paimon-python/pypaimon/tests/native_generic_index_build_test.py
@@ -24,6 +24,7 @@ import pyarrow as pa
import pytest
from pypaimon.globalindex.create_global_index import GlobalIndexBuilder
+from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
from pypaimon.execution.native.table import (_native_table)
from pypaimon.execution.native.read.builder import (_predicate_to_native)
from pypaimon.execution.native.runtime import (native_runtime_available)
@@ -82,8 +83,9 @@ def test_native_build_requires_row_tracking_not_read_optimization_options(rest_c
'global-index.enabled': str(enabled).lower()})
column = 'id' if kind in ('btree', 'bitmap') else 'name' if kind == 'full-text' else 'embedding'
options = ({'sorted-index.records-per-file': 100} if kind in ('btree', 'bitmap') else _options(kind))
- with patch.object(GlobalIndexBuilder, '_build_sorted_index', side_effect=AssertionError('Python sorted build ran')):
- with patch.object(GlobalIndexBuilder, '_build_generic_index',
+ with patch.object(PythonGlobalIndexBuilder, '_build_sorted_index',
+ side_effect=AssertionError('Python sorted build ran')):
+ with patch.object(PythonGlobalIndexBuilder, '_build_generic_index',
side_effect=AssertionError('Python generic build ran')):
messages = GlobalIndexBuilder(table, column, index_type=kind, options=options).build()
files = index_tests._message_files(messages)
@@ -173,7 +175,7 @@ def test_python_full_text_build_delegates_to_common_native_entry(rest_catalog):
catalog, _ = rest_catalog
table = _table(catalog)
predicate = table.new_read_builder().new_predicate_builder().equal('pt', 1)
- with patch.object(GlobalIndexBuilder, '_build_generic_index',
+ with patch.object(PythonGlobalIndexBuilder, '_build_generic_index',
side_effect=AssertionError('Python index build ran')):
messages = GlobalIndexBuilder(table, 'name', index_type='full-text',
partition_filter=predicate, options=_options('full-text', 4)).build()
diff --git a/paimon-python/pypaimon/tests/native_historical_write_commit_test.py b/paimon-python/pypaimon/tests/native_historical_write_commit_test.py
index 3c47251d70..77cd24de73 100644
--- a/paimon-python/pypaimon/tests/native_historical_write_commit_test.py
+++ b/paimon-python/pypaimon/tests/native_historical_write_commit_test.py
@@ -28,7 +28,7 @@ from pypaimon.execution.native.runtime import (native_runtime_available)
from pypaimon.schema.data_types import AtomicType
from pypaimon.schema.schema_change import SchemaChange
from pypaimon.tests import native_plan_rest_test
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = [pytest.mark.python_write, pytest.mark.python_commit]
diff --git a/paimon-python/pypaimon/tests/native_loaded_schema_io_test.py b/paimon-python/pypaimon/tests/native_loaded_schema_io_test.py
index 259b2f1492..375ac97498 100644
--- a/paimon-python/pypaimon/tests/native_loaded_schema_io_test.py
+++ b/paimon-python/pypaimon/tests/native_loaded_schema_io_test.py
@@ -31,7 +31,7 @@ from pypaimon.tests import native_plan_rest_test
from pypaimon.tests.native_authorized_update_test import (
_ROWS, _SCHEMA, _expected, _files, _native_only, _operate, _restricted, _rows, _table,
)
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = [pytest.mark.python_write, pytest.mark.python_commit]
diff --git a/paimon-python/pypaimon/tests/native_managed_blob_write_test.py b/paimon-python/pypaimon/tests/native_managed_blob_write_test.py
index 94882a4d5d..5f65a761ad 100644
--- a/paimon-python/pypaimon/tests/native_managed_blob_write_test.py
+++ b/paimon-python/pypaimon/tests/native_managed_blob_write_test.py
@@ -26,7 +26,7 @@ import pytest
from pypaimon import CatalogFactory
from pypaimon.schema.data_types import PyarrowFieldParser
from pypaimon.table.row.blob import Blob, BlobDescriptor
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
diff --git a/paimon-python/pypaimon/tests/native_map_shredding_write_test.py b/paimon-python/pypaimon/tests/native_map_shredding_write_test.py
index d18bf77960..97d6df426a 100644
--- a/paimon-python/pypaimon/tests/native_map_shredding_write_test.py
+++ b/paimon-python/pypaimon/tests/native_map_shredding_write_test.py
@@ -26,7 +26,7 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.data.map_shared_shredding import is_shared_shredding, parse_shared_shredding_metadata
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
from pypaimon.table.row.generic_row import GenericRow
pytestmark = pytest.mark.native_plan
diff --git a/paimon-python/pypaimon/tests/native_merge_into_test.py b/paimon-python/pypaimon/tests/native_merge_into_test.py
index ba0574c893..5918c5bfe1 100644
--- a/paimon-python/pypaimon/tests/native_merge_into_test.py
+++ b/paimon-python/pypaimon/tests/native_merge_into_test.py
@@ -442,7 +442,7 @@ def test_native_table_source_keeps_its_selected_branch(tmp_path):
@pytest.mark.parametrize('location', ['local', 'file-uri', 'directory', 'external'])
def test_python_written_partition_source_uses_java_paths_for_native_merge(tmp_path, stream, source_mode, location):
from pathlib import Path
- from pypaimon.execution.native.write.native_write import NativeTableWrite
+ from pypaimon.execution.write_dispatch import NativeTableWrite
target = _table(tmp_path / 'target', [dict(id=1, value=10, name='old')])
warehouse = tmp_path / 'source'
diff --git a/paimon-python/pypaimon/tests/native_mod_bucket_write_test.py b/paimon-python/pypaimon/tests/native_mod_bucket_write_test.py
index 0fac7bd839..d4c161f17c 100644
--- a/paimon-python/pypaimon/tests/native_mod_bucket_write_test.py
+++ b/paimon-python/pypaimon/tests/native_mod_bucket_write_test.py
@@ -28,7 +28,7 @@ from pypaimon.execution.native.runtime import (native_runtime_available)
from pypaimon.schema.table_schema import TableSchema
from pypaimon.table.row.generic_row import GenericRow
from pypaimon.tests import native_plan_rest_test
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
from pypaimon.execution.python.write.row_key_extractor import FixedBucketRowKeyExtractor
diff --git a/paimon-python/pypaimon/tests/native_nullable_primary_key_test.py b/paimon-python/pypaimon/tests/native_nullable_primary_key_test.py
index 53e41aea9c..0bb607c63d 100644
--- a/paimon-python/pypaimon/tests/native_nullable_primary_key_test.py
+++ b/paimon-python/pypaimon/tests/native_nullable_primary_key_test.py
@@ -29,7 +29,7 @@ from pypaimon import Schema
from pypaimon.table.data_evolution_merge_into import WhenMatched, WhenNotMatched, source_col
from pypaimon.table.row.generic_row import GenericRow
from pypaimon.tests import native_plan_rest_test
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
rest_catalog = native_plan_rest_test.rest_catalog
pytestmark = pytest.mark.native_plan
diff --git a/paimon-python/pypaimon/tests/native_overwrite_restore_test.py b/paimon-python/pypaimon/tests/native_overwrite_restore_test.py
index ee77d07bb9..f33c6b8bf0 100644
--- a/paimon-python/pypaimon/tests/native_overwrite_restore_test.py
+++ b/paimon-python/pypaimon/tests/native_overwrite_restore_test.py
@@ -27,7 +27,7 @@ from pypaimon import Schema
from pypaimon.index.index_file_handler import IndexFileHandler
from pypaimon.execution.native.runtime import (native_runtime_available)
from pypaimon.tests import native_plan_rest_test
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = [pytest.mark.python_write, pytest.mark.python_commit]
diff --git a/paimon-python/pypaimon/tests/native_parallel_index_build_test.py b/paimon-python/pypaimon/tests/native_parallel_index_build_test.py
index 092ec936b6..b8dde48cc9 100644
--- a/paimon-python/pypaimon/tests/native_parallel_index_build_test.py
+++ b/paimon-python/pypaimon/tests/native_parallel_index_build_test.py
@@ -27,6 +27,7 @@ import pytest
from pypaimon.api.api_response import GetTableSnapshotResponse
from pypaimon.globalindex.create_global_index import GlobalIndexBuilder
+from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
from pypaimon.execution.native.table import (_native_table)
from pypaimon.snapshot.table_snapshot import TableSnapshot
from pypaimon.tests import native_generic_index_build_test as generic
@@ -57,7 +58,7 @@ def _table(catalog, options=None, partitioned=False):
def _build(table, kind, options=None, **kwargs):
settings = generic._options(kind, 4)
settings.update(options or {})
- with patch.object(GlobalIndexBuilder, '_build_generic_index',
+ with patch.object(PythonGlobalIndexBuilder, '_build_generic_index',
side_effect=AssertionError('Python rows were materialized')):
return GlobalIndexBuilder(table, 'name' if kind == 'full-text' else 'embedding',
index_type=kind, options=settings, **kwargs).build()
diff --git a/paimon-python/pypaimon/tests/native_partition_types_test.py b/paimon-python/pypaimon/tests/native_partition_types_test.py
index 19564d2b00..37a3b0ae09 100644
--- a/paimon-python/pypaimon/tests/native_partition_types_test.py
+++ b/paimon-python/pypaimon/tests/native_partition_types_test.py
@@ -24,7 +24,7 @@ import pyarrow as pa
import pytest
from pypaimon import CatalogFactory, Schema
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
_PARTITIONS = [
diff --git a/paimon-python/pypaimon/tests/native_pk_blob_limit_test.py b/paimon-python/pypaimon/tests/native_pk_blob_limit_test.py
index d9c6262769..7414a7fe46 100644
--- a/paimon-python/pypaimon/tests/native_pk_blob_limit_test.py
+++ b/paimon-python/pypaimon/tests/native_pk_blob_limit_test.py
@@ -26,7 +26,7 @@ from pypaimon import CatalogFactory
from pypaimon.read.table_read import TableRead
from pypaimon.schema.data_types import PyarrowFieldParser
from pypaimon.table.row.blob import BlobDescriptor
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
diff --git a/paimon-python/pypaimon/tests/native_plan_test.py b/paimon-python/pypaimon/tests/native_plan_test.py
index 0f30bacfc1..0a7c6ad436 100644
--- a/paimon-python/pypaimon/tests/native_plan_test.py
+++ b/paimon-python/pypaimon/tests/native_plan_test.py
@@ -1427,17 +1427,6 @@ class NativePlanTest(unittest.TestCase):
with patch('pypaimon.execution.native.runtime.import_module', side_effect=ImportError('not installed')):
self.assertFalse(native_runtime_available())
- def test_installed_extension_requires_write_nullability_api(self):
- from pypaimon.schema.arrow_schema import normalize_arrow_write_nullability
-
- array = pyarrow.array([1], type=pyarrow.int32())
- data = pyarrow.table({'id': array})
- # Missing symbols in an installed extension are packaging errors. They
- # must not silently disable core NULL validation on write inputs.
- with patch.dict(sys.modules, {'pypaimon_rust.data': ModuleType('pypaimon_rust.data')}):
- with self.assertRaises(ImportError):
- normalize_arrow_write_nullability(data, data.schema)
-
def test_native_runtime_availability_does_not_probe_versioned_apis(self):
fake_df = ModuleType('pypaimon_rust.datafusion')
fake_mod = ModuleType('pypaimon_rust')
diff --git a/paimon-python/pypaimon/tests/native_postpone_fixed_write_test.py b/paimon-python/pypaimon/tests/native_postpone_fixed_write_test.py
index 72c6297eb9..68918c0f4e 100644
--- a/paimon-python/pypaimon/tests/native_postpone_fixed_write_test.py
+++ b/paimon-python/pypaimon/tests/native_postpone_fixed_write_test.py
@@ -24,7 +24,7 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.table.row.generic_row import GenericRow
-from pypaimon.execution.native.write.native_write import NativePostponeFixedBucketTableWrite
+from pypaimon.execution.write_dispatch import NativePostponeFixedBucketTableWrite
from pypaimon.write.postpone_bucket import PostponeBucketPlan, PostponeBucketPlanner
pytestmark = pytest.mark.native_plan
diff --git a/paimon-python/pypaimon/tests/native_postpone_write_test.py b/paimon-python/pypaimon/tests/native_postpone_write_test.py
index c14c2a4369..653c8f5b95 100644
--- a/paimon-python/pypaimon/tests/native_postpone_write_test.py
+++ b/paimon-python/pypaimon/tests/native_postpone_write_test.py
@@ -23,7 +23,7 @@ import pyarrow.parquet as pq
import pytest
from pypaimon import CatalogFactory, Schema
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
_SCHEMA = pa.schema([('id', pa.int32()), ('p', pa.string()),
diff --git a/paimon-python/pypaimon/tests/native_precomputed_bucket_test.py b/paimon-python/pypaimon/tests/native_precomputed_bucket_test.py
index 092f66d5e0..2d210526a3 100644
--- a/paimon-python/pypaimon/tests/native_precomputed_bucket_test.py
+++ b/paimon-python/pypaimon/tests/native_precomputed_bucket_test.py
@@ -24,7 +24,7 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.index.index_file_handler import IndexFileHandler
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
diff --git a/paimon-python/pypaimon/tests/native_read_type_test.py b/paimon-python/pypaimon/tests/native_read_type_test.py
index a1625f85d9..daa7087172 100644
--- a/paimon-python/pypaimon/tests/native_read_type_test.py
+++ b/paimon-python/pypaimon/tests/native_read_type_test.py
@@ -408,7 +408,8 @@ def test_map_projection_preserves_masked_key_nullability(tmp_path, native):
# A logical NOT NULL key becomes nullable after masking. Test the captured
# batch schema independently of the core's conservative read nullability.
with patch.object(FileSystemCatalog, 'auth_table_query', return_value=auth), \
- patch('pypaimon.read.table_read.read_arrow_schema', PyarrowFieldParser.from_paimon_schema):
+ patch('pypaimon.read.table_read.read_arrow_schema', PyarrowFieldParser.from_paimon_schema), \
+ patch('pypaimon.execution.read_support.read_arrow_schema', PyarrowFieldParser.from_paimon_schema):
splits = builder.new_scan().plan().splits()
result = builder.new_read().to_arrow_batch_reader(splits).read_all()
assert result.num_rows == 3
diff --git a/paimon-python/pypaimon/tests/native_row_sidecar_test.py b/paimon-python/pypaimon/tests/native_row_sidecar_test.py
index fbabb066bf..937fe40481 100644
--- a/paimon-python/pypaimon/tests/native_row_sidecar_test.py
+++ b/paimon-python/pypaimon/tests/native_row_sidecar_test.py
@@ -27,7 +27,8 @@ from pypaimon.data.generic_variant import GenericVariant
from pypaimon.schema.data_types import PyarrowFieldParser
from pypaimon.execution.native.write.native_update import NativeTableUpdateByRowId
from pypaimon.write.table_update_by_row_id import TableUpdateByRowId
-from pypaimon.execution.native.write.native_write import NativeTableWrite, native_runtime_available
+from pypaimon.execution.write_dispatch import NativeTableWrite
+from pypaimon.execution.native.write.native_write import native_runtime_available
pytestmark = [pytest.mark.native_plan, pytest.mark.skipif(
not native_runtime_available(), reason='pypaimon-rust not installed')]
diff --git a/paimon-python/pypaimon/tests/native_schema_cast_test.py b/paimon-python/pypaimon/tests/native_schema_cast_test.py
index c821f6cbd1..e71efd4f5e 100644
--- a/paimon-python/pypaimon/tests/native_schema_cast_test.py
+++ b/paimon-python/pypaimon/tests/native_schema_cast_test.py
@@ -232,7 +232,7 @@ def test_read_results_write_back_with_not_null_constraints(rest_catalog, native_
table = table.copy({'write.native.enabled': str(native_write).lower()})
builder = table.new_batch_write_builder()
writer, commit = builder.new_write(), builder.new_commit()
- from pypaimon.execution.native.write.native_write import NativeTableWrite
+ from pypaimon.execution.write_dispatch import NativeTableWrite
assert isinstance(writer, NativeTableWrite) == native_write
try:
writer.write_arrow(data)
diff --git a/paimon-python/pypaimon/tests/native_search_plan_test.py b/paimon-python/pypaimon/tests/native_search_plan_test.py
index 4bc486688a..38abe08128 100644
--- a/paimon-python/pypaimon/tests/native_search_plan_test.py
+++ b/paimon-python/pypaimon/tests/native_search_plan_test.py
@@ -26,7 +26,7 @@ import pytest
import pyarrow as pa
from pypaimon.execution.native.runtime import (native_runtime_available)
-from pypaimon.execution.native.read.native_search_plan import python_search_reader
+from pypaimon.execution.search_dispatch import python_search_reader
from pypaimon.execution.python.read.search.full_text_read import DataEvolutionFullTextRead
from pypaimon.execution.python.read.search.full_text_scan import DataEvolutionFullTextScan
from pypaimon.execution.python.read.search.primary_key_full_text_read import PrimaryKeyFullTextRead
diff --git a/paimon-python/pypaimon/tests/native_sequence_test.py b/paimon-python/pypaimon/tests/native_sequence_test.py
index 504e897586..757e93f102 100644
--- a/paimon-python/pypaimon/tests/native_sequence_test.py
+++ b/paimon-python/pypaimon/tests/native_sequence_test.py
@@ -27,7 +27,7 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.common.predicate_builder import PredicateBuilder
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
diff --git a/paimon-python/pypaimon/tests/native_sorted_index_build_test.py b/paimon-python/pypaimon/tests/native_sorted_index_build_test.py
index 108a98653e..204a95c563 100644
--- a/paimon-python/pypaimon/tests/native_sorted_index_build_test.py
+++ b/paimon-python/pypaimon/tests/native_sorted_index_build_test.py
@@ -28,6 +28,7 @@ import pytest
from pypaimon import Schema
from pypaimon.api.api_response import ErrorResponse, GetTableSnapshotResponse
from pypaimon.globalindex.create_global_index import GlobalIndexBuilder
+from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
from pypaimon.execution.python.globalindex.data_evolution_global_index_scanner import DataEvolutionGlobalIndexScanner
from pypaimon.index.index_file_handler import IndexFileHandler
from pypaimon.execution.native.table import (_native_table)
@@ -107,7 +108,8 @@ def _ids(table, predicate=None):
def _build(table, kind, column='name', **kwargs):
# A native build must never materialize or sort rows in Python.
- with patch.object(GlobalIndexBuilder, '_build_sorted_index', side_effect=AssertionError('Python index build ran')):
+ with patch.object(PythonGlobalIndexBuilder, '_build_sorted_index',
+ side_effect=AssertionError('Python index build ran')):
return GlobalIndexBuilder(table, column, index_type=kind, **kwargs).build()
@@ -410,8 +412,8 @@ def test_ineligible_tables_choose_python_before_building(rest_catalog, options):
catalog, _ = rest_catalog
table = _create(catalog, options=options)
_append(table, ROWS)
- original = GlobalIndexBuilder._build_sorted_index
- with patch.object(GlobalIndexBuilder, '_build_sorted_index', autospec=True, side_effect=original) as build:
+ original = PythonGlobalIndexBuilder._build_sorted_index
+ with patch.object(PythonGlobalIndexBuilder, '_build_sorted_index', autospec=True, side_effect=original) as build:
assert table.create_global_index('name') > 0
build.assert_called_once()
assert _ids(table, table.new_read_builder().new_predicate_builder().equal('name', 'a')) == [1, 5]
@@ -482,10 +484,10 @@ def test_query_authorization_remains_on_python_build_path(rest_catalog, restrict
table = table.copy({'query-auth.enabled': 'true'})
server.table_query_auth_store[table.identifier.get_full_name()] = AuthTableQueryResponse(
column_masking={'name': '{"name":"NULL"}'} if restricted else None)
- original = GlobalIndexBuilder._build_sorted_index
+ original = PythonGlobalIndexBuilder._build_sorted_index
with patch('pypaimon.execution.native.write.native_index_build._native_table',
side_effect=AssertionError('native auth build')), \
- patch.object(GlobalIndexBuilder, '_build_sorted_index', autospec=True, side_effect=original) as build:
+ patch.object(PythonGlobalIndexBuilder, '_build_sorted_index', autospec=True, side_effect=original) as build:
if restricted:
with pytest.raises(TableNoPermissionException):
GlobalIndexBuilder(table, 'name').build()
diff --git a/paimon-python/pypaimon/tests/native_stream_query_auth_test.py b/paimon-python/pypaimon/tests/native_stream_query_auth_test.py
index 083649174b..e0915e9dd2 100644
--- a/paimon-python/pypaimon/tests/native_stream_query_auth_test.py
+++ b/paimon-python/pypaimon/tests/native_stream_query_auth_test.py
@@ -108,7 +108,7 @@ def _rows(builder, plan, native):
def _frame(builder, native, snapshot=None):
- with patch('pypaimon.read.stream_read_builder.AsyncStreamingTableScan',
+ with patch('pypaimon.execution.python.read.streaming_table_scan.AsyncStreamingTableScan',
side_effect=AssertionError('Native stream fell back')) if native else ExitStack():
scan = builder.new_streaming_scan()
if native:
diff --git a/paimon-python/pypaimon/tests/native_timestamp_bucket_write_test.py b/paimon-python/pypaimon/tests/native_timestamp_bucket_write_test.py
index 832d597e9c..8999ea4763 100644
--- a/paimon-python/pypaimon/tests/native_timestamp_bucket_write_test.py
+++ b/paimon-python/pypaimon/tests/native_timestamp_bucket_write_test.py
@@ -35,7 +35,7 @@ from pypaimon.schema.table_schema import TableSchema
from pypaimon.table.row.generic_row import GenericRow, GenericRowDeserializer, GenericRowSerializer
from pypaimon.read.table_read import TableRead
from pypaimon.tests import native_plan_rest_test
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
from pypaimon.execution.python.write.row_key_extractor import FixedBucketRowKeyExtractor
diff --git a/paimon-python/pypaimon/tests/native_variant_update_test.py b/paimon-python/pypaimon/tests/native_variant_update_test.py
index 26b1ed3789..7fbb9e9cc1 100644
--- a/paimon-python/pypaimon/tests/native_variant_update_test.py
+++ b/paimon-python/pypaimon/tests/native_variant_update_test.py
@@ -30,7 +30,7 @@ from pypaimon.common.predicate_builder import PredicateBuilder
from pypaimon.data.generic_variant import GenericVariant
from pypaimon.schema.data_types import PyarrowFieldParser
from pypaimon.execution.native.write.native_update import NativeTableUpdateByRowId
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
from pypaimon.write.table_update import TableUpdate
from pypaimon.write.table_update_by_row_id import TableUpdateByRowId
from pypaimon.write.table_upsert_by_key import TableUpsertByKey
diff --git a/paimon-python/pypaimon/tests/native_vector_index_build_test.py b/paimon-python/pypaimon/tests/native_vector_index_build_test.py
index b3ca998eb2..34f5e28110 100644
--- a/paimon-python/pypaimon/tests/native_vector_index_build_test.py
+++ b/paimon-python/pypaimon/tests/native_vector_index_build_test.py
@@ -23,6 +23,7 @@ import pyarrow as pa
import pytest
from pypaimon.globalindex.create_global_index import GlobalIndexBuilder
+from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
from pypaimon.execution.vindex_input import (
PaimonVindexInput,
)
@@ -51,7 +52,7 @@ def _table(catalog, vectors, options=None, partitioned=False):
def _build(table, kind, options=None, **kwargs):
- with patch.object(GlobalIndexBuilder, '_build_generic_index',
+ with patch.object(PythonGlobalIndexBuilder, '_build_generic_index',
side_effect=AssertionError('Python vector rows were materialized')):
return GlobalIndexBuilder(table, 'embedding', index_type=kind, options=options, **kwargs).build()
diff --git a/paimon-python/pypaimon/tests/native_write_capabilities_test.py b/paimon-python/pypaimon/tests/native_write_capabilities_test.py
index 93e340b8da..87fe30d996 100644
--- a/paimon-python/pypaimon/tests/native_write_capabilities_test.py
+++ b/paimon-python/pypaimon/tests/native_write_capabilities_test.py
@@ -23,7 +23,7 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.table.row.generic_row import GenericRowDeserializer
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = pytest.mark.native_plan
diff --git a/paimon-python/pypaimon/tests/native_write_fork_safety_test.py b/paimon-python/pypaimon/tests/native_write_fork_safety_test.py
index 7c2e18aad6..3dfe03bf44 100644
--- a/paimon-python/pypaimon/tests/native_write_fork_safety_test.py
+++ b/paimon-python/pypaimon/tests/native_write_fork_safety_test.py
@@ -19,7 +19,7 @@
import sys
from contextlib import ExitStack, contextmanager
-from types import ModuleType, SimpleNamespace
+from types import ModuleType
from unittest.mock import Mock, patch
import pyarrow as pa
@@ -31,7 +31,7 @@ from pypaimon.write.table_upsert_by_key import TableUpsertByKey
pytestmark = [pytest.mark.python_write, pytest.mark.python_commit]
_OPERATIONS = ['batch-write', 'stream-write', 'commit', 'upsert', 'row-id-updater', 'update', 'batch-update',
- 'predicate-update', 'delete', 'predicate-delete', 'delete-match']
+ 'predicate-update', 'delete', 'predicate-delete']
class _ForkSafetyError(RuntimeError):
@@ -75,7 +75,7 @@ def _operation(tmp_path, operation):
factory = 'create_native_upsert'
invoke = lambda: upsert.upsert(data, ['id'])
elif operation == 'row-id-updater':
- context.enter_context(patch('pypaimon.write.table_update.TableUpdateByRowId', pure))
+ context.enter_context(patch('pypaimon.execution.update_dispatch.TableUpdateByRowId', pure))
factory, invoke = 'create_native_update_by_row_id', updater.new_update_by_row_id
elif operation in ('update', 'batch-update'):
data = data.append_column('_ROW_ID', pa.array([0], pa.int64()))
@@ -98,14 +98,6 @@ def _operation(tmp_path, operation):
context.enter_context(patch.object(updater, '_partition_only_delete_filter', pure))
factory = 'create_native_predicate_delete'
invoke = lambda: updater.delete_by_predicate(None)
- else:
- read_builder = Mock()
- read_builder.new_scan.return_value.plan_all_files.return_value.splits.return_value = [object()]
- context.enter_context(patch.object(updater, '_matched_update_scan_table', return_value=SimpleNamespace(
- new_read_builder=lambda: read_builder)))
- read_builder.new_read = pure
- factory = 'native_predicate_row_ids'
- invoke = lambda: updater._matched_delete_row_ids(None)
yield invoke, module + factory, pure
diff --git a/paimon-python/pypaimon/tests/native_write_inactive_options_test.py b/paimon-python/pypaimon/tests/native_write_inactive_options_test.py
index 1e4e790965..32be816ab9 100644
--- a/paimon-python/pypaimon/tests/native_write_inactive_options_test.py
+++ b/paimon-python/pypaimon/tests/native_write_inactive_options_test.py
@@ -25,7 +25,8 @@ import pytest
from pypaimon import Schema
from pypaimon.execution.native.runtime import (native_runtime_available)
-from pypaimon.execution.native.write.native_write import NativeTableWrite, create_native_write
+from pypaimon.execution.write_dispatch import NativeTableWrite
+from pypaimon.execution.native.write.native_write import create_native_write
pytestmark = [pytest.mark.python_write, pytest.mark.python_commit]
@@ -60,7 +61,8 @@ def test_inactive_changelog_format_qualifies_for_native_write(
'pypaimon.execution.native.write.native_write.create_native_write_table',
return_value=native_table) as create:
writer = create_native_write(table, 'job', static_partition=partition, stream=stream)
- assert isinstance(writer, NativeTableWrite)
+ from pypaimon.execution.native.write.native_write import NativeTableWrite as NativeWrite
+ assert isinstance(writer, NativeWrite)
create.assert_called_once_with(table)
builder = (native_table.new_stream_write_builder.return_value.with_commit_user.return_value if stream
else native_table.new_batch_write_builder.return_value._with_commit_user.return_value)
diff --git a/paimon-python/pypaimon/tests/native_write_path_completion_test.py b/paimon-python/pypaimon/tests/native_write_path_completion_test.py
index 2a30af350b..fb43f7a980 100644
--- a/paimon-python/pypaimon/tests/native_write_path_completion_test.py
+++ b/paimon-python/pypaimon/tests/native_write_path_completion_test.py
@@ -27,7 +27,7 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.execution.native.runtime import (native_runtime_available)
from pypaimon.execution.native.write.native_commit import _rest_catalog_supported
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = [pytest.mark.python_write, pytest.mark.python_commit]
native = pytest.mark.skipif(not native_runtime_available(), reason='pypaimon-rust not installed')
@@ -389,7 +389,7 @@ def test_consumed_blob_rows_are_released_without_losing_unplanned_inputs():
@pytest.mark.parametrize('use_native', [False, pytest.param(True, marks=native)])
def test_automatic_postpone_plan_counts_all_batches_per_partition(tmp_path, use_native):
- from pypaimon.execution.native.write.native_write import NativePostponeFixedBucketTableWrite
+ from pypaimon.execution.write_dispatch import NativePostponeFixedBucketTableWrite
schema = pa.schema([('id', pa.int64()), ('pt', pa.string())])
catalog = CatalogFactory.create({'warehouse': str(tmp_path / 'postpone')})
catalog.create_database('default', True)
@@ -465,7 +465,7 @@ def test_postpone_blob_row_bridge_keeps_sources_until_prepare(fail):
from pypaimon.schema.data_types import AtomicType, DataField
from pypaimon.table.row.blob import Blob, BlobDescriptor
from pypaimon.table.row.generic_row import GenericRow
- from pypaimon.execution.native.write.native_write import NativePostponeFixedBucketTableWrite
+ from pypaimon.execution.write_dispatch import NativePostponeFixedBucketTableWrite
# Exercise the bridge's deferred input lifetime without creating a table:
# Python catalog validation does not allow a primary-key BLOB schema.
diff --git a/paimon-python/pypaimon/tests/native_write_routing_test.py b/paimon-python/pypaimon/tests/native_write_routing_test.py
index 2d02fe588d..e886975ba3 100644
--- a/paimon-python/pypaimon/tests/native_write_routing_test.py
+++ b/paimon-python/pypaimon/tests/native_write_routing_test.py
@@ -31,7 +31,7 @@ from pypaimon.catalog.catalog_exception import TableNoPermissionException
from pypaimon.execution.native.runtime import (native_runtime_available)
from pypaimon.tests import native_plan_rest_test
from pypaimon.execution.native.write.native_commit import create_native_write_table
-from pypaimon.execution.native.write.native_write import NativePostponeFixedBucketTableWrite, NativeTableWrite
+from pypaimon.execution.write_dispatch import NativePostponeFixedBucketTableWrite, NativeTableWrite
from pypaimon.write.postpone_bucket import PostponeBucketPlan
rest_catalog = native_plan_rest_test.rest_catalog
diff --git a/paimon-python/pypaimon/tests/native_write_test.py b/paimon-python/pypaimon/tests/native_write_test.py
index c95bccc11c..2fb099a791 100644
--- a/paimon-python/pypaimon/tests/native_write_test.py
+++ b/paimon-python/pypaimon/tests/native_write_test.py
@@ -24,7 +24,7 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.common.options.core_options import CoreOptions
from pypaimon.common.options.options import Options
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
requires_native = pytest.mark.native_plan
@@ -84,7 +84,7 @@ def test_sequence_validation_precedes_native_selection(tmp_path, streaming, sequ
(pa.float64(), 'ascending', True),
])
def test_native_sequence_write_capabilities(tmp_path, type_, order, supported):
- from pypaimon.execution.native.write.native_write import create_native_write
+ from pypaimon.execution.native.write.native_write import NativeTableWrite as RawNativeWrite, create_native_write
table = _table(tmp_path, primary_key=True, id_type=type_, table_options={
'sequence.field': 'id', 'sequence.field.sort-order': order,
@@ -93,7 +93,7 @@ def test_native_sequence_write_capabilities(tmp_path, type_, order, supported):
patch('pypaimon.execution.native.write.native_write.create_native_write_table') as native:
writer = create_native_write(table, 'test')
if supported:
- assert isinstance(writer, NativeTableWrite)
+ assert isinstance(writer, RawNativeWrite)
native.assert_called_once()
writer.close()
else:
diff --git a/paimon-python/pypaimon/tests/native_write_type_test.py b/paimon-python/pypaimon/tests/native_write_type_test.py
index 010280bf2e..419a70c469 100644
--- a/paimon-python/pypaimon/tests/native_write_type_test.py
+++ b/paimon-python/pypaimon/tests/native_write_type_test.py
@@ -29,7 +29,7 @@ from pypaimon import Schema
from pypaimon.execution.native.runtime import (native_runtime_available)
from pypaimon.table.row.generic_row import GenericRow
from pypaimon.tests import native_plan_rest_test
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
rest_catalog = native_plan_rest_test.rest_catalog
diff --git a/paimon-python/pypaimon/tests/pure_writer_creation_time_test.py b/paimon-python/pypaimon/tests/pure_writer_creation_time_test.py
index 64a0cdf4d6..cc1ef2c7c0 100644
--- a/paimon-python/pypaimon/tests/pure_writer_creation_time_test.py
+++ b/paimon-python/pypaimon/tests/pure_writer_creation_time_test.py
@@ -28,7 +28,7 @@ from pypaimon import CatalogFactory, Schema
from pypaimon.data.timestamp import Timestamp
from pypaimon.manifest.manifest_file_manager import ManifestFileManager
from pypaimon.manifest.manifest_list_manager import ManifestListManager
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = [pytest.mark.python_write, pytest.mark.python_commit]
diff --git a/paimon-python/pypaimon/tests/py36/arrow_write_nullability_test.py b/paimon-python/pypaimon/tests/py36/arrow_write_nullability_test.py
new file mode 100644
index 0000000000..7e632465c1
--- /dev/null
+++ b/paimon-python/pypaimon/tests/py36/arrow_write_nullability_test.py
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Write validation also supports the Arrow 6 API used with Python 3.6."""
+
+import unittest
+
+import pyarrow as pa
+
+from pypaimon.schema.arrow_schema import normalize_arrow_write_nullability
+
+
+class ArrowWriteNullabilityTest(unittest.TestCase):
+
+ def test_map_null_parents_slices_and_required_values(self):
+ values = pa.array([None, None, 7, None], type=pa.int32())
+ original = pa.MapArray.from_arrays(
+ pa.array([0, 1, 2, 3, 4], type=pa.int32()),
+ pa.array(['outside', 'hidden', 'visible', 'outside']), values)
+ validity = pa.array([True, False, True, True]).buffers()[1]
+ array = pa.Array.from_buffers(
+ original.type, len(original), [validity, original.buffers()[1]],
+ children=[pa.StructArray.from_arrays(
+ [original.keys, original.items],
+ fields=[original.type.key_field, original.type.item_field])])
+ declared = pa.map_(pa.string(), pa.field('value', pa.int32(), nullable=False))
+ # Arrow 6's MAP type equality ignores item nullability. A required
+ # sibling field makes the ROW retargeting explicit on every version.
+ row_type = pa.struct([
+ pa.field('attrs', declared), pa.field('tag', pa.int32(), nullable=False)])
+ target = pa.schema([('nested', row_type)])
+
+ def source_data(factory, offset, length):
+ nested = pa.StructArray.from_arrays([
+ array.slice(offset, length), pa.array([1] * length, type=pa.int32())
+ ], names=['attrs', 'tag'])
+ return factory.from_arrays([nested], names=['nested'])
+
+ for factory in (pa.Table, pa.RecordBatch):
+ for offset, length in ((1, 2), (1, 1), (4, 0)):
+ source = source_data(factory, offset, length)
+ result = normalize_arrow_write_nullability(source, target)
+ self.assertEqual(result.to_pydict(), source.to_pydict())
+ self.assertEqual(result.schema.field('nested').type, row_type)
+ self.assertFalse(result.schema.field('nested').type[0].type.item_field.nullable)
+ result.validate(full=True)
+ source = source_data(factory, 2, 2)
+ with self.assertRaisesRegex(ValueError, 'nested.attrs.value.*NULL'):
+ normalize_arrow_write_nullability(source, target)
diff --git a/paimon-python/pypaimon/tests/ray_vector_index_build_test.py b/paimon-python/pypaimon/tests/ray_vector_index_build_test.py
index af5f6bb621..7d4647b4e1 100644
--- a/paimon-python/pypaimon/tests/ray_vector_index_build_test.py
+++ b/paimon-python/pypaimon/tests/ray_vector_index_build_test.py
@@ -31,7 +31,8 @@ pytest.importorskip("paimon_vindex")
from pypaimon.globalindex.create_global_index import GlobalIndexBuilder
from pypaimon.index.index_file_handler import IndexFileHandler
-from pypaimon.ray import vector_index_build
+from pypaimon.execution.python.globalindex import ray_build as vector_index_build
+from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder
from pypaimon.tests import ray_vector_search_test as fixtures
from pypaimon.tests.ray_vector_search_test import add_rows, VECTORS
@@ -154,6 +155,7 @@ def _inspect_jindo_context(context_refs):
# the SDK stand-in. Only Config/connect are replaced, not serialization.
with _jindo_sdk_stub():
builder, field, table_read, index_path = ray.get(context_refs[0])
+ assert isinstance(builder, PythonGlobalIndexBuilder)
handler = builder._table.file_io.filesystem.handler
assert table_read.table.file_io is builder._table.file_io
assert field.name == "embedding"
@@ -182,7 +184,7 @@ def test_jindo_build_context_reconstructs_on_worker(table, ray_cluster):
original.register_file_size("object", 123)
with patch.object(table.raw_table, "file_io", file_io):
# This is the exact tuple broadcast by build_vector_index.
- context = ray.put((builder, field, reader, "oss://bucket/index"))
+ context = ray.put((PythonGlobalIndexBuilder(builder), field, reader, "oss://bucket/index"))
result = ray.get(ray.remote(_inspect_jindo_context).remote([context]))
worker_pid, client_pid, size, root, token, index_path = result
assert worker_pid != os.getpid()
diff --git a/paimon-python/pypaimon/tests/read_execution_boundary_test.py b/paimon-python/pypaimon/tests/read_execution_boundary_test.py
new file mode 100644
index 0000000000..baf6833d36
--- /dev/null
+++ b/paimon-python/pypaimon/tests/read_execution_boundary_test.py
@@ -0,0 +1,268 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Read entry points load only the selected execution backend."""
+
+import subprocess
+import sys
+import textwrap
+from unittest.mock import Mock, patch
+
+import pytest
+
+
+def _run(script):
+ result = subprocess.run(
+ [sys.executable, '-c', textwrap.dedent(script)],
+ capture_output=True, text=True,
+ )
+ assert result.returncode == 0, result.stdout + result.stderr
+
+
+def test_native_read_scan_and_stream_preparation_do_not_import_python_execution():
+ _run("""
+ import importlib.abc
+ import sys
+ import tempfile
+ from types import SimpleNamespace
+ from unittest.mock import patch
+
+ class RejectPythonRead(importlib.abc.MetaPathFinder):
+ def find_spec(self, fullname, path=None, target=None):
+ if fullname.startswith('pypaimon.execution.python'):
+ raise AssertionError('Python execution imported: ' + fullname)
+
+ sys.meta_path.insert(0, RejectPythonRead())
+ import pyarrow as pa
+ from pypaimon import CatalogFactory, Schema
+ from pypaimon.read.plan import Plan
+ from pypaimon.read.table_read import TableRead
+ from pypaimon.read.table_scan import TableScan
+ from pypaimon.read.stream_read_builder import StreamReadBuilder
+
+ assert TableRead.__bases__ == (object,)
+ assert TableScan.__bases__ == (object,)
+ with tempfile.TemporaryDirectory() as path:
+ catalog = CatalogFactory.create({'warehouse': path})
+ catalog.create_database('default', True)
+ catalog.create_table('default.lazy', Schema.from_pyarrow_schema(
+ pa.schema([('id', pa.int32())]), options={
+ 'read.native.enabled': 'true', 'scan.native-plan.enabled': 'true',
+ }), False)
+ table = catalog.get_table('default.lazy')
+ builder = table.new_read_builder()
+ with patch('pypaimon.execution.native.runtime.native_runtime_available', return_value=True), \
+ patch('pypaimon.execution.native.read.plan.native_plan', return_value=Plan([], 7)):
+ assert builder.new_scan().plan().snapshot_id == 7
+ assert builder.new_scan().plan_all_files().snapshot_id == 7
+ assert builder.new_scan().scan_with_stats()[0].snapshot_id == 7
+ reader = builder.new_read()
+ assert reader.to_arrow([]).num_rows == 0
+ batches = reader.to_arrow_batch_reader([])
+ batches.close()
+ assert list(reader.to_iterator([])) == []
+ native = SimpleNamespace(checkpoint=lambda: 8)
+ with patch('pypaimon.execution.native.read.streaming_table_scan.native_stream_scan', return_value=native):
+ stream = StreamReadBuilder(table).new_streaming_scan()
+ assert stream.checkpoint() == 8
+ assert not any(name.startswith('pypaimon.execution.python') for name in sys.modules)
+ """)
+
+
+@pytest.mark.parametrize('failure', [False, True])
+def test_native_stream_failure_does_not_create_a_python_reader(failure):
+ from types import SimpleNamespace
+ import pyarrow as pa
+ from pypaimon.read.table_read import TableRead
+
+ schema = pa.schema([('id', pa.int32())])
+ closed = []
+
+ def batches():
+ try:
+ yield pa.record_batch([[1]], schema=schema)
+ if failure:
+ raise OSError('native stream failed')
+ finally:
+ closed.append(True)
+
+ read = TableRead.__new__(TableRead)
+ read.table = SimpleNamespace(options=SimpleNamespace(native_read_enabled=lambda: True))
+ read._read_parallelism = 1
+ read.limit = None
+ read.include_row_kind = False
+ read.predicate = None
+ read._deferred_blob_fields = set()
+ read._output_arrow_schema = lambda splits: schema
+ read._try_native_batches = Mock(return_value=batches())
+ with patch.dict(sys.modules, {'pypaimon.execution.python.read.table_read': None}):
+ reader = read.to_arrow_batch_reader([], parallelism=1)
+ assert reader.read_next_batch().to_pydict() == {'id': [1]}
+ if failure:
+ with pytest.raises(OSError, match='native stream failed'):
+ reader.read_next_batch()
+ reader.close()
+ assert closed == [True]
+
+
+def test_python_vector_executor_never_selects_the_native_vindex_facade():
+ from pypaimon.execution.python.read.search.vector_search_read import _create_vector_reader
+
+ expected = object()
+ with patch('pypaimon.execution.python.globalindex.vindex.vindex_vector_global_index_reader.'
+ 'PythonVindexVectorGlobalIndexReader', return_value=expected) as create, \
+ patch.dict(sys.modules, {'pypaimon.globalindex.vindex_reader': None}):
+ assert _create_vector_reader('ivf-flat', 'io', 'path', ['meta'], {'nprobe': '2'}) is expected
+ create.assert_called_once_with('io', 'path', ['meta'], {'nprobe': '2'})
+
+
+@pytest.mark.parametrize('outcome', ['success', 'schema-error', 'stream-error'])
+@pytest.mark.parametrize('close_error', [False, True])
+def test_native_materialization_always_closes_a_retained_batch_source(outcome, close_error):
+ import pyarrow as pa
+ from pypaimon.read.table_read import TableRead
+
+ schema = pa.schema([('id', pa.int32())])
+ events = []
+ primary = OSError('native stream failed')
+ cleanup = RuntimeError('native close failed')
+
+ def batches():
+ try:
+ batch_type = pa.int64() if outcome == 'schema-error' else pa.int32()
+ yield pa.record_batch([pa.array([1], type=batch_type)], names=['id'])
+ if outcome == 'stream-error':
+ raise primary
+ finally:
+ events.append('generator-closed')
+
+ class Source:
+ def __init__(self):
+ self.stream = batches()
+ self.close_count = 0
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return next(self.stream)
+
+ def close(self):
+ self.close_count += 1
+ self.stream.close()
+ if close_error:
+ raise cleanup
+
+ source = Source()
+ read = TableRead.__new__(TableRead)
+ read._read_parallelism = 1
+ read.include_row_kind = False
+ read._output_arrow_schema = lambda splits: schema
+ read._try_native_batches = Mock(return_value=source)
+ with patch.dict(sys.modules, {'pypaimon.execution.python.read.table_read': None}):
+ if outcome == 'schema-error':
+ with pytest.raises(TypeError, match="has type int64, expected int32"):
+ read.to_arrow([], parallelism=1)
+ elif outcome == 'stream-error':
+ with pytest.raises(OSError) as caught:
+ read.to_arrow([], parallelism=1)
+ assert caught.value is primary
+ elif close_error:
+ with pytest.raises(RuntimeError) as caught:
+ read.to_arrow([], parallelism=1)
+ assert caught.value is cleanup
+ else:
+ assert read.to_arrow([], parallelism=1).to_pydict() == {'id': [1]}
+ # Keep source and its generator reachable: cleanup must not depend on GC.
+ assert source.close_count == 1
+ assert source.stream.gi_frame is None
+ assert events == ['generator-closed']
+
+
+@pytest.mark.parametrize('failure', ['schema', 'stream'])
+def test_native_converter_preserves_primary_failure_when_retained_source_close_fails(failure):
+ import pyarrow as pa
+ from pypaimon.read.table_read import TableRead
+
+ schema = pa.schema([('id', pa.int32())])
+ primary = OSError('primary native stream failure')
+ closed = []
+
+ class Source:
+ close_count = 0
+
+ def __init__(self):
+ self.stream = self.batches()
+
+ def batches(self):
+ try:
+ if failure == 'stream':
+ raise primary
+ yield pa.record_batch([pa.array([1], type=pa.int64())], names=['id'])
+ finally:
+ closed.append(True)
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return next(self.stream)
+
+ def close(self):
+ self.close_count += 1
+ self.stream.close()
+ raise RuntimeError('secondary close failure')
+
+ source = Source()
+ read = TableRead.__new__(TableRead)
+ read._prepare_native_batch = lambda batch, schema: batch
+ read._project_batch_to_output = lambda batch: batch
+ converted = read._convert_native_batches(source, schema)
+ with patch.dict(sys.modules, {'pypaimon.execution.python.read.table_read': None}):
+ if failure == 'schema':
+ with pytest.raises(TypeError, match='has type int64, expected int32'):
+ read._batches_to_arrow(converted, schema)
+ else:
+ with pytest.raises(OSError) as caught:
+ read._batches_to_arrow(converted, schema)
+ assert caught.value is primary
+ assert source.close_count == 1
+ assert source.stream.gi_frame is None
+ assert closed == [True]
+
+
+def test_native_row_conversion_keeps_its_failure_when_closing_batches_also_fails():
+ from types import SimpleNamespace
+ import pyarrow as pa
+ from pypaimon.read.table_read import TableRead
+ from pypaimon.schema.data_types import AtomicType, DataField
+
+ primary = ValueError('invalid native row batch')
+ source = Mock(close=Mock(side_effect=OSError('secondary close failure')))
+ read = TableRead.__new__(TableRead)
+ read.table = SimpleNamespace(options=SimpleNamespace(native_read_enabled=lambda: True))
+ read.read_type = [DataField(0, 'id', AtomicType('INT'))]
+ read.output_projection = None
+ read._output_arrow_schema = lambda splits: pa.schema([('id', pa.int32())])
+ read._try_native_batches = Mock(return_value=source)
+ with patch('pypaimon.execution.native.read.row_iterator.native_rows', side_effect=primary), \
+ patch.dict(sys.modules, {'pypaimon.execution.python.read.table_read': None}):
+ rows = read._try_native_rows([])
+ with pytest.raises(ValueError) as caught:
+ next(rows)
+ assert caught.value is primary
+ source.close.assert_called_once_with()
diff --git a/paimon-python/pypaimon/tests/reader_parallel_test.py b/paimon-python/pypaimon/tests/reader_parallel_test.py
index b31ba265e4..3e99e6fb77 100644
--- a/paimon-python/pypaimon/tests/reader_parallel_test.py
+++ b/paimon-python/pypaimon/tests/reader_parallel_test.py
@@ -27,7 +27,8 @@ import pyarrow as pa
import pytest
from pypaimon import CatalogFactory, Schema
-from pypaimon.read.table_read import TableRead, _RemainingRows
+from pypaimon.read.table_read import TableRead
+from pypaimon.execution.python.read.table_read import _RemainingRows
class ResolveParallelismTest(unittest.TestCase):
diff --git a/paimon-python/pypaimon/tests/shard_table_updator_test.py b/paimon-python/pypaimon/tests/shard_table_updator_test.py
index dba16afc56..9a5e077f0e 100644
--- a/paimon-python/pypaimon/tests/shard_table_updator_test.py
+++ b/paimon-python/pypaimon/tests/shard_table_updator_test.py
@@ -39,7 +39,7 @@ class ShardUpdateValidationTest(unittest.TestCase):
field_names=['value'],
options=SimpleNamespace(native_write_enabled=lambda: native))
update = update_type(table, 'writer').with_update_type([])
- with patch('pypaimon.write.table_update.prepare_native_write_operation',
+ with patch('pypaimon.execution.update_dispatch.prepare_native_write_operation',
side_effect=AssertionError('Prepared a backend for invalid columns')) as prepare, \
patch.object(update, '_new_python_shard_updator',
side_effect=AssertionError('Created a Python shard updater')) as python:
diff --git a/paimon-python/pypaimon/tests/shared_execution_import_test.py b/paimon-python/pypaimon/tests/shared_execution_import_test.py
new file mode 100644
index 0000000000..ce90b68b42
--- /dev/null
+++ b/paimon-python/pypaimon/tests/shared_execution_import_test.py
@@ -0,0 +1,114 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Storage and table metadata must not eagerly load Python write engines."""
+
+import subprocess
+import sys
+from types import ModuleType
+from unittest.mock import patch
+
+import pytest
+
+
+def test_native_table_preparation_does_not_load_python_write_implementations():
+ source = """
+import importlib.abc
+import sys
+from types import SimpleNamespace
+
+class RejectPythonWriters(importlib.abc.MetaPathFinder):
+ def find_spec(self, fullname, path=None, target=None):
+ if fullname.startswith('pypaimon.execution.python'):
+ raise AssertionError('Native preparation loaded a Python writer: ' + fullname)
+
+sys.meta_path.insert(0, RejectPythonWriters())
+from pypaimon.catalog.catalog_environment import CatalogEnvironment
+from pypaimon.filesystem.local_file_io import LocalFileIO
+from pypaimon.filesystem.hdfs_native_file_io import HdfsNativeFileIO
+from pypaimon.table.file_store_table import FileStoreTable
+from pypaimon.execution.native.table import _resolved_schema_file_io_options
+
+file_io = LocalFileIO()
+table = SimpleNamespace(file_io=file_io, catalog_environment=CatalogEnvironment.empty())
+assert _resolved_schema_file_io_options(table) == {}
+assert FileStoreTable.__module__ == 'pypaimon.table.file_store_table'
+"""
+ result = subprocess.run([sys.executable, '-c', source], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ universal_newlines=True, timeout=30)
+ assert result.returncode == 0, result.stdout + result.stderr
+
+
+@pytest.mark.parametrize('link', ['direct', 'cause', 'context'])
+def test_loaded_fork_error_is_propagated_without_loading_native(link):
+ from pypaimon.execution.dispatch import propagate_fork_safety_error
+
+ class ForkSafetyError(RuntimeError):
+ pass
+
+ sdk = ModuleType('pypaimon_rust')
+ sdk.ForkSafetyError = ForkSafetyError
+ failure = ForkSafetyError('inherited SDK state')
+ wrapped = failure if link == 'direct' else RuntimeError('prefetch failed')
+ if link == 'cause':
+ wrapped.__cause__ = failure
+ elif link == 'context':
+ wrapped.__context__ = failure
+ with patch.dict(sys.modules, {'pypaimon_rust': sdk, 'pypaimon.execution.native.runtime': None}):
+ with pytest.raises(ForkSafetyError) as caught:
+ propagate_fork_safety_error(wrapped)
+ assert caught.value is failure
+
+
+def test_loaded_fork_error_check_terminates_on_cyclic_exception_chain():
+ from pypaimon.execution.dispatch import propagate_fork_safety_error
+
+ sdk = ModuleType('pypaimon_rust')
+ sdk.ForkSafetyError = type('ForkSafetyError', (RuntimeError,), {})
+ first, second = ValueError('first'), RuntimeError('second')
+ first.__cause__, second.__context__ = second, first
+ with patch.dict(sys.modules, {'pypaimon_rust': sdk, 'pypaimon.execution.native.runtime': None}):
+ assert propagate_fork_safety_error(first) is None
+
+
+def test_python_streaming_prefetch_failure_never_loads_native():
+ source = """
+import importlib.abc
+import sys
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+class BlockNative(importlib.abc.MetaPathFinder):
+ def find_spec(self, fullname, path=None, target=None):
+ if fullname.startswith(('pypaimon.execution.native', 'pypaimon_rust')):
+ raise AssertionError('Python prefetch loaded Native: ' + fullname)
+
+sys.meta_path.insert(0, BlockNative())
+from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan
+
+scan = object.__new__(AsyncStreamingTableScan)
+scan._snapshot_manager = SimpleNamespace(find_next_scannable=Mock(side_effect=OSError('prefetch unavailable')))
+scan.follow_up_scanner = SimpleNamespace(should_scan=Mock())
+scan._lookahead_size = 4
+assert scan._fetch_plan_with_lookahead(1) is None
+scan._snapshot_manager.find_next_scannable.assert_called_once_with(
+ 1, scan.follow_up_scanner.should_scan, lookahead_size=4)
+assert not any(name.startswith(('pypaimon.execution.native', 'pypaimon_rust')) for name in sys.modules)
+"""
+ result = subprocess.run([sys.executable, '-c', source], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ universal_newlines=True, timeout=30)
+ assert result.returncode == 0, result.stdout + result.stderr
diff --git a/paimon-python/pypaimon/tests/stream_read_builder_test.py b/paimon-python/pypaimon/tests/stream_read_builder_test.py
index ca7caae9f7..0e4f74b5a0 100644
--- a/paimon-python/pypaimon/tests/stream_read_builder_test.py
+++ b/paimon-python/pypaimon/tests/stream_read_builder_test.py
@@ -173,7 +173,7 @@ class TestNativeStreamFactory:
def test_optional_native_initialization_falls_back(self, mock_scan_table):
mock_scan_table.options.native_plan_enabled.return_value = True
builder = StreamReadBuilder(mock_scan_table)
- with patch('pypaimon.read.stream_read_builder.StreamTableScan',
+ with patch('pypaimon.execution.native.read.streaming_table_scan.StreamTableScan',
side_effect=ImportError('optional native runtime unavailable')):
assert isinstance(builder.new_streaming_scan(), AsyncStreamingTableScan)
diff --git a/paimon-python/pypaimon/tests/streaming_scan_import_boundary_test.py b/paimon-python/pypaimon/tests/streaming_scan_import_boundary_test.py
new file mode 100644
index 0000000000..a54c437b50
--- /dev/null
+++ b/paimon-python/pypaimon/tests/streaming_scan_import_boundary_test.py
@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Importing a public streaming class loads only its own execution backend."""
+
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+pytestmark = pytest.mark.skipif(
+ sys.version_info < (3, 7), reason='Module-level lazy exports require Python 3.7')
+
+
+def _run(source, *args):
+ result = subprocess.run(
+ [sys.executable, '-c', textwrap.dedent(source)] + list(args),
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ universal_newlines=True, timeout=30)
+ assert result.returncode == 0, result.stdout + result.stderr
+
+
+@pytest.mark.parametrize('export, backend, forbidden', [
+ ('StreamTableScan', 'native', 'python'),
+ ('AsyncStreamingTableScan', 'python', 'native'),
+])
+def test_public_streaming_class_imports_only_its_backend(export, backend, forbidden):
+ _run("""
+ import importlib
+ import importlib.abc
+ import inspect
+ import pickle
+ import sys
+
+ export, backend, forbidden = sys.argv[1:]
+ blocked = 'pypaimon.execution.' + forbidden
+
+ class RejectOtherBackend(importlib.abc.MetaPathFinder):
+ def find_spec(self, fullname, path=None, target=None):
+ if fullname == blocked or fullname.startswith(blocked + '.'):
+ raise AssertionError('Unexpected backend import: ' + fullname)
+
+ sys.meta_path.insert(0, RejectOtherBackend())
+ if backend == 'native':
+ from pypaimon.read.streaming_table_scan import StreamTableScan as public_class
+ else:
+ from pypaimon.read.streaming_table_scan import AsyncStreamingTableScan as public_class
+ module = importlib.import_module('pypaimon.read.streaming_table_scan')
+ direct = importlib.import_module(
+ 'pypaimon.execution.' + backend + '.read.streaming_table_scan')
+ canonical_class = getattr(direct, export)
+ assert public_class is canonical_class
+ assert getattr(module, export) is canonical_class
+ assert inspect.signature(public_class) == inspect.signature(canonical_class)
+ assert pickle.loads(pickle.dumps(public_class)) is canonical_class
+ assert not any(name == blocked or name.startswith(blocked + '.')
+ for name in sys.modules)
+ """, export, backend, forbidden)
+
+
+def test_streaming_exports_are_discoverable_without_loading_either_backend():
+ _run("""
+ import importlib
+ import importlib.abc
+ import sys
+
+ class RejectBackends(importlib.abc.MetaPathFinder):
+ def find_spec(self, fullname, path=None, target=None):
+ if fullname.startswith(('pypaimon.execution.python',
+ 'pypaimon.execution.native')):
+ raise AssertionError('Unexpected backend import: ' + fullname)
+
+ sys.meta_path.insert(0, RejectBackends())
+ module = importlib.import_module('pypaimon.read.streaming_table_scan')
+ exports = ['AsyncStreamingTableScan', 'StreamTableScan']
+ assert module.__all__ == exports
+ assert all(name in dir(module) for name in exports)
+ assert all(name not in module.__dict__ for name in exports)
+ try:
+ module.unknown_export
+ except AttributeError as error:
+ assert 'unknown_export' in str(error)
+ assert 'pypaimon.read.streaming_table_scan' in str(error)
+ else:
+ raise AssertionError('An unknown export must raise AttributeError')
+ """)
diff --git a/paimon-python/pypaimon/tests/system/system_string_filter_test.py b/paimon-python/pypaimon/tests/system/system_string_filter_test.py
index f2bf1026d3..bc200fb407 100644
--- a/paimon-python/pypaimon/tests/system/system_string_filter_test.py
+++ b/paimon-python/pypaimon/tests/system/system_string_filter_test.py
@@ -270,6 +270,9 @@ def test_string_filter_applies_to_all_outputs(table, output):
def test_native_result_handoff_uses_string_filter(table):
+ from types import SimpleNamespace
+
+ table.base_table.options = SimpleNamespace(native_read_enabled=lambda: True)
builder = table.new_read_builder()
predicate = builder.new_predicate_builder().contains("name", "parquet")
rows = table._build_arrow_table()
diff --git a/paimon-python/pypaimon/tests/test_read_builder_nested_projection.py b/paimon-python/pypaimon/tests/test_read_builder_nested_projection.py
index 1d7bd673ac..8fc2cd7dea 100644
--- a/paimon-python/pypaimon/tests/test_read_builder_nested_projection.py
+++ b/paimon-python/pypaimon/tests/test_read_builder_nested_projection.py
@@ -28,7 +28,7 @@ from pypaimon.read.datasource.split_provider import (
CatalogSplitProvider, PreResolvedSplitProvider)
from pypaimon.read.read_builder import ReadBuilder
from pypaimon.read.stream_read_builder import StreamReadBuilder
-from pypaimon.read.table_read import _RemainingRows
+from pypaimon.execution.python.read.table_read import _RemainingRows
from pypaimon.schema.data_types import AtomicType, DataField, RowType
diff --git a/paimon-python/pypaimon/tests/torch_read_test.py b/paimon-python/pypaimon/tests/torch_read_test.py
index 9316b4b748..28bfd889b6 100644
--- a/paimon-python/pypaimon/tests/torch_read_test.py
+++ b/paimon-python/pypaimon/tests/torch_read_test.py
@@ -398,6 +398,8 @@ class TorchDistributedShardingTest(unittest.TestCase):
"-m",
"torch.distributed.run",
"--standalone",
+ # Bootstrap locally without resolving the host's FQDN.
+ "--local-addr=127.0.0.1",
"--nproc-per-node=2",
script,
output_dir,
diff --git a/paimon-python/pypaimon/tests/variant_test.py b/paimon-python/pypaimon/tests/variant_test.py
index 701f5c1674..3f0f9b6c19 100644
--- a/paimon-python/pypaimon/tests/variant_test.py
+++ b/paimon-python/pypaimon/tests/variant_test.py
@@ -1563,7 +1563,6 @@ class TestVariantPaimonTable(unittest.TestCase):
def test_native_writer_preserves_shredded_variant_types(self):
try:
import pypaimon_rust # noqa: F401
- from pypaimon.execution.native.write.native_write import NativeTableWrite # noqa: F401
except ImportError:
self.skipTest('native writer is not available')
self._check_shredding_type_preservation(native=True)
diff --git a/paimon-python/pypaimon/tests/vindex_batch_write_test.py b/paimon-python/pypaimon/tests/vindex_batch_write_test.py
index 4c5b52c743..1fd4a8ab17 100644
--- a/paimon-python/pypaimon/tests/vindex_batch_write_test.py
+++ b/paimon-python/pypaimon/tests/vindex_batch_write_test.py
@@ -22,7 +22,7 @@ from unittest.mock import Mock, patch
import numpy as np
import pyarrow as pa
-from pypaimon.globalindex.create_global_index import GlobalIndexBuilder, _write_vector_batch
+from pypaimon.execution.python.globalindex.create_global_index import PythonGlobalIndexBuilder, _write_vector_batch
from pypaimon.execution.python.globalindex.vindex.vindex_vector_index_writer import VindexVectorIndexWriter
from pypaimon.schema.data_types import ArrayType, AtomicType
from pypaimon.utils.range import Range
@@ -168,7 +168,7 @@ class VindexBatchWriteTest(unittest.TestCase):
_write_vector_batch(self._writer(), batch, 'embedding', Range(10, 19))
def test_null_row_ids_are_rejected_before_writing_source_batch(self):
- builder = object.__new__(GlobalIndexBuilder)
+ builder = object.__new__(PythonGlobalIndexBuilder)
builder._table = Mock()
builder._core_options = Mock()
builder._core_options.global_index_row_count_per_shard.return_value = 10
diff --git a/paimon-python/pypaimon/tests/write/backend_boundary_test.py b/paimon-python/pypaimon/tests/write/backend_boundary_test.py
new file mode 100644
index 0000000000..e74ab7ac1b
--- /dev/null
+++ b/paimon-python/pypaimon/tests/write/backend_boundary_test.py
@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Execution ownership and public writer hook contracts."""
+
+import builtins
+from unittest.mock import Mock, patch
+
+import pyarrow as pa
+import pytest
+
+from pypaimon import CatalogFactory, Schema
+from pypaimon.execution.native.write.native_write import NativeTableWrite as NativeWrite
+from pypaimon.write.table_write import BatchTableWrite
+
+
+def _table(tmp_path, options=None):
+ catalog = CatalogFactory.create({'warehouse': str(tmp_path)})
+ catalog.create_database('default', True)
+ catalog.create_table('default.data', Schema.from_pyarrow_schema(
+ pa.schema([('id', pa.int32()), ('value', pa.string())]),
+ options=options or {}), False)
+ return catalog.get_table('default.data')
+
+
+@pytest.mark.parametrize('input_method', ['write_arrow', 'write_pandas'])
+@pytest.mark.parametrize('override', ['subclass', 'instance'])
+def test_public_writer_keeps_dynamic_batch_hooks(tmp_path, input_method, override):
+ table = _table(tmp_path)
+ builder = table.new_batch_write_builder()
+ batches = []
+
+ class ObservedWriter(BatchTableWrite):
+ def write_arrow_batch(self, data, bucket=None):
+ batches.append(data)
+ return super().write_arrow_batch(data, bucket)
+
+ if override == 'subclass':
+ writer = ObservedWriter(table, builder.commit_user)
+ else:
+ writer = builder.new_write()
+ original = writer.write_arrow_batch
+
+ def observed(data, bucket=None):
+ batches.append(data)
+ return original(data, bucket)
+
+ writer.write_arrow_batch = observed
+ data = pa.table({'id': pa.array([1, 2], pa.int32()), 'value': ['one', 'two']})
+ getattr(writer, input_method)(data if input_method == 'write_arrow' else data.to_pandas())
+ assert len(batches) == 1
+ assert batches[0].to_pydict() == data.to_pydict()
+ commit = builder.new_commit()
+ try:
+ commit.commit(writer.prepare_commit())
+ finally:
+ writer.close()
+ commit.close()
+ read = table.new_read_builder()
+ assert read.new_read().to_arrow(read.new_scan().plan().splits()).to_pylist() == data.to_pylist()
+
+
+def test_native_adapter_owns_only_one_backend(tmp_path):
+ core = Mock()
+ writer = NativeWrite(_table(tmp_path), 'job', None, False, core)
+ for name in ('_python_writer', '_new_python_writer', '_switch_to_python',
+ 'file_store_write', 'row_key_extractor'):
+ assert not hasattr(writer, name)
+ writer.write_arrow_batch(pa.record_batch(
+ [pa.array([1], pa.int32()), ['one']], names=['id', 'value']))
+ assert writer._written
+ core.write_arrow.assert_called_once()
+ writer.close()
+ core.close.assert_called_once_with()
+
+
+def test_python_delete_matching_respects_independent_read_backend(tmp_path):
+ table = _table(tmp_path, {'row-tracking.enabled': 'true', 'data-evolution.enabled': 'true',
+ 'deletion-vectors.enabled': 'true'})
+ builder = table.new_batch_write_builder()
+ writer, commit = builder.new_write(), builder.new_commit()
+ try:
+ writer.write_arrow(pa.table({'id': pa.array([1, 2], pa.int32()), 'value': ['one', 'two']}))
+ commit.commit(writer.prepare_commit())
+ finally:
+ writer.close()
+ commit.close()
+
+ table = table.copy({'write.native.enabled': 'true', 'read.native.enabled': 'false',
+ 'scan.native-plan.enabled': 'false'})
+ updater = table.new_batch_write_builder().new_update()
+ predicate = table.new_read_builder().new_predicate_builder().equal('id', 2)
+ original_import = builtins.__import__
+
+ def no_native(name, *args, **kwargs):
+ if name.startswith(('pypaimon.execution.native', 'pypaimon_rust')):
+ raise AssertionError('Python delete matching selected Native execution: ' + name)
+ return original_import(name, *args, **kwargs)
+
+ with patch('builtins.__import__', side_effect=no_native):
+ assert updater._matched_delete_row_ids(predicate) == [1]
diff --git a/paimon-python/pypaimon/tests/write/execution_lifecycle_test.py b/paimon-python/pypaimon/tests/write/execution_lifecycle_test.py
index 1a6b655d55..e1e3bb161a 100644
--- a/paimon-python/pypaimon/tests/write/execution_lifecycle_test.py
+++ b/paimon-python/pypaimon/tests/write/execution_lifecycle_test.py
@@ -217,7 +217,7 @@ def test_invalid_commit_configuration_fails_before_allocating_either_backend(
def test_native_writer_does_not_construct_python_writer_or_file_index(
tmp_path):
import pyarrow as pa
- from pypaimon.execution.native.write.native_write import NativeTableWrite
+ from pypaimon.execution.write_dispatch import NativeTableWrite
table = _data_evolution_table(tmp_path)
native_table, builder = _core_builder()
@@ -377,7 +377,7 @@ def test_native_callback_error_keeps_publication_outside_fallback_and_closes_onc
@pytest.mark.parametrize('written', [False, True])
def test_native_writer_attribute_queries_never_select_python(tmp_path, stream, written):
import pyarrow as pa
- from pypaimon.execution.native.write.native_write import NativeTableWrite
+ from pypaimon.execution.write_dispatch import NativeTableWrite
from pypaimon.write.table_write import TableWrite
table = _data_evolution_table(tmp_path)
@@ -408,7 +408,7 @@ def test_native_writer_attribute_queries_never_select_python(tmp_path, stream, w
def test_native_writer_explicit_python_diagnostics_keep_lazy_fallback(tmp_path):
- from pypaimon.execution.native.write.native_write import NativeTableWrite
+ from pypaimon.execution.write_dispatch import NativeTableWrite
table = _data_evolution_table(tmp_path)
core = Mock()
@@ -441,7 +441,7 @@ def test_native_writer_explicit_python_diagnostics_keep_lazy_fallback(tmp_path):
'file_store_write', 'row_key_extractor', 'table_pyarrow_schema', 'restore_snapshot_id', 'batch_committed',
])
def test_native_writer_diagnostic_assignment_selects_python_before_changing_configuration(tmp_path, attribute):
- from pypaimon.execution.native.write.native_write import NativeTableWrite
+ from pypaimon.execution.write_dispatch import NativeTableWrite
table = _data_evolution_table(tmp_path)
core = Mock()
@@ -456,7 +456,7 @@ def test_native_writer_diagnostic_assignment_selects_python_before_changing_conf
def test_native_batch_preparation_failure_retains_public_lifecycle_state_without_python(tmp_path):
- from pypaimon.execution.native.write.native_write import NativeTableWrite
+ from pypaimon.execution.write_dispatch import NativeTableWrite
core = Mock()
failure = OSError('batch preparation failed')
@@ -475,7 +475,7 @@ def test_native_batch_preparation_failure_retains_public_lifecycle_state_without
@pytest.mark.parametrize('failed', [False, True])
def test_explicit_python_diagnostics_preserve_empty_native_batch_preparation_state(tmp_path, failed):
- from pypaimon.execution.native.write.native_write import NativeTableWrite
+ from pypaimon.execution.write_dispatch import NativeTableWrite
core = Mock()
if failed:
diff --git a/paimon-python/pypaimon/tests/write/object_row_kind_test.py b/paimon-python/pypaimon/tests/write/object_row_kind_test.py
index ab83d27c8c..cd6cd2971c 100644
--- a/paimon-python/pypaimon/tests/write/object_row_kind_test.py
+++ b/paimon-python/pypaimon/tests/write/object_row_kind_test.py
@@ -27,7 +27,7 @@ from pypaimon.table.file_store_table import FileStoreTable
from pypaimon.table.row.generic_row import GenericRow
from pypaimon.table.row.projected_row import ProjectedRow
from pypaimon.table.row.row_kind import RowKind
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
@pytest.fixture(
diff --git a/paimon-python/pypaimon/tests/write/postpone_execution_boundary_test.py b/paimon-python/pypaimon/tests/write/postpone_execution_boundary_test.py
index f70dc2b9c6..74af623787 100644
--- a/paimon-python/pypaimon/tests/write/postpone_execution_boundary_test.py
+++ b/paimon-python/pypaimon/tests/write/postpone_execution_boundary_test.py
@@ -17,13 +17,16 @@
"""Ray input grouping must not allocate a Python file execution backend."""
+from pathlib import Path
+import subprocess
+import sys
from unittest.mock import Mock, patch
import pyarrow as pa
import pytest
from pypaimon import CatalogFactory, Schema
-from pypaimon.execution.native.write.native_write import NativePostponeFixedBucketTableWrite
+from pypaimon.execution.write_dispatch import NativePostponeFixedBucketTableWrite
from pypaimon.read.plan import Plan
from pypaimon.read.split import DataSplit
from pypaimon.read.split_serializer import deserialize_split_v1, serialize_split_v1
@@ -31,6 +34,57 @@ from pypaimon.table.row.generic_row import GenericRow
from pypaimon.write.bucket_plan import PostponeBucketPlan
+def test_public_postpone_input_planning_does_not_load_either_backend(tmp_path):
+ source = r'''
+import importlib.abc
+import sys
+
+class BlockExecution(importlib.abc.MetaPathFinder):
+ def find_spec(self, fullname, path=None, target=None):
+ if fullname.startswith(('pypaimon.execution.python', 'pypaimon.execution.native', 'pypaimon_rust')):
+ raise AssertionError('Input planning loaded an execution backend: ' + fullname)
+
+sys.meta_path.insert(0, BlockExecution())
+import pyarrow as pa
+from pypaimon import CatalogFactory, Schema
+from pypaimon.write.postpone_bucket import PostponeBucketPlan, PostponeBucketPlanner
+
+catalog = CatalogFactory.create({'warehouse': sys.argv[1]})
+catalog.create_database('default', True)
+catalog.create_table('default.data', Schema.from_pyarrow_schema(
+ pa.schema([('id', pa.int32()), ('p', pa.string()), ('value', pa.int64())]),
+ primary_keys=['id', 'p'], partition_keys=['p'], options={
+ 'bucket': '-2', 'postpone.target-row-num-per-bucket': '2',
+ 'postpone.batch-write-fixed-bucket.max-parallelism': '8',
+ }), False)
+table = catalog.get_table('default.data')
+data = pa.record_batch([
+ pa.array(range(7), pa.int32()), ['first'] * 5 + ['second'] * 2,
+ pa.array(range(7), pa.int64()),
+], names=['id', 'p', 'value'])
+planner = PostponeBucketPlanner(table, known_num_buckets={})
+assert planner.current_plan().as_dict() == {}
+stats = planner.input_partition_stats(data)
+assert stats == {('first',): (5, 0), ('second',): (2, 0)}
+plan = planner.plan(stats)
+assert isinstance(plan, PostponeBucketPlan)
+assert plan.as_dict() == {('first',): 4, ('second',): 1}
+assert planner.current_plan().as_dict() == plan.as_dict()
+
+existing = PostponeBucketPlanner(table, known_num_buckets={('first',): 3})
+assert existing.input_partition_stats(data) == {('second',): (2, 0)}
+assert existing.plan(existing.input_partition_stats(data)).as_dict() == {
+ ('first',): 3, ('second',): 1}
+assert not any(name.startswith(('pypaimon.execution.python', 'pypaimon.execution.native', 'pypaimon_rust'))
+ for name in sys.modules)
+'''
+ result = subprocess.run(
+ [sys.executable, '-c', source, str(tmp_path / 'warehouse')],
+ cwd=str(Path(__file__).resolve().parents[3]),
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=30)
+ assert result.returncode == 0, result.stdout + result.stderr
+
+
def _table(tmp_path):
catalog = CatalogFactory.create({'warehouse': str(tmp_path)})
catalog.create_database('default', True)
diff --git a/paimon-python/pypaimon/tests/write/retained_delete_nullability_test.py b/paimon-python/pypaimon/tests/write/retained_delete_nullability_test.py
index 33e9fe59ff..2931cd03cc 100644
--- a/paimon-python/pypaimon/tests/write/retained_delete_nullability_test.py
+++ b/paimon-python/pypaimon/tests/write/retained_delete_nullability_test.py
@@ -27,7 +27,7 @@ from pypaimon import CatalogFactory, Schema
from pypaimon.execution.native.runtime import (native_runtime_available)
from pypaimon.table.row.generic_row import GenericRow
from pypaimon.table.row.row_kind import RowKind
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
pytestmark = [pytest.mark.python_write, pytest.mark.python_commit]
diff --git a/paimon-python/pypaimon/tests/write/row_kind_nullability_test.py b/paimon-python/pypaimon/tests/write/row_kind_nullability_test.py
index a7e8cbc0d7..2ee66bf6dc 100644
--- a/paimon-python/pypaimon/tests/write/row_kind_nullability_test.py
+++ b/paimon-python/pypaimon/tests/write/row_kind_nullability_test.py
@@ -27,7 +27,7 @@ from pypaimon import CatalogFactory, Schema
from pypaimon.execution.native.runtime import (native_runtime_available)
from pypaimon.table.row.generic_row import GenericRow
from pypaimon.table.row.row_kind import RowKind
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
from pypaimon.write.row_kind import skip_write_row
@@ -233,6 +233,8 @@ def test_internal_row_kinds_check_required_nulls_before_filtering(tmp_path, kind
def test_native_bridge_leaves_nested_value_validation_to_core(tmp_path):
+ from pypaimon.execution.native.write.native_write import NativeTableWrite
+
transport = pa.schema([('items', pa.list_(pa.int32()))])
declared = pa.schema([('items', pa.list_(pa.field('element', pa.int32(), nullable=False)))])
data = pa.RecordBatch.from_arrays([pa.array([[None]], type=transport.field(0).type)], schema=transport)
diff --git a/paimon-python/pypaimon/tests/write/row_kind_write_test.py b/paimon-python/pypaimon/tests/write/row_kind_write_test.py
index a3670853f8..f4fa2b61cb 100644
--- a/paimon-python/pypaimon/tests/write/row_kind_write_test.py
+++ b/paimon-python/pypaimon/tests/write/row_kind_write_test.py
@@ -25,7 +25,7 @@ from unittest.mock import patch
from pypaimon import CatalogFactory, Schema
from pypaimon.table.row.generic_row import GenericRow
from pypaimon.table.row.row_kind import RowKind
-from pypaimon.execution.native.write.native_write import NativeTableWrite
+from pypaimon.execution.write_dispatch import NativeTableWrite
@pytest.fixture(params=[False, pytest.param(True, marks=pytest.mark.native_plan)],
diff --git a/paimon-python/pypaimon/tests/write/table_write_test.py b/paimon-python/pypaimon/tests/write/table_write_test.py
index 8f37397231..762f630aa4 100644
--- a/paimon-python/pypaimon/tests/write/table_write_test.py
+++ b/paimon-python/pypaimon/tests/write/table_write_test.py
@@ -810,7 +810,7 @@ class TableWriteTest(unittest.TestCase):
PostponeFixedBucketBatchTableWrite,
PostponeFixedBucketWriteBuilder,
)
- from pypaimon.execution.native.write.native_write import NativePostponeFixedBucketTableWrite
+ from pypaimon.execution.write_dispatch import NativePostponeFixedBucketTableWrite
self.assertIsInstance(
write_builder, PostponeFixedBucketWriteBuilder)
self.assertIsInstance(
diff --git a/paimon-python/pypaimon/write/postpone_batch_table_write.py b/paimon-python/pypaimon/write/postpone_batch_table_write.py
index 54bf83b092..9440feeb0b 100644
--- a/paimon-python/pypaimon/write/postpone_batch_table_write.py
+++ b/paimon-python/pypaimon/write/postpone_batch_table_write.py
@@ -15,10 +15,13 @@
# specific language governing permissions and limitations
# under the License.
-from typing import Optional
+from typing import List, Optional
+
+import pyarrow as pa
+
+from pypaimon.write.commit_message import CommitMessage
from pypaimon.execution.dispatch import create_write
-from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
from pypaimon.table.bucket_mode import BucketMode
from pypaimon.write.table_write import BatchTableWrite
from pypaimon.write.write_builder import BatchWriteBuilder
@@ -47,9 +50,63 @@ class PostponeFixedBucketWriteBuilder(BatchWriteBuilder):
restore_snapshot_id=self.restore_snapshot_id)
-class PostponeFixedBucketBatchTableWrite(PythonPostponeFixedBucketWriteMixin, BatchTableWrite):
+class PostponeFixedBucketBatchTableWrite(BatchTableWrite):
"""Batch writer which plans postpone rows before routing new partitions."""
- def __init__(self, table, commit_user, static_partition: Optional[dict] = None,
- bucket_plan=None):
- super().__init__(table, commit_user, static_partition, bucket_plan)
+ def __init__(self, table, commit_user, static_partition: Optional[dict] = None, bucket_plan=None):
+ from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
+ PythonPostponeFixedBucketWriteMixin.__init__(self, table, commit_user, static_partition, bucket_plan)
+ self.batch_committed = False
+
+ def _create_file_store_write(self, commit_user):
+ from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
+ return PythonPostponeFixedBucketWriteMixin._create_file_store_write(self, commit_user)
+
+ def _create_row_key_extractor(self, static_partition):
+ from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
+ return PythonPostponeFixedBucketWriteMixin._create_row_key_extractor(self, static_partition)
+
+ def _write_partition_bucket_batch(self, partition, bucket, data):
+ from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
+ return PythonPostponeFixedBucketWriteMixin._write_partition_bucket_batch(self, partition, bucket, data)
+
+ def _write_partition_bucket_row(self, partition, bucket, row, values_by_name):
+ from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
+ return PythonPostponeFixedBucketWriteMixin._write_partition_bucket_row(
+ self, partition, bucket, row, values_by_name)
+
+ def _buffer_input(self, data):
+ from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
+ return PythonPostponeFixedBucketWriteMixin._buffer_input(self, data)
+
+ def write_arrow(self, table: pa.Table, bucket=None):
+ from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
+ return PythonPostponeFixedBucketWriteMixin.write_arrow(self, table, bucket)
+
+ def write_arrow_batch(self, data: pa.RecordBatch, bucket=None):
+ from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
+ return PythonPostponeFixedBucketWriteMixin.write_arrow_batch(self, data, bucket)
+
+ def write_row(self, row):
+ from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
+ return PythonPostponeFixedBucketWriteMixin.write_row(self, row)
+
+ def _flush_pending_inputs(self):
+ from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
+ return PythonPostponeFixedBucketWriteMixin._flush_pending_inputs(self)
+
+ def prepare_commit(self) -> List[CommitMessage]:
+ from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
+ return PythonPostponeFixedBucketWriteMixin.prepare_commit(self)
+
+ def _distributed_write_options(self, *, overwrite=False):
+ from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
+ return PythonPostponeFixedBucketWriteMixin._distributed_write_options(self, overwrite=overwrite)
+
+ def close(self):
+ from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
+ return PythonPostponeFixedBucketWriteMixin.close(self)
+
+ def abort(self):
+ from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWriteMixin
+ return PythonPostponeFixedBucketWriteMixin.abort(self)
diff --git a/paimon-python/pypaimon/write/postpone_bucket.py b/paimon-python/pypaimon/write/postpone_bucket.py
index 0fc7b42670..28b646a4e9 100644
--- a/paimon-python/pypaimon/write/postpone_bucket.py
+++ b/paimon-python/pypaimon/write/postpone_bucket.py
@@ -15,11 +15,16 @@
# specific language governing permissions and limitations
# under the License.
-from pypaimon.execution.python.write.postpone_bucket import PythonPostponeBucketPlanner
from pypaimon.write.bucket_plan import PostponeBucketPlan
+from pypaimon.write.bucket_planning import PostponeBucketPlanning
__all__ = ["PostponeBucketPlan", "PostponeBucketPlanner"]
-class PostponeBucketPlanner(PythonPostponeBucketPlanner):
+class PostponeBucketPlanner(PostponeBucketPlanning):
"""Plan fixed bucket counts for postpone batch writes."""
+
+ @staticmethod
+ def _load_bucket_metadata(table):
+ from pypaimon.execution.python.write.postpone_bucket import PythonPostponeBucketPlanner
+ return PythonPostponeBucketPlanner._load_bucket_metadata(table)
diff --git a/paimon-python/pypaimon/write/table_delete.py b/paimon-python/pypaimon/write/table_delete.py
index 02c6d09d5d..7ba3644ab8 100644
--- a/paimon-python/pypaimon/write/table_delete.py
+++ b/paimon-python/pypaimon/write/table_delete.py
@@ -17,22 +17,60 @@
from typing import TYPE_CHECKING, List, Optional, Sequence
-from pypaimon.execution.python.write.table_delete import PythonTableDeleteMixin
from pypaimon.write.commit_message import CommitMessage
if TYPE_CHECKING:
from pypaimon.execution.python.write.table_delete import _AnchorRangesInfo
-class TableDeleteByRowId(PythonTableDeleteMixin):
+class TableDeleteByRowId:
"""Prepare row deletion messages for a table."""
- def __init__(
- self,
- table,
- _precomputed_anchor_ranges: Optional["_AnchorRangesInfo"] = None,
- ):
- return super().__init__(table, _precomputed_anchor_ranges)
+ def __init__(self, table, _precomputed_anchor_ranges: Optional['_AnchorRangesInfo'] = None):
+ from pypaimon.execution.python.write.table_delete import PythonTableDeleteMixin
+ return PythonTableDeleteMixin.__init__(self, table, _precomputed_anchor_ranges)
def delete(self, row_ids: Sequence[int]) -> List[CommitMessage]:
- return super().delete(row_ids)
+ from pypaimon.execution.python.write.table_delete import PythonTableDeleteMixin
+ return PythonTableDeleteMixin.delete(self, row_ids)
+
+ def _snapshot_anchor_ranges(self):
+ from pypaimon.execution.python.write.table_delete import PythonTableDeleteMixin
+ return PythonTableDeleteMixin._snapshot_anchor_ranges(self)
+
+ def _validate_delete(self):
+ from pypaimon.execution.python.write.table_delete import PythonTableDeleteMixin
+ return PythonTableDeleteMixin._validate_delete(self)
+
+ @staticmethod
+ def _deduplicate_row_ids(row_ids):
+ from pypaimon.execution.python.write.table_delete import PythonTableDeleteMixin
+ return PythonTableDeleteMixin._deduplicate_row_ids(row_ids)
+
+ def _load_anchor_ranges(self):
+ from pypaimon.execution.python.write.table_delete import PythonTableDeleteMixin
+ return PythonTableDeleteMixin._load_anchor_ranges(self)
+
+ def _scan_anchor_ranges(self):
+ from pypaimon.execution.python.write.table_delete import PythonTableDeleteMixin
+ return PythonTableDeleteMixin._scan_anchor_ranges(self)
+
+ def _group_delete_targets(self, row_ids, anchors):
+ from pypaimon.execution.python.write.table_delete import PythonTableDeleteMixin
+ return PythonTableDeleteMixin._group_delete_targets(self, row_ids, anchors)
+
+ def _build_commit_message(self, partition, bucket, file_positions, snapshot_id):
+ from pypaimon.execution.python.write.table_delete import PythonTableDeleteMixin
+ return PythonTableDeleteMixin._build_commit_message(self, partition, bucket, file_positions, snapshot_id)
+
+ def _read_existing_deletion_vectors(self, partition, bucket, snapshot_id):
+ from pypaimon.execution.python.write.table_delete import PythonTableDeleteMixin
+ return PythonTableDeleteMixin._read_existing_deletion_vectors(self, partition, bucket, snapshot_id)
+
+ def _write_deletion_vector_index(self, partition, bucket, deletion_vectors):
+ from pypaimon.execution.python.write.table_delete import PythonTableDeleteMixin
+ return PythonTableDeleteMixin._write_deletion_vector_index(self, partition, bucket, deletion_vectors)
+
+ def _index_file_path(self, index_file, partition, bucket):
+ from pypaimon.execution.python.write.table_delete import PythonTableDeleteMixin
+ return PythonTableDeleteMixin._index_file_path(self, index_file, partition, bucket)
diff --git a/paimon-python/pypaimon/write/table_update.py b/paimon-python/pypaimon/write/table_update.py
index 887f6f6bea..f68e33b577 100644
--- a/paimon-python/pypaimon/write/table_update.py
+++ b/paimon-python/pypaimon/write/table_update.py
@@ -15,26 +15,27 @@
# specific language governing permissions and limitations
# under the License.
-from typing import Any, Iterable, List, Mapping, Optional, Sequence
+from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Sequence, Tuple
+import pyarrow
import pyarrow as pa
from pypaimon.common.predicate import Predicate
+from pypaimon.manifest.schema.data_file_meta import DataFileMeta
+from pypaimon.utils.range import Range
+
+if TYPE_CHECKING:
+ from pypaimon.execution.python.write.writer.data_writer import DataWriter
from pypaimon.common.predicate_builder import PredicateBuilder
-from pypaimon.execution.dispatch import prepare_native_write_operation
-from pypaimon.execution.python.write.table_update import (
- PythonTableUpdateMixin, ShardTableUpdator, SingleWriter)
from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER
-from pypaimon.table.special_fields import SpecialFields
from pypaimon.write.commit_message import CommitMessage
from pypaimon.write.table_update_by_row_id import TableUpdateByRowId
-from pypaimon.write.table_upsert_by_key import TableUpsertByKey
-from pypaimon.write.update_validation import _authorize_update, predicate_fields
+from pypaimon.write.update_validation import predicate_fields
__all__ = ["TableUpdate", "BatchTableUpdate", "StreamTableUpdate", "ShardTableUpdator", "SingleWriter"]
-class TableUpdate(PythonTableUpdateMixin):
+class TableUpdate:
"""Public batch and stream update configuration and execution entry points."""
def __init__(self, table, commit_user):
@@ -45,18 +46,9 @@ class TableUpdate(PythonTableUpdateMixin):
self.update_cols = None
self.projection = None
- def _new_row_id_updater(
- self, commit_identifier: int, _precomputed_files_info=None
- ) -> TableUpdateByRowId:
- _authorize_update(self.table)
- if _precomputed_files_info is None:
- native = prepare_native_write_operation(
- 'update_by_row_id', self.table, self.commit_user, commit_identifier)
- if native is not None:
- return native
- return TableUpdateByRowId(
- self.table, self.commit_user, commit_identifier,
- _precomputed_files_info=_precomputed_files_info)
+ def _new_row_id_updater(self, commit_identifier: int, _precomputed_files_info=None) -> TableUpdateByRowId:
+ from pypaimon.execution.update_dispatch import TableUpdateOperations
+ return TableUpdateOperations._new_row_id_updater(self, commit_identifier, _precomputed_files_info)
def with_update_type(self, update_cols: List[str]):
update_cols = list(dict.fromkeys(update_cols))
@@ -76,41 +68,20 @@ class TableUpdate(PythonTableUpdateMixin):
def new_shard_updator(self, shard_num: int, total_shard_count: int):
"""Create a shard updater for scan and rewrite style updates."""
- if total_shard_count <= 0 or shard_num < 0 or shard_num >= total_shard_count:
- raise ValueError('Shard index must be smaller than a positive shard count')
- if self.update_cols == []:
- raise ValueError('Shard update columns must not be empty')
- native = prepare_native_write_operation(
- 'shard_update', self.table, self.commit_user, self.projection,
- self.update_cols, shard_num, total_shard_count,
- stream=isinstance(self, StreamTableUpdate))
- if native is not None:
- return native
- return self._new_python_shard_updator(shard_num, total_shard_count)
-
- def _update_by_arrow_with_row_id(
- self, table: pa.Table, commit_identifier: int
- ) -> List[CommitMessage]:
- cols = self.update_cols if self.update_cols is not None else [
- c for c in table.column_names if c != SpecialFields.ROW_ID.name]
- if table.num_rows and cols and SpecialFields.ROW_ID.name in table.column_names:
- native = prepare_native_write_operation(
- 'update', self.table, self.commit_user, cols)
- if native is not None:
- return native.update_by_arrow_with_row_id(table)
- return self._new_row_id_updater(commit_identifier).update_columns(table, cols)
+ from pypaimon.execution.update_dispatch import TableUpdateOperations
+ return TableUpdateOperations.new_shard_updator(self, shard_num, total_shard_count)
+
+ def _update_by_arrow_with_row_id(self, table: pa.Table, commit_identifier: int) -> List[CommitMessage]:
+ from pypaimon.execution.update_dispatch import TableUpdateOperations
+ return TableUpdateOperations._update_by_arrow_with_row_id(self, table, commit_identifier)
def _update_by_arrow_batches_with_row_id(
- self, tables: Iterable[pa.Table], commit_identifier: int
- ) -> List[CommitMessage]:
- return super()._update_by_arrow_batches_with_row_id(tables, commit_identifier)
+ self, tables: Iterable[pa.Table], commit_identifier: int) -> List[CommitMessage]:
+ from pypaimon.execution.update_dispatch import TableUpdateOperations
+ return TableUpdateOperations._update_by_arrow_batches_with_row_id(self, tables, commit_identifier)
def _upsert_by_arrow_with_key(
- self,
- table: pa.Table,
- upsert_keys: List[str],
- commit_identifier: int,
- ) -> List[CommitMessage]:
+ self, table: pa.Table, upsert_keys: List[str], commit_identifier: int) -> List[CommitMessage]:
"""Shared implementation for ``upsert_by_arrow_with_key``.
For each row in the input Arrow table:
@@ -130,97 +101,105 @@ class TableUpdate(PythonTableUpdateMixin):
Returns:
List of :class:`CommitMessage` objects to be committed.
"""
- return TableUpsertByKey(
- self.table, self.commit_user, commit_identifier
- ).upsert(table, upsert_keys, self.update_cols)
-
- def _upsert_by_key(
- self,
- rows,
- upsert_keys: List[str],
- commit_identifier: int,
- ) -> List[CommitMessage]:
- return TableUpsertByKey(
- self.table, self.commit_user, commit_identifier
- ).upsert_rows(rows, upsert_keys, self.update_cols)
+ from pypaimon.execution.update_dispatch import TableUpdateOperations
+ return TableUpdateOperations._upsert_by_arrow_with_key(self, table, upsert_keys, commit_identifier)
- def _merge_into(
- self,
- source: Any,
- on,
- when_matched: Sequence,
- when_not_matched: Sequence,
- commit_identifier: int,
- ) -> List[CommitMessage]:
- from pypaimon.table.data_evolution_merge_into import merge_into
+ def _upsert_by_key(self, rows, upsert_keys: List[str], commit_identifier: int) -> List[CommitMessage]:
+ from pypaimon.execution.update_dispatch import TableUpdateOperations
+ return TableUpdateOperations._upsert_by_key(self, rows, upsert_keys, commit_identifier)
- return merge_into(
- self.table,
- source,
- on=on,
- when_matched=when_matched,
- when_not_matched=when_not_matched,
- commit_user=self.commit_user,
- commit_identifier=commit_identifier,
- )
+ def _merge_into(self, source: Any, on, when_matched: Sequence, when_not_matched: Sequence,
+ commit_identifier: int) -> List[CommitMessage]:
+ from pypaimon.execution.update_dispatch import TableUpdateOperations
+ return TableUpdateOperations._merge_into(self, source, on, when_matched, when_not_matched, commit_identifier)
def _update_by_predicate(
- self,
- predicate: Optional[Predicate],
- assignments: Mapping[str, Any],
- commit_identifier: int,
- read_columns: Optional[Sequence[str]] = None,
- ) -> List[CommitMessage]:
+ self, predicate: Optional[Predicate], assignments: Mapping[str, Any],
+ commit_identifier: int, read_columns: Optional[Sequence[str]] = None) -> List[CommitMessage]:
"""Shared implementation for SQL-like ``UPDATE ... WHERE ...``.
``predicate`` identifies the target rows. Assignment values may be
literals or callables receiving one matched logical file group as an
Arrow table.
"""
- has_callable = any(callable(value) for value in assignments.values())
- has_array = any(
- isinstance(value, (pa.Array, pa.ChunkedArray))
- for value in assignments.values()
- )
- read_columns = tuple(read_columns or ())
- self._validate_predicate_update(
- assignments, read_columns, has_callable, has_array
- )
+ from pypaimon.execution.update_dispatch import TableUpdateOperations
+ return TableUpdateOperations._update_by_predicate(self, predicate, assignments, commit_identifier, read_columns)
- if self.table.options.native_write_enabled():
- native = prepare_native_write_operation(
- 'predicate_update', self.table, self.commit_user,
- predicate, list(assignments))
- if native is not None:
- return native.update(assignments, read_columns)
- return self._execute_predicate_update(
- predicate, assignments, commit_identifier, read_columns,
- has_callable, has_array)
-
- def _delete_by_predicate(
- self, predicate: Optional[Predicate], commit_identifier: int,
- ) -> List[CommitMessage]:
- native = prepare_native_write_operation(
- 'predicate_delete', self.table, self.commit_user, predicate)
- if native is not None:
- return native.delete()
- return self._execute_predicate_delete(predicate, commit_identifier)
+ def _delete_by_predicate(self, predicate: Optional[Predicate], commit_identifier: int) -> List[CommitMessage]:
+ from pypaimon.execution.update_dispatch import TableUpdateOperations
+ return TableUpdateOperations._delete_by_predicate(self, predicate, commit_identifier)
- def _delete_by_row_id(
- self, row_ids: Sequence[int], commit_identifier: int,
- ) -> List[CommitMessage]:
- if len(row_ids):
- native = prepare_native_write_operation(
- 'delete', self.table, self.commit_user)
- if native is not None:
- return native.delete_by_row_id(row_ids)
- return self._delete_rows_with_python(row_ids)
+ def _delete_by_row_id(self, row_ids: Sequence[int], commit_identifier: int) -> List[CommitMessage]:
+ from pypaimon.execution.update_dispatch import TableUpdateOperations
+ return TableUpdateOperations._delete_by_row_id(self, row_ids, commit_identifier)
def _matched_delete_row_ids(self, predicate: Optional[Predicate]) -> List[int]:
- return super()._matched_delete_row_ids(predicate)
+ from pypaimon.execution.update_dispatch import TableUpdateOperations
+ return TableUpdateOperations._matched_delete_row_ids(self, predicate)
_predicate_fields = staticmethod(predicate_fields)
+ @staticmethod
+ def _predicate_update_file_groups(splits):
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._predicate_update_file_groups(splits)
+
+ def _matched_update_scan_table(self):
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._matched_update_scan_table(self)
+
+ def _validate_predicate_update(self, assignments, read_columns, has_callable, has_array):
+ from pypaimon.write.update_validation import _validate_predicate_update
+ return _validate_predicate_update(self, assignments, read_columns, has_callable, has_array)
+
+ def _build_predicate_update_table(self, assignments, matched):
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._build_predicate_update_table(self, assignments, matched)
+
+ @staticmethod
+ def _assignment_to_array(value, data_type, row_count):
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._assignment_to_array(value, data_type, row_count)
+
+ def _delete_by_partition_filter(self, partition_filter):
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._delete_by_partition_filter(self, partition_filter)
+
+ @staticmethod
+ def _partition_delete_message(messages, partition, bucket, snapshot_id):
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._partition_delete_message(messages, partition, bucket, snapshot_id)
+
+ def _partition_index_entries(self, snapshot, partition_filter):
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._partition_index_entries(self, snapshot, partition_filter)
+
+ def _partition_only_delete_filter(self, predicate):
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._partition_only_delete_filter(self, predicate)
+
+ def _rewrite_predicate_to_partition_indices(self, predicate, partition_index):
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._rewrite_predicate_to_partition_indices(self, predicate, partition_index)
+
+ def _execute_predicate_update(self, predicate, assignments, commit_identifier,
+ read_columns, has_callable, has_array):
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._execute_predicate_update(
+ self, predicate, assignments, commit_identifier, read_columns, has_callable, has_array)
+
+ def _execute_predicate_delete(self, predicate, commit_identifier):
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._execute_predicate_delete(self, predicate, commit_identifier)
+
+ def _new_python_shard_updator(self, shard_num, total_shard_count):
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._new_python_shard_updator(self, shard_num, total_shard_count)
+
+ def _delete_rows_with_python(self, row_ids):
+ from pypaimon.execution.python.write.table_update import PythonTableUpdateMixin
+ return PythonTableUpdateMixin._delete_rows_with_python(self, row_ids)
+
class BatchTableUpdate(TableUpdate):
"""Batch-mode table update; commit messages always use
@@ -236,21 +215,15 @@ class BatchTableUpdate(TableUpdate):
"""Apply column updates keyed by ``_ROW_ID`` to existing rows."""
return self._update_by_arrow_with_row_id(table, BATCH_COMMIT_IDENTIFIER)
- def update_by_arrow_batches_with_row_id(
- self, tables: Iterable[pa.Table]
- ) -> List[CommitMessage]:
+ def update_by_arrow_batches_with_row_id(self, tables: Iterable[pa.Table]) -> List[CommitMessage]:
"""Apply row-id updates from batches using one target-file index.
For each updated column, batches must target disjoint ``first_row_id``
file groups. Conflicting overlap is rejected and all files staged by
earlier batches are aborted.
"""
- native = prepare_native_write_operation(
- 'update', self.table, self.commit_user, self.update_cols)
- if native is not None:
- return native.update_by_arrow_batches_with_row_id(tables)
- return self._update_by_arrow_batches_with_row_id(
- tables, BATCH_COMMIT_IDENTIFIER)
+ from pypaimon.execution.update_dispatch import TableUpdateOperations
+ return TableUpdateOperations.update_by_arrow_batches_with_row_id(self, tables)
def upsert_by_arrow_with_key(
self, table: pa.Table, upsert_keys: List[str]
@@ -406,3 +379,56 @@ class StreamTableUpdate(TableUpdate):
when_not_matched,
commit_identifier,
)
+
+
+class ShardTableUpdator:
+
+ def __init__(self, table, projection: Optional[List[str]],
+ write_cols: List[str], commit_user, shard_num: int, total_shard_count: int):
+ from pypaimon.execution.python.write.table_update import ShardTableUpdator
+ return ShardTableUpdator.__init__(self, table, projection, write_cols,
+ commit_user, shard_num, total_shard_count)
+
+ @staticmethod
+ def compute_from_files(files: List[DataFileMeta]) -> List[Range]:
+ from pypaimon.execution.python.write.table_update import ShardTableUpdator
+ return ShardTableUpdator.compute_from_files(files)
+
+ def arrow_reader(self) -> pyarrow.ipc.RecordBatchReader:
+ from pypaimon.execution.python.write.table_update import ShardTableUpdator
+ return ShardTableUpdator.arrow_reader(self)
+
+ def prepare_commit(self) -> List[CommitMessage]:
+ from pypaimon.execution.python.write.table_update import ShardTableUpdator
+ return ShardTableUpdator.prepare_commit(self)
+
+ def update_by_arrow_batch(self, data: pa.RecordBatch):
+ from pypaimon.execution.python.write.table_update import ShardTableUpdator
+ return ShardTableUpdator.update_by_arrow_batch(self, data)
+
+ def _init_writer(self):
+ from pypaimon.execution.python.write.table_update import ShardTableUpdator
+ return ShardTableUpdator._init_writer(self)
+
+
+class SingleWriter:
+
+ def __init__(self, writer: 'DataWriter', partition, first_row_id: int, row_count: int):
+ from pypaimon.execution.python.write.table_update import SingleWriter
+ return SingleWriter.__init__(self, writer, partition, first_row_id, row_count)
+
+ def capacity(self) -> int:
+ from pypaimon.execution.python.write.table_update import SingleWriter
+ return SingleWriter.capacity(self)
+
+ def write(self, data: pa.RecordBatch):
+ from pypaimon.execution.python.write.table_update import SingleWriter
+ return SingleWriter.write(self, data)
+
+ def partition(self) -> Tuple:
+ from pypaimon.execution.python.write.table_update import SingleWriter
+ return SingleWriter.partition(self)
+
+ def end(self) -> DataFileMeta:
+ from pypaimon.execution.python.write.table_update import SingleWriter
+ return SingleWriter.end(self)
diff --git a/paimon-python/pypaimon/write/table_update_by_row_id.py b/paimon-python/pypaimon/write/table_update_by_row_id.py
index ff56bc134a..485f162b88 100644
--- a/paimon-python/pypaimon/write/table_update_by_row_id.py
+++ b/paimon-python/pypaimon/write/table_update_by_row_id.py
@@ -20,7 +20,6 @@ from typing import TYPE_CHECKING, List, Optional
import pyarrow as pa
-from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
from pypaimon.write.commit_message import CommitMessage
from pypaimon.write.row_input import RowIdUpdateInputMixin
@@ -28,14 +27,15 @@ if TYPE_CHECKING:
from pypaimon.execution.python.write.row_id_file_index import RowIdFileIndex
-class TableUpdateByRowId(PythonRowIdUpdateMixin, RowIdUpdateInputMixin, metaclass=ABCMeta):
+class TableUpdateByRowId(RowIdUpdateInputMixin, metaclass=ABCMeta):
"""Incrementally prepare updates addressed by stable row IDs."""
- def __init__(
- self, table, commit_user: str, commit_identifier: int,
- _precomputed_files_info: Optional["RowIdFileIndex"] = None,
- ):
- return super().__init__(table, commit_user, commit_identifier, _precomputed_files_info)
+ FIRST_ROW_ID_COLUMN = '_FIRST_ROW_ID'
+
+ def __init__(self, table, commit_user: str, commit_identifier: int,
+ _precomputed_files_info: Optional['RowIdFileIndex'] = None):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin.__init__(self, table, commit_user, commit_identifier, _precomputed_files_info)
def update_columns(self, data: pa.Table, column_names: List[str]) -> List[CommitMessage]:
"""
@@ -48,4 +48,101 @@ class TableUpdateByRowId(PythonRowIdUpdateMixin, RowIdUpdateInputMixin, metaclas
Returns:
List of commit messages
"""
- return super().update_columns(data, column_names)
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin.update_columns(self, data, column_names)
+
+ def _snapshot_files_info(self):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._snapshot_files_info(self)
+
+ def _load_existing_files_info(self):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._load_existing_files_info(self)
+
+ def _write_row_columns(self, update_data, column_names, blob_object_columns):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._write_row_columns(self, update_data, column_names, blob_object_columns)
+
+ def _calculate_first_row_id(self, data):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._calculate_first_row_id(self, data)
+
+ def _write_by_first_row_id(self, data, column_names, blob_object_columns=None):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._write_by_first_row_id(self, data, column_names, blob_object_columns)
+
+ def _read_original_file_data(self, first_row_id, column_names):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._read_original_file_data(self, first_row_id, column_names)
+
+ def _original_file_read(self, first_row_id, column_names):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._original_file_read(self, first_row_id, column_names)
+
+ def _merged_batches(self, first_row_id, data, column_names):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._merged_batches(self, first_row_id, data, column_names)
+
+ def _write_group_streaming(self, partition, first_row_id, data, column_names):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._write_group_streaming(self, partition, first_row_id, data, column_names)
+
+ def _merge_update_with_original(self, original_data, update_data, column_names,
+ first_row_id, blob_object_columns=None, blob_columns_with_baseline=None):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._merge_update_with_original(
+ self, original_data, update_data, column_names, first_row_id,
+ blob_object_columns, blob_columns_with_baseline)
+
+ def _normalize_write_data(self, data):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._normalize_write_data(self, data)
+
+ def _check_write_not_null(self, column_name, has_null):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._check_write_not_null(self, column_name, has_null)
+
+ @classmethod
+ def _merge_chunked_column(cls, original_col, update_col, sorted_updates):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._merge_chunked_column.__func__(cls, original_col, update_col, sorted_updates)
+
+ @classmethod
+ def _merge_chunk_with_updates(cls, original, update_col, update_chunk_offsets, updates):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._merge_chunk_with_updates.__func__(
+ cls, original, update_col, update_chunk_offsets, updates)
+
+ @staticmethod
+ def _chunk_offsets(column):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._chunk_offsets(column)
+
+ @staticmethod
+ def _take_from_chunked_array(column, chunk_offsets, indices):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._take_from_chunked_array(column, chunk_offsets, indices)
+
+ @staticmethod
+ def _is_offset_overflow(error):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._is_offset_overflow(error)
+
+ @staticmethod
+ def _coerce_column(col, target_type):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._coerce_column(col, target_type)
+
+ def _blob_placeholder(self, column_name):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._blob_placeholder(self, column_name)
+
+ def _write_group(self, partition, first_row_id, data, column_names, blob_object_columns=None):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._write_group(
+ self, partition, first_row_id, data, column_names, blob_object_columns)
+
+ @staticmethod
+ def _assign_update_file_metadata(new_files, first_row_id, column_names, blob_columns):
+ from pypaimon.execution.python.write.table_update_by_row_id import PythonRowIdUpdateMixin
+ return PythonRowIdUpdateMixin._assign_update_file_metadata(new_files, first_row_id, column_names, blob_columns)
diff --git a/paimon-python/pypaimon/write/table_upsert_by_key.py b/paimon-python/pypaimon/write/table_upsert_by_key.py
index 55f0594758..200358d73a 100644
--- a/paimon-python/pypaimon/write/table_upsert_by_key.py
+++ b/paimon-python/pypaimon/write/table_upsert_by_key.py
@@ -19,13 +19,10 @@ from typing import List, Optional
import pyarrow as pa
-from pypaimon.execution.dispatch import prepare_native_write_operation
-from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
from pypaimon.write.commit_message import CommitMessage
-from pypaimon.write.row_utils import row_to_named_values
-class TableUpsertByKey(PythonTableUpsertMixin):
+class TableUpsertByKey:
"""Upsert append-table rows using one or more user-selected key columns."""
def __init__(self, table, commit_user: str, commit_identifier: int):
@@ -62,49 +59,85 @@ class TableUpsertByKey(PythonTableUpsertMixin):
Returns:
List of CommitMessages to be committed.
"""
- self._validate_inputs(data, upsert_keys, update_cols)
-
- # Determine which columns to update
- if update_cols is None or len(update_cols) == len(self.table.field_names):
- effective_update_cols = None # means all columns
- else:
- effective_update_cols = update_cols
-
- columns = (list(effective_update_cols) if effective_update_cols
- else list(self.table.field_names))
- native = self._create_native_upsert(data, upsert_keys, columns)
- if native is not None:
- return native.upsert()
-
- return self._upsert_arrow(data, upsert_keys, effective_update_cols)
-
- def upsert_rows(self, rows, upsert_keys: List[str],
- update_cols: Optional[List[str]] = None) -> List[CommitMessage]:
- row_list = self._normalize_rows(rows)
- if not row_list:
- raise ValueError("rows must not be empty.")
-
- row_items = [
- (row, row_to_named_values(row, self.table.table_schema.fields))
- for row in row_list
- ]
- for _, values_by_name in row_items:
- self._validate_row_inputs(values_by_name, upsert_keys, update_cols)
-
- if update_cols is None or len(update_cols) == len(self.table.field_names):
- effective_update_cols = None
- else:
- effective_update_cols = update_cols
-
- columns = (list(effective_update_cols) if effective_update_cols is not None
- else list(self.table.field_names))
- native = self._create_native_upsert(
- [values for _, values in row_items], upsert_keys, columns)
- if native is not None:
- return native.upsert()
-
- return self._upsert_rows(row_items, upsert_keys, effective_update_cols)
+ from pypaimon.execution.update_dispatch import UpsertOperations
+ return UpsertOperations.upsert(self, data, upsert_keys, update_cols)
+
+ def upsert_rows(self, rows, upsert_keys: List[str], update_cols: Optional[List[str]] = None) -> List[CommitMessage]:
+ from pypaimon.execution.update_dispatch import UpsertOperations
+ return UpsertOperations.upsert_rows(self, rows, upsert_keys, update_cols)
def _create_native_upsert(self, data, upsert_keys, columns):
- return prepare_native_write_operation(
- 'upsert', self.table, self.commit_user, data, upsert_keys, columns)
+ from pypaimon.execution.update_dispatch import UpsertOperations
+ return UpsertOperations._create_native_upsert(self, data, upsert_keys, columns)
+
+ def _new_row_id_updater(self):
+ from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
+ return PythonTableUpsertMixin._new_row_id_updater(self)
+
+ @staticmethod
+ def _normalize_rows(rows):
+ from pypaimon.write.update_validation import _normalize_rows
+ return _normalize_rows(rows)
+
+ def _group_rows_by_partition(self, row_items):
+ from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
+ return PythonTableUpsertMixin._group_rows_by_partition(self, row_items)
+
+ def _upsert_row_partition(self, row_items, upsert_keys, partition_spec, update_cols):
+ from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
+ return PythonTableUpsertMixin._upsert_row_partition(self, row_items, upsert_keys, partition_spec, update_cols)
+
+ @staticmethod
+ def _dedup_row_items_last_write_wins(row_items, input_key_tuples, partition_spec):
+ from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
+ return PythonTableUpsertMixin._dedup_row_items_last_write_wins(row_items, input_key_tuples, partition_spec)
+
+ def _group_by_partition(self, data):
+ from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
+ return PythonTableUpsertMixin._group_by_partition(self, data)
+
+ def _upsert_partition(self, partition_data, upsert_keys, partition_spec, update_cols):
+ from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
+ return PythonTableUpsertMixin._upsert_partition(self, partition_data, upsert_keys, partition_spec, update_cols)
+
+ @staticmethod
+ def _dedup_last_write_wins(partition_data, input_key_tuples, partition_spec):
+ from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
+ return PythonTableUpsertMixin._dedup_last_write_wins(partition_data, input_key_tuples, partition_spec)
+
+ def _validate_inputs(self, data, upsert_keys, update_cols):
+ from pypaimon.write.update_validation import _validate_inputs
+ return _validate_inputs(self, data, upsert_keys, update_cols)
+
+ def _validate_row_inputs(self, values_by_name, upsert_keys, update_cols):
+ from pypaimon.write.update_validation import _validate_row_inputs
+ return _validate_row_inputs(self, values_by_name, upsert_keys, update_cols)
+
+ def _build_key_to_row_ids_map(self, match_keys, partition_spec, input_key_set):
+ from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
+ return PythonTableUpsertMixin._build_key_to_row_ids_map(self, match_keys, partition_spec, input_key_set)
+
+ def _do_updates(self, data, matched_indices, input_key_tuples, key_to_row_ids, update_cols):
+ from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
+ return PythonTableUpsertMixin._do_updates(
+ self, data, matched_indices, input_key_tuples, key_to_row_ids, update_cols)
+
+ def _do_appends(self, data, new_indices):
+ from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
+ return PythonTableUpsertMixin._do_appends(self, data, new_indices)
+
+ def _append_rows(self, row_items):
+ from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
+ return PythonTableUpsertMixin._append_rows(self, row_items)
+
+ def _append_row_column_names(self, row_items):
+ from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
+ return PythonTableUpsertMixin._append_row_column_names(self, row_items)
+
+ def _upsert_arrow(self, data, upsert_keys, effective_update_cols):
+ from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
+ return PythonTableUpsertMixin._upsert_arrow(self, data, upsert_keys, effective_update_cols)
+
+ def _upsert_rows(self, row_items, upsert_keys, effective_update_cols):
+ from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsertMixin
+ return PythonTableUpsertMixin._upsert_rows(self, row_items, upsert_keys, effective_update_cols)
diff --git a/paimon-python/pypaimon/write/table_write.py b/paimon-python/pypaimon/write/table_write.py
index 27548f026b..b9bd2515b2 100644
--- a/paimon-python/pypaimon/write/table_write.py
+++ b/paimon-python/pypaimon/write/table_write.py
@@ -19,7 +19,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional
import pyarrow as pa
-from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER
from pypaimon.table.row.blob import BlobConsumer
from pypaimon.write.commit_message import CommitMessage
@@ -28,38 +27,47 @@ if TYPE_CHECKING:
from ray.data import Dataset
-class TableWrite(PythonTableWriteMixin):
+class TableWrite:
"""Public writer API backed by the selected execution implementation."""
- def __init__(self, table, commit_user, static_partition: Optional[dict] = None,
- *, restore_snapshot_id=None):
- return super().__init__(table, commit_user, static_partition, restore_snapshot_id=restore_snapshot_id)
+ def __init__(self, table, commit_user, static_partition: Optional[dict] = None, *, restore_snapshot_id=None):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin.__init__(
+ self, table, commit_user, static_partition, restore_snapshot_id=restore_snapshot_id)
def write_arrow(self, data, bucket: Optional[int] = None):
"""Write Arrow data, optionally to a bucket assigned upstream."""
- return super().write_arrow(data, bucket)
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin.write_arrow(self, data, bucket)
def write_arrow_batch(self, data: pa.RecordBatch, bucket: Optional[int] = None):
- return super().write_arrow_batch(data, bucket)
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin.write_arrow_batch(self, data, bucket)
def roll_before_group_if_needed(self, row_count: int):
"""Roll current files before the next logical write group if needed."""
- return super().roll_before_group_if_needed(row_count)
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin.roll_before_group_if_needed(self, row_count)
def write_row(self, row):
- return super().write_row(row)
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin.write_row(self, row)
def write_pandas(self, dataframe):
- return super().write_pandas(dataframe)
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin.write_pandas(self, dataframe)
def with_write_type(self, write_cols: List[str]):
- return super().with_write_type(write_cols)
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin.with_write_type(self, write_cols)
def with_blob_consumer(self, blob_consumer: BlobConsumer):
- return super().with_blob_consumer(blob_consumer)
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin.with_blob_consumer(self, blob_consumer)
def with_blob_uri_reader_factory(self, uri_reader_factory):
- return super().with_blob_uri_reader_factory(uri_reader_factory)
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin.with_blob_uri_reader_factory(self, uri_reader_factory)
def write_ray(
self,
@@ -89,14 +97,79 @@ class TableWrite(PythonTableWriteMixin):
the Ray write runs in overwrite mode for this partition and
overrides any builder-level partition spec.
"""
- return super().write_ray(
- dataset, overwrite, concurrency, ray_remote_args, hash_fixed_precluster, static_partition)
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin.write_ray(self, dataset, overwrite, concurrency,
+ ray_remote_args, hash_fixed_precluster, static_partition)
def close(self):
- return super().close()
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin.close(self)
def abort(self):
- return super().abort()
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin.abort(self)
+
+ def _create_file_store_write(self, commit_user):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._create_file_store_write(self, commit_user)
+
+ def _create_row_key_extractor(self, static_partition):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._create_row_key_extractor(self, static_partition)
+
+ def _write_partition_bucket_batch(self, partition, bucket, data):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._write_partition_bucket_batch(self, partition, bucket, data)
+
+ def _write_precomputed_bucket(self, data, bucket):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._write_precomputed_bucket(self, data, bucket)
+
+ def _write_partition_bucket_row(self, partition, bucket, row, values_by_name):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._write_partition_bucket_row(self, partition, bucket, row, values_by_name)
+
+ def _distributed_write_options(self, *, overwrite=False):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._distributed_write_options(self, overwrite=overwrite)
+
+ def _prepare_commit(self, commit_identifier):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._prepare_commit(self, commit_identifier)
+
+ def _release_prepared_indexes(self):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._release_prepared_indexes(self)
+
+ def _prepare_arrow_data(self, data):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._prepare_arrow_data(self, data)
+
+ def _prepare_arrow_schema(self, data):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._prepare_arrow_schema(self, data)
+
+ def _normalize_arrow_write_schema(self, data):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._normalize_arrow_write_schema(self, data)
+
+ def _validate_pyarrow_schema(self, data_schema, *, check_nested_nullability=True):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._validate_pyarrow_schema(
+ self, data_schema, check_nested_nullability=check_nested_nullability)
+
+ def _is_compatible_pyarrow_schema(self, data_schema, expected_schema, *, check_nested_nullability=True):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._is_compatible_pyarrow_schema(
+ self, data_schema, expected_schema, check_nested_nullability=check_nested_nullability)
+
+ def _write_cols_pyarrow_schema(self, write_cols):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._write_cols_pyarrow_schema(self, write_cols)
+
+ def _raise_inconsistent_schema(self, data_schema):
+ from pypaimon.execution.python.write.table_write import PythonTableWriteMixin
+ return PythonTableWriteMixin._raise_inconsistent_schema(self, data_schema)
class BatchTableWrite(TableWrite):
diff --git a/paimon-python/pypaimon/write/update_validation.py b/paimon-python/pypaimon/write/update_validation.py
index 06fb98a398..b07c1fc27c 100644
--- a/paimon-python/pypaimon/write/update_validation.py
+++ b/paimon-python/pypaimon/write/update_validation.py
@@ -15,7 +15,13 @@
# specific language governing permissions and limitations
# under the License.
+from typing import Any, Dict, List, Mapping, Optional, Sequence
+
+import pyarrow as pa
+
from pypaimon.common.predicate import Predicate
+from pypaimon.table.row.internal_row import InternalRow
+from pypaimon.write.row_utils import require_columns
def _authorize_update(table):
@@ -35,3 +41,90 @@ def predicate_fields(predicate: Predicate) -> set:
for child in predicate.literals or []:
fields.update(predicate_fields(child))
return fields
+
+
+def _validate_predicate_update(
+ self, assignments: Mapping[str, Any], read_columns: Optional[Sequence[str]],
+ has_callable: bool, has_array: bool):
+ if not self.table.options.data_evolution_enabled():
+ raise ValueError("update_by_predicate requires 'data-evolution.enabled' = 'true'.")
+ if not self.table.options.row_tracking_enabled():
+ raise ValueError("update_by_predicate requires 'row-tracking.enabled' = 'true'.")
+ if not assignments:
+ raise ValueError('assignments must not be empty.')
+ if read_columns and (not has_callable):
+ raise ValueError('read_columns requires a callable assignment.')
+ if has_callable:
+ if has_array:
+ raise ValueError('Callable assignments cannot be combined with Arrow array assignments.')
+ if not read_columns:
+ raise ValueError('Callable assignments require read_columns.')
+ for col in read_columns:
+ if col not in self.table.field_names:
+ raise ValueError(f'Read column {col} is not in table schema.')
+ partition_keys = set(self.table.partition_keys)
+ for col in assignments:
+ if col not in self.table.field_names:
+ raise ValueError(f'Column {col} is not in table schema.')
+ if col in partition_keys:
+ raise ValueError(f"update_by_predicate does not support updating partition column '{col}'.")
+
+
+def _validate_inputs(self, data: pa.Table, upsert_keys: List[str], update_cols: Optional[List[str]]):
+ """Validate inputs before processing."""
+ if not self.table.options.data_evolution_enabled():
+ raise ValueError("upsert_by_arrow_with_key requires 'data-evolution.enabled' = 'true'.")
+ if not self.table.options.row_tracking_enabled():
+ raise ValueError("upsert_by_arrow_with_key requires 'row-tracking.enabled' = 'true'.")
+ if not upsert_keys:
+ raise ValueError('upsert_keys must not be empty.')
+ for key in upsert_keys:
+ if key not in self.table.field_names:
+ raise ValueError(f"upsert_key '{key}' is not in table schema fields: {self.table.field_names}")
+ if key not in data.column_names:
+ raise ValueError(f"upsert_key '{key}' is not in input data columns: {data.column_names}")
+ partition_keys = self.table.partition_keys
+ if partition_keys:
+ missing_in_data = [pk for pk in partition_keys if pk not in data.column_names]
+ if missing_in_data:
+ raise ValueError(
+ 'For partitioned tables, input data must contain all partition key '
+ f'columns. Missing: {missing_in_data}')
+ if update_cols is not None:
+ for col in update_cols:
+ if col not in self.table.field_names:
+ raise ValueError(
+ f"Column '{col}' in update_cols is not in table schema fields: {self.table.field_names}")
+ if data.num_rows == 0:
+ raise ValueError('Input data is empty.')
+
+
+def _validate_row_inputs(self, values_by_name: Dict[str, Any],
+ upsert_keys: List[str], update_cols: Optional[List[str]]):
+ if not self.table.options.data_evolution_enabled():
+ raise ValueError("upsert_by_key requires 'data-evolution.enabled' = 'true'.")
+ if not self.table.options.row_tracking_enabled():
+ raise ValueError("upsert_by_key requires 'row-tracking.enabled' = 'true'.")
+ if not upsert_keys:
+ raise ValueError('upsert_keys must not be empty.')
+ for key in upsert_keys:
+ if key not in self.table.field_names:
+ raise ValueError(f"upsert_key '{key}' is not in table schema fields: {self.table.field_names}")
+ unknown_fields = [field_name for field_name in values_by_name if field_name not in self.table.field_names]
+ if unknown_fields:
+ raise ValueError(
+ f'upsert_by_key got row field(s) {unknown_fields} '
+ f'that are not in table schema fields: {self.table.field_names}')
+ require_columns(values_by_name, upsert_keys, 'upsert_by_key')
+ require_columns(values_by_name, self.table.partition_keys, 'upsert_by_key')
+ if update_cols is not None:
+ for col in update_cols:
+ if col not in self.table.field_names:
+ raise ValueError(
+ f"Column '{col}' in update_cols is not in table schema fields: {self.table.field_names}")
+
+
+def _normalize_rows(rows) -> List:
+ if isinstance(rows, InternalRow):
+ return [rows]
+ return list(rows)