You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/10/22 05:43:05 UTC
(paimon) branch master updated: [Python] Move PK index planning into execution and remove bundled samples (#10741)
This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 2526367dd8 [Python] Move PK index planning into execution and remove bundled samples (#10741)
2526367dd8 is described below
commit 2526367dd8e1f32795b1b5801f7eb1ca6910a38d
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Thu Oct 22 13:42:59 2026 +0800
[Python] Move PK index planning into execution and remove bundled samples (#10741)
---
.github/workflows/release-python.yml | 5 -
paimon-python/MANIFEST.in | 1 -
paimon-python/dev/test_packaging.py | 9 +-
paimon-python/pypaimon/execution/README.md | 7 +
.../python/index}/__init__.py | 2 +
.../python/index/pk}/__init__.py | 2 +
.../index/pk/primary_key_index_source_policy.py | 0
.../python}/index/pksorted/__init__.py | 0
.../index/pksorted/pk_sorted_bucket_index_state.py | 4 +-
.../index/pksorted/pk_sorted_index_group.py | 0
.../read/search/primary_key_full_text_scan.py | 2 +-
.../read/search/primary_key_sorted_index_scan.py | 4 +-
.../python/read/search/primary_key_vector_scan.py | 2 +-
paimon-python/pypaimon/sample/__init__.py | 17 --
paimon-python/pypaimon/sample/data/__init__.py | 17 --
paimon-python/pypaimon/sample/data/data.jsonl | 100 ---------
.../pypaimon/sample/oss_read_and_write.py | 70 ------
.../rest_catalog_blob_as_descriptor_sample.py | 154 -------------
.../sample/rest_catalog_ray_data_sample.py | 245 ---------------------
.../sample/rest_catalog_ray_lance_sample.py | 212 ------------------
.../sample/rest_catalog_ray_sink_sample.py | 153 -------------
.../sample/rest_catalog_read_write_sample.py | 108 ---------
.../tests/index_execution_boundary_test.py | 175 +++++++++++++++
.../tests/primary_key_sorted_index_scan_test.py | 2 +-
pom.xml | 1 -
25 files changed, 200 insertions(+), 1092 deletions(-)
diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml
index fe32d56998..336dabb077 100644
--- a/.github/workflows/release-python.yml
+++ b/.github/workflows/release-python.yml
@@ -114,9 +114,6 @@ jobs:
"pypaimon/_version.py", "pypaimon/_full_version"):
if prefix + required not in names:
raise SystemExit("Missing {} from sdist".format(required))
- sample_data = "pypaimon/sample/data/data.jsonl"
- if prefix + sample_data not in names:
- raise SystemExit("Missing sample data from sdist")
for forbidden in ("pypaimon/tests/",):
if any(name.startswith(prefix + forbidden) for name in names):
raise SystemExit(
@@ -135,8 +132,6 @@ jobs:
]
if not license_files or not notice_files:
raise SystemExit("Wheel is missing LICENSE or NOTICE")
- if "pypaimon/sample/data/data.jsonl" not in names:
- raise SystemExit("Wheel is missing sample data")
for forbidden in ("pypaimon/tests/",):
if any(name.startswith(forbidden) for name in names):
raise SystemExit(
diff --git a/paimon-python/MANIFEST.in b/paimon-python/MANIFEST.in
index a0d8cabd27..f554ed80ab 100644
--- a/paimon-python/MANIFEST.in
+++ b/paimon-python/MANIFEST.in
@@ -20,5 +20,4 @@ include NOTICE
include pyproject.toml
include README.md
recursive-include pypaimon *.py
-recursive-include pypaimon/sample/data *.jsonl
recursive-exclude pypaimon/tests *
diff --git a/paimon-python/dev/test_packaging.py b/paimon-python/dev/test_packaging.py
index 89fabc9003..428fbac0ee 100644
--- a/paimon-python/dev/test_packaging.py
+++ b/paimon-python/dev/test_packaging.py
@@ -119,8 +119,7 @@ class PackagingTest(unittest.TestCase):
self.assertFalse(any(name.startswith("pypaimon/tests/")
for name in names))
self.assertFalse(any(name.endswith((".so", ".pyd", ".jar")) for name in names))
- for name in ("pypaimon/sample/data/data.jsonl", "pypaimon/_version.py"):
- self.assertIn(name, names)
+ self.assertIn("pypaimon/_version.py", names)
execution_files = (
"pypaimon/table/source/search_execution.py",
"pypaimon/ray/search_execution.py",
@@ -189,6 +188,9 @@ class PackagingTest(unittest.TestCase):
"pypaimon/execution/python/globalindex/btree/block_handle.py",
"pypaimon/execution/python/globalindex/block_compression.py",
"pypaimon/execution/python/globalindex/key_serializer.py",
+ "pypaimon/execution/python/index/pk/primary_key_index_source_policy.py",
+ "pypaimon/execution/python/index/pksorted/pk_sorted_bucket_index_state.py",
+ "pypaimon/execution/python/index/pksorted/pk_sorted_index_group.py",
"pypaimon/execution/python/globalindex/vindex/vindex_vector_global_index_reader.py",
"pypaimon/execution/python/write/table_write.py",
"pypaimon/execution/python/write/table_commit.py",
@@ -254,6 +256,9 @@ class PackagingTest(unittest.TestCase):
"pypaimon/globalindex/btree_format/block_handle.py",
"pypaimon/globalindex/bitmap/bitmap_index_writer.py",
"pypaimon/globalindex/block_compression.py",
+ "pypaimon/index/pk/primary_key_index_source_policy.py",
+ "pypaimon/index/pksorted/pk_sorted_bucket_index_state.py",
+ "pypaimon/index/pksorted/pk_sorted_index_group.py",
)
for name in internal_aliases:
self.assertNotIn(name, names)
diff --git a/paimon-python/pypaimon/execution/README.md b/paimon-python/pypaimon/execution/README.md
index ddc9cc06cd..1ab8e23ed1 100644
--- a/paimon-python/pypaimon/execution/README.md
+++ b/paimon-python/pypaimon/execution/README.md
@@ -57,6 +57,7 @@ pypaimon/
globalindex/ Public index contracts and backend coordination
index_dispatch.py Index selection and Vindex resource ownership
vindex_input.py Python SDK/Native callback streams and ownership
+ index/ Shared index metadata, definitions and input routing
table/source/
search_dispatch.py Search selection and transported-plan coordination
table/system/ SystemSplit metadata and common Arrow output
@@ -77,6 +78,7 @@ pypaimon/
write/ File writers, committers and update execution
external_path_provider.py Per-writer external file placement strategies
globalindex/ Index file codecs, readers, writers and build algorithms
+ index/ Primary-key index source selection and sorted-index grouping
merge_engine_dispatch.py Python merge-function selection
native/
runtime.py Lazy SDK discovery and process safety
@@ -104,6 +106,11 @@ execution, corresponding to Rust Core's deletion-vector implementation.
`DeletionFile` and `DeletionVectorMeta` remain shared metadata in `table/source`
and `index`; importing or transporting them does not load either backend.
+Primary-key index source selection, payload eligibility and sorted-index coverage
+planning belong to `execution/python/index`. Index definitions, source metadata
+and their wire formats remain in `index`, shared with public builders and Native
+adapters. Native index planning performs source selection and grouping in Rust.
+
Arrow read schemas and logical type-change rules stay in `schema`. Shared
numeric, calendar and metadata text helpers stay in `utils`; Native read and
commit adapters use them without importing Python cast executors. Array value
diff --git a/paimon-python/pypaimon/index/pksorted/__init__.py b/paimon-python/pypaimon/execution/python/index/__init__.py
similarity index 94%
copy from paimon-python/pypaimon/index/pksorted/__init__.py
copy to paimon-python/pypaimon/execution/python/index/__init__.py
index aa678ede1d..d40cb1108a 100644
--- a/paimon-python/pypaimon/index/pksorted/__init__.py
+++ b/paimon-python/pypaimon/execution/python/index/__init__.py
@@ -13,3 +13,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+"""Python index planning implementations."""
diff --git a/paimon-python/pypaimon/index/pksorted/__init__.py b/paimon-python/pypaimon/execution/python/index/pk/__init__.py
similarity index 92%
copy from paimon-python/pypaimon/index/pksorted/__init__.py
copy to paimon-python/pypaimon/execution/python/index/pk/__init__.py
index aa678ede1d..eb18162abc 100644
--- a/paimon-python/pypaimon/index/pksorted/__init__.py
+++ b/paimon-python/pypaimon/execution/python/index/pk/__init__.py
@@ -13,3 +13,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+"""Primary-key index source selection for Python scans."""
diff --git a/paimon-python/pypaimon/index/pk/primary_key_index_source_policy.py b/paimon-python/pypaimon/execution/python/index/pk/primary_key_index_source_policy.py
similarity index 100%
rename from paimon-python/pypaimon/index/pk/primary_key_index_source_policy.py
rename to paimon-python/pypaimon/execution/python/index/pk/primary_key_index_source_policy.py
diff --git a/paimon-python/pypaimon/index/pksorted/__init__.py b/paimon-python/pypaimon/execution/python/index/pksorted/__init__.py
similarity index 100%
rename from paimon-python/pypaimon/index/pksorted/__init__.py
rename to paimon-python/pypaimon/execution/python/index/pksorted/__init__.py
diff --git a/paimon-python/pypaimon/index/pksorted/pk_sorted_bucket_index_state.py b/paimon-python/pypaimon/execution/python/index/pksorted/pk_sorted_bucket_index_state.py
similarity index 95%
rename from paimon-python/pypaimon/index/pksorted/pk_sorted_bucket_index_state.py
rename to paimon-python/pypaimon/execution/python/index/pksorted/pk_sorted_bucket_index_state.py
index c541c9d798..a66bd0746c 100644
--- a/paimon-python/pypaimon/index/pksorted/pk_sorted_bucket_index_state.py
+++ b/paimon-python/pypaimon/execution/python/index/pksorted/pk_sorted_bucket_index_state.py
@@ -18,8 +18,8 @@ from dataclasses import dataclass
from pypaimon.index.pk.primary_key_index_source_file import PrimaryKeyIndexSourceFile
from pypaimon.index.pk.primary_key_index_source_meta import PrimaryKeyIndexSourceMeta
-from pypaimon.index.pk.primary_key_index_source_policy import should_read
-from pypaimon.index.pksorted.pk_sorted_index_group import PkSortedIndexGroup
+from pypaimon.execution.python.index.pk.primary_key_index_source_policy import should_read
+from pypaimon.execution.python.index.pksorted.pk_sorted_index_group import PkSortedIndexGroup
@dataclass(frozen=True)
diff --git a/paimon-python/pypaimon/index/pksorted/pk_sorted_index_group.py b/paimon-python/pypaimon/execution/python/index/pksorted/pk_sorted_index_group.py
similarity index 100%
rename from paimon-python/pypaimon/index/pksorted/pk_sorted_index_group.py
rename to paimon-python/pypaimon/execution/python/index/pksorted/pk_sorted_index_group.py
diff --git a/paimon-python/pypaimon/execution/python/read/search/primary_key_full_text_scan.py b/paimon-python/pypaimon/execution/python/read/search/primary_key_full_text_scan.py
index a08ba3bb10..0e0b9007a2 100644
--- a/paimon-python/pypaimon/execution/python/read/search/primary_key_full_text_scan.py
+++ b/paimon-python/pypaimon/execution/python/read/search/primary_key_full_text_scan.py
@@ -24,7 +24,7 @@ from pypaimon.globalindex.indexed_split import IndexedSplit
from pypaimon.index.index_file_handler import IndexFileHandler
from pypaimon.index.pk.primary_key_index_source_meta import (
PrimaryKeyIndexSourceMeta)
-from pypaimon.index.pk.primary_key_index_source_policy import (
+from pypaimon.execution.python.index.pk.primary_key_index_source_policy import (
should_read as _should_read_source)
from pypaimon.read.query_auth_split import QueryAuthSplit
from pypaimon.read.split import DataSplit
diff --git a/paimon-python/pypaimon/execution/python/read/search/primary_key_sorted_index_scan.py b/paimon-python/pypaimon/execution/python/read/search/primary_key_sorted_index_scan.py
index 290394f64b..18678214ec 100644
--- a/paimon-python/pypaimon/execution/python/read/search/primary_key_sorted_index_scan.py
+++ b/paimon-python/pypaimon/execution/python/read/search/primary_key_sorted_index_scan.py
@@ -24,8 +24,8 @@ from pypaimon.globalindex.global_index_result import GlobalIndexResult
from pypaimon.execution.python.globalindex.data_evolution_global_index_scanner import _create_inner_readers
from pypaimon.common.options.core_options import CoreOptions
from pypaimon.index.pk.primary_key_index_source_file import PrimaryKeyIndexSourceFile
-from pypaimon.index.pk.primary_key_index_source_policy import should_read
-from pypaimon.index.pksorted.pk_sorted_bucket_index_state import PkSortedBucketIndexState
+from pypaimon.execution.python.index.pk.primary_key_index_source_policy import should_read
+from pypaimon.execution.python.index.pksorted.pk_sorted_bucket_index_state import PkSortedBucketIndexState
from pypaimon.utils.roaring_bitmap import RoaringBitmap64
diff --git a/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py b/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py
index ffe67e5fc0..7b472a33cf 100644
--- a/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py
+++ b/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py
@@ -24,7 +24,7 @@ from pypaimon.common.options.core_options import CoreOptions
from pypaimon.common.options.options import Options
from pypaimon.index.index_file_handler import IndexFileHandler
from pypaimon.index.pk.primary_key_index_source_meta import PrimaryKeyIndexSourceMeta
-from pypaimon.index.pk.primary_key_index_source_policy import (
+from pypaimon.execution.python.index.pk.primary_key_index_source_policy import (
should_read as _should_read_source)
from pypaimon.read.query_auth_split import QueryAuthSplit
from pypaimon.read.split import DataSplit
diff --git a/paimon-python/pypaimon/sample/__init__.py b/paimon-python/pypaimon/sample/__init__.py
deleted file mode 100644
index 65b48d4d79..0000000000
--- a/paimon-python/pypaimon/sample/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
diff --git a/paimon-python/pypaimon/sample/data/__init__.py b/paimon-python/pypaimon/sample/data/__init__.py
deleted file mode 100644
index 65b48d4d79..0000000000
--- a/paimon-python/pypaimon/sample/data/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
diff --git a/paimon-python/pypaimon/sample/data/data.jsonl b/paimon-python/pypaimon/sample/data/data.jsonl
deleted file mode 100644
index f33de19c57..0000000000
--- a/paimon-python/pypaimon/sample/data/data.jsonl
+++ /dev/null
@@ -1,100 +0,0 @@
-{"id": 1, "name": "Item_1", "category": "B", "value": 12.8, "score": 55, "timestamp": "2024-01-02T10:00:00Z"}
-{"id": 2, "name": "Item_2", "category": "C", "value": 15.1, "score": 60, "timestamp": "2024-01-03T10:00:00Z"}
-{"id": 3, "name": "Item_3", "category": "A", "value": 17.4, "score": 65, "timestamp": "2024-01-04T10:00:00Z"}
-{"id": 4, "name": "Item_4", "category": "B", "value": 19.7, "score": 70, "timestamp": "2024-01-05T10:00:00Z"}
-{"id": 5, "name": "Item_5", "category": "C", "value": 22.0, "score": 75, "timestamp": "2024-01-06T10:00:00Z"}
-{"id": 6, "name": "Item_6", "category": "A", "value": 24.299999999999997, "score": 80, "timestamp": "2024-01-07T10:00:00Z"}
-{"id": 7, "name": "Item_7", "category": "B", "value": 26.599999999999998, "score": 85, "timestamp": "2024-01-08T10:00:00Z"}
-{"id": 8, "name": "Item_8", "category": "C", "value": 28.9, "score": 90, "timestamp": "2024-01-09T10:00:00Z"}
-{"id": 9, "name": "Item_9", "category": "A", "value": 31.2, "score": 95, "timestamp": "2024-01-10T10:00:00Z"}
-{"id": 10, "name": "Item_10", "category": "B", "value": 33.5, "score": 100, "timestamp": "2024-01-11T10:00:00Z"}
-{"id": 11, "name": "Item_11", "category": "C", "value": 35.8, "score": 105, "timestamp": "2024-01-12T10:00:00Z"}
-{"id": 12, "name": "Item_12", "category": "A", "value": 38.099999999999994, "score": 110, "timestamp": "2024-01-13T10:00:00Z"}
-{"id": 13, "name": "Item_13", "category": "B", "value": 40.4, "score": 115, "timestamp": "2024-01-14T10:00:00Z"}
-{"id": 14, "name": "Item_14", "category": "C", "value": 42.699999999999996, "score": 120, "timestamp": "2024-01-15T10:00:00Z"}
-{"id": 15, "name": "Item_15", "category": "A", "value": 45.0, "score": 125, "timestamp": "2024-01-16T10:00:00Z"}
-{"id": 16, "name": "Item_16", "category": "B", "value": 47.3, "score": 130, "timestamp": "2024-01-17T10:00:00Z"}
-{"id": 17, "name": "Item_17", "category": "C", "value": 49.599999999999994, "score": 135, "timestamp": "2024-01-18T10:00:00Z"}
-{"id": 18, "name": "Item_18", "category": "A", "value": 51.9, "score": 140, "timestamp": "2024-01-19T10:00:00Z"}
-{"id": 19, "name": "Item_19", "category": "B", "value": 54.199999999999996, "score": 145, "timestamp": "2024-01-20T10:00:00Z"}
-{"id": 20, "name": "Item_20", "category": "C", "value": 56.5, "score": 150, "timestamp": "2024-01-21T10:00:00Z"}
-{"id": 21, "name": "Item_21", "category": "A", "value": 58.8, "score": 155, "timestamp": "2024-01-22T10:00:00Z"}
-{"id": 22, "name": "Item_22", "category": "B", "value": 61.099999999999994, "score": 160, "timestamp": "2024-01-23T10:00:00Z"}
-{"id": 23, "name": "Item_23", "category": "C", "value": 63.4, "score": 165, "timestamp": "2024-01-24T10:00:00Z"}
-{"id": 24, "name": "Item_24", "category": "A", "value": 65.69999999999999, "score": 170, "timestamp": "2024-01-25T10:00:00Z"}
-{"id": 25, "name": "Item_25", "category": "B", "value": 68.0, "score": 175, "timestamp": "2024-01-26T10:00:00Z"}
-{"id": 26, "name": "Item_26", "category": "C", "value": 70.3, "score": 180, "timestamp": "2024-01-27T10:00:00Z"}
-{"id": 27, "name": "Item_27", "category": "A", "value": 72.6, "score": 185, "timestamp": "2024-01-28T10:00:00Z"}
-{"id": 28, "name": "Item_28", "category": "B", "value": 74.89999999999999, "score": 190, "timestamp": "2024-01-01T10:00:00Z"}
-{"id": 29, "name": "Item_29", "category": "C", "value": 77.19999999999999, "score": 195, "timestamp": "2024-01-02T10:00:00Z"}
-{"id": 30, "name": "Item_30", "category": "A", "value": 79.5, "score": 200, "timestamp": "2024-01-03T10:00:00Z"}
-{"id": 31, "name": "Item_31", "category": "B", "value": 81.8, "score": 205, "timestamp": "2024-01-04T10:00:00Z"}
-{"id": 32, "name": "Item_32", "category": "C", "value": 84.1, "score": 210, "timestamp": "2024-01-05T10:00:00Z"}
-{"id": 33, "name": "Item_33", "category": "A", "value": 86.39999999999999, "score": 215, "timestamp": "2024-01-06T10:00:00Z"}
-{"id": 34, "name": "Item_34", "category": "B", "value": 88.69999999999999, "score": 220, "timestamp": "2024-01-07T10:00:00Z"}
-{"id": 35, "name": "Item_35", "category": "C", "value": 91.0, "score": 225, "timestamp": "2024-01-08T10:00:00Z"}
-{"id": 36, "name": "Item_36", "category": "A", "value": 93.3, "score": 230, "timestamp": "2024-01-09T10:00:00Z"}
-{"id": 37, "name": "Item_37", "category": "B", "value": 95.6, "score": 235, "timestamp": "2024-01-10T10:00:00Z"}
-{"id": 38, "name": "Item_38", "category": "C", "value": 97.89999999999999, "score": 240, "timestamp": "2024-01-11T10:00:00Z"}
-{"id": 39, "name": "Item_39", "category": "A", "value": 100.19999999999999, "score": 245, "timestamp": "2024-01-12T10:00:00Z"}
-{"id": 40, "name": "Item_40", "category": "B", "value": 102.5, "score": 250, "timestamp": "2024-01-13T10:00:00Z"}
-{"id": 41, "name": "Item_41", "category": "C", "value": 104.8, "score": 255, "timestamp": "2024-01-14T10:00:00Z"}
-{"id": 42, "name": "Item_42", "category": "A", "value": 107.1, "score": 260, "timestamp": "2024-01-15T10:00:00Z"}
-{"id": 43, "name": "Item_43", "category": "B", "value": 109.39999999999999, "score": 265, "timestamp": "2024-01-16T10:00:00Z"}
-{"id": 44, "name": "Item_44", "category": "C", "value": 111.69999999999999, "score": 270, "timestamp": "2024-01-17T10:00:00Z"}
-{"id": 45, "name": "Item_45", "category": "A", "value": 113.99999999999999, "score": 275, "timestamp": "2024-01-18T10:00:00Z"}
-{"id": 46, "name": "Item_46", "category": "B", "value": 116.3, "score": 280, "timestamp": "2024-01-19T10:00:00Z"}
-{"id": 47, "name": "Item_47", "category": "C", "value": 118.6, "score": 285, "timestamp": "2024-01-20T10:00:00Z"}
-{"id": 48, "name": "Item_48", "category": "A", "value": 120.89999999999999, "score": 290, "timestamp": "2024-01-21T10:00:00Z"}
-{"id": 49, "name": "Item_49", "category": "B", "value": 123.19999999999999, "score": 295, "timestamp": "2024-01-22T10:00:00Z"}
-{"id": 50, "name": "Item_50", "category": "C", "value": 125.49999999999999, "score": 300, "timestamp": "2024-01-23T10:00:00Z"}
-{"id": 51, "name": "Item_51", "category": "A", "value": 127.8, "score": 305, "timestamp": "2024-01-24T10:00:00Z"}
-{"id": 52, "name": "Item_52", "category": "B", "value": 130.1, "score": 310, "timestamp": "2024-01-25T10:00:00Z"}
-{"id": 53, "name": "Item_53", "category": "C", "value": 132.39999999999998, "score": 315, "timestamp": "2024-01-26T10:00:00Z"}
-{"id": 54, "name": "Item_54", "category": "A", "value": 134.7, "score": 320, "timestamp": "2024-01-27T10:00:00Z"}
-{"id": 55, "name": "Item_55", "category": "B", "value": 137.0, "score": 325, "timestamp": "2024-01-28T10:00:00Z"}
-{"id": 56, "name": "Item_56", "category": "C", "value": 139.29999999999998, "score": 330, "timestamp": "2024-01-01T10:00:00Z"}
-{"id": 57, "name": "Item_57", "category": "A", "value": 141.6, "score": 335, "timestamp": "2024-01-02T10:00:00Z"}
-{"id": 58, "name": "Item_58", "category": "B", "value": 143.89999999999998, "score": 340, "timestamp": "2024-01-03T10:00:00Z"}
-{"id": 59, "name": "Item_59", "category": "C", "value": 146.2, "score": 345, "timestamp": "2024-01-04T10:00:00Z"}
-{"id": 60, "name": "Item_60", "category": "A", "value": 148.5, "score": 350, "timestamp": "2024-01-05T10:00:00Z"}
-{"id": 61, "name": "Item_61", "category": "B", "value": 150.79999999999998, "score": 355, "timestamp": "2024-01-06T10:00:00Z"}
-{"id": 62, "name": "Item_62", "category": "C", "value": 153.1, "score": 360, "timestamp": "2024-01-07T10:00:00Z"}
-{"id": 63, "name": "Item_63", "category": "A", "value": 155.39999999999998, "score": 365, "timestamp": "2024-01-08T10:00:00Z"}
-{"id": 64, "name": "Item_64", "category": "B", "value": 157.7, "score": 370, "timestamp": "2024-01-09T10:00:00Z"}
-{"id": 65, "name": "Item_65", "category": "C", "value": 160.0, "score": 375, "timestamp": "2024-01-10T10:00:00Z"}
-{"id": 66, "name": "Item_66", "category": "A", "value": 162.29999999999998, "score": 380, "timestamp": "2024-01-11T10:00:00Z"}
-{"id": 67, "name": "Item_67", "category": "B", "value": 164.6, "score": 385, "timestamp": "2024-01-12T10:00:00Z"}
-{"id": 68, "name": "Item_68", "category": "C", "value": 166.89999999999998, "score": 390, "timestamp": "2024-01-13T10:00:00Z"}
-{"id": 69, "name": "Item_69", "category": "A", "value": 169.2, "score": 395, "timestamp": "2024-01-14T10:00:00Z"}
-{"id": 70, "name": "Item_70", "category": "B", "value": 171.5, "score": 400, "timestamp": "2024-01-15T10:00:00Z"}
-{"id": 71, "name": "Item_71", "category": "C", "value": 173.79999999999998, "score": 405, "timestamp": "2024-01-16T10:00:00Z"}
-{"id": 72, "name": "Item_72", "category": "A", "value": 176.1, "score": 410, "timestamp": "2024-01-17T10:00:00Z"}
-{"id": 73, "name": "Item_73", "category": "B", "value": 178.39999999999998, "score": 415, "timestamp": "2024-01-18T10:00:00Z"}
-{"id": 74, "name": "Item_74", "category": "C", "value": 180.7, "score": 420, "timestamp": "2024-01-19T10:00:00Z"}
-{"id": 75, "name": "Item_75", "category": "A", "value": 183.0, "score": 425, "timestamp": "2024-01-20T10:00:00Z"}
-{"id": 76, "name": "Item_76", "category": "B", "value": 185.29999999999998, "score": 430, "timestamp": "2024-01-21T10:00:00Z"}
-{"id": 77, "name": "Item_77", "category": "C", "value": 187.6, "score": 435, "timestamp": "2024-01-22T10:00:00Z"}
-{"id": 78, "name": "Item_78", "category": "A", "value": 189.89999999999998, "score": 440, "timestamp": "2024-01-23T10:00:00Z"}
-{"id": 79, "name": "Item_79", "category": "B", "value": 192.2, "score": 445, "timestamp": "2024-01-24T10:00:00Z"}
-{"id": 80, "name": "Item_80", "category": "C", "value": 194.5, "score": 450, "timestamp": "2024-01-25T10:00:00Z"}
-{"id": 81, "name": "Item_81", "category": "A", "value": 196.79999999999998, "score": 455, "timestamp": "2024-01-26T10:00:00Z"}
-{"id": 82, "name": "Item_82", "category": "B", "value": 199.1, "score": 460, "timestamp": "2024-01-27T10:00:00Z"}
-{"id": 83, "name": "Item_83", "category": "C", "value": 201.39999999999998, "score": 465, "timestamp": "2024-01-28T10:00:00Z"}
-{"id": 84, "name": "Item_84", "category": "A", "value": 203.7, "score": 470, "timestamp": "2024-01-01T10:00:00Z"}
-{"id": 85, "name": "Item_85", "category": "B", "value": 205.99999999999997, "score": 475, "timestamp": "2024-01-02T10:00:00Z"}
-{"id": 86, "name": "Item_86", "category": "C", "value": 208.29999999999998, "score": 480, "timestamp": "2024-01-03T10:00:00Z"}
-{"id": 87, "name": "Item_87", "category": "A", "value": 210.6, "score": 485, "timestamp": "2024-01-04T10:00:00Z"}
-{"id": 88, "name": "Item_88", "category": "B", "value": 212.89999999999998, "score": 490, "timestamp": "2024-01-05T10:00:00Z"}
-{"id": 89, "name": "Item_89", "category": "C", "value": 215.2, "score": 495, "timestamp": "2024-01-06T10:00:00Z"}
-{"id": 90, "name": "Item_90", "category": "A", "value": 217.49999999999997, "score": 500, "timestamp": "2024-01-07T10:00:00Z"}
-{"id": 91, "name": "Item_91", "category": "B", "value": 219.79999999999998, "score": 505, "timestamp": "2024-01-08T10:00:00Z"}
-{"id": 92, "name": "Item_92", "category": "C", "value": 222.1, "score": 510, "timestamp": "2024-01-09T10:00:00Z"}
-{"id": 93, "name": "Item_93", "category": "A", "value": 224.39999999999998, "score": 515, "timestamp": "2024-01-10T10:00:00Z"}
-{"id": 94, "name": "Item_94", "category": "B", "value": 226.7, "score": 520, "timestamp": "2024-01-11T10:00:00Z"}
-{"id": 95, "name": "Item_95", "category": "C", "value": 228.99999999999997, "score": 525, "timestamp": "2024-01-12T10:00:00Z"}
-{"id": 96, "name": "Item_96", "category": "A", "value": 231.29999999999998, "score": 530, "timestamp": "2024-01-13T10:00:00Z"}
-{"id": 97, "name": "Item_97", "category": "B", "value": 233.6, "score": 535, "timestamp": "2024-01-14T10:00:00Z"}
-{"id": 98, "name": "Item_98", "category": "C", "value": 235.89999999999998, "score": 540, "timestamp": "2024-01-15T10:00:00Z"}
-{"id": 99, "name": "Item_99", "category": "A", "value": 238.2, "score": 545, "timestamp": "2024-01-16T10:00:00Z"}
-{"id": 100, "name": "Item_100", "category": "B", "value": 240.49999999999997, "score": 550, "timestamp": "2024-01-17T10:00:00Z"}
diff --git a/paimon-python/pypaimon/sample/oss_read_and_write.py b/paimon-python/pypaimon/sample/oss_read_and_write.py
deleted file mode 100644
index bd7310be25..0000000000
--- a/paimon-python/pypaimon/sample/oss_read_and_write.py
+++ /dev/null
@@ -1,70 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-import pyarrow as pa
-
-from pypaimon.catalog.catalog_factory import CatalogFactory
-from pypaimon.schema.schema import Schema
-
-
-def oss_read_and_write():
- warehouse = 'oss://<your-bucket>/<warehouse-path>'
- catalog = CatalogFactory.create({
- 'warehouse': warehouse,
- 'fs.oss.endpoint': 'oss-<your-region>.aliyuncs.com',
- 'fs.oss.accessKeyId': '<your-ak>',
- 'fs.oss.accessKeySecret': '<your-sk>',
- 'fs.oss.region': '<your-region>'
- })
- simple_pa_schema = pa.schema([
- ('f0', pa.int32()),
- ('f1', pa.string()),
- ('f2', pa.string())
- ])
- data = {
- 'f0': [1, 2, 3],
- 'f1': ['a', 'b', 'c'],
- 'f2': ['X', 'Y', 'Z']
- }
- catalog.create_database("test_db", False)
- catalog.create_table("test_db.test_table", Schema.from_pyarrow_schema(simple_pa_schema), False)
- table = catalog.get_table("test_db.test_table")
-
- # write data
- write_builder = table.new_batch_write_builder()
- table_write = write_builder.new_write()
- table_commit = write_builder.new_commit()
- write_data = pa.Table.from_pydict(data, simple_pa_schema)
- table_write.write_arrow(write_data)
- commit_messages = table_write.prepare_commit()
- table_commit.commit(commit_messages)
- table_write.close()
- table_commit.close()
-
- # read data
- read_builder = table.new_read_builder()
- table_scan = read_builder.new_scan()
- table_read = read_builder.new_read()
- splits = table_scan.plan().splits()
-
- result = table_read.to_arrow(splits)
- print(result)
-
-
-if __name__ == '__main__':
- oss_read_and_write()
diff --git a/paimon-python/pypaimon/sample/rest_catalog_blob_as_descriptor_sample.py b/paimon-python/pypaimon/sample/rest_catalog_blob_as_descriptor_sample.py
deleted file mode 100644
index 1e90fa1bdf..0000000000
--- a/paimon-python/pypaimon/sample/rest_catalog_blob_as_descriptor_sample.py
+++ /dev/null
@@ -1,154 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-"""
-Sample demonstrating descriptor-stored blob fields with REST catalog.
-"""
-import pyarrow as pa
-
-from pypaimon import CatalogFactory, Schema
-from pypaimon.common.file_io import FileIO
-from pypaimon.common.options import Options
-from pypaimon.table.row.blob import Blob, BlobDescriptor
-
-
-def write_table_with_blob(catalog, video_file_path: str, external_oss_options: dict):
- database_name = 'blob_demo'
- table_name = 'test_table_blob_' + str(int(__import__('time').time()))
-
- catalog.create_database(
- name=database_name,
- ignore_if_exists=True,
- )
-
- pa_schema = pa.schema([
- ('text', pa.string()),
- ('names', pa.list_(pa.string())),
- ('video', pa.large_binary()) # Blob column
- ])
-
- schema = Schema.from_pyarrow_schema(
- pa_schema=pa_schema,
- partition_keys=None,
- primary_keys=None,
- options={
- 'row-tracking.enabled': 'true',
- 'data-evolution.enabled': 'true',
- 'blob-field': 'video',
- 'blob-descriptor-field': 'video'
- },
- comment='Table with blob column using descriptor-stored field')
-
- table_identifier = f'{database_name}.{table_name}'
- catalog.create_table(
- identifier=table_identifier,
- schema=schema,
- ignore_if_exists=True
- )
-
- table = catalog.get_table(table_identifier)
- print(f"✓ Table created: {table_identifier}")
-
- # Access external OSS file to get file size
- try:
- external_file_io = FileIO.get(video_file_path, Options(external_oss_options))
- video_file_size = external_file_io.get_file_size(video_file_path)
- except Exception as e:
- raise FileNotFoundError(
- f"Failed to access external OSS file: {video_file_path}\n"
- f"Error: {e}\n"
- f"Please check your external_oss_options credentials."
- ) from e
-
- # Create BlobDescriptor
- blob_descriptor = BlobDescriptor(video_file_path, 0, video_file_size)
- descriptor_bytes = blob_descriptor.serialize()
- write_builder = table.new_batch_write_builder()
- table_write = write_builder.new_write()
- table_commit = write_builder.new_commit()
-
- table_write.write_arrow(pa.Table.from_pydict({
- 'text': ['Sample video'],
- 'names': [['video1.mp4']],
- 'video': [descriptor_bytes]
- }, schema=pa_schema))
-
- table_commit.commit(table_write.prepare_commit())
- print("✓ Data committed successfully")
- table_write.close()
- table_commit.close()
-
- return f'{database_name}.{table_name}'
-
-
-def read_table_with_blob(catalog, table_name: str):
- table = catalog.get_table(table_name)
-
- read_builder = table.new_read_builder()
- table_scan = read_builder.new_scan()
- splits = table_scan.plan().splits()
- table_read = read_builder.new_read()
-
- result = table_read.to_arrow(splits)
- print(f"✓ Read {result.num_rows} rows")
-
- video_bytes_list = result.column('video').to_pylist()
- for video_bytes in video_bytes_list:
- if video_bytes is None:
- continue
- blob_descriptor = BlobDescriptor.deserialize(video_bytes)
- from pypaimon.common.uri_reader import FileUriReader
- uri_reader = FileUriReader(table.file_io)
- blob = Blob.from_descriptor(uri_reader, blob_descriptor)
- blob_data = blob.to_data()
- print(f"✓ Blob data verified: {len(blob_data) / 1024 / 1024:.2f} MB")
- break
-
- return result
-
-
-if __name__ == '__main__':
- external_oss_options = {
- 'fs.oss.accessKeyId': "YOUR_EXTERNAL_OSS_ACCESS_KEY_ID",
- 'fs.oss.accessKeySecret': "YOUR_EXTERNAL_OSS_ACCESS_KEY_SECRET",
- 'fs.oss.endpoint': "oss-cn-hangzhou.aliyuncs.com",
- 'fs.oss.region': "cn-hangzhou",
- }
-
- video_file_path = "oss://your-bucket/blob_test/video.mov"
-
- catalog_options = {
- 'metastore': 'rest',
- 'uri': "http://your-rest-catalog-uri",
- 'warehouse': "your_warehouse",
- 'dlf.region': 'cn-hangzhou',
- "token.provider": "dlf",
- 'dlf.access-key-id': "YOUR_DLF_ACCESS_KEY_ID",
- 'dlf.access-key-secret': "YOUR_DLF_ACCESS_KEY_SECRET",
- 'dlf.oss-endpoint': "oss-cn-hangzhou.aliyuncs.com",
- **external_oss_options
- }
-
- catalog = CatalogFactory.create(catalog_options)
-
- try:
- table_name = write_table_with_blob(catalog, video_file_path, external_oss_options)
- result = read_table_with_blob(catalog, table_name)
- print("✓ Test completed successfully!")
- except Exception as e:
- print(f'✗ Error: {e}')
- raise
diff --git a/paimon-python/pypaimon/sample/rest_catalog_ray_data_sample.py b/paimon-python/pypaimon/sample/rest_catalog_ray_data_sample.py
deleted file mode 100644
index 280322aabb..0000000000
--- a/paimon-python/pypaimon/sample/rest_catalog_ray_data_sample.py
+++ /dev/null
@@ -1,245 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-"""
-Example: REST Catalog with Ray Data Integration
-
-Demonstrates:
-1. REST catalog basic read/write operations
-2. Reading Paimon tables into Ray Dataset for distributed processing
-3. Using Ray Data operations (map, filter, etc.) on Paimon data
-4. Performance comparison between simple and distributed reads
-
-Prerequisites:
- pip install pypaimon ray pandas pyarrow
-"""
-
-import tempfile
-import uuid
-
-import pandas as pd
-import pyarrow as pa
-import ray
-
-from pypaimon import CatalogFactory, Schema
-from pypaimon.tests.rest.rest_server import RESTCatalogServer
-from pypaimon.api.api_response import ConfigResponse
-from pypaimon.api.auth import BearTokenAuthProvider
-
-
-def main():
- """REST catalog with Ray Data integration example."""
-
- # Initialize Ray
- if not ray.is_initialized():
- ray.init(ignore_reinit_error=True, num_cpus=2)
- print("Ray initialized successfully")
-
- # Setup mock REST server
- temp_dir = tempfile.mkdtemp()
- token = str(uuid.uuid4())
- server = RESTCatalogServer(
- data_path=temp_dir,
- auth_provider=BearTokenAuthProvider(token),
- config=ConfigResponse(defaults={"prefix": "mock-test"}),
- warehouse="warehouse"
- )
- server.start()
- print(f"REST server started at: {server.get_url()}")
-
- try:
- # Create REST catalog
- catalog = CatalogFactory.create({
- 'metastore': 'rest',
- 'uri': f"http://localhost:{server.port}",
- 'warehouse': "warehouse", # Must match server's warehouse parameter
- 'token.provider': 'bear',
- 'token': token,
- })
- catalog.create_database("default", True)
-
- # Create table schema
- schema = Schema.from_pyarrow_schema(pa.schema([
- ('id', pa.int64()),
- ('name', pa.string()),
- ('category', pa.string()),
- ('value', pa.float64()),
- ('score', pa.int32()),
- ]))
- table_name = 'default.ray_example_table'
- catalog.create_table(table_name, schema, True)
- table = catalog.get_table(table_name)
-
- print(f"\nTable created: {table_name}")
- print(f"Table path: {table.table_path}")
-
- # Write data
- print("\n" + "="*60)
- print("Step 1: Writing data to Paimon table")
- print("="*60)
- write_builder = table.new_batch_write_builder()
- table_write = write_builder.new_write()
- table_commit = write_builder.new_commit()
-
- # Generate sample data
- data = pd.DataFrame({
- 'id': list(range(1, 21)), # 20 rows
- 'name': [f'Item_{i}' for i in range(1, 21)],
- 'category': ['A', 'B', 'C'] * 6 + ['A', 'B'],
- 'value': [10.5 + i * 2.3 for i in range(20)],
- 'score': [50 + i * 5 for i in range(20)],
- })
- table_write.write_pandas(data)
- table_commit.commit(table_write.prepare_commit())
- table_write.close()
- table_commit.close()
- print(f"✓ Successfully wrote {len(data)} rows to table")
-
- # Read with Ray Data (distributed mode)
- print("\n" + "="*60)
- print("Step 2: Reading data with Ray Data (distributed mode)")
- print("="*60)
- read_builder = table.new_read_builder()
- table_read = read_builder.new_read()
- table_scan = read_builder.new_scan()
- splits = table_scan.plan().splits()
-
- print(f"Number of splits: {len(splits)}")
-
- # Convert to Ray Dataset
- ray_dataset = table_read.to_ray(splits, override_num_blocks=2)
- print("✓ Ray Dataset created successfully")
- print(f" - Total rows: {ray_dataset.count()}")
- # Note: num_blocks() requires materialized dataset, so we skip it for simplicity
-
- # Convert to pandas for display
- df_ray = ray_dataset.to_pandas()
- print("\nFirst 5 rows from Ray Dataset:")
- print(df_ray.head().to_string())
-
- # Compare with simple read
- print("\n" + "="*60)
- print("Step 3: Comparison with simple read mode")
- print("="*60)
- ray_dataset_simple = table_read.to_ray(splits, override_num_blocks=1)
- df_simple = ray_dataset_simple.to_pandas()
-
- print(f"Distributed mode rows: {ray_dataset.count()}")
- print(f"Simple mode rows: {ray_dataset_simple.count()}")
- print("✓ Both modes return the same number of rows")
-
- # Verify data consistency (sort by id for comparison)
- df_ray_sorted = df_ray.sort_values(by='id').reset_index(drop=True)
- df_simple_sorted = df_simple.sort_values(by='id').reset_index(drop=True)
- pd.testing.assert_frame_equal(df_ray_sorted, df_simple_sorted)
- print("✓ Data content matches between distributed and simple modes")
-
- # Ray Data operations
- print("\n" + "="*60)
- print("Step 4: Ray Data operations on Paimon data")
- print("="*60)
-
- # Filter operation
- print("\n4.1 Filter: Get items with score >= 80")
- filtered_dataset = ray_dataset.filter(lambda row: row['score'] >= 80)
- df_filtered = filtered_dataset.to_pandas()
- print(f" - Filtered rows: {len(df_filtered)}")
- print(f" - IDs: {sorted(df_filtered['id'].tolist())}")
-
- # Map operation
- print("\n4.2 Map: Double the value column")
-
- def double_value(row):
- row['value'] = row['value'] * 2
- return row
-
- mapped_dataset = ray_dataset.map(double_value)
- df_mapped = mapped_dataset.to_pandas()
- print(f" - Original first value: {df_ray.iloc[0]['value']:.2f}")
- print(f" - Mapped first value: {df_mapped.iloc[0]['value']:.2f}")
-
- # Group by operation (using pandas after conversion)
- print("\n4.3 Group by: Aggregate by category")
- df_grouped = df_ray.groupby('category').agg({
- 'value': 'sum',
- 'score': 'mean',
- 'id': 'count'
- }).round(2)
- print(df_grouped.to_string())
-
- # Predicate filtering at read time
- print("\n" + "="*60)
- print("Step 5: Predicate filtering at read time")
- print("="*60)
- predicate_builder = read_builder.new_predicate_builder()
- predicate = predicate_builder.equal('category', 'A')
- read_builder_filtered = read_builder.with_filter(predicate)
-
- table_read_filtered = read_builder_filtered.new_read()
- table_scan_filtered = read_builder_filtered.new_scan()
- splits_filtered = table_scan_filtered.plan().splits()
-
- ray_dataset_filtered = table_read_filtered.to_ray(splits_filtered, override_num_blocks=2)
- df_filtered_at_read = ray_dataset_filtered.to_pandas()
- print(f"✓ Filtered at read time: {ray_dataset_filtered.count()} rows")
- print(f" - All categories are 'A': {all(df_filtered_at_read['category'] == 'A')}")
-
- # Projection (select specific columns)
- print("\n" + "="*60)
- print("Step 6: Column projection")
- print("="*60)
- read_builder_projected = read_builder.with_projection(['id', 'name', 'value'])
- table_read_projected = read_builder_projected.new_read()
- table_scan_projected = read_builder_projected.new_scan()
- splits_projected = table_scan_projected.plan().splits()
-
- ray_dataset_projected = table_read_projected.to_ray(splits_projected, override_num_blocks=2)
- df_projected = ray_dataset_projected.to_pandas()
- print(f"✓ Projected columns: {list(df_projected.columns)}")
- print(" - Expected: ['id', 'name', 'value']")
- print(f" - Match: {set(df_projected.columns) == {'id', 'name', 'value'}}")
-
- # Optional: Blob data example (uncomment to demonstrate blob data handling)
- # print("\n" + "="*60)
- # print("Step 7: Blob data handling (Optional)")
- # print("="*60)
- # print("Paimon supports storing blob data (images, audio, etc.)")
- # print("For blob data examples, see: pypaimon/sample/rest_catalog_blob_as_descriptor_sample.py")
-
- print("\n" + "="*60)
- print("Summary")
- print("="*60)
- print("✓ Successfully demonstrated Ray Data integration with REST catalog")
- print("✓ Distributed reading works correctly")
- print("✓ Ray Data operations (filter, map, group by) work on Paimon data")
- print("✓ Predicate filtering and projection work at read time")
- print("\nRay Data enables:")
- print(" - Distributed parallel reading from Paimon tables")
- print(" - Scalable data processing with Ray operations")
- print(" - Integration with Ray ecosystem (Ray Train, Ray Serve, etc.)")
- print("\nFor blob data (images, audio) examples, see:")
- print(" - pypaimon/sample/rest_catalog_blob_as_descriptor_sample.py")
-
- finally:
- server.shutdown()
- if ray.is_initialized():
- ray.shutdown()
- print("\n✓ Server stopped and Ray shutdown")
-
-
-if __name__ == '__main__':
- main()
diff --git a/paimon-python/pypaimon/sample/rest_catalog_ray_lance_sample.py b/paimon-python/pypaimon/sample/rest_catalog_ray_lance_sample.py
deleted file mode 100644
index 18b3f769e3..0000000000
--- a/paimon-python/pypaimon/sample/rest_catalog_ray_lance_sample.py
+++ /dev/null
@@ -1,212 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-"""
-Example: REST Catalog + Ray Data + Lance Format Integration
-
-Demonstrates:
-1. REST catalog with Lance file format
-2. Writing data to Paimon table using Lance format
-3. Reading data using Ray Data for distributed processing
-4. Integration of REST catalog, Ray Data, and Lance format
-"""
-
-import tempfile
-import uuid
-
-import pandas as pd
-import pyarrow as pa
-import ray
-
-
-from pypaimon import CatalogFactory, Schema
-from pypaimon.common.options.core_options import CoreOptions
-
-from pypaimon.tests.rest.rest_server import RESTCatalogServer
-from pypaimon.api.api_response import ConfigResponse
-from pypaimon.api.auth import BearTokenAuthProvider
-
-
-def main():
- """REST catalog + Ray Data + Lance format integration example."""
-
- # Initialize Ray
- ray.init(ignore_reinit_error=True)
- print("Ray initialized successfully")
-
- # Setup mock REST server
- temp_dir = tempfile.mkdtemp()
- token = str(uuid.uuid4())
- server = RESTCatalogServer(
- data_path=temp_dir,
- auth_provider=BearTokenAuthProvider(token),
- config=ConfigResponse(defaults={"prefix": "mock-test"}),
- warehouse="warehouse"
- )
- server.start()
- print(f"REST server started at: {server.get_url()}")
-
- try:
- # Create REST catalog
- catalog = CatalogFactory.create({
- 'metastore': 'rest',
- 'uri': f"http://localhost:{server.port}",
- 'warehouse': "warehouse", # Must match server's warehouse parameter
- 'token.provider': 'bear',
- 'token': token,
- })
- catalog.create_database("default", True)
-
- # Create table schema with Lance format
- pa_schema = pa.schema([
- ('id', pa.int64()),
- ('name', pa.string()),
- ('value', pa.float64()),
- ('category', pa.string()),
- ])
-
- # Create schema with Lance file format option
- schema = Schema.from_pyarrow_schema(
- pa_schema=pa_schema,
- primary_keys=['id'],
- options={
- 'bucket': '2',
- CoreOptions.FILE_FORMAT.key(): CoreOptions.FILE_FORMAT_LANCE,
- }
- )
-
- table_name = 'default.ray_lance_example'
- catalog.create_table(table_name, schema, False)
- table = catalog.get_table(table_name)
-
- table_path = table.table_path
- print(f"\nTable path: {table_path}")
- print(f"File format: {CoreOptions.FILE_FORMAT_LANCE}")
-
- # Write data using Lance format
- print("\nWriting data with Lance format...")
- write_builder = table.new_batch_write_builder()
- table_write = write_builder.new_write()
- table_commit = write_builder.new_commit()
-
- # Write multiple batches to demonstrate Lance format
- data1 = pd.DataFrame({
- 'id': [1, 2, 3, 4, 5],
- 'name': ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],
- 'value': [10.5, 20.3, 30.7, 40.1, 50.9],
- 'category': ['A', 'B', 'A', 'B', 'A'],
- })
- table_write.write_pandas(data1)
-
- data2 = pd.DataFrame({
- 'id': [6, 7, 8],
- 'name': ['Frank', 'Grace', 'Henry'],
- 'value': [60.2, 70.4, 80.6],
- 'category': ['B', 'A', 'B'],
- })
- table_write.write_pandas(data2)
-
- table_commit.commit(table_write.prepare_commit())
- table_write.close()
- table_commit.close()
- print("Data written successfully with Lance format!")
-
- # Read data using standard methods
- print("\nReading data using standard methods...")
- read_builder = table.new_read_builder()
- table_scan = read_builder.new_scan()
- table_read = read_builder.new_read()
- splits = table_scan.plan().splits()
-
- # Read to Pandas
- result_pandas = table_read.to_pandas(splits)
- print("\nPandas DataFrame result:")
- print(result_pandas)
-
- # Read to Arrow
- result_arrow = table_read.to_arrow(splits)
- print(f"\nArrow Table result: {result_arrow.num_rows} rows")
-
- # Read using Ray Data
- print("\nReading data using Ray Data for distributed processing...")
- ray_dataset = table_read.to_ray(splits)
-
- print(f"Ray Dataset: {ray_dataset}")
- print(f"Number of rows: {ray_dataset.count()}")
-
- # Sample Ray Data operations
- print("\nRay Data operations:")
-
- # Take first few rows
- sample_data = ray_dataset.take(3)
- print(f"First 3 rows: {sample_data}")
-
- # Filter data
- filtered_dataset = ray_dataset.filter(lambda row: row['value'] > 30.0)
- print(f"Filtered rows (value > 30): {filtered_dataset.count()}")
-
- # Map operation
- def double_value(row):
- row['value'] = row['value'] * 2
- return row
-
- mapped_dataset = ray_dataset.map(double_value)
- print(f"Mapped dataset (doubled values): {mapped_dataset.count()} rows")
-
- # Convert to Pandas
- ray_pandas = ray_dataset.to_pandas()
- print("\nRay Dataset converted to Pandas:")
- print(ray_pandas)
-
- # Group by category
- print("\nGrouping by category using Ray Data:")
- grouped = ray_dataset.groupby("category").sum("value")
- print(grouped)
-
- # Demonstrate predicate pushdown with Lance format
- print("\nDemonstrating predicate pushdown with Lance format...")
- predicate_builder = read_builder.new_predicate_builder()
- predicate = predicate_builder.greater_than('value', 30.0)
- read_builder_filtered = read_builder.with_filter(predicate)
-
- table_scan_filtered = read_builder_filtered.new_scan()
- table_read_filtered = read_builder_filtered.new_read()
- splits_filtered = table_scan_filtered.plan().splits()
-
- result_filtered = table_read_filtered.to_pandas(splits_filtered)
- print("\nFiltered result (value > 30):")
- print(result_filtered)
-
- # Demonstrate projection with Lance format
- print("\nDemonstrating column projection with Lance format...")
- read_builder_projected = read_builder.with_projection(['id', 'name', 'value'])
- table_scan_projected = read_builder_projected.new_scan()
- table_read_projected = read_builder_projected.new_read()
- splits_projected = table_scan_projected.plan().splits()
-
- result_projected = table_read_projected.to_pandas(splits_projected)
- print("\nProjected result (id, name, value only):")
- print(result_projected)
-
- finally:
- server.shutdown()
- ray.shutdown()
- print("\nServer stopped and Ray shutdown")
-
-
-if __name__ == '__main__':
- main()
diff --git a/paimon-python/pypaimon/sample/rest_catalog_ray_sink_sample.py b/paimon-python/pypaimon/sample/rest_catalog_ray_sink_sample.py
deleted file mode 100644
index 83264854d7..0000000000
--- a/paimon-python/pypaimon/sample/rest_catalog_ray_sink_sample.py
+++ /dev/null
@@ -1,153 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-"""
-Example: REST Catalog + Ray Sink with Local File + Mock Server
-
-Demonstrates reading JSON data from local file using Ray Data and writing to Paimon table
-using Ray Sink (write_ray) with a mock REST catalog server.
-"""
-
-import os
-import tempfile
-import uuid
-
-import pyarrow as pa
-import ray
-
-from pypaimon import CatalogFactory, Schema
-from pypaimon.common.options.core_options import CoreOptions
-from pypaimon.schema.data_types import PyarrowFieldParser
-from pypaimon.tests.rest.rest_server import RESTCatalogServer
-from pypaimon.api.api_response import ConfigResponse
-from pypaimon.api.auth import BearTokenAuthProvider
-
-
-def _get_sample_data_path(filename: str) -> str:
- sample_dir = os.path.dirname(os.path.abspath(__file__))
- return os.path.join(sample_dir, 'data', filename)
-
-
-def main():
- ray.init(ignore_reinit_error=True, num_cpus=2)
-
- temp_dir = tempfile.mkdtemp()
- token = str(uuid.uuid4())
- server = RESTCatalogServer(
- data_path=temp_dir,
- auth_provider=BearTokenAuthProvider(token),
- config=ConfigResponse(defaults={"prefix": "mock-test"}),
- warehouse="warehouse"
- )
- server.start()
- print(f"REST server started at: {server.get_url()}")
-
- json_file_path = _get_sample_data_path('data.jsonl')
-
- table_write = None
- try:
- catalog = CatalogFactory.create({
- 'metastore': 'rest',
- 'uri': f"http://localhost:{server.port}",
- 'warehouse': "warehouse",
- 'token.provider': 'bear',
- 'token': token,
- })
- catalog.create_database("default", ignore_if_exists=True)
-
- schema = Schema.from_pyarrow_schema(pa.schema([
- ('id', pa.int64()),
- ('name', pa.string()),
- ('category', pa.string()),
- ('value', pa.float64()),
- ('score', pa.int64()),
- ('timestamp', pa.timestamp('s')),
- ]), primary_keys=['id'], options={
- CoreOptions.BUCKET.key(): '4'
- })
-
- table_name = 'default.oss_json_import_table'
- catalog.create_table(table_name, schema, ignore_if_exists=True)
- table = catalog.get_table(table_name)
-
- print(f"Reading JSON from local file: {json_file_path}")
- ray_dataset = ray.data.read_json(
- json_file_path,
- concurrency=2,
- )
-
- print(f"Ray Dataset: {ray_dataset.count()} rows")
- print(f"Schema: {ray_dataset.schema()}")
-
- table_pa_schema = PyarrowFieldParser.from_paimon_schema(table.table_schema.fields)
-
- def cast_batch_to_table_schema(batch: pa.RecordBatch) -> pa.Table:
- arrays = []
- for field in table_pa_schema:
- col_name = field.name
- col_array = batch.column(col_name)
- if isinstance(col_array, pa.ChunkedArray):
- col_array = col_array.combine_chunks()
- if col_array.type != field.type:
- col_array = col_array.cast(field.type)
- arrays.append(col_array)
- record_batch = pa.RecordBatch.from_arrays(arrays, schema=table_pa_schema)
- return pa.Table.from_batches([record_batch])
-
- ray_dataset = ray_dataset.map_batches(
- cast_batch_to_table_schema,
- batch_format="pyarrow",
- )
-
- write_builder = table.new_batch_write_builder()
- table_write = write_builder.new_write()
-
- table_write.write_ray(
- ray_dataset,
- overwrite=False,
- concurrency=2,
- ray_remote_args={"num_cpus": 1}
- )
-
- # write_ray() has already committed the data, just close the writer
- table_write.close()
- table_write = None
-
- print(f"Successfully wrote {ray_dataset.count()} rows to table")
-
- read_builder = table.new_read_builder()
- table_read = read_builder.new_read()
- table_scan = read_builder.new_scan()
- splits = table_scan.plan().splits()
-
- result_df = table_read.to_pandas(splits)
- print(f"Read back {len(result_df)} rows from table")
- print(result_df.head())
-
- finally:
- if table_write is not None:
- try:
- table_write.close()
- except Exception:
- pass
- server.shutdown()
- if ray.is_initialized():
- ray.shutdown()
-
-
-if __name__ == '__main__':
- main()
diff --git a/paimon-python/pypaimon/sample/rest_catalog_read_write_sample.py b/paimon-python/pypaimon/sample/rest_catalog_read_write_sample.py
deleted file mode 100644
index 543e63dd09..0000000000
--- a/paimon-python/pypaimon/sample/rest_catalog_read_write_sample.py
+++ /dev/null
@@ -1,108 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-"""
-Example: REST Catalog Read and Write
-
-Demonstrates:
-1. REST catalog basic read/write operations
-2. Table paths include URI schemes (file://, oss://, s3://)
-3. Paths with schemes can be used directly with FileIO
-"""
-
-import tempfile
-import uuid
-
-import pandas as pd
-import pyarrow as pa
-
-from pypaimon import CatalogFactory, Schema
-from pypaimon.tests.rest.rest_server import RESTCatalogServer
-from pypaimon.api.api_response import ConfigResponse
-from pypaimon.api.auth import BearTokenAuthProvider
-
-
-def main():
- """REST catalog read/write example with path scheme demonstration."""
-
- # Setup mock REST server
- temp_dir = tempfile.mkdtemp()
- token = str(uuid.uuid4())
- server = RESTCatalogServer(
- data_path=temp_dir,
- auth_provider=BearTokenAuthProvider(token),
- config=ConfigResponse(defaults={"prefix": "mock-test"}),
- warehouse="warehouse"
- )
- server.start()
- print(f"REST server started at: {server.get_url()}")
-
- try:
- # Note: warehouse must match server's warehouse parameter for config endpoint
- catalog = CatalogFactory.create({
- 'metastore': 'rest',
- 'uri': f"http://localhost:{server.port}",
- 'warehouse': "warehouse", # Must match server's warehouse parameter
- 'token.provider': 'bear',
- 'token': token,
- })
- catalog.create_database("default", False)
-
- schema = Schema.from_pyarrow_schema(pa.schema([
- ('id', pa.int64()),
- ('name', pa.string()),
- ('value', pa.float64()),
- ]))
- table_name = 'default.example_table'
- catalog.create_table(table_name, schema, False)
- table = catalog.get_table(table_name)
-
- table_path = table.table_path
- print(f"\nTable path: {table_path}")
-
- print("\nWriting data...")
- write_builder = table.new_batch_write_builder()
- table_write = write_builder.new_write()
- table_commit = write_builder.new_commit()
-
- data = pd.DataFrame({
- 'id': [1, 2, 3],
- 'name': ['Alice', 'Bob', 'Charlie'],
- 'value': [10.5, 20.3, 30.7]
- })
- table_write.write_pandas(data)
- table_commit.commit(table_write.prepare_commit())
- table_write.close()
- table_commit.close()
- print("Data written successfully!")
-
- # Read data
- print("\nReading data...")
- read_builder = table.new_read_builder()
- table_scan = read_builder.new_scan()
- table_read = read_builder.new_read()
- splits = table_scan.plan().splits()
- result = table_read.to_pandas(splits)
- print(result)
-
- finally:
- server.shutdown()
- print("\nServer stopped")
-
-
-if __name__ == '__main__':
- main()
diff --git a/paimon-python/pypaimon/tests/index_execution_boundary_test.py b/paimon-python/pypaimon/tests/index_execution_boundary_test.py
new file mode 100644
index 0000000000..93fffb9fe5
--- /dev/null
+++ b/paimon-python/pypaimon/tests/index_execution_boundary_test.py
@@ -0,0 +1,175 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Index contracts and Native planning remain independent of Python execution."""
+
+from pathlib import Path
+import subprocess
+import sys
+import textwrap
+
+
+def test_public_index_contracts_and_native_plans_do_not_load_python_execution():
+ script = r"""
+import importlib.abc
+import pickle
+import sys
+from contextlib import ExitStack
+from tempfile import TemporaryDirectory
+from types import SimpleNamespace
+from unittest.mock import Mock, patch
+
+class RejectPythonExecution(importlib.abc.MetaPathFinder):
+ def find_spec(self, fullname, path=None, target=None):
+ if fullname.startswith('pypaimon.execution.python'):
+ raise AssertionError('Native index planning imported Python execution: ' + fullname)
+
+sys.meta_path.insert(0, RejectPythonExecution())
+from pypaimon.catalog.catalog_environment import CatalogEnvironment
+from pypaimon.common.file_io import FileIO
+from pypaimon.common.identifier import Identifier
+from pypaimon.common.predicate_builder import PredicateBuilder
+from pypaimon.globalindex.global_index_meta import GlobalIndexMeta
+from pypaimon.index.data_evolution_index_source_meta import DataEvolutionIndexSourceMeta
+from pypaimon.index.deletion_vector_meta import DeletionVectorMeta
+from pypaimon.index.index_file_meta import IndexFileMeta
+from pypaimon.index.pk.primary_key_index_definition import PrimaryKeyIndexFamily
+from pypaimon.index.pk.primary_key_index_definitions import PrimaryKeyIndexDefinitions
+from pypaimon.index.pk.primary_key_index_source_file import PrimaryKeyIndexSourceFile
+from pypaimon.index.pk.primary_key_index_source_meta import PrimaryKeyIndexSourceMeta
+from pypaimon.schema.data_types import ArrayType, AtomicType, DataField
+from pypaimon.schema.table_schema import TableSchema
+from pypaimon.table.file_store_table import FileStoreTable
+from pypaimon.table.source.search_types import PrimaryKeyFullTextScanPlan, PrimaryKeyVectorScanPlan
+
+# These are transport contracts, so retaining their original class identity
+# matters independently of where the two executors keep their internal state.
+source = PrimaryKeyIndexSourceFile('data.parquet', 3)
+metadata = (
+ PrimaryKeyIndexSourceMeta(1, [source]),
+ DataEvolutionIndexSourceMeta(7),
+ IndexFileMeta('btree', 'index', 17, 3,
+ {'data.parquet': DeletionVectorMeta('data.parquet', 4, 9, 1)},
+ global_index_meta=GlobalIndexMeta(0, 2, 0)),
+)
+transported = pickle.loads(pickle.dumps(metadata))
+assert all(type(actual) is type(expected) for actual, expected in zip(transported, metadata))
+assert transported[0].source_files == (source,)
+assert transported[1].scan_snapshot_id == 7
+assert transported[2].dv_ranges == metadata[2].dv_ranges
+assert type(transported[2].global_index_meta) is GlobalIndexMeta
+
+schema = TableSchema(fields=[
+ DataField(0, 'id', AtomicType('INT')),
+ DataField(1, 'embedding', ArrayType('ARRAY', AtomicType('FLOAT'))),
+ DataField(2, 'body', AtomicType('STRING')),
+], primary_keys=['id'], options={
+ 'bucket': '1', 'scan.native-plan.enabled': 'true', 'read.native.enabled': 'true',
+ 'pk-btree.index.columns': 'id', 'pk-vector.index.columns': 'embedding',
+ 'fields.embedding.pk-vector.index.type': 'ivf-flat',
+ 'pk-full-text.index.columns': 'body',
+})
+definitions = PrimaryKeyIndexDefinitions.create(schema).definitions
+assert [(value.column, value.field_id, value.family) for value in definitions] == [
+ ('id', 0, PrimaryKeyIndexFamily.BTREE),
+ ('embedding', 1, PrimaryKeyIndexFamily.VECTOR),
+ ('body', 2, PrimaryKeyIndexFamily.FULL_TEXT),
+]
+
+with TemporaryDirectory() as directory, ExitStack() as stack:
+ table = FileStoreTable(FileIO.get(directory), Identifier('default', 'indexed'), directory,
+ schema, CatalogEnvironment(catalog_loader=object()))
+ predicate = PredicateBuilder(table.fields).greater_than('id', 0)
+ native_plan = SimpleNamespace(splits=lambda: [], snapshot_id=lambda: 7, stats=lambda: None)
+ core_scan = Mock()
+ core_scan.plan.return_value = native_plan
+ core_builder = Mock()
+ core_builder.with_filter.return_value = core_builder
+ core_builder.with_projection.return_value = core_builder
+ core_builder.new_scan.return_value = core_scan
+ stack.enter_context(patch('pypaimon.execution.native.runtime.native_runtime_available', return_value=True))
+ stack.enter_context(patch('pypaimon.execution.native.table._resolved_schema_file_io_options', return_value={}))
+ stack.enter_context(patch('pypaimon.execution.native.read.builder._native_read_builder',
+ return_value=core_builder))
+ scan = table.new_read_builder().with_filter(predicate).new_scan()
+ plan = scan.plan()
+ assert plan.snapshot_id == 7 and plan.splits() == []
+ assert scan._file_scanner is None
+ core_builder.with_filter.assert_called_once_with(
+ {'method': 'greaterThan', 'field': 'id', 'literals': [0]})
+ core_scan.plan.assert_called_once_with()
+
+ # Use the same Core Scan/Plan/Read protocol as the Native search adapters.
+ search_plan = SimpleNamespace(snapshot=lambda: None, splits=lambda: [])
+ result = SimpleNamespace(snapshot_id=lambda: None, positions=lambda: [], splits=lambda: [])
+ reader = SimpleNamespace(read_plan=Mock(return_value=result))
+ search_builder = Mock()
+ search_builder.with_limit.return_value = search_builder
+ search_builder.new_vector_search_scan.return_value.scan.return_value = search_plan
+ search_builder.new_full_text_scan.return_value.scan.return_value = search_plan
+ search_builder.new_vector_search_read.return_value = reader
+ search_builder.new_full_text_read.return_value = reader
+ native_table = Mock()
+ native_table.new_vector_search_builder.return_value = search_builder
+ native_table.new_full_text_search_builder.return_value = search_builder
+ for module in ('native_vector_search', 'native_full_text_search'):
+ prefix = 'pypaimon.execution.native.read.' + module
+ stack.enter_context(patch(prefix + '.native_runtime_available', return_value=True))
+ stack.enter_context(patch(prefix + '._native_table', return_value=native_table))
+ stack.enter_context(patch('pypaimon.execution.native.read.native_vector_search._catalog_metastore',
+ return_value='rest'))
+ stack.enter_context(patch('pypaimon.execution.native.read.native_full_text_search._rest_catalog_supported',
+ return_value=True))
+ wire_plan = object()
+ convert = stack.enter_context(patch(
+ 'pypaimon.execution.native.read.native_search_plan._plan_to_native', return_value=wire_plan))
+ for family in ('vector', 'text'):
+ if family == 'vector':
+ builder = table.new_vector_search_builder().with_vector_column('embedding').with_limit(2)
+ builder.with_query_vector([1., 0.])
+ scan, read = builder.new_vector_search_scan(), builder.new_vector_search_read()
+ expected_plan = PrimaryKeyVectorScanPlan
+ else:
+ builder = table.new_full_text_search_builder().with_query('body', 'needle').with_limit(2)
+ scan, read = builder.new_full_text_scan(), builder.new_full_text_read()
+ expected_plan = PrimaryKeyFullTextScanPlan
+ plan = scan.scan()
+ assert type(plan) is expected_plan and plan.splits() == []
+ answer = read.read_plan(plan)
+ assert answer.snapshot_id == 0 and answer.splits == ()
+ assert scan._python_executor is None and read._python_executor is None
+ assert convert.call_args[0][0] is plan
+ reader.read_plan.assert_called_with(wire_plan)
+ search_builder.with_vector_column.assert_called_with('embedding')
+ search_builder.with_query.assert_called_with('body', 'needle')
+ failure = OSError('Core query failed')
+ reader.read_plan.side_effect = failure
+ try:
+ read.read_plan(plan)
+ except OSError as error:
+ assert error is failure
+ else:
+ raise AssertionError('Native query failure was hidden')
+ assert read._python_executor is None
+assert not any(name.startswith('pypaimon.execution.python') for name in sys.modules)
+"""
+ result = subprocess.run(
+ [sys.executable, '-c', textwrap.dedent(script)],
+ cwd=str(Path(__file__).resolve().parents[2]),
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
+ )
+ assert result.returncode == 0, result.stdout + result.stderr
diff --git a/paimon-python/pypaimon/tests/primary_key_sorted_index_scan_test.py b/paimon-python/pypaimon/tests/primary_key_sorted_index_scan_test.py
index 9e325b7d78..cfb1a17e0e 100644
--- a/paimon-python/pypaimon/tests/primary_key_sorted_index_scan_test.py
+++ b/paimon-python/pypaimon/tests/primary_key_sorted_index_scan_test.py
@@ -28,7 +28,7 @@ from pypaimon.index.pk.primary_key_index_definition import (
PrimaryKeyIndexDefinition, PrimaryKeyIndexFamily)
from pypaimon.index.pk.primary_key_index_source_file import PrimaryKeyIndexSourceFile
from pypaimon.index.pk.primary_key_index_source_meta import PrimaryKeyIndexSourceMeta
-from pypaimon.index.pksorted.pk_sorted_bucket_index_state import PkSortedBucketIndexState
+from pypaimon.execution.python.index.pksorted.pk_sorted_bucket_index_state import PkSortedBucketIndexState
from pypaimon.manifest.index_manifest_entry import IndexManifestEntry
from pypaimon.read.split import DataSplit
from pypaimon.schema.data_types import AtomicType, DataField
diff --git a/pom.xml b/pom.xml
index d6255a5597..e339255489 100644
--- a/pom.xml
+++ b/pom.xml
@@ -677,7 +677,6 @@ under the License.
<exclude>paimon-common/src/test/resources/fmindex-v1-golden.base64</exclude>
<exclude>paimon-core/src/test/resources/compatibility/**</exclude>
<exclude>paimon-python/pypaimon/tests/resources/calendar_extract_java.jsonl</exclude>
- <exclude>paimon-python/pypaimon/sample/data/**</exclude>
<exclude>**/*.proto</exclude>
<exclude>**/Cargo*</exclude>
</excludes>