You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by ho...@apache.org on 2026/09/20 04:10:54 UTC
(paimon) branch release-2.1 updated (eaf7c8744b -> 6bfd8418dc)
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a change to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
from eaf7c8744b [core][python] Add optional Bloom filters to BTree indexes (#9946)
new 7590416537 [core][spark][docs] Support row filters in full-text search (#9855)
new 0a5678bfbe [core] Use a single BTree scan for bounded range predicates (#9844)
new 48d2fc60ef [flink] Fix rollback_to procedure throwing NullPointerException when neither snapshot_id nor tag is set (#9944)
new fa8ae00346 [core] Reject a non-parquet per-level or changelog format for variant columns at DDL time (#9942)
new 512f285d1c [common] Infer a variant shredding schema regardless of the writer's key order (#9940)
new 23633c3c44 [common] Convert variant timestamps with the requested zone instead of the JVM default (#9939)
new 1e9c720f96 [python] Share final lookups across batch vector queries (#9935)
new b9e8d12f6d [python] Keep batch vector scoring vectorized for null rows (#9934)
new 0952d147bd [core] Preserve query authorization when narrowing chain table splits (#9929)
new c0398c9d7e [python] Support distributed vector search with Ray (#9930)
new 2af7567e5f [core] Apply deletion vectors during data evolution file index pushdown (#9922)
new a27b548472 [format] Use buffered encoder for Avro writes (#9928)
new f97545a0a6 [api] Sign x-acs-action in DLFOpenApiV4Signer (#9937)
new f8c9bb2dc8 [format] Simplify buffered Avro encoder selection
new 93b899b6b2 [core] Separate commit history bounds from strict mode (#9960)
new e4f49ec969 [core] Request only top-k candidates from data-evolution full-text indexes (#9955)
new 9d93f1bac8 [core] Fix OOM error handling in MergeTreeWriter and improve resource cleanup (#9933)
new c2b6a0435c [python] Distribute vector search refinement with Ray (#9951)
new a74d86960b [python] Add opt-in Rust native data reads (#9954)
new 6bfd8418dc [core] Optimize tag reads during snapshot expiration (#9967)
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci-python.yml | 2 +-
docs/docs/concepts/rest/dlf.md | 6 +-
docs/docs/flink/procedures/snapshots.md | 6 +-
.../multimodal-table/global-index/full-text.mdx | 62 ++
.../global-index/hybrid-search.mdx | 12 +
docs/docs/primary-key-table/global-index.mdx | 4 +-
docs/docs/pypaimon/multimodal-search.md | 71 +++
docs/generated/core_configuration.html | 14 +-
.../main/java/org/apache/paimon/CoreOptions.java | 41 +-
.../apache/paimon/rest/auth/DLFOpenApiActions.java | 184 ++++++
.../paimon/rest/auth/DLFOpenApiV4Signer.java | 13 +
.../paimon/rest/auth/DLFOpenApiActionsTest.java | 141 +++++
.../paimon/rest/auth/DLFOpenApiV4SignerTest.java | 92 +++
.../paimon/data/variant/BaseVariantReader.java | 3 +-
.../data/variant/InferVariantShreddingSchema.java | 24 +-
.../org/apache/paimon/data/variant/VariantGet.java | 88 ++-
.../paimon/globalindex/GlobalIndexEvaluator.java | 69 ++-
.../paimon/globalindex/GlobalIndexReader.java | 22 +
.../globalindex/OffsetGlobalIndexReader.java | 7 +
.../globalindex/SortedFileGlobalIndexReader.java | 13 +-
.../paimon/globalindex/UnionGlobalIndexReader.java | 7 +
.../paimon/globalindex/btree/BTreeIndexReader.java | 5 +
.../globalindex/btree/LazyFilteredBTreeReader.java | 10 +
.../paimon/data/variant/GenericVariantTest.java | 81 +++
.../variant/InferVariantShreddingSchemaTest.java | 47 ++
.../data/variant/PaimonShreddingUtilsTest.java | 44 +-
.../globalindex/GlobalIndexEvaluatorTest.java | 48 ++
.../btree/LazyFilteredBTreeIndexReaderTest.java | 90 +++
.../TestFullTextGlobalIndexReader.java | 5 +
.../paimon/catalog/TableQueryAuthResult.java | 33 +
.../org/apache/paimon/io/FileIndexEvaluator.java | 47 +-
.../apache/paimon/io/RollingFileWriterImpl.java | 11 +-
.../apache/paimon/manifest/ManifestAvroReader.java | 12 +
.../org/apache/paimon/manifest/ManifestFile.java | 33 +-
.../paimon/operation/DataEvolutionSplitRead.java | 25 +-
.../apache/paimon/operation/FileDeletionBase.java | 145 ++++-
.../paimon/operation/FileStoreCommitImpl.java | 9 +-
.../apache/paimon/operation/SnapshotDeletion.java | 6 +-
.../paimon/operation/commit/StrictModeChecker.java | 18 +-
.../org/apache/paimon/schema/SchemaValidation.java | 46 ++
.../apache/paimon/table/ChainGroupReadTable.java | 93 ++-
.../paimon/table/ChainTableFileStoreTable.java | 5 +-
.../apache/paimon/table/ChainTableStreamScan.java | 49 +-
.../apache/paimon/table/ExpireSnapshotsImpl.java | 81 ++-
.../org/apache/paimon/table/PostponeUtils.java | 4 +-
.../paimon/table/source/AbstractDataTableScan.java | 22 +-
.../table/source/DataEvolutionFullTextRead.java | 221 ++++++-
.../table/source/DataEvolutionFullTextScan.java | 88 ++-
.../apache/paimon/table/source/DataTableScan.java | 12 +
.../paimon/table/source/FilteredRowIdReader.java | 132 ++++
.../paimon/table/source/FullTextSearchBuilder.java | 11 +
.../table/source/FullTextSearchBuilderImpl.java | 42 +-
.../table/source/HybridSearchBuilderImpl.java | 13 +-
.../table/source/IndexFullTextSearchSplit.java | 46 +-
.../apache/paimon/table/source/QueryAuthSplit.java | 10 +
.../paimon/table/source/RawFullTextReadImpl.java | 33 +-
.../java/org/apache/paimon/CoreOptionsTest.java | 32 +
.../paimon/catalog/TableQueryAuthResultTest.java | 85 +++
.../apache/paimon/io/FileIndexEvaluatorTest.java | 83 ++-
.../apache/paimon/io/RollingFileWriterTest.java | 57 ++
.../apache/paimon/manifest/ManifestFileTest.java | 44 ++
.../paimon/operation/ExpireSnapshotsTest.java | 153 ++++-
.../paimon/operation/FileStoreCommitTest.java | 39 ++
.../apache/paimon/schema/SchemaValidationTest.java | 63 ++
.../paimon/table/ChainTableFileStoreTableTest.java | 314 +++++++++-
.../paimon/table/DataEvolutionFileIndexTest.java | 68 +-
.../org/apache/paimon/table/PostponeUtilsTest.java | 2 +-
.../apache/paimon/table/sink/TableCommitTest.java | 67 +-
.../table/source/FullTextSearchBuilderTest.java | 687 ++++++++++++++++++++-
.../table/source/PrimaryKeyFullTextSearchTest.java | 32 +
.../flink/action/DataEvolutionMergeIntoAction.java | 2 +-
.../apache/paimon/flink/action/RescaleAction.java | 3 +-
.../dataevolution/DataEvolutionDeleteSink.java | 3 +-
.../flink/procedure/RollbackToProcedure.java | 7 +-
.../action/DeleteActionDataEvolutionITCase.java | 2 +-
.../flink/procedure/RollbackProcedureITCase.java | 74 +++
.../apache/paimon/format/avro/AvroFileFormat.java | 19 +-
.../paimon/format/avro/AvroFileFormatTest.java | 153 +++++
.../parquet/reader/VariantShreddingReadTest.java | 62 ++
.../writer/InferVariantShreddingWriteTest.java | 41 ++
.../index/NativeFullTextRowFilterTest.java | 517 ++++++++++++++++
paimon-python/README.md | 21 +
.../pypaimon/api/auth/dlf_openapi_actions.py | 103 +++
paimon-python/pypaimon/api/auth/dlf_signer.py | 8 +
.../pypaimon/common/options/core_options.py | 12 +
.../lumina/lumina_vector_global_index_reader.py | 2 +-
.../vindex/vindex_vector_global_index_reader.py | 2 +-
paimon-python/pypaimon/multimodal/query.py | 67 +-
paimon-python/pypaimon/ray/vector_search.py | 176 ++++++
paimon-python/pypaimon/read/native_plan.py | 81 ++-
paimon-python/pypaimon/read/table_read.py | 321 +++++++++-
paimon-python/pypaimon/read/table_scan.py | 13 +-
.../pypaimon/table/source/vector_search_read.py | 31 +-
.../pypaimon/tests/batch_vector_lookup_test.py | 181 ++++++
.../pypaimon/tests/data_evolution_formats_test.py | 4 +-
.../pypaimon/tests/data_evolution_test.py | 4 +-
.../pypaimon/tests/native_plan_integration_test.py | 373 ++++++++++-
paimon-python/pypaimon/tests/native_plan_test.py | 30 +-
paimon-python/pypaimon/tests/native_read_test.py | 349 +++++++++++
.../pypaimon/tests/ray_vector_refinement_test.py | 134 ++++
.../pypaimon/tests/ray_vector_search_test.py | 409 ++++++++++++
.../pypaimon/tests/rest/dlf_signer_test.py | 127 ++++
paimon-python/pypaimon/tests/table_update_test.py | 3 +-
.../tests/vector_metric_consistency_test.py | 23 +-
.../pypaimon/tests/vector_scoring_test.py | 50 ++
paimon-python/pypaimon/tests/vindex_input_test.py | 11 +-
.../paimon/spark/procedure/RescaleProcedure.java | 2 +-
.../org/apache/paimon/spark/PaimonBaseScan.scala | 4 +-
.../spark/SparkPostponeStagedCommitter.scala | 4 +-
.../paimon/spark/commands/PaimonSparkWriter.scala | 4 +-
.../paimon/spark/sql/FullTextSearchTest.scala | 228 +++++++
.../apache/paimon/spark/sql/HybridSearchTest.scala | 57 +-
.../apache/paimon/spark/sql/VariantTestBase.scala | 47 ++
113 files changed, 7558 insertions(+), 390 deletions(-)
create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java
create mode 100644 paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java
create mode 100644 paimon-core/src/main/java/org/apache/paimon/table/source/FilteredRowIdReader.java
create mode 100644 paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextRowFilterTest.java
create mode 100644 paimon-python/pypaimon/api/auth/dlf_openapi_actions.py
create mode 100644 paimon-python/pypaimon/ray/vector_search.py
create mode 100644 paimon-python/pypaimon/tests/batch_vector_lookup_test.py
create mode 100644 paimon-python/pypaimon/tests/native_read_test.py
create mode 100644 paimon-python/pypaimon/tests/ray_vector_refinement_test.py
create mode 100644 paimon-python/pypaimon/tests/ray_vector_search_test.py
(paimon) 19/20: [python] Add opt-in Rust native data reads (#9954)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit a74d86960bf6b949670db99edea392997bf259ed
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Fri Sep 18 21:45:01 2026 +0800
[python] Add opt-in Rust native data reads (#9954)
---
paimon-python/README.md | 21 ++
.../pypaimon/common/options/core_options.py | 12 +
paimon-python/pypaimon/read/native_plan.py | 81 ++++-
paimon-python/pypaimon/read/table_read.py | 321 ++++++++++++++++--
paimon-python/pypaimon/read/table_scan.py | 13 +-
.../pypaimon/tests/data_evolution_formats_test.py | 4 +-
.../pypaimon/tests/data_evolution_test.py | 4 +-
.../pypaimon/tests/native_plan_integration_test.py | 373 ++++++++++++++++++++-
paimon-python/pypaimon/tests/native_plan_test.py | 30 +-
paimon-python/pypaimon/tests/native_read_test.py | 349 +++++++++++++++++++
paimon-python/pypaimon/tests/table_update_test.py | 3 +-
11 files changed, 1151 insertions(+), 60 deletions(-)
diff --git a/paimon-python/README.md b/paimon-python/README.md
index b3541b1bdb..c8ee07f1d3 100644
--- a/paimon-python/README.md
+++ b/paimon-python/README.md
@@ -124,6 +124,27 @@ Python planner for unsupported scans. New bindings preserve `plan.snapshot_id`
even when pruning removes every split. Native explain output includes snapshot
and split metadata; native pruning counters are not exposed.
+To run both split planning and data-file reading in Rust, enable the independent
+native-read option:
+
+```python
+native_table = table.copy({"read.native.enabled": "true"})
+builder = native_table.new_read_builder().with_projection(["id", "name"])
+plan = builder.new_scan().plan()
+rows = builder.new_read().to_arrow(plan.splits())
+```
+
+Native reads return PyArrow batches through the Arrow C Data interface. They
+currently require untouched splits produced by the native planner and top-level
+projection. Query authorization, nested projection, row-kind output, and
+explicit blob parallelism retain the Python reader. For materialized
+`to_arrow()` reads, the effective split parallelism (the method argument,
+`read.parallelism`, or the automatic default) runs independent Rust readers
+over contiguous split groups. Streaming `to_arrow_batch_reader()` keeps one
+lazy Rust reader. A missing reader capability, unsupported route, or
+native-reader construction failure falls back to Python; I/O and data errors
+raised after streaming starts surface to the caller.
+
With Rust main's `Table.from_resolved_schema()` binding, filesystem and JDBC catalog
tables preserve the Python table's resolved schema and complete effective
options. Stale table objects, historical schemas, and `copy()` overrides or
diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py
index 7df8ee0ef2..50e48ab3ca 100644
--- a/paimon-python/pypaimon/common/options/core_options.py
+++ b/paimon-python/pypaimon/common/options/core_options.py
@@ -686,6 +686,15 @@ class CoreOptions:
"reader still reads the files.")
)
+ READ_NATIVE_ENABLED: ConfigOption[bool] = (
+ ConfigOptions.key("read.native.enabled")
+ .boolean_type()
+ .default_value(False)
+ .with_description("Read data via pypaimon_rust and return PyArrow batches. "
+ "This also enables native split planning; unsupported "
+ "routes fall back to pypaimon.")
+ )
+
CHANGELOG_PRODUCER: ConfigOption[ChangelogProducer] = (
ConfigOptions.key("changelog-producer")
.enum_type(ChangelogProducer)
@@ -1582,6 +1591,9 @@ class CoreOptions:
def native_plan_enabled(self, default=None):
return self.options.get(CoreOptions.SCAN_NATIVE_PLAN_ENABLED, default)
+ def native_read_enabled(self, default=None):
+ return self.options.get(CoreOptions.READ_NATIVE_ENABLED, default)
+
def changelog_producer(self, default=None):
return self.options.get(CoreOptions.CHANGELOG_PRODUCER, default)
diff --git a/paimon-python/pypaimon/read/native_plan.py b/paimon-python/pypaimon/read/native_plan.py
index ed12a98948..6a15b41939 100644
--- a/paimon-python/pypaimon/read/native_plan.py
+++ b/paimon-python/pypaimon/read/native_plan.py
@@ -49,6 +49,13 @@ def native_runtime_available() -> bool:
return hasattr(PaimonCatalog, 'get_table') and hasattr(Split, 'serialize')
+def native_reader_available() -> bool:
+ """Whether pypaimon-rust exposes the batch-read API used by PyPaimon."""
+ return (native_runtime_available()
+ and native_method_available('ReadBuilder', 'new_read')
+ and native_method_available('TableRead', 'read'))
+
+
def native_family_search_modes_available() -> bool:
"""Whether Rust supports family-specific global-index search modes."""
return native_version_at_least(0, 4)
@@ -260,25 +267,13 @@ def _restore_python_partition_paths(table, splits: List[Split]) -> None:
for data_file, python_path in candidates:
if data_file.file_name in bucket_files[bucket_path]:
data_file.file_path = python_path
+ # The retained Rust split still points at its canonical path.
+ # Invalidate it so native reading cannot bypass this repair.
+ split._native_split = None
-def native_plan(
- table,
- predicate: Optional[Predicate] = None,
- limit: Optional[int] = None,
- projection: Optional[List[str]] = None,
- row_ranges: Optional[List[Tuple[int, int]]] = None,
- incremental_range: Optional[Tuple[int, int]] = None,
- row_position_slice: Optional[Tuple[int, int]] = None,
- row_position_shard: Optional[Tuple[int, int]] = None) -> Plan:
- """Plan with pypaimon_rust, preserving snapshot metadata.
-
- Native conversion or planning failures are handled by TableScan, which
- falls back to the Python planner.
- """
- if not native_runtime_available():
- raise RuntimeError(
- "scan.native-plan.enabled needs pypaimon-rust>=0.3.0 (split planning API)")
+def _native_read_builder(table):
+ """Reconstruct the Rust table and return a builder for the same schema."""
file_io_options = _resolved_schema_file_io_options(table)
if file_io_options is not None:
from pypaimon_rust.datafusion import Table
@@ -307,12 +302,56 @@ def native_plan(
branch = getattr(rt, 'branch', None)
if not callable(branch) or branch() != table.current_branch():
raise RuntimeError("Native table did not resolve the requested branch")
+ return builder
+
+
+def _configure_native_read_builder(builder, predicate, limit, projection):
if projection is not None:
builder = builder.with_projection(projection)
if predicate is not None:
builder = builder.with_filter(_predicate_to_native(predicate))
if limit is not None:
builder = builder.with_limit(limit)
+ return builder
+
+
+def native_read(table, splits, predicate: Optional[Predicate] = None,
+ limit: Optional[int] = None,
+ projection: Optional[List[str]] = None,
+ blob_parallelism: Optional[int] = None):
+ """Read Rust ``Split`` objects into PyArrow ``RecordBatch`` objects."""
+ if not native_reader_available():
+ raise RuntimeError(
+ "read.native.enabled needs the pypaimon-rust native reader API")
+ builder = _configure_native_read_builder(
+ _native_read_builder(table), predicate, limit, projection)
+ if blob_parallelism is not None:
+ builder = builder.with_blob_parallelism(blob_parallelism)
+ reader = builder.new_read()
+ read_arrow = getattr(reader, 'read_arrow', None)
+ return (read_arrow(splits) if callable(read_arrow)
+ else reader.read(splits))
+
+
+def native_plan(
+ table,
+ predicate: Optional[Predicate] = None,
+ limit: Optional[int] = None,
+ projection: Optional[List[str]] = None,
+ row_ranges: Optional[List[Tuple[int, int]]] = None,
+ incremental_range: Optional[Tuple[int, int]] = None,
+ row_position_slice: Optional[Tuple[int, int]] = None,
+ row_position_shard: Optional[Tuple[int, int]] = None) -> Plan:
+ """Plan with pypaimon_rust, preserving snapshot metadata.
+
+ Native conversion or planning failures are handled by TableScan, which
+ falls back to the Python planner.
+ """
+ if not native_runtime_available():
+ raise RuntimeError(
+ "scan.native-plan.enabled needs pypaimon-rust>=0.3.0 (split planning API)")
+ builder = _configure_native_read_builder(
+ _native_read_builder(table), predicate, limit, projection)
if row_ranges is not None:
builder = builder.with_row_ranges(row_ranges)
scan = (builder.new_scan() if incremental_range is None
@@ -327,6 +366,14 @@ def native_plan(
# Trimmed primary keys decode per-file min/max keys (PK merge-on-read).
kfields = table.trimmed_primary_keys_fields
splits = [deserialize_split_v1(split.serialize(), pfields, kfields) for split in rust_splits]
+ if table.options.native_read_enabled():
+ # Retain the opaque Rust split next to the Python metadata view. The
+ # normal planner/reader contract remains a Python Split list, while
+ # native reads can consume the exact Rust split without a second lossy
+ # conversion. Any Python split transformation creates a fresh object
+ # without this marker and thus safely falls back to the Python reader.
+ for split, rust_split in zip(splits, rust_splits):
+ split._native_split = rust_split
_restore_python_partition_paths(table, splits)
snapshot_id = getattr(rust_plan, 'snapshot_id', None)
if callable(snapshot_id):
diff --git a/paimon-python/pypaimon/read/table_read.py b/paimon-python/pypaimon/read/table_read.py
index 5b8407874e..166b8d4aac 100644
--- a/paimon-python/pypaimon/read/table_read.py
+++ b/paimon-python/pypaimon/read/table_read.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import logging
import os
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -23,6 +24,7 @@ from typing import Any, Callable, Dict, Iterator, List, Optional
import pandas
import pyarrow
+from pypaimon.common.options.core_options import CoreOptions
from pypaimon.common.predicate import Predicate
from pypaimon.common.predicate_json_parser import extract_referenced_fields
from pypaimon.data.map_shared_shredding import map_selected_keys_field
@@ -42,8 +44,19 @@ from pypaimon.schema.data_types import (
from pypaimon.table.row.offset_row import OffsetRow
ROW_KIND_COLUMN = "_row_kind"
+logger = logging.getLogger(__name__)
_RECORD_BATCH_READER_FROM_STREAM = getattr(
pyarrow.ipc.RecordBatchReader, "from_stream", None)
+_NATIVE_READ_FILE_FORMATS = frozenset({
+ CoreOptions.FILE_FORMAT_PARQUET,
+ CoreOptions.FILE_FORMAT_ORC,
+ CoreOptions.FILE_FORMAT_AVRO,
+ CoreOptions.FILE_FORMAT_ROW,
+ CoreOptions.FILE_FORMAT_MOSAIC,
+})
+_NATIVE_READ_FILE_SUFFIXES = tuple(
+ '.%s' % file_format for file_format in _NATIVE_READ_FILE_FORMATS)
+_NATIVE_BLOB_FILE_SUFFIX = '.blob'
class _ClosableArrowBatchReader:
@@ -113,12 +126,16 @@ class _RemainingRows:
return self._remaining <= 0
+class _NativeReadSetupError(Exception):
+ """Native reader construction failed before a batch was exposed."""
+
+
class TableRead:
"""Implementation of TableRead for native Python reading."""
- # Cap on peak concurrent blob reads across the whole parallel read: split
- # workers (P) each spin up blob_parallelism (B) blob threads, so peak
- # connections ~= P*B. Shrink per-split B to keep the product bounded.
+ # Cap peak concurrent BLOB range reads across the whole parallel read:
+ # split workers (P) each allow blob_parallelism (B) in-flight reads, so
+ # peak connections ~= P*B. Shrink per-reader B to keep the product bounded.
_MAX_TOTAL_BLOB_WORKERS = 64
def __init__(
@@ -213,11 +230,17 @@ class TableRead:
self,
splits: List[Split],
blob_parallelism: Optional[int] = None):
- effective_bp = self._resolve_blob_parallelism(blob_parallelism)
schema = PyarrowFieldParser.from_paimon_schema(self.read_type)
if self.include_row_kind:
schema = self._add_row_kind_to_schema(schema)
- batch_iterator = self._arrow_batch_generator(splits, schema, effective_bp)
+ effective_bp = self._resolve_blob_parallelism(blob_parallelism)
+ native_batches = self._try_native_batches(
+ splits, schema, blob_parallelism=effective_bp)
+ if native_batches is not None:
+ batch_iterator = iter(native_batches)
+ else:
+ batch_iterator = self._arrow_batch_generator(
+ splits, schema, effective_bp)
reader_type = pyarrow.ipc.RecordBatchReader
reader = reader_type.from_batches(schema, batch_iterator)
return reader, batch_iterator
@@ -229,8 +252,18 @@ class TableRead:
return pyarrow.schema([row_kind_field] + list(schema))
@staticmethod
- def _try_to_pad_batch_by_schema(batch: pyarrow.RecordBatch, target_schema):
- if batch.schema.names == target_schema.names:
+ def _try_to_pad_batch_by_schema(
+ batch: pyarrow.RecordBatch,
+ target_schema,
+ allow_type_cast: bool = False):
+ """Align a batch with the requested schema.
+
+ Native batches stay strict so an outdated Rust BLOB representation
+ cannot be silently converted. Python query-auth masking is allowed
+ to cast internal dependency columns because masking intentionally may
+ change their logical type.
+ """
+ if batch.schema.equals(target_schema, check_metadata=True):
return batch
columns = []
@@ -239,6 +272,12 @@ class TableRead:
for field in target_schema:
if field.name in batch.schema.names:
col = batch.column(field.name)
+ if col.type != field.type:
+ if not allow_type_cast:
+ raise TypeError(
+ "Batch field '%s' has type %s, expected %s" % (
+ field.name, col.type, field.type))
+ col = col.cast(field.type)
else:
col = pyarrow.nulls(num_rows, type=field.type)
columns.append(col)
@@ -265,12 +304,11 @@ class TableRead:
and a ``limit`` set, the returned rows are an arbitrary
subset of the requested size, since which splits fill the row
quota first is non-deterministic. Data-evolution reads with
- deferred BLOB resolution run serially when a limit may discard
- rows, so payloads are not materialized from discarded splits.
- blob_parallelism: number of threads for concurrent blob reads
- within each batch. ``None`` or ``1`` (default) reads blobs
- serially; ``>= 2`` uses a thread pool with ``pread`` for
- concurrent ranged reads. GIL is released during I/O. On the
+ deferred BLOB resolution falls back to the Python reader when
+ a limit may discard rows, so discarded payloads are not read.
+ blob_parallelism: maximum concurrent blob range reads within each
+ split reader. ``None`` or ``1`` (default) reads blobs serially;
+ ``>= 2`` enables concurrent ranged reads. On the
parallel path, peak blob threads (``parallelism`` *
``blob_parallelism``) are capped at
``_MAX_TOTAL_BLOB_WORKERS``; per-split ``blob_parallelism`` is
@@ -282,21 +320,202 @@ class TableRead:
if self.include_row_kind:
schema = self._add_row_kind_to_schema(schema)
+ native_parallel = self._should_run_parallel(splits, effective)
+ native_bp = effective_bp
+ if native_parallel:
+ native_bp = self._cap_blob_parallelism(
+ min(effective, len(splits)), effective_bp)
+ native_batches = self._try_native_batches(
+ splits,
+ schema,
+ parallelism=effective,
+ blob_parallelism=native_bp,
+ )
+ if native_batches is not None:
+ return self._batches_to_arrow(native_batches, schema)
+
if self._should_run_parallel(splits, effective):
return self._to_arrow_parallel(splits, schema, effective, effective_bp)
- batch_reader = self.to_arrow_batch_reader(splits, blob_parallelism=effective_bp)
+ batch_iterator = self._arrow_batch_generator(
+ splits, schema, effective_bp)
+ batch_reader = pyarrow.ipc.RecordBatchReader.from_batches(
+ schema, batch_iterator)
table_list = []
for batch in iter(batch_reader.read_next_batch, None):
if batch.num_rows == 0:
continue
- table_list.append(self._try_to_pad_batch_by_schema(batch, schema))
+ table_list.append(self._try_to_pad_batch_by_schema(
+ batch, schema, allow_type_cast=True))
- if not table_list:
- return pyarrow.Table.from_arrays([pyarrow.array([], type=field.type) for field in schema], schema=schema)
- else:
- return pyarrow.Table.from_batches(table_list)
+ return self._batches_to_arrow(
+ table_list, schema, allow_type_cast=True)
+
+ @staticmethod
+ def _batches_to_arrow(batches, schema, allow_type_cast: bool = False):
+ batches = [
+ TableRead._try_to_pad_batch_by_schema(
+ batch, schema, allow_type_cast=allow_type_cast)
+ for batch in batches
+ if batch.num_rows > 0
+ ]
+ if not batches:
+ return pyarrow.Table.from_arrays(
+ [pyarrow.array([], type=field.type) for field in schema],
+ schema=schema)
+ return pyarrow.Table.from_batches(batches)
+
+ def _try_native_batches(
+ self,
+ splits: List[Split],
+ schema: pyarrow.Schema,
+ parallelism: Optional[int] = None,
+ blob_parallelism: Optional[int] = None):
+ """Return Rust-read batches, or ``None`` when this read must fall back."""
+ if not self.table.options.native_read_enabled():
+ return None
+ # These Python-only output controls do not yet have native equivalents.
+ if self.include_row_kind or self.nested_name_paths:
+ return None
+ if self.table.options.file_format() not in _NATIVE_READ_FILE_FORMATS:
+ return None
+ if not splits:
+ return []
+ if self._deferred_blob_limit_may_prune(splits):
+ return None
+ if self._native_schema_needs_python_fallback(schema):
+ return None
+ try:
+ from pypaimon.read.native_plan import native_read
+ except Exception as e:
+ logger.warning(
+ "Native read failed, falling back to the Python reader: %s", e)
+ return None
+ rust_splits = []
+ for split in splits:
+ if isinstance(split, QueryAuthSplit):
+ return None
+ if not self._native_split_files_supported(split):
+ return None
+ rust_split = getattr(split, '_native_split', None)
+ if rust_split is None:
+ return None
+ rust_splits.append(rust_split)
+ if (parallelism is not None
+ and self._should_run_parallel(splits, parallelism)):
+ try:
+ return self._native_batches_parallel(
+ native_read, rust_splits, schema, parallelism,
+ blob_parallelism)
+ except _NativeReadSetupError as e:
+ logger.warning(
+ "Native read failed, falling back to the Python reader: %s", e)
+ return None
+ try:
+ read_kwargs = {
+ 'predicate': self.predicate,
+ 'limit': self.limit,
+ 'projection': [field.name for field in self.read_type],
+ }
+ if blob_parallelism is not None:
+ read_kwargs['blob_parallelism'] = blob_parallelism
+ batches = native_read(self.table, rust_splits, **read_kwargs)
+ except Exception as e:
+ logger.warning(
+ "Native read failed, falling back to the Python reader: %s", e)
+ return None
+ return self._convert_native_batches(batches, schema)
+
+ def _native_batches_parallel(
+ self, native_read, rust_splits, schema, effective,
+ blob_parallelism):
+ """Read contiguous split groups with independent Rust readers."""
+ workers = min(effective, len(rust_splits))
+ base_size, larger_groups = divmod(len(rust_splits), workers)
+ groups = []
+ offset = 0
+ for index in range(workers):
+ size = base_size + (1 if index < larger_groups else 0)
+ groups.append(rust_splits[offset:offset + size])
+ offset += size
+
+ remaining_state = _RemainingRows(self.limit)
+ results = [None] * len(groups)
+ with ThreadPoolExecutor(
+ max_workers=workers,
+ thread_name_prefix="pypaimon-native-read") as executor:
+ futures = {
+ executor.submit(
+ self._read_native_split_group,
+ native_read,
+ group,
+ schema,
+ remaining_state,
+ blob_parallelism,
+ ): index
+ for index, group in enumerate(groups)
+ }
+ for future in as_completed(futures):
+ results[futures[future]] = future.result()
+
+ return [batch for group_batches in results for batch in group_batches]
+
+ def _read_native_split_group(
+ self, native_read, rust_splits, schema, remaining_state,
+ blob_parallelism):
+ if remaining_state.exhausted():
+ return []
+ try:
+ read_kwargs = {
+ 'predicate': self.predicate,
+ 'limit': self.limit,
+ 'projection': [field.name for field in self.read_type],
+ }
+ if blob_parallelism is not None:
+ read_kwargs['blob_parallelism'] = blob_parallelism
+ batches = native_read(self.table, rust_splits, **read_kwargs)
+ except Exception as e:
+ raise _NativeReadSetupError(str(e)) from e
+ result = []
+ for batch in batches:
+ if batch.num_rows == 0:
+ continue
+ allowed = remaining_state.try_consume(batch.num_rows)
+ if allowed == 0:
+ break
+ if allowed < batch.num_rows:
+ batch = batch.slice(0, allowed)
+ batch = self._project_batch_to_output(batch)
+ result.append(self._try_to_pad_batch_by_schema(batch, schema))
+ if remaining_state.exhausted():
+ break
+ return result
+
+ @staticmethod
+ def _native_split_files_supported(split):
+ for data_file in split.files:
+ file_name = data_file.file_name.lower()
+ if ('.vector.' not in file_name
+ and not file_name.endswith(_NATIVE_READ_FILE_SUFFIXES)
+ and not file_name.endswith(_NATIVE_BLOB_FILE_SUFFIX)):
+ return False
+ return True
+
+ def _convert_native_batches(self, batches, schema):
+ """Apply PyPaimon's exact output limit lazily to native batches."""
+ remaining = self.limit
+ for batch in batches:
+ if batch.num_rows == 0:
+ continue
+ if remaining is not None and batch.num_rows > remaining:
+ batch = batch.slice(0, remaining)
+ batch = self._project_batch_to_output(batch)
+ yield self._try_to_pad_batch_by_schema(batch, schema)
+ if remaining is not None:
+ remaining -= batch.num_rows
+ if remaining <= 0:
+ break
def _arrow_batch_generator(self, splits: List[Split], schema: pyarrow.Schema,
blob_parallelism: int = 1) -> Iterator[pyarrow.RecordBatch]:
@@ -400,9 +619,9 @@ class TableRead:
return max(1, cls._MAX_TOTAL_BLOB_WORKERS // workers)
def _should_run_parallel(
- self,
- splits: List[Split],
- effective: int,
+ self,
+ splits: List[Split],
+ effective: int,
) -> bool:
"""Decide whether to take the parallel read path.
@@ -410,14 +629,59 @@ class TableRead:
overhead, no behavior change). A single split is never
parallelized since there is nothing to fan out across.
"""
- deferred_limit_may_prune = (
- self.limit is not None
- and self._deferred_blob_fields
- and not self._limit_covers_all_splits(splits)
- )
+ deferred_limit_may_prune = self._deferred_blob_limit_may_prune(splits)
return (effective >= 2 and len(splits) >= 2
and not deferred_limit_may_prune)
+ def _deferred_blob_limit_may_prune(self, splits: List[Split]) -> bool:
+ return (self.limit is not None
+ and (self._deferred_blob_fields
+ or self._native_inline_blob_fields())
+ and not self._limit_covers_all_splits(splits))
+
+ def _native_inline_blob_fields(self) -> set:
+ """Return configured BLOB fields that native reads resolve eagerly."""
+ options = self.table.options
+ if options.blob_as_descriptor():
+ return set()
+ inline_fields = (
+ options.blob_descriptor_fields()
+ | options.blob_view_fields()
+ )
+ read_names = {
+ field.name
+ for field in getattr(self, '_scan_read_type', self.read_type)
+ }
+ return inline_fields & read_names
+
+ @classmethod
+ def _native_schema_needs_python_fallback(cls, schema: pyarrow.Schema) -> bool:
+ """Rust currently reads precision-zero timestamps as milliseconds."""
+ return any(
+ cls._native_type_needs_python_fallback(field.type)
+ for field in schema
+ )
+
+ @classmethod
+ def _native_type_needs_python_fallback(cls, data_type) -> bool:
+ if pyarrow.types.is_timestamp(data_type):
+ return data_type.unit == 's'
+ if pyarrow.types.is_struct(data_type):
+ return any(
+ cls._native_type_needs_python_fallback(field.type)
+ for field in data_type
+ )
+ if (pyarrow.types.is_list(data_type)
+ or pyarrow.types.is_large_list(data_type)
+ or pyarrow.types.is_fixed_size_list(data_type)):
+ return cls._native_type_needs_python_fallback(data_type.value_type)
+ if pyarrow.types.is_map(data_type):
+ return (
+ cls._native_type_needs_python_fallback(data_type.key_type)
+ or cls._native_type_needs_python_fallback(data_type.item_type)
+ )
+ return False
+
def _limit_covers_all_splits(self, splits: List[Split]) -> bool:
"""Return whether split metadata proves that LIMIT cannot drop rows."""
total_rows = 0
@@ -473,7 +737,8 @@ class TableRead:
for batch in split_batches:
if batch.num_rows == 0:
continue
- table_list.append(self._try_to_pad_batch_by_schema(batch, schema))
+ table_list.append(self._try_to_pad_batch_by_schema(
+ batch, schema, allow_type_cast=True))
if not table_list:
return pyarrow.Table.from_arrays(
diff --git a/paimon-python/pypaimon/read/table_scan.py b/paimon-python/pypaimon/read/table_scan.py
index 2a21527242..6d269957e6 100755
--- a/paimon-python/pypaimon/read/table_scan.py
+++ b/paimon-python/pypaimon/read/table_scan.py
@@ -57,6 +57,7 @@ _NATIVE_FORWARDED_OPTIONS = frozenset({
}) | _NATIVE_SEARCH_MODE_OPTIONS
_NATIVE_PLAN_INDEPENDENT_OPTIONS = frozenset({
CoreOptions.BLOB_AS_DESCRIPTOR.key(),
+ CoreOptions.READ_NATIVE_ENABLED.key(),
CoreOptions.READ_BATCH_SIZE.key(),
CoreOptions.READ_PARALLELISM.key(),
})
@@ -95,7 +96,7 @@ class TableScan:
auth_result = self.__auth_query()
# The native planner bypasses the auth-aware file scanner. Resolve auth
# before selecting a planning backend.
- if (auth_result is None and self.table.options.native_plan_enabled()
+ if (auth_result is None and self._native_requested()
and self._native_plan_supported()):
native = self._try_native_plan()
if native is not None:
@@ -105,6 +106,14 @@ class TableScan:
plan = self.file_scanner.scan()
return wrap_plan_with_auth(auth_result, plan)
+ def _native_requested(self) -> bool:
+ if self.table.options.native_plan_enabled():
+ return True
+ if not self.table.options.native_read_enabled():
+ return False
+ from pypaimon.read.native_plan import native_reader_available
+ return native_reader_available()
+
def _native_plan_supported(self) -> bool:
# Any probe failure (e.g. a remote schema/metadata read) must fall back, not fail the scan.
try:
@@ -393,7 +402,7 @@ class TableScan:
explain reports their split metadata without pruning counters.
"""
auth_result = self.__auth_query()
- if (auth_result is None and self.table.options.native_plan_enabled()
+ if (auth_result is None and self._native_requested()
and self._native_plan_supported()):
native = self._try_native_plan()
if native is not None:
diff --git a/paimon-python/pypaimon/tests/data_evolution_formats_test.py b/paimon-python/pypaimon/tests/data_evolution_formats_test.py
index c970387995..73057665ee 100644
--- a/paimon-python/pypaimon/tests/data_evolution_formats_test.py
+++ b/paimon-python/pypaimon/tests/data_evolution_formats_test.py
@@ -494,7 +494,9 @@ class DataEvolutionFormatsTest(unittest.TestCase):
{'a': [1, 2, 3, 4], 'b': ['x', 'y', 'z', 'w'],
'c': [0.1, 0.2, 0.3, 0.4]},
schema=pa_schema)
- self.assertEqual(actual, expect)
+ # Table scans are unordered; native planning may emit merged groups
+ # before raw groups.
+ self.assertEqual(actual.sort_by('a'), expect.sort_by('a'))
# ------------------------------------------------------------------
# Blob-format data evolution
diff --git a/paimon-python/pypaimon/tests/data_evolution_test.py b/paimon-python/pypaimon/tests/data_evolution_test.py
index 8460f6ef2f..99af2e4933 100644
--- a/paimon-python/pypaimon/tests/data_evolution_test.py
+++ b/paimon-python/pypaimon/tests/data_evolution_test.py
@@ -497,7 +497,9 @@ class DataEvolutionTest(unittest.TestCase):
'f1': ['a'] * 100 + ['x'] + ['c'],
'f2': ['b'] * 100 + ['y'] + ['d'],
}, schema=simple_pa_schema)
- self.assertEqual(actual, expect)
+ # A table scan does not guarantee row-id order. Native planning may
+ # pack merge-required and raw groups in a different split order.
+ self.assertEqual(actual.sort_by('f0'), expect.sort_by('f0'))
self.assertEqual(len(actual.schema), len(expect.schema), 'Merge read output column count must match schema')
self.assertEqual(actual.schema.names, expect.schema.names, 'Merge read output column names must match schema')
diff --git a/paimon-python/pypaimon/tests/native_plan_integration_test.py b/paimon-python/pypaimon/tests/native_plan_integration_test.py
index 10cc5d3471..bbface6217 100644
--- a/paimon-python/pypaimon/tests/native_plan_integration_test.py
+++ b/paimon-python/pypaimon/tests/native_plan_integration_test.py
@@ -15,6 +15,9 @@
# specific language governing permissions and limitations
# under the License.
+import datetime
+import os
+import pickle
import tempfile
import unittest
from unittest.mock import patch
@@ -25,9 +28,10 @@ import pytest
from pypaimon import CatalogFactory, Schema
from pypaimon.globalindex.global_index_result import GlobalIndexResult
from pypaimon.read.native_plan import (
- native_family_search_modes_available, native_method_available,
+ native_family_search_modes_available, native_method_available, native_read,
+ native_reader_available,
)
-from pypaimon.table.row.blob import BlobDescriptor
+from pypaimon.table.row.blob import BlobDescriptor, BlobRef
from pypaimon.utils.range import Range
@@ -92,6 +96,13 @@ class NativePlanIntegrationTest(unittest.TestCase):
self.assertEqual(
native_table.new_read_builder().explain().native_planned, expect_native)
+ @staticmethod
+ def _native_rows(builder, plan):
+ with patch(
+ 'pypaimon.read.table_read.TableRead._create_split_read',
+ side_effect=AssertionError('Python reader was used')):
+ return builder.new_read().to_arrow(plan.splits()).to_pylist()
+
def test_primary_key_matches_normal_plan(self):
self.cat.create_table('default.pk_t', Schema.from_pyarrow_schema(
self.schema, primary_keys=['k'], options={'bucket': '1'}), False)
@@ -145,6 +156,102 @@ class NativePlanIntegrationTest(unittest.TestCase):
self._write('ap_t', [{'k': 3, 'v': 'c'}])
self._assert_matches('ap_t')
+ @unittest.skipUnless(native_reader_available(),
+ "pypaimon-rust native reader API not installed")
+ def test_native_read_bypasses_python_split_reader(self):
+ self.cat.create_table(
+ 'default.native_read_t', Schema.from_pyarrow_schema(self.schema), False)
+ self._write('native_read_t', [
+ {'k': 1, 'v': 'a'}, {'k': 2, 'v': 'b'}, {'k': 3, 'v': 'c'}])
+
+ table = self.cat.get_table('default.native_read_t').copy(
+ {'read.native.enabled': 'true'})
+ builder = table.new_read_builder().with_projection(['k']).with_limit(2)
+ builder.with_filter(builder.new_predicate_builder().greater_or_equal('k', 2))
+ plan = builder.new_scan().plan()
+ splits = [pickle.loads(pickle.dumps(split)) for split in plan.splits()]
+
+ with patch(
+ 'pypaimon.read.table_read.TableRead._create_split_read',
+ side_effect=AssertionError('Python reader was used')):
+ read = builder.new_read()
+ rows = read.to_arrow(splits).to_pylist()
+ streamed = pa.Table.from_batches(
+ list(read.to_arrow_batch_reader(splits))).to_pylist()
+
+ self.assertEqual(rows, [{'k': 2}, {'k': 3}])
+ self.assertEqual(streamed, rows)
+ self.assertTrue(builder.explain().native_planned)
+
+ @unittest.skipUnless(native_reader_available(),
+ "pypaimon-rust native reader API not installed")
+ def test_native_read_uses_effective_split_parallelism(self):
+ schema = pa.schema([
+ ('k', pa.int64()),
+ ('v', pa.string()),
+ ('dt', pa.string()),
+ ])
+ self.cat.create_table('default.native_parallel_t',
+ Schema.from_pyarrow_schema(
+ schema,
+ partition_keys=['dt']), False)
+ table = self.cat.get_table('default.native_parallel_t')
+ write_builder = table.new_batch_write_builder()
+ write = write_builder.new_write()
+ write.write_arrow(pa.Table.from_pylist([
+ {'k': 1, 'v': 'a', 'dt': 'p1'},
+ {'k': 2, 'v': 'b', 'dt': 'p2'},
+ {'k': 3, 'v': 'c', 'dt': 'p3'},
+ {'k': 4, 'v': 'd', 'dt': 'p4'},
+ ], schema=schema))
+ write_builder.new_commit().commit(write.prepare_commit())
+ write.close()
+
+ native_table = table.copy({
+ 'read.native.enabled': 'true',
+ 'read.parallelism': '2',
+ })
+ builder = native_table.new_read_builder()
+ plan = builder.new_scan().plan()
+ self.assertEqual(len(plan.splits()), 4)
+
+ with patch('pypaimon.read.native_plan.native_read',
+ wraps=native_read) as rust_reads, \
+ patch(
+ 'pypaimon.read.table_read.TableRead._create_split_read',
+ side_effect=AssertionError('Python reader was used')):
+ rows = builder.new_read().to_arrow(plan.splits()).to_pylist()
+
+ self.assertEqual(sorted(rows, key=lambda row: row['k']), [
+ {'k': 1, 'v': 'a', 'dt': 'p1'},
+ {'k': 2, 'v': 'b', 'dt': 'p2'},
+ {'k': 3, 'v': 'c', 'dt': 'p3'},
+ {'k': 4, 'v': 'd', 'dt': 'p4'},
+ ])
+ self.assertEqual(rust_reads.call_count, 2)
+
+ @unittest.skipUnless(native_reader_available(),
+ "pypaimon-rust native reader API not installed")
+ def test_native_read_primary_key_matches_python(self):
+ self.cat.create_table('default.native_read_pk', Schema.from_pyarrow_schema(
+ self.schema, primary_keys=['k'], options={'bucket': '1'}), False)
+ self._write('native_read_pk', [{'k': 1, 'v': 'a'}, {'k': 2, 'v': 'old'}])
+ self._write('native_read_pk', [{'k': 2, 'v': 'new'}, {'k': 3, 'v': 'c'}])
+
+ normal = self._plan_and_read('native_read_pk', native=False)[1]
+ table = self.cat.get_table('default.native_read_pk').copy(
+ {'read.native.enabled': 'true'})
+ builder = table.new_read_builder()
+ plan = builder.new_scan().plan()
+ with patch(
+ 'pypaimon.read.table_read.TableRead._create_split_read',
+ side_effect=AssertionError('Python reader was used')):
+ native = sorted(
+ builder.new_read().to_arrow(plan.splits()).to_pylist(),
+ key=lambda row: row['k'])
+
+ self.assertEqual(native, normal)
+
def test_append_distribution_matches_interleaved_partition_buckets(self):
self.schema = pa.schema([('k', pa.int64()), ('v', pa.string()), ('p', pa.string())])
self.cat.create_table('default.interleaved_t', Schema.from_pyarrow_schema(
@@ -238,7 +345,7 @@ class NativePlanIntegrationTest(unittest.TestCase):
self._assert_matches('de_t')
native_table = self.cat.get_table('default.de_t').copy(
- {'scan.native-plan.enabled': 'true'})
+ {'read.native.enabled': 'true'})
predicate = native_table.new_read_builder().new_predicate_builder().equal(
'v', 'b2')
builder = (native_table.new_read_builder()
@@ -246,7 +353,7 @@ class NativePlanIntegrationTest(unittest.TestCase):
.with_filter(predicate)
.with_limit(1))
plan = builder.new_scan().plan()
- rows = builder.new_read().to_arrow(plan.splits()).to_pylist()
+ rows = self._native_rows(builder, plan)
self.assertEqual(rows, [{'k': 2}])
self.assertTrue(builder.explain().native_planned)
@@ -255,6 +362,9 @@ class NativePlanIntegrationTest(unittest.TestCase):
.with_projection(['media.camera'])
.with_limit(1))
blob_plan = blob_builder.new_scan().plan()
+ # A dotted top-level field currently uses PyPaimon's nested-projection
+ # machinery, so this part deliberately exercises the documented
+ # Python fallback while the scalar projection above is native.
blob_rows = blob_builder.new_read().to_arrow(
blob_plan.splits()).to_pylist()
self.assertEqual(blob_rows, [{'media.camera': b'a'}])
@@ -279,6 +389,241 @@ class NativePlanIntegrationTest(unittest.TestCase):
)
self.assertTrue(descriptor_builder.explain().native_planned)
+ @unittest.skipUnless(native_reader_available(),
+ "pypaimon-rust native reader API not installed")
+ def test_native_read_data_evolution_partial_appends(self):
+ schema = pa.schema([
+ ('f0', pa.int32()),
+ ('f1', pa.string()),
+ ('f2', pa.string()),
+ ])
+ self.cat.create_table('default.de_order_t', Schema.from_pyarrow_schema(
+ schema, options={
+ 'row-tracking.enabled': 'true',
+ 'data-evolution.enabled': 'true',
+ }), False)
+ table = self.cat.get_table('default.de_order_t')
+ write_builder = table.new_batch_write_builder()
+
+ base = write_builder.new_write()
+ base.write_arrow(pa.Table.from_pydict({
+ 'f0': [0, 1],
+ 'f1': ['z', 'a'],
+ 'f2': ['q', 'b'],
+ }, schema=schema))
+ write_builder.new_commit().commit(base.prepare_commit())
+ base.close()
+
+ left = write_builder.new_write().with_write_type(['f0', 'f1'])
+ right = write_builder.new_write().with_write_type(['f2'])
+ left.write_arrow(pa.Table.from_pydict(
+ {'f0': [2], 'f1': ['x']},
+ schema=pa.schema([('f0', pa.int32()), ('f1', pa.string())])))
+ right.write_arrow(pa.Table.from_pydict(
+ {'f2': ['y']}, schema=pa.schema([('f2', pa.string())])))
+ messages = left.prepare_commit() + right.prepare_commit()
+ for message in messages:
+ for data_file in message.new_files:
+ data_file.first_row_id = 2
+ write_builder.new_commit().commit(messages)
+ left.close()
+ right.close()
+
+ native_table = table.copy({'read.native.enabled': 'true'})
+ builder = native_table.new_read_builder()
+ plan = builder.new_scan().plan()
+ rows = self._native_rows(builder, plan)
+
+ self.assertEqual(sorted(rows, key=lambda row: row['f0']), [
+ {'f0': 0, 'f1': 'z', 'f2': 'q'},
+ {'f0': 1, 'f1': 'a', 'f2': 'b'},
+ {'f0': 2, 'f1': 'x', 'f2': 'y'},
+ ])
+ self.assertTrue(builder.explain().native_planned)
+
+ @unittest.skipUnless(native_reader_available(),
+ "pypaimon-rust native reader API not installed")
+ def test_native_read_data_evolution_blob_parallelism(self):
+ schema = pa.schema([
+ ('id', pa.int32()),
+ ('img', pa.large_binary()),
+ ])
+ self.cat.create_table('default.native_blob_t', Schema.from_pyarrow_schema(
+ schema, options={
+ 'row-tracking.enabled': 'true',
+ 'data-evolution.enabled': 'true',
+ }), False)
+ table = self.cat.get_table('default.native_blob_t')
+ payloads = [b'a', b'bb', b'ccc']
+ write = table.new_batch_write_builder().new_write()
+ write.write_arrow(pa.Table.from_pydict({
+ 'id': [1, 2, 3],
+ 'img': payloads,
+ }, schema=schema))
+ table.new_batch_write_builder().new_commit().commit(write.prepare_commit())
+ write.close()
+
+ native_table = table.copy({'read.native.enabled': 'true'})
+ builder = native_table.new_read_builder().with_projection(['id', 'img'])
+ plan = builder.new_scan().plan()
+ self.assertTrue(any(
+ data_file.file_name.endswith('.blob')
+ for split in plan.splits()
+ for data_file in split.files
+ ))
+
+ with patch(
+ 'pypaimon.read.table_read.TableRead._create_split_read',
+ side_effect=AssertionError('Python reader was used')):
+ table_result = builder.new_read().to_arrow(
+ plan.splits(), parallelism=1, blob_parallelism=3)
+ batch_table = builder.new_read().to_arrow_batch_reader(
+ plan.splits(), blob_parallelism=2).read_all()
+
+ expected = {'id': [1, 2, 3], 'img': payloads}
+ self.assertEqual(table_result.schema, schema)
+ self.assertEqual(batch_table.schema, schema)
+ self.assertEqual(table_result.to_pydict(), expected)
+ self.assertEqual(batch_table.to_pydict(), expected)
+ self.assertTrue(builder.explain().native_planned)
+
+ @unittest.skipUnless(native_reader_available(),
+ "pypaimon-rust native reader API not installed")
+ def test_native_read_pruning_limit_defers_blob_payload_io(self):
+ schema = pa.schema([
+ ('id', pa.int32()),
+ ('payload', pa.large_binary()),
+ ])
+ self.cat.create_table('default.native_blob_limit_t',
+ Schema.from_pyarrow_schema(schema, options={
+ 'row-tracking.enabled': 'true',
+ 'data-evolution.enabled': 'true',
+ }), False)
+ table = self.cat.get_table('default.native_blob_limit_t')
+ write = table.new_batch_write_builder().new_write()
+ write.write_arrow(pa.Table.from_pydict({
+ 'id': [1, 2, 3],
+ 'payload': [b'a', b'bb', b'ccc'],
+ }, schema=schema))
+ table.new_batch_write_builder().new_commit().commit(write.prepare_commit())
+ write.close()
+
+ native_table = table.copy({'read.native.enabled': 'true'})
+ builder = native_table.new_read_builder().with_limit(1)
+ plan = builder.new_scan().plan()
+ fetched = []
+ original_to_data = BlobRef.to_data
+
+ def tracked_to_data(blob):
+ fetched.append(blob)
+ return original_to_data(blob)
+
+ with patch.object(BlobRef, 'to_data', tracked_to_data), patch(
+ 'pypaimon.read.native_plan.native_read',
+ return_value=[]) as native:
+ result = builder.new_read().to_arrow(
+ plan.splits(), parallelism=1)
+
+ native.assert_not_called()
+ self.assertEqual(result.to_pydict(), {'id': [1], 'payload': [b'a']})
+ self.assertEqual(len(fetched), 1)
+
+ @unittest.skipUnless(native_reader_available(),
+ "pypaimon-rust native reader API not installed")
+ def test_native_read_pruning_limit_defers_descriptor_blob_payload_io(self):
+ schema = pa.schema([
+ ('id', pa.int32()),
+ ('payload', pa.large_binary()),
+ ])
+ with tempfile.TemporaryDirectory() as payload_dir:
+ first_path = os.path.join(payload_dir, 'first')
+ with open(first_path, 'wb') as output:
+ output.write(b'first')
+ missing_path = os.path.join(payload_dir, 'missing')
+ descriptors = [
+ BlobDescriptor(
+ 'file://' + first_path, 0, 5).serialize(),
+ BlobDescriptor(
+ 'file://' + missing_path, 0, 7).serialize(),
+ ]
+
+ self.cat.create_table(
+ 'default.native_descriptor_limit_t',
+ Schema.from_pyarrow_schema(schema, options={
+ 'row-tracking.enabled': 'true',
+ 'data-evolution.enabled': 'true',
+ 'blob-descriptor-field': 'payload',
+ }), False)
+ table = self.cat.get_table('default.native_descriptor_limit_t')
+ write = table.new_batch_write_builder().new_write()
+ write.write_arrow(pa.Table.from_pydict({
+ 'id': [1, 2],
+ 'payload': descriptors,
+ }, schema=schema))
+ table.new_batch_write_builder().new_commit().commit(
+ write.prepare_commit())
+ write.close()
+
+ native_table = table.copy({'read.native.enabled': 'true'})
+ builder = native_table.new_read_builder().with_limit(1)
+ plan = builder.new_scan().plan()
+ fetched = []
+ original_to_data = BlobRef.to_data
+
+ def tracked_to_data(blob):
+ fetched.append(blob)
+ return original_to_data(blob)
+
+ with patch.object(BlobRef, 'to_data', tracked_to_data), patch(
+ 'pypaimon.read.native_plan.native_read',
+ return_value=[]) as native:
+ result = builder.new_read().to_arrow(plan.splits())
+
+ native.assert_not_called()
+ self.assertEqual(
+ result.to_pydict(), {'id': [1], 'payload': [b'first']})
+ self.assertEqual(len(fetched), 1)
+
+ @unittest.skipUnless(native_reader_available(),
+ "pypaimon-rust native reader API not installed")
+ def test_native_read_falls_back_for_precision_zero_timestamps(self):
+ cases = [
+ ('timestamp', pa.timestamp('s'), datetime.datetime(1970, 1, 1)),
+ ('timestamp_ltz', pa.timestamp('s', tz='UTC'),
+ datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)),
+ ]
+ for name, timestamp_type, first_value in cases:
+ with self.subTest(name=name):
+ schema = pa.schema([
+ ('id', pa.int32()),
+ ('ts', timestamp_type),
+ ])
+ table_name = 'native_%s_zero_t' % name
+ self.cat.create_table(
+ 'default.%s' % table_name,
+ Schema.from_pyarrow_schema(schema), False)
+ table = self.cat.get_table('default.%s' % table_name)
+ write = table.new_batch_write_builder().new_write()
+ write.write_arrow(pa.Table.from_pydict({
+ 'id': [1, 2],
+ 'ts': [first_value, first_value],
+ }, schema=schema))
+ table.new_batch_write_builder().new_commit().commit(
+ write.prepare_commit())
+ write.close()
+
+ native_table = table.copy({'read.native.enabled': 'true'})
+ builder = native_table.new_read_builder()
+ plan = builder.new_scan().plan()
+ with patch(
+ 'pypaimon.read.native_plan.native_read',
+ return_value=[]) as native:
+ result = builder.new_read().to_arrow(plan.splits())
+
+ native.assert_not_called()
+ self.assertEqual(result.schema, schema)
+ self.assertEqual(result.num_rows, 2)
+
@unittest.skipUnless(_has_native_row_ranges(),
"pypaimon_rust row-range API not installed")
def test_data_evolution_global_index_row_ranges(self):
@@ -293,7 +638,7 @@ class NativePlanIntegrationTest(unittest.TestCase):
{'k': 3, 'v': 'c'},
])
table = self.cat.get_table('default.de_range_t').copy(
- {'scan.native-plan.enabled': 'true'})
+ {'read.native.enabled': 'true'})
builder = table.new_read_builder()
scan = builder.new_scan().with_global_index_result(
GlobalIndexResult.from_range(Range(1, 1)))
@@ -302,7 +647,7 @@ class NativePlanIntegrationTest(unittest.TestCase):
with patch.object(
scan.file_scanner, 'scan', side_effect=AssertionError("fallback")):
plan = scan.plan()
- rows = builder.new_read().to_arrow(plan.splits()).to_pylist()
+ rows = self._native_rows(builder, plan)
self.assertEqual(rows, [{'k': 2, 'v': 'b'}])
self.assertEqual(
@@ -471,6 +816,22 @@ class NativePlanIntegrationTest(unittest.TestCase):
self._assert_matches('pt_t')
+ native_table = t.copy({
+ 'scan.native-plan.enabled': 'true',
+ 'read.native.enabled': 'true',
+ })
+ builder = native_table.new_read_builder()
+ plan = builder.new_scan().plan()
+ with patch('pypaimon.read.native_plan.native_read',
+ wraps=native_read) as read:
+ rows = builder.new_read().to_arrow(plan.splits()).to_pylist()
+ read.assert_not_called()
+ self.assertEqual(sorted(rows, key=lambda row: row['k']), [
+ {'k': 1, 'p': 'a/b'},
+ {'k': 2, 'p': 'a/b'},
+ {'k': 3, 'p': 'c'},
+ ])
+
def test_explain_reflects_native_plan(self):
self.cat.create_table(
'default.ex_t', Schema.from_pyarrow_schema(self.schema), False)
diff --git a/paimon-python/pypaimon/tests/native_plan_test.py b/paimon-python/pypaimon/tests/native_plan_test.py
index b7892c1185..b62177d8bd 100644
--- a/paimon-python/pypaimon/tests/native_plan_test.py
+++ b/paimon-python/pypaimon/tests/native_plan_test.py
@@ -50,6 +50,7 @@ def _scan(native_enabled, file_scanner):
scan = TableScan.__new__(TableScan)
scan.table = Mock()
scan.table.options.native_plan_enabled.return_value = native_enabled
+ scan.table.options.native_read_enabled.return_value = False
scan.table.options.options.contains_key.return_value = False # no time-travel
scan.table.options.options.contains.return_value = False # no incremental
scan.table.options.merge_engine.return_value = None # not first-row
@@ -145,9 +146,13 @@ class NativePlanTest(unittest.TestCase):
self.assertIsNone(_resolved_schema_file_io_options(table))
def test_switch_defaults_off(self):
- self.assertFalse(CoreOptions(Options({})).native_plan_enabled())
+ defaults = CoreOptions(Options({}))
+ self.assertFalse(defaults.native_plan_enabled())
+ self.assertFalse(defaults.native_read_enabled())
self.assertTrue(
CoreOptions(Options({"scan.native-plan.enabled": "true"})).native_plan_enabled())
+ self.assertTrue(
+ CoreOptions(Options({"read.native.enabled": "true"})).native_read_enabled())
def test_catalogless_standard_file_io_options_are_preserved(self):
from pypaimon.catalog.catalog_environment import CatalogEnvironment
@@ -173,6 +178,19 @@ class NativePlanTest(unittest.TestCase):
self.assertIs(scan.plan(), sentinel)
fs.scan.assert_called_once_with()
+ def test_native_read_switch_also_requests_native_plan(self):
+ fs = Mock(partition_key_predicate=None)
+ scan = _scan(native_enabled=False, file_scanner=fs)
+ scan.table.options.native_read_enabled.return_value = True
+ expected = Plan([], 1)
+ with patch('pypaimon.read.native_plan.native_reader_available',
+ return_value=True), \
+ patch('pypaimon.read.native_plan.native_plan',
+ return_value=expected) as np:
+ self.assertEqual(scan.plan(), expected)
+ np.assert_called_once()
+ fs.scan.assert_not_called()
+
def test_plan_routes_to_native_and_prunes_partitions(self):
# Native planner returns every partition; the predicate keeps only [2026, 7].
keep = Mock(partition=Mock(values=[2026, 7]))
@@ -664,6 +682,7 @@ class NativePlanTest(unittest.TestCase):
)
split = Mock(
partition=Mock(values=['a/b']), bucket=0, files=[data_file])
+ split._native_split = object()
_restore_python_partition_paths(table, [split])
@@ -671,6 +690,7 @@ class NativePlanTest(unittest.TestCase):
data_file.file_path,
'/warehouse/t/p=a/b/bucket-0/data.parquet',
)
+ self.assertIsNone(split._native_split)
def test_partition_path_keeps_existing_rust_path(self):
table = Mock(partition_keys=['p'])
@@ -762,11 +782,13 @@ class NativePlanTest(unittest.TestCase):
with patch.dict(sys.modules,
{'pypaimon_rust': fake_mod, 'pypaimon_rust.datafusion': fake_df}), \
patch('pypaimon.read.native_plan._catalog_options', return_value={}), \
- patch('pypaimon.read.native_plan.deserialize_split_v1',
- return_value='decoded') as des:
+ patch('pypaimon.read.native_plan.deserialize_split_v1') as des:
+ decoded = Mock()
+ des.return_value = decoded
result = native_plan(table, row_ranges=[(1, 2)])
- self.assertEqual(result.splits(), ['decoded'])
+ self.assertEqual(result.splits(), [decoded])
+ self.assertIs(decoded._native_split, split)
self.assertEqual(result.snapshot_id, 3)
rt.new_read_builder.assert_called_once_with({
CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(): '1024',
diff --git a/paimon-python/pypaimon/tests/native_read_test.py b/paimon-python/pypaimon/tests/native_read_test.py
new file mode 100644
index 0000000000..11282480b0
--- /dev/null
+++ b/paimon-python/pypaimon/tests/native_read_test.py
@@ -0,0 +1,349 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import threading
+from unittest.mock import Mock, patch
+
+import pyarrow as pa
+import pytest
+
+from pypaimon.read.table_read import TableRead
+from pypaimon.schema.data_types import AtomicType, DataField
+
+
+class _Split:
+ def __init__(self, file_name='data.parquet'):
+ self.files = [Mock(file_name=file_name)]
+
+
+def _table_read(limit=None):
+ read = TableRead.__new__(TableRead)
+ read.table = Mock()
+ read.table.options.native_read_enabled.return_value = True
+ read.table.options.file_format.return_value = 'parquet'
+ read.table.options.blob_as_descriptor.return_value = False
+ read.table.options.blob_descriptor_fields.return_value = set()
+ read.table.options.blob_view_fields.return_value = set()
+ read.predicate = None
+ read.read_type = [DataField(0, 'id', AtomicType('INT'))]
+ read.include_row_kind = False
+ read.nested_name_paths = None
+ read.limit = limit
+ read._read_parallelism = 1
+ read._deferred_blob_fields = set()
+ read._predicate_extra_fields = []
+ read._output_column_names = ['id']
+ return read
+
+
+def _blob_table_read(limit=None):
+ read = _table_read(limit)
+ read.read_type = [DataField(0, 'payload', AtomicType('BLOB'))]
+ read._output_column_names = ['payload']
+ return read
+
+
+def _id_batch(values):
+ return pa.record_batch(
+ [pa.array(values, type=pa.int32())], names=['id'])
+
+
+def test_native_read_consumes_retained_rust_splits_and_enforces_limit():
+ read = _table_read(limit=2)
+ first, second = _Split(), _Split()
+ first._native_split = object()
+ second._native_split = object()
+ batches = [_id_batch([1, 2, 3])]
+
+ with patch(
+ 'pypaimon.read.native_plan.native_read',
+ return_value=batches) as native:
+ result = read.to_arrow([first, second])
+
+ assert result.to_pydict() == {'id': [1, 2]}
+ native.assert_called_once_with(
+ read.table,
+ [first._native_split, second._native_split],
+ predicate=None,
+ limit=2,
+ projection=['id'],
+ blob_parallelism=1,
+ )
+
+
+def test_native_read_falls_back_for_transformed_python_split():
+ read = _table_read()
+ schema = pa.schema([('id', pa.int32())])
+ split = _Split()
+
+ with patch('pypaimon.read.native_plan.native_read') as native:
+ assert read._try_native_batches([split], schema) is None
+
+ native.assert_not_called()
+
+
+def test_native_read_uses_effective_parallelism_from_table_option():
+ read = _table_read()
+ read._read_parallelism = 2
+ splits = [_Split() for _ in range(4)]
+ for index, split in enumerate(splits):
+ split._native_split = index
+
+ barrier = threading.Barrier(2)
+ worker_names = set()
+ worker_names_lock = threading.Lock()
+
+ def read_group(table, rust_splits, **kwargs):
+ with worker_names_lock:
+ worker_names.add(threading.current_thread().name)
+ barrier.wait(timeout=5)
+ return [_id_batch(rust_splits)]
+
+ with patch('pypaimon.read.native_plan.native_read',
+ side_effect=read_group) as native:
+ result = read.to_arrow(splits)
+
+ assert result.to_pydict() == {'id': [0, 1, 2, 3]}
+ assert native.call_count == 2
+ groups = sorted(call.args[1] for call in native.call_args_list)
+ assert groups == [[0, 1], [2, 3]]
+ assert len(worker_names) == 2
+
+
+def test_native_read_runtime_parallelism_overrides_table_option():
+ read = _table_read()
+ read._read_parallelism = 4
+ splits = [_Split() for _ in range(4)]
+ for index, split in enumerate(splits):
+ split._native_split = index
+
+ with patch(
+ 'pypaimon.read.native_plan.native_read',
+ side_effect=lambda table, group, **kwargs: [
+ _id_batch(group)]) as native:
+ result = read.to_arrow(splits, parallelism=2)
+
+ assert result.num_rows == 4
+ assert native.call_count == 2
+
+
+def test_native_read_caps_blob_parallelism_across_split_readers():
+ read = _table_read()
+ splits = [_Split() for _ in range(16)]
+ for index, split in enumerate(splits):
+ split._native_split = index
+
+ with patch(
+ 'pypaimon.read.native_plan.native_read',
+ side_effect=lambda table, group, **kwargs: [
+ _id_batch(group)]) as native:
+ result = read.to_arrow(
+ splits, parallelism=16, blob_parallelism=16)
+
+ assert result.num_rows == 16
+ assert native.call_count == 16
+ assert {call.kwargs['blob_parallelism']
+ for call in native.call_args_list} == {4}
+
+
+def test_parallel_native_read_shares_limit_across_readers():
+ read = _table_read(limit=3)
+ read._read_parallelism = 2
+ splits = [_Split() for _ in range(4)]
+ for index, split in enumerate(splits):
+ split._native_split = index
+
+ with patch(
+ 'pypaimon.read.native_plan.native_read',
+ side_effect=lambda table, group, **kwargs: [
+ _id_batch(group)]):
+ result = read.to_arrow(splits)
+
+ assert result.num_rows == 3
+
+
+def test_parallel_native_reader_setup_failure_falls_back():
+ read = _table_read()
+ splits = [_Split() for _ in range(2)]
+ for split in splits:
+ split._native_split = object()
+
+ with patch('pypaimon.read.native_plan.native_read',
+ side_effect=RuntimeError('setup failed')):
+ assert read._try_native_batches(
+ splits, pa.schema([('id', pa.int32())]), parallelism=2) is None
+
+
+def test_parallel_native_stream_error_propagates():
+ read = _table_read()
+ splits = [_Split() for _ in range(2)]
+ for split in splits:
+ split._native_split = object()
+
+ def broken_stream(table, group, **kwargs):
+ def batches():
+ raise RuntimeError('stream failed')
+ yield
+ return batches()
+
+ with patch('pypaimon.read.native_plan.native_read',
+ side_effect=broken_stream), \
+ pytest.raises(RuntimeError, match='stream failed'):
+ read._try_native_batches(
+ splits, pa.schema([('id', pa.int32())]), parallelism=2)
+
+
+def test_native_read_failure_falls_back():
+ read = _table_read()
+ schema = pa.schema([('id', pa.int32())])
+ split = _Split()
+ split._native_split = object()
+
+ with patch('pypaimon.read.native_plan.native_read',
+ side_effect=RuntimeError('unsupported')):
+ assert read._try_native_batches([split], schema) is None
+
+
+def test_native_read_falls_back_for_unsupported_file_format():
+ read = _table_read()
+ read.table.options.file_format.return_value = 'vortex'
+ schema = pa.schema([('id', pa.int32())])
+ split = _Split()
+ split._native_split = object()
+
+ with patch('pypaimon.read.native_plan.native_read') as native:
+ assert read._try_native_batches([split], schema) is None
+
+ native.assert_not_called()
+
+
+def test_native_read_falls_back_for_unsupported_dedicated_file():
+ read = _table_read()
+ schema = pa.schema([('id', pa.int32())])
+ split = _Split('camera.video')
+ split._native_split = object()
+
+ with patch('pypaimon.read.native_plan.native_read') as native:
+ assert read._try_native_batches([split], schema) is None
+
+ native.assert_not_called()
+
+
+def test_native_read_supports_blob_file_and_forwards_parallelism():
+ read = _table_read()
+ schema = pa.schema([('id', pa.int32())])
+ split = _Split('picture.blob')
+ split._native_split = object()
+
+ with patch(
+ 'pypaimon.read.native_plan.native_read',
+ return_value=[_id_batch([1])]) as native:
+ batches = list(read._try_native_batches(
+ [split], schema, blob_parallelism=3))
+
+ assert batches[0].column('id').to_pylist() == [1]
+ assert native.call_args.kwargs['blob_parallelism'] == 3
+
+
+def test_native_read_preserves_large_binary_blob_schema_for_batch_reader():
+ read = _blob_table_read()
+ split = _Split('payload.blob')
+ split._native_split = object()
+ native_batch = pa.record_batch(
+ [pa.array([b'a'], type=pa.large_binary())], names=['payload'])
+
+ with patch(
+ 'pypaimon.read.native_plan.native_read',
+ return_value=[native_batch]):
+ result = read.to_arrow_batch_reader([split]).read_all()
+
+ assert result.schema == pa.schema([('payload', pa.large_binary())])
+ assert result.to_pydict() == {'payload': [b'a']}
+
+
+def test_native_read_does_not_cast_legacy_binary_blob_schema():
+ batch = pa.record_batch(
+ [pa.array([b'a'], type=pa.binary())], names=['payload'])
+
+ with pytest.raises(
+ TypeError,
+ match="Batch field 'payload' has type binary, expected large_binary"):
+ TableRead._try_to_pad_batch_by_schema(
+ batch, pa.schema([('payload', pa.large_binary())]))
+
+
+def test_native_read_preserves_parallel_large_binary_blob_schema():
+ read = _blob_table_read()
+ splits = [_Split('payload.blob') for _ in range(2)]
+ for index, split in enumerate(splits):
+ split._native_split = index
+
+ with patch(
+ 'pypaimon.read.native_plan.native_read',
+ side_effect=lambda table, group, **kwargs: [pa.record_batch(
+ [pa.array([bytes(group)], type=pa.large_binary())],
+ names=['payload'])]):
+ result = read.to_arrow(splits, parallelism=2)
+
+ assert result.schema == pa.schema([('payload', pa.large_binary())])
+
+
+def test_native_read_defers_to_python_for_pruning_blob_limit():
+ read = _blob_table_read(limit=1)
+ read._deferred_blob_fields = {'payload'}
+ split = _Split('payload.blob')
+ split._native_split = object()
+ split.merged_row_count = Mock(return_value=2)
+
+ with patch(
+ 'pypaimon.read.native_plan.native_read') as native:
+ assert read._try_native_batches(
+ [split], pa.schema([('payload', pa.large_binary())]),
+ blob_parallelism=1) is None
+
+ native.assert_not_called()
+
+
+def test_native_read_defers_to_python_for_pruning_descriptor_blob_limit():
+ read = _blob_table_read(limit=1)
+ read.table.options.blob_descriptor_fields.return_value = {'payload'}
+ split = _Split('payload.parquet')
+ split._native_split = object()
+ split.merged_row_count = Mock(return_value=2)
+
+ with patch('pypaimon.read.native_plan.native_read') as native:
+ assert read._try_native_batches(
+ [split], pa.schema([('payload', pa.large_binary())]),
+ blob_parallelism=1) is None
+
+ native.assert_not_called()
+
+
+@pytest.mark.parametrize('data_type', [
+ pa.timestamp('s'),
+ pa.timestamp('s', tz='UTC'),
+])
+def test_native_read_falls_back_for_precision_zero_timestamps(data_type):
+ read = _table_read()
+ split = _Split()
+ split._native_split = object()
+
+ with patch('pypaimon.read.native_plan.native_read') as native:
+ assert read._try_native_batches(
+ [split], pa.schema([('ts', data_type)])) is None
+
+ native.assert_not_called()
diff --git a/paimon-python/pypaimon/tests/table_update_test.py b/paimon-python/pypaimon/tests/table_update_test.py
index 130dd53686..092990ab98 100644
--- a/paimon-python/pypaimon/tests/table_update_test.py
+++ b/paimon-python/pypaimon/tests/table_update_test.py
@@ -288,7 +288,8 @@ class _TableUpdateTestBase(DataEvolutionTestBase):
self._apply_commit(tc, msgs, cid)
tc.close()
- result = self._read_all(table)
+ # Table scans are unordered; compare in stable business-key order.
+ result = self._read_all(table).sort_by('id')
self.assertEqual(
[25, 30, 99, 99, 99],
result['age'].to_pylist(),
(paimon) 02/20: [core] Use a single BTree scan for bounded range predicates (#9844)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 0a5678bfbe2a680ea634b79bc5c8747a4bcd1da6
Author: jerry <ji...@gmail.com>
AuthorDate: Fri Sep 18 11:55:09 2026 +0800
[core] Use a single BTree scan for bounded range predicates (#9844)
---
.../paimon/globalindex/GlobalIndexEvaluator.java | 69 ++++++++++++++++-
.../paimon/globalindex/GlobalIndexReader.java | 22 ++++++
.../globalindex/OffsetGlobalIndexReader.java | 7 ++
.../globalindex/SortedFileGlobalIndexReader.java | 13 +++-
.../paimon/globalindex/UnionGlobalIndexReader.java | 7 ++
.../paimon/globalindex/btree/BTreeIndexReader.java | 5 ++
.../globalindex/btree/LazyFilteredBTreeReader.java | 10 +++
.../globalindex/GlobalIndexEvaluatorTest.java | 48 ++++++++++++
.../btree/LazyFilteredBTreeIndexReaderTest.java | 90 ++++++++++++++++++++++
9 files changed, 265 insertions(+), 6 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexEvaluator.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexEvaluator.java
index 224bc2805a..48bfb7780a 100644
--- a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexEvaluator.java
+++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexEvaluator.java
@@ -18,8 +18,11 @@
package org.apache.paimon.globalindex;
+import org.apache.paimon.predicate.And;
import org.apache.paimon.predicate.CompoundPredicate;
import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.GreaterOrEqual;
+import org.apache.paimon.predicate.GreaterThan;
import org.apache.paimon.predicate.IsNaN;
import org.apache.paimon.predicate.IsNotNull;
import org.apache.paimon.predicate.LeafBinaryFunction;
@@ -27,6 +30,8 @@ import org.apache.paimon.predicate.LeafFunction;
import org.apache.paimon.predicate.LeafNAryFunction;
import org.apache.paimon.predicate.LeafPredicate;
import org.apache.paimon.predicate.LeafTernaryFunction;
+import org.apache.paimon.predicate.LessOrEqual;
+import org.apache.paimon.predicate.LessThan;
import org.apache.paimon.predicate.Or;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.TopN;
@@ -49,6 +54,7 @@ import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
+import java.util.function.Function;
import java.util.function.IntFunction;
import java.util.stream.Collectors;
@@ -142,6 +148,14 @@ public class GlobalIndexEvaluator implements Closeable {
return CompletableFuture.completedFuture(Optional.empty());
}
FieldRef fieldRef = fieldRefOptional.get();
+ return visitFieldAsync(
+ fieldRef,
+ reader -> predicate.function().visit(reader, fieldRef, predicate.literals()));
+ }
+
+ private CompletableFuture<Optional<Evaluation>> visitFieldAsync(
+ FieldRef fieldRef,
+ Function<GlobalIndexReader, CompletableFuture<Optional<GlobalIndexResult>>> visitor) {
int fieldId = rowType.getField(fieldRef.name()).id();
Collection<GlobalIndexReader> readers =
indexReadersCache.computeIfAbsent(fieldId, readersFunction::apply);
@@ -149,7 +163,7 @@ public class GlobalIndexEvaluator implements Closeable {
List<CompletableFuture<Optional<GlobalIndexResult>>> readerFutures =
new ArrayList<>(readers.size());
for (GlobalIndexReader reader : readers) {
- readerFutures.add(predicate.function().visit(reader, fieldRef, predicate.literals()));
+ readerFutures.add(visitor.apply(reader));
}
return CompletableFuture.allOf(readerFutures.toArray(new CompletableFuture[0]))
@@ -189,8 +203,41 @@ public class GlobalIndexEvaluator implements Closeable {
}
List<CompletableFuture<Optional<Evaluation>>> childFutures =
new ArrayList<>(children.size());
- for (Predicate child : children) {
- childFutures.add(visitAsync(child));
+ for (int i = 0; i < children.size(); i++) {
+ Predicate child = children.get(i);
+ CompletableFuture<Optional<Evaluation>> range = null;
+ if (predicate.function() instanceof And && isRangeBound(child)) {
+ LeafPredicate first = (LeafPredicate) child;
+ // ponytail: pair search is quadratic in filter count; group by field for large
+ // conjunctions.
+ for (int j = i + 1; j < children.size(); j++) {
+ Predicate other = children.get(j);
+ if (!isRangeBound(other)) {
+ continue;
+ }
+ LeafPredicate second = (LeafPredicate) other;
+ if (isLowerBound(first) == isLowerBound(second)
+ || !first.fieldRefOptional().equals(second.fieldRefOptional())) {
+ continue;
+ }
+ LeafPredicate lower = isLowerBound(first) ? first : second;
+ LeafPredicate upper = isLowerBound(first) ? second : first;
+ FieldRef field = first.fieldRefOptional().get();
+ range =
+ visitFieldAsync(
+ field,
+ reader ->
+ reader.visitRange(
+ field,
+ lower.literals().get(0),
+ upper.literals().get(0),
+ lower.function() instanceof GreaterOrEqual,
+ upper.function() instanceof LessOrEqual));
+ children.remove(j);
+ break;
+ }
+ }
+ childFutures.add(range == null ? visitAsync(child) : range);
}
return CompletableFuture.allOf(childFutures.toArray(new CompletableFuture[0]))
@@ -204,6 +251,22 @@ public class GlobalIndexEvaluator implements Closeable {
});
}
+ private static boolean isRangeBound(Predicate predicate) {
+ if (!(predicate instanceof LeafPredicate)) {
+ return false;
+ }
+ LeafPredicate leaf = (LeafPredicate) predicate;
+ return leaf.fieldRefOptional().isPresent()
+ && (isLowerBound(leaf)
+ || leaf.function() instanceof LessThan
+ || leaf.function() instanceof LessOrEqual)
+ && leaf.literals().get(0) != null;
+ }
+
+ private static boolean isLowerBound(LeafPredicate leaf) {
+ return leaf.function() instanceof GreaterThan || leaf.function() instanceof GreaterOrEqual;
+ }
+
private Optional<Evaluation> combineResults(
List<Optional<Evaluation>> results, CompoundPredicate predicate) {
Set<Integer> contributingFieldIds = new HashSet<>();
diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java
index c6901c176b..f5d6ab9fad 100644
--- a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java
+++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java
@@ -77,6 +77,28 @@ public interface GlobalIndexReader
return CompletableFuture.completedFuture(Optional.empty());
}
+ /**
+ * Evaluate a bounded range. Flags specify whether each endpoint is included. Readers without a
+ * combined scan retain the supported bounds as candidate filters.
+ */
+ default CompletableFuture<Optional<GlobalIndexResult>> visitRange(
+ FieldRef fieldRef, Object from, Object to, boolean fromInclusive, boolean toInclusive) {
+ CompletableFuture<Optional<GlobalIndexResult>> lower =
+ fromInclusive
+ ? visitGreaterOrEqual(fieldRef, from)
+ : visitGreaterThan(fieldRef, from);
+ CompletableFuture<Optional<GlobalIndexResult>> upper =
+ toInclusive ? visitLessOrEqual(fieldRef, to) : visitLessThan(fieldRef, to);
+ return lower.thenCombine(
+ upper,
+ (left, right) -> {
+ if (!left.isPresent()) {
+ return right;
+ }
+ return right.isPresent() ? Optional.of(left.get().and(right.get())) : left;
+ });
+ }
+
@Override
default CompletableFuture<Optional<GlobalIndexResult>> visitNotLike(
FieldRef fieldRef, Object literal) {
diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/OffsetGlobalIndexReader.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/OffsetGlobalIndexReader.java
index 9c29bb918d..70283659ec 100644
--- a/paimon-common/src/main/java/org/apache/paimon/globalindex/OffsetGlobalIndexReader.java
+++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/OffsetGlobalIndexReader.java
@@ -184,6 +184,13 @@ public class OffsetGlobalIndexReader implements ContainsRefiningGlobalIndexReade
return wrapped.visitNotIn(fieldRef, literals).thenApply(this::applyOffset);
}
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitRange(
+ FieldRef fieldRef, Object from, Object to, boolean fromInclusive, boolean toInclusive) {
+ return wrapped.visitRange(fieldRef, from, to, fromInclusive, toInclusive)
+ .thenApply(this::applyOffset);
+ }
+
@Override
public CompletableFuture<Optional<GlobalIndexResult>> visitBetween(
FieldRef fieldRef, Object from, Object to) {
diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/SortedFileGlobalIndexReader.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/SortedFileGlobalIndexReader.java
index 814ec92414..fc78ce190d 100644
--- a/paimon-common/src/main/java/org/apache/paimon/globalindex/SortedFileGlobalIndexReader.java
+++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/SortedFileGlobalIndexReader.java
@@ -236,12 +236,19 @@ public abstract class SortedFileGlobalIndexReader<R extends Closeable>
@Override
public CompletableFuture<Optional<GlobalIndexResult>> visitBetween(
FieldRef fieldRef, Object from, Object to) {
+ return visitRange(fieldRef, from, to, reader -> visitBetween(reader, from, to));
+ }
+
+ /** Prune files using enclosing inclusive bounds, then apply the exact range in the reader. */
+ protected CompletableFuture<Optional<GlobalIndexResult>> visitRange(
+ FieldRef fieldRef,
+ Object from,
+ Object to,
+ Function<R, Optional<GlobalIndexResult>> visitor) {
if (!canFallbackScan(from) || to == null) {
return unsupported();
}
- return visitFallbackParallel(
- () -> fileSelector.visitBetween(fieldRef, from, to),
- reader -> visitBetween(reader, from, to));
+ return visitFallbackParallel(() -> fileSelector.visitBetween(fieldRef, from, to), visitor);
}
@Override
diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/UnionGlobalIndexReader.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/UnionGlobalIndexReader.java
index f55986c694..dd2cb1eeaa 100644
--- a/paimon-common/src/main/java/org/apache/paimon/globalindex/UnionGlobalIndexReader.java
+++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/UnionGlobalIndexReader.java
@@ -181,6 +181,13 @@ public class UnionGlobalIndexReader implements ContainsRefiningGlobalIndexReader
return unionAsync(reader -> reader.visitNotIn(fieldRef, literals));
}
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitRange(
+ FieldRef fieldRef, Object from, Object to, boolean fromInclusive, boolean toInclusive) {
+ return unionAsync(
+ reader -> reader.visitRange(fieldRef, from, to, fromInclusive, toInclusive));
+ }
+
@Override
public CompletableFuture<Optional<GlobalIndexResult>> visitBetween(
FieldRef fieldRef, Object from, Object to) {
diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexReader.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexReader.java
index d7d6017b5c..b42224106d 100644
--- a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexReader.java
+++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexReader.java
@@ -358,6 +358,11 @@ public class BTreeIndexReader implements Closeable {
});
}
+ public Optional<GlobalIndexResult> visitRange(
+ Object from, Object to, boolean fromInclusive, boolean toInclusive) {
+ return createResult(() -> rangeQuery(from, to, fromInclusive, toInclusive));
+ }
+
public Optional<GlobalIndexResult> visitBetween(Object from, Object to) {
return createResult(() -> rangeQuery(from, to, true, true));
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeReader.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeReader.java
index fdf9b513a4..f47c218350 100644
--- a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeReader.java
+++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeReader.java
@@ -229,6 +229,16 @@ public class LazyFilteredBTreeReader extends SortedFileGlobalIndexReader<BTreeIn
return reader.visitIn(literals);
}
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitRange(
+ FieldRef fieldRef, Object from, Object to, boolean fromInclusive, boolean toInclusive) {
+ return visitRange(
+ fieldRef,
+ from,
+ to,
+ reader -> reader.visitRange(from, to, fromInclusive, toInclusive));
+ }
+
@Override
protected Optional<GlobalIndexResult> visitBetween(
BTreeIndexReader reader, Object from, Object to) {
diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexEvaluatorTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexEvaluatorTest.java
index 09c593dc86..16d5e61584 100644
--- a/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexEvaluatorTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexEvaluatorTest.java
@@ -87,6 +87,54 @@ class GlobalIndexEvaluatorTest {
return new StubGlobalIndexReader(result);
}
+ @Test
+ void testRangeFallbackAndOrBoundary() {
+ AtomicInteger rangeCalls = new AtomicInteger();
+ GlobalIndexReader reader =
+ new StubGlobalIndexReader(null) {
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitRange(
+ FieldRef field,
+ Object from,
+ Object to,
+ boolean fromInclusive,
+ boolean toInclusive) {
+ rangeCalls.incrementAndGet();
+ return super.visitRange(field, from, to, fromInclusive, toInclusive);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitGreaterOrEqual(
+ FieldRef field, Object literal) {
+ return CompletableFuture.completedFuture(Optional.of(resultOf(2, 3)));
+ }
+ };
+ PredicateBuilder builder = new PredicateBuilder(rowType());
+ try (GlobalIndexEvaluator evaluator =
+ new GlobalIndexEvaluator(rowType(), fieldId -> Collections.singletonList(reader))) {
+ assertBitmapContainsExactly(
+ evaluator
+ .evaluate(
+ PredicateBuilder.and(
+ builder.greaterOrEqual(0, 30),
+ builder.lessThan(0, 120)))
+ .get()
+ .results(),
+ 2L,
+ 3L);
+ assertThat(rangeCalls).hasValue(1);
+ assertThat(
+ evaluator.evaluate(
+ PredicateBuilder.or(
+ builder.greaterOrEqual(0, 30),
+ builder.lessThan(0, 120))))
+ .isEmpty();
+ evaluator.evaluate(
+ PredicateBuilder.and(builder.greaterOrEqual(0, 30), builder.lessThan(1, 120)));
+ assertThat(rangeCalls).hasValue(1);
+ }
+ }
+
@Test
void testSingleFieldSequential() {
RowType rowType = rowType();
diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeIndexReaderTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeIndexReaderTest.java
index 7cc643f3ec..af7d553979 100644
--- a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeIndexReaderTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeIndexReaderTest.java
@@ -20,21 +20,28 @@ package org.apache.paimon.globalindex.btree;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.globalindex.GlobalIndexEvaluator;
import org.apache.paimon.globalindex.GlobalIndexIOMeta;
import org.apache.paimon.globalindex.GlobalIndexReader;
import org.apache.paimon.globalindex.GlobalIndexResult;
import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter;
import org.apache.paimon.globalindex.OffsetGlobalIndexReader;
import org.apache.paimon.globalindex.ResultEntry;
+import org.apache.paimon.globalindex.UnionGlobalIndexReader;
import org.apache.paimon.globalindex.btree.BTreeIndexReader.KeyRowIds;
import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.options.Options;
+import org.apache.paimon.predicate.And;
+import org.apache.paimon.predicate.CompoundPredicate;
import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.predicate.TopN;
import org.apache.paimon.testutils.junit.parameterized.ParameterizedTestExtension;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.IntType;
+import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.SemaphoredDelegatingExecutor;
@@ -66,6 +73,11 @@ import static org.apache.paimon.predicate.SortValue.SortDirection.ASCENDING;
import static org.apache.paimon.predicate.SortValue.SortDirection.DESCENDING;
import static org.apache.paimon.shade.guava30.com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.clearInvocations;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
/** Test for {@link LazyFilteredBTreeReader} to read multiple files. */
@ExtendWith(ParameterizedTestExtension.class)
@@ -96,6 +108,84 @@ public class LazyFilteredBTreeIndexReaderTest extends AbstractIndexReaderTest {
return written;
}
+ @TestTemplate
+ public void testFusedRangeThroughEvaluatorAndWrappers() throws Exception {
+ List<Pair<Object, Long>> values = new ArrayList<>(data);
+ values.add(Pair.of(null, (long) dataNum));
+ List<GlobalIndexIOMeta> written = Collections.singletonList(writeData(values));
+ RowType rowType =
+ new RowType(Collections.singletonList(new DataField(1, "testField", dataType)));
+ PredicateBuilder builder = new PredicateBuilder(rowType);
+ LazyFilteredBTreeReader indexReader =
+ spy(
+ (LazyFilteredBTreeReader)
+ globalIndexer.createReader(
+ fileReader,
+ written,
+ dataNum + 1,
+ newDirectExecutorService()));
+ BTreeIndexReader btreeReader = spy(indexReader.openReader(written.get(0)));
+ doReturn(btreeReader).when(indexReader).openReader(written.get(0));
+ GlobalIndexReader wrapped =
+ new UnionGlobalIndexReader(
+ Collections.singletonList(
+ new OffsetGlobalIndexReader(indexReader, 1000L, 1000L + dataNum)));
+ try (GlobalIndexEvaluator evaluator =
+ new GlobalIndexEvaluator(rowType, fieldId -> Collections.singletonList(wrapped))) {
+ Object from = data.get(dataNum / 4).getKey();
+ Object to = data.get(dataNum * 3 / 4).getKey();
+ // Include empty and singleton intervals, and both predicate orders.
+ for (Object[] bounds :
+ Arrays.asList(
+ new Object[] {from, to},
+ new Object[] {from, from},
+ new Object[] {to, from})) {
+ for (boolean lowerInclusive : Arrays.asList(false, true)) {
+ for (boolean upperInclusive : Arrays.asList(false, true)) {
+ Predicate lower =
+ lowerInclusive
+ ? builder.greaterOrEqual(0, bounds[0])
+ : builder.greaterThan(0, bounds[0]);
+ Predicate upper =
+ upperInclusive
+ ? builder.lessOrEqual(0, bounds[1])
+ : builder.lessThan(0, bounds[1]);
+ for (List<Predicate> order :
+ Arrays.asList(
+ Arrays.asList(lower, upper), Arrays.asList(upper, lower))) {
+ clearInvocations(btreeReader);
+ GlobalIndexEvaluator.Evaluation result =
+ evaluator
+ .evaluateWithContributingFields(
+ new CompoundPredicate(And.INSTANCE, order))
+ .get();
+ List<Long> expected = new ArrayList<>();
+ for (Pair<Object, Long> value : data) {
+ int low = comparator.compare(value.getKey(), bounds[0]);
+ int high = comparator.compare(value.getKey(), bounds[1]);
+ if ((low > 0 || lowerInclusive && low == 0)
+ && (high < 0 || upperInclusive && high == 0)) {
+ expected.add(value.getValue() + 1000L);
+ }
+ }
+ assertResult(result.result(), expected);
+ assertThat(result.contributingFieldIds()).containsExactly(1);
+ if (comparator.compare(bounds[0], bounds[1]) <= 0) {
+ verify(btreeReader)
+ .visitRange(
+ bounds[0],
+ bounds[1],
+ lowerInclusive,
+ upperInclusive);
+ }
+ verifyNoMoreInteractions(btreeReader);
+ }
+ }
+ }
+ }
+ }
+ }
+
private int firstStrictlyGreaterKeyIndex() {
for (int i = 1; i < dataNum; i++) {
if (comparator.compare(data.get(i - 1).getKey(), data.get(i).getKey()) < 0) {
(paimon) 14/20: [format] Simplify buffered Avro encoder selection
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit f8c9bb2dc8dfd104c254e9fb74a8fa5850436d24
Author: JingsongLi <ji...@gmail.com>
AuthorDate: Fri Sep 18 13:15:57 2026 +0800
[format] Simplify buffered Avro encoder selection
---
.../java/org/apache/paimon/format/FileFormat.java | 9 ++------
.../apache/paimon/format/FileFormatFactory.java | 25 ----------------------
.../apache/paimon/format/avro/AvroFileFormat.java | 4 +---
.../paimon/format/avro/AvroFileFormatTest.java | 4 ++--
4 files changed, 5 insertions(+), 37 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
index 7a06e544ea..f17c23b255 100644
--- a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
+++ b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
@@ -74,10 +74,6 @@ public abstract class FileFormat {
}
public static FileFormat fromIdentifier(String identifier, Options options) {
- return fromIdentifier(identifier, options, false);
- }
-
- private static FileFormat fromIdentifier(String identifier, Options options, boolean manifest) {
return fromIdentifier(
normalizeFileFormat(identifier),
new FormatContext(
@@ -86,8 +82,7 @@ public abstract class FileFormat {
options.get(CoreOptions.WRITE_BATCH_SIZE),
options.get(CoreOptions.WRITE_BATCH_MEMORY),
options.get(CoreOptions.FILE_COMPRESSION_ZSTD_LEVEL),
- options.get(CoreOptions.FILE_BLOCK_SIZE),
- manifest));
+ options.get(CoreOptions.FILE_BLOCK_SIZE)));
}
/** Create a {@link FileFormat} from format identifier and format options. */
@@ -125,6 +120,6 @@ public abstract class FileFormat {
// Manifest blocks retain the Avro default independently of data-file block sizing.
Options manifestOptions = new Options(options.toMap());
manifestOptions.remove(CoreOptions.FILE_BLOCK_SIZE);
- return FileFormat.fromIdentifier(CoreOptions.FILE_FORMAT_AVRO, manifestOptions, true);
+ return FileFormat.fromIdentifier(CoreOptions.FILE_FORMAT_AVRO, manifestOptions);
}
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java b/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java
index 157f151ad2..b5cba150fe 100644
--- a/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java
+++ b/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java
@@ -41,7 +41,6 @@ public interface FileFormatFactory {
private final MemorySize writeBatchMemory;
private final int zstdLevel;
@Nullable private final MemorySize blockSize;
- private final boolean manifest;
@VisibleForTesting
public FormatContext(Options options, int readBatchSize, int writeBatchSize) {
@@ -64,31 +63,12 @@ public interface FileFormatFactory {
MemorySize writeBatchMemory,
int zstdLevel,
@Nullable MemorySize blockSize) {
- this(
- options,
- readBatchSize,
- writeBatchSize,
- writeBatchMemory,
- zstdLevel,
- blockSize,
- false);
- }
-
- public FormatContext(
- Options options,
- int readBatchSize,
- int writeBatchSize,
- MemorySize writeBatchMemory,
- int zstdLevel,
- @Nullable MemorySize blockSize,
- boolean manifest) {
this.options = options;
this.readBatchSize = readBatchSize;
this.writeBatchSize = writeBatchSize;
this.writeBatchMemory = writeBatchMemory;
this.zstdLevel = zstdLevel;
this.blockSize = blockSize;
- this.manifest = manifest;
}
public Options options() {
@@ -116,11 +96,6 @@ public interface FileFormatFactory {
return blockSize;
}
- /** Whether the format is used for manifest metadata rather than data files. */
- public boolean isManifest() {
- return manifest;
- }
-
public boolean caseSensitive() {
return options.getOptional(CatalogOptions.CASE_SENSITIVE).orElse(true);
}
diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
index 845cce65d5..ad243b6132 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
@@ -68,7 +68,6 @@ public class AvroFileFormat extends FileFormat {
private final Options options;
private final int zstdLevel;
@Nullable private final MemorySize blockSize;
- private final boolean useBufferedEncoder;
public AvroFileFormat(FormatContext context) {
super(IDENTIFIER);
@@ -76,7 +75,6 @@ public class AvroFileFormat extends FileFormat {
this.options = getIdentifierPrefixOptions(context.options());
this.zstdLevel = context.zstdLevel();
this.blockSize = context.blockSize();
- this.useBufferedEncoder = !context.isManifest();
}
@Override
@@ -160,7 +158,7 @@ public class AvroFileFormat extends FileFormat {
@Override
public FormatWriter create(PositionOutputStream out, String compression)
throws IOException {
- return createBlockWriter(out, rowType, compression, useBufferedEncoder);
+ return createBlockWriter(out, rowType, compression, true);
}
}
}
diff --git a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
index 251f3ca974..bef9ef8873 100644
--- a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
+++ b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
@@ -261,7 +261,7 @@ public class AvroFileFormatTest {
@ParameterizedTest
@ValueSource(strings = {"avro", "parquet", "orc"})
- void testManifestWriterUsesDirectEncoder(String dataFileFormat) throws IOException {
+ void testManifestRowWriterUsesBufferedEncoder(String dataFileFormat) throws IOException {
Options options = new Options();
options.set(CoreOptions.FILE_FORMAT, dataFileFormat);
FileFormat manifestFormat = FileFormat.manifestFormat(new CoreOptions(options));
@@ -275,7 +275,7 @@ public class AvroFileFormatTest {
assertThat(writer)
.extracting("writer")
.extracting("bufOut")
- .isInstanceOf(DirectBinaryEncoder.class);
+ .isInstanceOf(BufferedBinaryEncoder.class);
writer.addElement(GenericRow.of(42));
}
}
(paimon) 01/20: [core][spark][docs] Support row filters in full-text search (#9855)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 759041653725c4aa0b927d9fd586b5e37f41891d
Author: Xiangyi Zhu <82...@users.noreply.github.com>
AuthorDate: Fri Sep 18 11:46:06 2026 +0800
[core][spark][docs] Support row filters in full-text search (#9855)
---
.../multimodal-table/global-index/full-text.mdx | 62 +++
.../global-index/hybrid-search.mdx | 12 +
docs/docs/primary-key-table/global-index.mdx | 4 +-
.../table/source/DataEvolutionFullTextRead.java | 208 ++++++-
.../table/source/DataEvolutionFullTextScan.java | 88 ++-
.../paimon/table/source/FilteredRowIdReader.java | 132 +++++
.../paimon/table/source/FullTextSearchBuilder.java | 11 +
.../table/source/FullTextSearchBuilderImpl.java | 42 +-
.../table/source/HybridSearchBuilderImpl.java | 13 +-
.../table/source/IndexFullTextSearchSplit.java | 46 +-
.../paimon/table/source/RawFullTextReadImpl.java | 33 +-
.../table/source/FullTextSearchBuilderTest.java | 617 ++++++++++++++++++++-
.../table/source/PrimaryKeyFullTextSearchTest.java | 32 ++
.../index/NativeFullTextRowFilterTest.java | 517 +++++++++++++++++
.../org/apache/paimon/spark/PaimonBaseScan.scala | 4 +-
.../paimon/spark/sql/FullTextSearchTest.scala | 228 ++++++++
.../apache/paimon/spark/sql/HybridSearchTest.scala | 57 +-
17 files changed, 2026 insertions(+), 80 deletions(-)
diff --git a/docs/docs/multimodal-table/global-index/full-text.mdx b/docs/docs/multimodal-table/global-index/full-text.mdx
index 8edef0e54d..af6ad4916b 100644
--- a/docs/docs/multimodal-table/global-index/full-text.mdx
+++ b/docs/docs/multimodal-table/global-index/full-text.mdx
@@ -206,6 +206,17 @@ FROM full_text_search(
10
)
ORDER BY __paimon_search_score DESC, id;
+
+-- Filter rows before ranking: the top 10 is taken among rows with category = 'lake'.
+SELECT id, content, __paimon_search_score
+FROM full_text_search(
+ 'my_table',
+ 'content',
+ '{"match":{"query":"paimon lake format"}}',
+ 10
+)
+WHERE category = 'lake' AND dt >= '2026-09-01'
+ORDER BY __paimon_search_score DESC, id;
```
Spark exposes the score as the `__paimon_search_score` metadata column.
@@ -230,6 +241,19 @@ try (RecordReader<InternalRow> reader = readBuilder.newRead().createReader(plan)
}
```
+Add a row filter with `withFilter` to rank only matching rows:
+
+```java
+Predicate filter = new PredicateBuilder(table.rowType()).equal(1, BinaryString.fromString("lake"));
+
+GlobalIndexResult result =
+ table.newFullTextSearchBuilder()
+ .withQuery("content", "{\"match\":{\"query\":\"paimon lake\"}}")
+ .withFilter(filter)
+ .withLimit(10)
+ .executeLocal();
+```
+
</TabItem>
<TabItem value="python-sdk" label="Python SDK">
@@ -259,6 +283,44 @@ When `full-text-index.search-mode` is `full` or `detail`, Paimon also covers
unindexed row ranges by reading raw rows and building a temporary native
full-text index for the searched column.
+### Row Filters
+
+A full-text search can carry a row filter: a Spark `WHERE` clause on the
+`full_text_search` result, `withFilter` on the Java builder, or `withFilter` on
+a hybrid search. Partition predicates in the filter prune partitions; the
+remaining predicates are evaluated **before** top-k ranking, so the result is
+the top-k among matching rows rather than a filtered subset of the unfiltered
+top-k.
+
+Non-partition predicates are resolved into an exact set of matching rows
+**before** ranking, and that set is handed to the existing full-text index, so
+BM25 statistics are always those of the full corpus:
+
+- Rows whose filter columns are covered by a scalar global index (BTree,
+ Bitmap, Multivalue, FM) are decided by the index, the same way
+ [vector search pre-filters](./vector#vector-search) rows. When the index can
+ only produce candidates (a conjunct no index could evaluate, or `contains`,
+ ends-with and `LIKE` on a BTree index), the candidates are verified by reading
+ their filter columns.
+- Rows whose filter columns are not covered by a scalar index follow
+ `scalar-index.search-mode`:
+
+| `scalar-index.search-mode` | Rows without an index on the filter columns |
+|---|---|
+| `fast` (default) | Excluded from the search; a warning is logged |
+| `full` / `detail` | Decided by reading their filter columns |
+
+Build a scalar index on the columns you filter by to avoid the data read.
+With `full-text-index.search-mode=fast`, only rows covered by the full-text
+index are considered in either case; rows outside the coverage are searched
+through a temporary index in `full` mode, where the filter is applied the same
+way. Predicates the engine cannot push down (for example, UDFs) are applied by
+the engine after the search, so such queries may return fewer than `limit`
+rows.
+
+Primary-key full-text indexes do not support row filters yet; see
+[Primary-Key Indexes](../../primary-key-table/global-index#limitations).
+
## Query DSL
The query DSL is a JSON object with one top-level query type. Use the examples
diff --git a/docs/docs/multimodal-table/global-index/hybrid-search.mdx b/docs/docs/multimodal-table/global-index/hybrid-search.mdx
index 09ef65e11b..81bfae13e0 100644
--- a/docs/docs/multimodal-table/global-index/hybrid-search.mdx
+++ b/docs/docs/multimodal-table/global-index/hybrid-search.mdx
@@ -397,3 +397,15 @@ increase route limits when a small candidate set restricts final recall.
For Java and Python, the subsequent table scan does not preserve ranking order.
See [Read Scored Results](./manage-indexes#read-scored-results) to access scores by row ID.
+
+## Row Filters
+
+A row filter given to a hybrid search (`WHERE` on the Spark `hybrid_search`
+result, or `withFilter` on the Java builder) is applied to every route before
+its own top-k, so vector and full-text candidates are drawn from the same
+filtered row set before the ranker merges them. Full-text routes follow the same
+rules as a standalone full-text search; see
+[Full-Text Row Filters](./full-text#row-filters). In particular, under
+`scalar-index.search-mode=fast` a full-text route whose filter columns have no
+scalar index contributes no candidates, so the fused result comes from the
+other routes only.
diff --git a/docs/docs/primary-key-table/global-index.mdx b/docs/docs/primary-key-table/global-index.mdx
index 82ca5f097b..a5c2a8d986 100644
--- a/docs/docs/primary-key-table/global-index.mdx
+++ b/docs/docs/primary-key-table/global-index.mdx
@@ -620,6 +620,8 @@ create a table with the desired definition and migrate the data.
- FAST is the default search mode and excludes uncovered files from primary-key Vector and Full
Text results. Vector supports exact fallback for uncovered files in FULL and DETAIL modes;
primary-key Full Text supports only FAST until compaction creates persistent archives.
-- Full Text routes support partition pruning but not arbitrary row predicates before Top-K.
+- Primary-key Full Text routes support partition pruning but not arbitrary row predicates
+ before Top-K. Row filters on full-text search are supported for Data Evolution tables only; see
+ [Row Filters](../multimodal-table/global-index/full-text#row-filters).
- Hybrid search cannot mix source-backed physical routes with global row-ID routes.
- Online replacement between two definitions on the same column is not supported.
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java
index 40e492968c..9a8227b1b9 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java
@@ -18,7 +18,11 @@
package org.apache.paimon.table.source;
+import org.apache.paimon.CoreOptions.GlobalIndexSearchMode;
import org.apache.paimon.Snapshot;
+import org.apache.paimon.globalindex.DataEvolutionGlobalIndexCoverage;
+import org.apache.paimon.globalindex.DataEvolutionGlobalIndexScanner;
+import org.apache.paimon.globalindex.GlobalIndexEvaluator;
import org.apache.paimon.globalindex.GlobalIndexIOMeta;
import org.apache.paimon.globalindex.GlobalIndexReadThreadPool;
import org.apache.paimon.globalindex.GlobalIndexReader;
@@ -32,31 +36,49 @@ import org.apache.paimon.index.GlobalIndexMeta;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.index.IndexPathFactory;
import org.apache.paimon.partition.PartitionPredicate;
+import org.apache.paimon.predicate.CompoundPredicate;
+import org.apache.paimon.predicate.Contains;
+import org.apache.paimon.predicate.EndsWith;
import org.apache.paimon.predicate.FullTextSearch;
+import org.apache.paimon.predicate.LeafFunction;
+import org.apache.paimon.predicate.LeafPredicate;
+import org.apache.paimon.predicate.Like;
+import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.types.DataField;
import org.apache.paimon.utils.IOUtils;
import org.apache.paimon.utils.Range;
import org.apache.paimon.utils.RoaringNavigableMap64;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import javax.annotation.Nullable;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import static org.apache.paimon.CoreOptions.GLOBAL_INDEX_THREAD_NUM;
+import static org.apache.paimon.predicate.PredicateVisitor.collectFieldIds;
import static org.apache.paimon.utils.Preconditions.checkNotNull;
/** Implementation for {@link FullTextRead}. */
public class DataEvolutionFullTextRead implements FullTextRead {
+ private static final Logger LOG = LoggerFactory.getLogger(DataEvolutionFullTextRead.class);
+
private final FileStoreTable table;
@Nullable private final PartitionPredicate partitionFilter;
+ @Nullable private final Predicate filter;
private final int limit;
private final DataField textColumn;
private final String query;
@@ -67,8 +89,19 @@ public class DataEvolutionFullTextRead implements FullTextRead {
int limit,
List<DataField> textColumns,
String query) {
+ this(table, partitionFilter, null, limit, textColumns, query);
+ }
+
+ public DataEvolutionFullTextRead(
+ FileStoreTable table,
+ @Nullable PartitionPredicate partitionFilter,
+ @Nullable Predicate filter,
+ int limit,
+ List<DataField> textColumns,
+ String query) {
this.table = table;
this.partitionFilter = partitionFilter;
+ this.filter = filter;
this.limit = limit;
if (textColumns.size() != 1) {
throw new IllegalArgumentException(
@@ -100,10 +133,12 @@ public class DataEvolutionFullTextRead implements FullTextRead {
ExecutorService executor = GlobalIndexReadThreadPool.getExecutorService(parallelism);
Map<String, List<IndexFullTextSearchSplit>> splitsByColumn = new HashMap<>();
+ List<IndexFullTextSearchSplit> indexSplits = new ArrayList<>();
List<Range> rawRowRanges = new ArrayList<>();
for (FullTextSearchSplit split : splits) {
if (split instanceof IndexFullTextSearchSplit) {
IndexFullTextSearchSplit indexSplit = (IndexFullTextSearchSplit) split;
+ indexSplits.add(indexSplit);
splitsByColumn
.computeIfAbsent(indexSplit.columnName(), k -> new ArrayList<>())
.add(indexSplit);
@@ -115,14 +150,22 @@ public class DataEvolutionFullTextRead implements FullTextRead {
GlobalIndexFileReader indexFileReader = m -> table.fileIO().newInputStream(m.filePath());
RoaringNavigableMap64 liveRows =
GlobalIndexLiveRowFilter.liveRows(table, planSnapshot, partitionFilter, null);
+ RoaringNavigableMap64 matchedRows = matchedRows(indexSplits, planSnapshot, liveRows);
ScoredGlobalIndexResult result =
- evalQuery(splitsByColumn, indexPathFactory, indexFileReader, executor, liveRows);
+ evalQuery(
+ splitsByColumn,
+ indexPathFactory,
+ indexFileReader,
+ executor,
+ liveRows,
+ matchedRows);
if (!rawRowRanges.isEmpty()) {
result =
new RawFullTextReadImpl(
table,
planSnapshot,
partitionFilter,
+ filter,
limit,
textColumn,
this::evalQuery)
@@ -131,12 +174,153 @@ public class DataEvolutionFullTextRead implements FullTextRead {
return result.topK(limit);
}
+ /**
+ * Rows of the indexed splits that satisfy {@link #filter}, or {@code null} when there is no
+ * filter. The set is exact: rows enter top-k ranking only when the whole predicate holds.
+ *
+ * <ul>
+ * <li>Rows whose filter columns are covered by a scalar global index are decided by the
+ * index. When the index answer may be a superset (a conjunct it could not evaluate was
+ * dropped, or a {@code contains} / {@code endsWith} / {@code like} leaf, which BTree
+ * answers with every non-null row), the candidates are refined by reading their filter
+ * columns.
+ * <li>Rows whose filter columns are not covered follow {@code scalar-index.search-mode}:
+ * excluded in {@code fast}, otherwise decided by reading their filter columns.
+ * </ul>
+ *
+ * <p>Either way the rows stay in the existing full-text index and are handed to it as an
+ * include bitmap, so BM25 statistics are those of the full corpus.
+ */
+ @Nullable
+ private RoaringNavigableMap64 matchedRows(
+ List<IndexFullTextSearchSplit> indexSplits,
+ @Nullable Snapshot planSnapshot,
+ @Nullable RoaringNavigableMap64 liveRows) {
+ if (filter == null || indexSplits.isEmpty()) {
+ return null;
+ }
+
+ RoaringNavigableMap64 covered = new RoaringNavigableMap64();
+ Set<IndexFileMeta> scalarIndexFiles =
+ new TreeSet<>(Comparator.comparing(IndexFileMeta::fileName));
+ for (IndexFullTextSearchSplit split : indexSplits) {
+ for (Range range : split.searchRowRanges()) {
+ covered.addRange(range);
+ }
+ scalarIndexFiles.addAll(split.scalarIndexFiles());
+ }
+ if (liveRows != null) {
+ covered.and(liveRows);
+ }
+
+ RoaringNavigableMap64 unindexed = new RoaringNavigableMap64();
+ for (Range range :
+ new DataEvolutionGlobalIndexCoverage(
+ table,
+ planSnapshot,
+ partitionFilter,
+ scalarIndexFiles,
+ table.coreOptions().scalarIndexSearchMode())
+ .unindexedRanges(table.rowType(), filter)) {
+ unindexed.addRange(range);
+ }
+ unindexed.and(covered);
+ RoaringNavigableMap64 decidedByIndex =
+ RoaringNavigableMap64.or(new RoaringNavigableMap64(), covered);
+ decidedByIndex.andNot(unindexed);
+
+ RoaringNavigableMap64 matched = new RoaringNavigableMap64();
+ if (!decidedByIndex.isEmpty()) {
+ Optional<GlobalIndexEvaluator.Evaluation> evaluation =
+ evaluateWithIndexes(scalarIndexFiles, planSnapshot);
+ if (evaluation.isPresent()) {
+ RoaringNavigableMap64 fromIndex =
+ RoaringNavigableMap64.and(
+ evaluation.get().result().results(), decidedByIndex);
+ if (!isExact(evaluation.get()) && !fromIndex.isEmpty()) {
+ fromIndex =
+ new FilteredRowIdReader(table, planSnapshot, partitionFilter, filter)
+ .matchingRowIds(fromIndex);
+ }
+ matched.or(fromIndex);
+ } else {
+ warnUnindexedFilter();
+ }
+ }
+ if (!unindexed.isEmpty()) {
+ matched.or(
+ new FilteredRowIdReader(table, planSnapshot, partitionFilter, filter)
+ .matchingRowIds(unindexed));
+ }
+ return matched;
+ }
+
+ private Optional<GlobalIndexEvaluator.Evaluation> evaluateWithIndexes(
+ Set<IndexFileMeta> scalarIndexFiles, @Nullable Snapshot planSnapshot) {
+ Optional<DataEvolutionGlobalIndexScanner> optionalScanner =
+ DataEvolutionGlobalIndexScanner.create(
+ table, planSnapshot, partitionFilter, scalarIndexFiles);
+ if (!optionalScanner.isPresent()) {
+ return Optional.empty();
+ }
+ try (DataEvolutionGlobalIndexScanner scanner = optionalScanner.get()) {
+ return scanner.scanWithCoverage(filter);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Whether an index evaluation is an exact match set. The global index contract only promises
+ * candidates: a conjunct no index could evaluate is dropped, and BTree answers substring
+ * predicates with every non-null row. Both cases are refined from the data.
+ */
+ private boolean isExact(GlobalIndexEvaluator.Evaluation evaluation) {
+ Set<Integer> filterFieldIds = collectFieldIds(table.rowType(), filter);
+ if (!evaluation.contributingFieldIds().containsAll(filterFieldIds)) {
+ return false;
+ }
+ return !hasCandidateOnlyLeaf(filter);
+ }
+
+ private static boolean hasCandidateOnlyLeaf(Predicate predicate) {
+ if (predicate instanceof CompoundPredicate) {
+ for (Predicate child : ((CompoundPredicate) predicate).children()) {
+ if (hasCandidateOnlyLeaf(child)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ if (predicate instanceof LeafPredicate) {
+ LeafFunction function = ((LeafPredicate) predicate).function();
+ return function instanceof Contains
+ || function instanceof EndsWith
+ || function instanceof Like;
+ }
+ return false;
+ }
+
+ private void warnUnindexedFilter() {
+ if (table.coreOptions().scalarIndexSearchMode() == GlobalIndexSearchMode.FAST) {
+ LOG.warn(
+ "Full-text search on table {} has a row filter {} that no scalar global index "
+ + "can evaluate; indexed rows are excluded from the result because "
+ + "scalar-index.search-mode is fast. Build a scalar index on the "
+ + "filtered columns or use search mode full.",
+ table.name(),
+ filter);
+ }
+ }
+
ScoredGlobalIndexResult evalQuery(
Map<String, List<IndexFullTextSearchSplit>> splitsByColumn,
IndexPathFactory indexPathFactory,
GlobalIndexFileReader indexFileReader,
- ExecutorService executor) {
- return evalQuery(splitsByColumn, indexPathFactory, indexFileReader, executor, null);
+ ExecutorService executor,
+ @Nullable RoaringNavigableMap64 includeRowIds) {
+ return evalQuery(
+ splitsByColumn, indexPathFactory, indexFileReader, executor, null, includeRowIds);
}
private ScoredGlobalIndexResult evalQuery(
@@ -144,14 +328,16 @@ public class DataEvolutionFullTextRead implements FullTextRead {
IndexPathFactory indexPathFactory,
GlobalIndexFileReader indexFileReader,
ExecutorService executor,
- @Nullable RoaringNavigableMap64 liveRows) {
+ @Nullable RoaringNavigableMap64 liveRows,
+ @Nullable RoaringNavigableMap64 matchedRows) {
return evalColumnQuery(
textColumn.name(),
splitsByColumn,
indexPathFactory,
indexFileReader,
executor,
- liveRows);
+ liveRows,
+ matchedRows);
}
private ScoredGlobalIndexResult evalColumnQuery(
@@ -160,7 +346,8 @@ public class DataEvolutionFullTextRead implements FullTextRead {
IndexPathFactory indexPathFactory,
GlobalIndexFileReader indexFileReader,
ExecutorService executor,
- @Nullable RoaringNavigableMap64 liveRows) {
+ @Nullable RoaringNavigableMap64 liveRows,
+ @Nullable RoaringNavigableMap64 matchedRows) {
List<IndexFullTextSearchSplit> columnSplits = splitsByColumn.get(column);
if (columnSplits == null || columnSplits.isEmpty()) {
return ScoredGlobalIndexResult.createEmpty();
@@ -183,7 +370,7 @@ public class DataEvolutionFullTextRead implements FullTextRead {
split.fullTextIndexFiles(),
indexFileReader,
executor,
- includeRowIds(split, liveRows)));
+ includeRowIds(split, liveRows, matchedRows)));
}
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
@@ -201,7 +388,9 @@ public class DataEvolutionFullTextRead implements FullTextRead {
@Nullable
private static RoaringNavigableMap64 includeRowIds(
- IndexFullTextSearchSplit split, @Nullable RoaringNavigableMap64 liveRows) {
+ IndexFullTextSearchSplit split,
+ @Nullable RoaringNavigableMap64 liveRows,
+ @Nullable RoaringNavigableMap64 matchedRows) {
RoaringNavigableMap64 include = new RoaringNavigableMap64();
for (Range range : split.searchRowRanges()) {
include.addRange(range);
@@ -209,6 +398,9 @@ public class DataEvolutionFullTextRead implements FullTextRead {
if (liveRows != null) {
include.and(liveRows);
}
+ if (matchedRows != null) {
+ include.and(matchedRows);
+ }
long physicalRowCount = split.rowRangeEnd() - split.rowRangeStart() + 1;
return include.getLongCardinality() == physicalRowCount ? null : include;
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextScan.java
index fbd1bd83d1..b909a87007 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextScan.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextScan.java
@@ -28,6 +28,7 @@ import org.apache.paimon.index.IndexFileHandler;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.manifest.IndexManifestEntry;
import org.apache.paimon.partition.PartitionPredicate;
+import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.source.snapshot.TimeTravelUtil;
import org.apache.paimon.types.DataField;
@@ -49,6 +50,7 @@ import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;
+import static org.apache.paimon.predicate.PredicateVisitor.collectFieldIds;
import static org.apache.paimon.utils.Preconditions.checkNotNull;
/** Implementation for {@link FullTextScan}. */
@@ -56,6 +58,7 @@ public class DataEvolutionFullTextScan implements FullTextScan {
private final FileStoreTable table;
@Nullable private final PartitionPredicate partitionFilter;
+ @Nullable private final Predicate filter;
private final List<DataField> textColumns;
@Nullable private final Snapshot pinnedSnapshot;
@@ -78,8 +81,18 @@ public class DataEvolutionFullTextScan implements FullTextScan {
@Nullable PartitionPredicate partitionFilter,
List<DataField> textColumns,
@Nullable Snapshot pinnedSnapshot) {
+ this(table, partitionFilter, null, textColumns, pinnedSnapshot);
+ }
+
+ public DataEvolutionFullTextScan(
+ FileStoreTable table,
+ @Nullable PartitionPredicate partitionFilter,
+ @Nullable Predicate filter,
+ List<DataField> textColumns,
+ @Nullable Snapshot pinnedSnapshot) {
this.table = table;
this.partitionFilter = partitionFilter;
+ this.filter = filter;
this.textColumns = textColumns;
this.pinnedSnapshot = pinnedSnapshot;
}
@@ -98,6 +111,7 @@ public class DataEvolutionFullTextScan implements FullTextScan {
idToColumn.put(textColumn.id(), textColumn.name());
}
+ Set<Integer> filterFieldIds = collectFieldIds(table.rowType(), filter);
@Nullable
Snapshot snapshot =
pinnedSnapshot != null ? pinnedSnapshot : TimeTravelUtil.tryTravelOrLatest(table);
@@ -111,34 +125,54 @@ public class DataEvolutionFullTextScan implements FullTextScan {
if (globalIndex == null) {
return false;
}
- return !matchedTextColumnIds(globalIndex, textColumnIds).isEmpty()
- && supportsFullTextSearch(entry.indexFile().indexType());
+ return isFullTextIndexFile(entry.indexFile(), textColumnIds)
+ || containsAnyField(globalIndex, filterFieldIds);
};
List<IndexFileMeta> allIndexFiles =
indexFileHandler.scan(snapshot, indexFileFilter).stream()
.map(IndexManifestEntry::indexFile)
.collect(Collectors.toList());
+ List<IndexFileMeta> fullTextIndexFiles = new ArrayList<>();
+ List<IndexFileMeta> scalarIndexFiles = new ArrayList<>();
+ for (IndexFileMeta indexFile : allIndexFiles) {
+ boolean fullText = isFullTextIndexFile(indexFile, textColumnIds);
+ if (fullText) {
+ fullTextIndexFiles.add(indexFile);
+ }
+ // A dedicated full-text index cannot evaluate scalar predicates; a multi-field index
+ // (text carried next to scalar extra fields) can serve both roles.
+ GlobalIndexMeta meta = checkNotNull(indexFile.globalIndexMeta());
+ if (containsAnyField(meta, filterFieldIds) && (!fullText || hasExtraFields(meta))) {
+ scalarIndexFiles.add(indexFile);
+ }
+ }
+ // Build splits: for each chosen full-text range, attach the scalar index files that can
+ // pre-filter its rows. Rows the full-text index does not cover go to the raw split; rows
+ // it covers but no scalar index does are resolved from the data at read time.
List<FullTextSearchSplit> splits = new ArrayList<>();
for (IndexRangeSelection selection :
- chooseIndexRanges(allIndexFiles, textColumnIds, idToColumn)) {
+ chooseIndexRanges(fullTextIndexFiles, textColumnIds, idToColumn)) {
splits.add(
new IndexFullTextSearchSplit(
selection.columnName,
selection.fileRange.from,
selection.fileRange.to,
selection.files,
- selection.searchRanges));
+ selection.searchRanges,
+ scalarIndexFiles(
+ scalarIndexFiles,
+ Collections.singletonList(selection.fileRange))));
}
- if (!allIndexFiles.isEmpty()) {
+ if (!fullTextIndexFiles.isEmpty()) {
List<Range> rawRowRanges =
new DataEvolutionGlobalIndexCoverage(
table,
snapshot,
partitionFilter,
- allIndexFiles,
+ fullTextIndexFiles,
table.coreOptions().fullTextIndexSearchMode())
.unindexedRanges(textColumnIds);
if (!rawRowRanges.isEmpty()) {
@@ -408,6 +442,48 @@ public class DataEvolutionFullTextScan implements FullTextScan {
}
}
+ private static boolean isFullTextIndexFile(
+ IndexFileMeta indexFile, Set<Integer> textColumnIds) {
+ GlobalIndexMeta globalIndex = checkNotNull(indexFile.globalIndexMeta());
+ return !matchedTextColumnIds(globalIndex, textColumnIds).isEmpty()
+ && supportsFullTextSearch(indexFile.indexType());
+ }
+
+ private static boolean containsAnyField(GlobalIndexMeta meta, Set<Integer> fieldIds) {
+ if (fieldIds.contains(meta.indexFieldId())) {
+ return true;
+ }
+ int[] extraFieldIds = meta.extraFieldIds();
+ if (extraFieldIds != null) {
+ for (int extraFieldId : extraFieldIds) {
+ if (fieldIds.contains(extraFieldId)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static boolean hasExtraFields(GlobalIndexMeta meta) {
+ int[] extraFieldIds = meta.extraFieldIds();
+ return extraFieldIds != null && extraFieldIds.length > 0;
+ }
+
+ private static List<IndexFileMeta> scalarIndexFiles(
+ List<IndexFileMeta> scalarIndexFiles, List<Range> rowRanges) {
+ List<IndexFileMeta> result = new ArrayList<>();
+ for (IndexFileMeta indexFile : scalarIndexFiles) {
+ Range indexRange = checkNotNull(indexFile.globalIndexMeta()).rowRange();
+ for (Range rowRange : rowRanges) {
+ if (rowRange.hasIntersection(indexRange)) {
+ result.add(indexFile);
+ break;
+ }
+ }
+ }
+ return result;
+ }
+
private static boolean supportsFullTextSearch(String indexType) {
GlobalIndexerFactory factory;
try {
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/FilteredRowIdReader.java b/paimon-core/src/main/java/org/apache/paimon/table/source/FilteredRowIdReader.java
new file mode 100644
index 0000000000..8f88c511c2
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FilteredRowIdReader.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.partition.PartitionPredicate;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateVisitor;
+import org.apache.paimon.reader.RecordReader;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.SpecialFields;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Range;
+import org.apache.paimon.utils.RoaringNavigableMap64;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Evaluates a row predicate on the data for a given set of row ids and returns the ids that satisfy
+ * it. Only the filter columns and the row id are read, so this is the exact counterpart of a scalar
+ * global index lookup: a search can hand the result to its index as an include bitmap without
+ * touching the index corpus.
+ */
+class FilteredRowIdReader {
+
+ private final FileStoreTable table;
+ @Nullable private final Snapshot planSnapshot;
+ @Nullable private final PartitionPredicate partitionFilter;
+ private final Predicate filter;
+
+ FilteredRowIdReader(
+ FileStoreTable table,
+ @Nullable Snapshot planSnapshot,
+ @Nullable PartitionPredicate partitionFilter,
+ Predicate filter) {
+ this.table = table;
+ this.planSnapshot = planSnapshot;
+ this.partitionFilter = partitionFilter;
+ this.filter = filter;
+ }
+
+ /** The subset of {@code rows} whose data satisfies the filter. */
+ RoaringNavigableMap64 matchingRowIds(RoaringNavigableMap64 rows) {
+ RoaringNavigableMap64 matching = new RoaringNavigableMap64();
+ if (rows.isEmpty()) {
+ return matching;
+ }
+ List<Range> rowRanges = Range.sortAndMergeOverlap(rows.toRangeList(), true);
+ RowType readType = readType();
+ int rowIdIndex = readType.getFieldIndex(SpecialFields.ROW_ID.name());
+
+ TableScan.Plan plan =
+ readBuilder(readType, false).withRowRanges(rowRanges).newScan().plan();
+ try (RecordReader<InternalRow> reader =
+ readBuilder(readType, true).newRead().executeFilter().createReader(plan)) {
+ reader.forEachRemaining(
+ row -> {
+ long rowId = row.getLong(rowIdIndex);
+ if (rows.contains(rowId)) {
+ matching.add(rowId);
+ }
+ });
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to evaluate the row filter on table data.", e);
+ }
+ return matching;
+ }
+
+ private RowType readType() {
+ RowType tableRowType = table.rowType();
+ Set<String> filterFields = PredicateVisitor.collectFieldNames(filter);
+ List<String> readFields = new ArrayList<>();
+ for (String field : tableRowType.getFieldNames()) {
+ if (filterFields.contains(field)) {
+ readFields.add(field);
+ }
+ }
+ return SpecialFields.rowTypeWithRowId(tableRowType.project(readFields));
+ }
+
+ private ReadBuilder readBuilder(RowType readType, boolean withFilter) {
+ ReadBuilder readBuilder = readTable().newReadBuilder().withReadType(readType);
+ if (partitionFilter != null) {
+ readBuilder.withPartitionFilter(partitionFilter);
+ }
+ if (withFilter) {
+ readBuilder.withFilter(filter);
+ }
+ return readBuilder;
+ }
+
+ private FileStoreTable readTable() {
+ if (planSnapshot == null) {
+ return table;
+ }
+ Map<String, String> pinOptions = new HashMap<>();
+ pinOptions.put(
+ CoreOptions.SCAN_MODE.key(), CoreOptions.StartupMode.FROM_SNAPSHOT.toString());
+ pinOptions.put(CoreOptions.SCAN_SNAPSHOT_ID.key(), String.valueOf(planSnapshot.id()));
+ pinOptions.put(CoreOptions.SCAN_VERSION.key(), null);
+ pinOptions.put(CoreOptions.SCAN_TAG_NAME.key(), null);
+ pinOptions.put(CoreOptions.SCAN_WATERMARK.key(), null);
+ pinOptions.put(CoreOptions.SCAN_TIMESTAMP.key(), null);
+ pinOptions.put(CoreOptions.SCAN_TIMESTAMP_MILLIS.key(), null);
+ return table.copyWithoutTimeTravel(pinOptions);
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilder.java b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilder.java
index 5b091670da..9e07fcdbd9 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilder.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilder.java
@@ -20,6 +20,7 @@ package org.apache.paimon.table.source;
import org.apache.paimon.globalindex.GlobalIndexResult;
import org.apache.paimon.partition.PartitionPredicate;
+import org.apache.paimon.predicate.Predicate;
import java.io.Serializable;
@@ -32,6 +33,16 @@ public interface FullTextSearchBuilder extends Serializable {
"This full-text search builder does not support partition filters.");
}
+ /**
+ * Push a row filter. Rows that do not satisfy the predicate are excluded before top-k ranking,
+ * so the returned top-k is the top-k among matching rows. Partition predicates contained in the
+ * filter are extracted and applied as partition filters.
+ */
+ default FullTextSearchBuilder withFilter(Predicate predicate) {
+ throw new UnsupportedOperationException(
+ "This full-text search builder does not support row filters.");
+ }
+
/** The top k results to return. */
FullTextSearchBuilder withLimit(int limit);
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java
index 77d74c9a5f..baa6dab5f9 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java
@@ -23,15 +23,21 @@ import org.apache.paimon.catalog.TableQueryAuthResult;
import org.apache.paimon.index.pk.PrimaryKeyIndexDefinition;
import org.apache.paimon.index.pk.PrimaryKeyIndexDefinitions;
import org.apache.paimon.partition.PartitionPredicate;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.InnerTable;
import org.apache.paimon.types.DataField;
+import org.apache.paimon.utils.Pair;
import javax.annotation.Nullable;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.List;
import java.util.Optional;
+import static org.apache.paimon.partition.PartitionPredicate.splitPartitionPredicatesAndDataPredicates;
import static org.apache.paimon.utils.Preconditions.checkArgument;
import static org.apache.paimon.utils.Preconditions.checkNotNull;
@@ -46,6 +52,7 @@ public class FullTextSearchBuilderImpl implements FullTextSearchBuilder {
private String fieldName;
private String query;
private PartitionPredicate partitionFilter;
+ @Nullable private Predicate filter;
@Nullable private Snapshot pinnedSnapshot;
public FullTextSearchBuilderImpl(InnerTable table) {
@@ -54,10 +61,37 @@ public class FullTextSearchBuilderImpl implements FullTextSearchBuilder {
@Override
public FullTextSearchBuilder withPartitionFilter(PartitionPredicate partitionFilter) {
- this.partitionFilter = partitionFilter;
+ addPartitionFilter(partitionFilter);
return this;
}
+ @Override
+ public FullTextSearchBuilder withFilter(Predicate predicate) {
+ Pair<Optional<PartitionPredicate>, List<Predicate>> pair =
+ splitPartitionPredicatesAndDataPredicates(
+ predicate, table.rowType(), table.partitionKeys());
+ if (pair.getLeft().isPresent()) {
+ addPartitionFilter(pair.getLeft().get());
+ }
+ if (!pair.getRight().isEmpty()) {
+ Predicate dataFilter = PredicateBuilder.and(pair.getRight());
+ this.filter =
+ this.filter == null ? dataFilter : PredicateBuilder.and(filter, dataFilter);
+ }
+ return this;
+ }
+
+ private void addPartitionFilter(@Nullable PartitionPredicate partitionFilter) {
+ if (partitionFilter == null) {
+ return;
+ }
+ this.partitionFilter =
+ this.partitionFilter == null
+ ? partitionFilter
+ : PartitionPredicate.and(
+ Arrays.asList(this.partitionFilter, partitionFilter));
+ }
+
@Override
public FullTextSearchBuilder withLimit(int limit) {
this.limit = limit;
@@ -82,6 +116,7 @@ public class FullTextSearchBuilderImpl implements FullTextSearchBuilder {
: new DataEvolutionFullTextScan(
table,
partitionFilter,
+ filter,
Collections.singletonList(textColumn),
pinnedSnapshot);
}
@@ -97,6 +132,7 @@ public class FullTextSearchBuilderImpl implements FullTextSearchBuilder {
: new DataEvolutionFullTextRead(
table,
partitionFilter,
+ filter,
limit,
Collections.singletonList(textColumn),
query);
@@ -121,6 +157,10 @@ public class FullTextSearchBuilderImpl implements FullTextSearchBuilder {
PrimaryKeyIndexDefinitions.create(table.schema()).definitions()) {
if (definition.family() == PrimaryKeyIndexDefinition.Family.FULL_TEXT
&& definition.fieldId() == textColumn.id()) {
+ if (filter != null) {
+ throw new UnsupportedOperationException(
+ "Primary-key full-text search does not support non-partition filters yet.");
+ }
return Optional.of(definition);
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/HybridSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/HybridSearchBuilderImpl.java
index 35460de7b4..4183af92bb 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/HybridSearchBuilderImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/HybridSearchBuilderImpl.java
@@ -165,16 +165,6 @@ public class HybridSearchBuilderImpl implements HybridSearchBuilder {
if (limit <= 0) {
throw new IllegalArgumentException("Limit must be positive, got: " + limit);
}
- if (filter != null) {
- for (HybridSearchRoute route : routes) {
- if (!route.isVector()) {
- throw new UnsupportedOperationException(
- "Hybrid search with full-text routes does not support non-partition "
- + "filters because full-text indexes cannot apply row-id "
- + "pre-filters before top-k ranking.");
- }
- }
- }
}
@Override
@@ -377,6 +367,9 @@ public class HybridSearchBuilderImpl implements HybridSearchBuilder {
if (partitionFilter != null) {
fullTextSearchBuilder.withPartitionFilter(partitionFilter);
}
+ if (filter != null) {
+ fullTextSearchBuilder.withFilter(filter);
+ }
return fullTextSearchBuilder;
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/IndexFullTextSearchSplit.java b/paimon-core/src/main/java/org/apache/paimon/table/source/IndexFullTextSearchSplit.java
index 54a55536f1..a7e93bfefa 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/IndexFullTextSearchSplit.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/IndexFullTextSearchSplit.java
@@ -37,7 +37,8 @@ public class IndexFullTextSearchSplit extends FullTextSearchSplit {
private static final long serialVersionUID = 1L;
- private static final int VERSION = 1;
+ /** Version 2 appends the scalar index files usable as a row pre-filter. */
+ private static final int VERSION = 2;
private static final ThreadLocal<IndexFileMetaSerializer> INDEX_SERIALIZER =
ThreadLocal.withInitial(IndexFileMetaSerializer::new);
@@ -47,6 +48,7 @@ public class IndexFullTextSearchSplit extends FullTextSearchSplit {
private long rowRangeEnd;
private List<Range> searchRowRanges;
private transient List<IndexFileMeta> fullTextIndexFiles;
+ private transient List<IndexFileMeta> scalarIndexFiles;
public IndexFullTextSearchSplit(
long rowRangeStart, long rowRangeEnd, List<IndexFileMeta> fullTextIndexFiles) {
@@ -72,6 +74,22 @@ public class IndexFullTextSearchSplit extends FullTextSearchSplit {
long rowRangeEnd,
List<IndexFileMeta> fullTextIndexFiles,
List<Range> searchRowRanges) {
+ this(
+ columnName,
+ rowRangeStart,
+ rowRangeEnd,
+ fullTextIndexFiles,
+ searchRowRanges,
+ Collections.emptyList());
+ }
+
+ public IndexFullTextSearchSplit(
+ String columnName,
+ long rowRangeStart,
+ long rowRangeEnd,
+ List<IndexFileMeta> fullTextIndexFiles,
+ List<Range> searchRowRanges,
+ List<IndexFileMeta> scalarIndexFiles) {
this.columnName = columnName;
this.rowRangeStart = rowRangeStart;
this.rowRangeEnd = rowRangeEnd;
@@ -86,6 +104,7 @@ public class IndexFullTextSearchSplit extends FullTextSearchSplit {
}
}
this.searchRowRanges = Collections.unmodifiableList(ranges);
+ this.scalarIndexFiles = Collections.unmodifiableList(new ArrayList<>(scalarIndexFiles));
}
public String columnName() {
@@ -108,24 +127,35 @@ public class IndexFullTextSearchSplit extends FullTextSearchSplit {
return fullTextIndexFiles;
}
+ /** Scalar global index files intersecting this split, used to pre-filter rows. */
+ public List<IndexFileMeta> scalarIndexFiles() {
+ return scalarIndexFiles;
+ }
+
private void writeObject(ObjectOutputStream out) throws IOException {
out.defaultWriteObject();
out.writeInt(VERSION);
IndexFileMetaSerializer serializer = INDEX_SERIALIZER.get();
DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(out);
serializer.serializeList(fullTextIndexFiles, view);
+ serializer.serializeList(scalarIndexFiles, view);
}
private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
in.defaultReadObject();
int version = in.readInt();
- if (version != VERSION) {
+ if (version < 1 || version > VERSION) {
throw new IOException("Unsupported IndexFullTextSearchSplit version: " + version);
}
IndexFileMetaSerializer serializer = INDEX_SERIALIZER.get();
DataInputViewStreamWrapper view = new DataInputViewStreamWrapper(in);
this.fullTextIndexFiles =
Collections.unmodifiableList(new ArrayList<>(serializer.deserializeList(view)));
+ this.scalarIndexFiles =
+ version >= 2
+ ? Collections.unmodifiableList(
+ new ArrayList<>(serializer.deserializeList(view)))
+ : Collections.emptyList();
if (searchRowRanges == null) {
searchRowRanges = Collections.singletonList(new Range(rowRangeStart, rowRangeEnd));
} else {
@@ -143,13 +173,19 @@ public class IndexFullTextSearchSplit extends FullTextSearchSplit {
&& rowRangeEnd == that.rowRangeEnd
&& Objects.equals(columnName, that.columnName)
&& Objects.equals(searchRowRanges, that.searchRowRanges)
- && Objects.equals(fullTextIndexFiles, that.fullTextIndexFiles);
+ && Objects.equals(fullTextIndexFiles, that.fullTextIndexFiles)
+ && Objects.equals(scalarIndexFiles, that.scalarIndexFiles);
}
@Override
public int hashCode() {
return Objects.hash(
- columnName, rowRangeStart, rowRangeEnd, searchRowRanges, fullTextIndexFiles);
+ columnName,
+ rowRangeStart,
+ rowRangeEnd,
+ searchRowRanges,
+ fullTextIndexFiles,
+ scalarIndexFiles);
}
@Override
@@ -166,6 +202,8 @@ public class IndexFullTextSearchSplit extends FullTextSearchSplit {
+ searchRowRanges
+ ", fullTextIndexFiles="
+ fullTextIndexFiles
+ + ", scalarIndexFiles="
+ + scalarIndexFiles
+ '}';
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java
index 86c6d2a1d2..44a7e63a36 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java
@@ -37,6 +37,7 @@ import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.index.IndexPathFactory;
import org.apache.paimon.options.Options;
import org.apache.paimon.partition.PartitionPredicate;
+import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.SpecialFields;
@@ -68,6 +69,7 @@ class RawFullTextReadImpl {
private final FileStoreTable table;
@Nullable private final Snapshot planSnapshot;
@Nullable private final PartitionPredicate partitionFilter;
+ @Nullable private final Predicate filter;
private final int limit;
private final DataField textColumn;
private final IndexSearch indexSearch;
@@ -79,9 +81,21 @@ class RawFullTextReadImpl {
int limit,
DataField textColumn,
IndexSearch indexSearch) {
+ this(table, planSnapshot, partitionFilter, null, limit, textColumn, indexSearch);
+ }
+
+ RawFullTextReadImpl(
+ FileStoreTable table,
+ @Nullable Snapshot planSnapshot,
+ @Nullable PartitionPredicate partitionFilter,
+ @Nullable Predicate filter,
+ int limit,
+ DataField textColumn,
+ IndexSearch indexSearch) {
this.table = table;
this.planSnapshot = planSnapshot;
this.partitionFilter = partitionFilter;
+ this.filter = filter;
this.limit = limit;
this.textColumn = textColumn;
this.indexSearch = indexSearch;
@@ -111,22 +125,33 @@ class RawFullTextReadImpl {
int rowIdIndex = readType.getFieldIndex(SpecialFields.ROW_ID.name());
Map<String, RawFullTextIndex> rawIndexes =
createRawFullTextIndexes(splitsByColumn, readType, rawRowRanges);
+ // Every raw row is indexed so the temporary index scores against the full raw corpus;
+ // the row filter only decides which rows the search may return.
+ RoaringNavigableMap64 matchingRows = filter == null ? null : new RoaringNavigableMap64();
try {
try (RecordReader<InternalRow> reader = readBuilder.newRead().createReader(plan);
CloseableIterator<InternalRow> iterator = reader.toCloseableIterator()) {
while (iterator.hasNext()) {
InternalRow row = iterator.next();
+ long rowId = row.getLong(rowIdIndex);
for (RawFullTextIndex rawIndex : rawIndexes.values()) {
- long rowId = row.getLong(rowIdIndex);
rawIndex.write(row, rowId);
}
+ if (matchingRows != null && filter.test(row)) {
+ matchingRows.add(rowId);
+ }
}
}
} catch (Exception e) {
throw new RuntimeException("Failed to read raw rows for full-text search.", e);
}
+ if (matchingRows != null && matchingRows.isEmpty()) {
+ IOUtils.closeAllQuietly(rawIndexes.values());
+ return ScoredGlobalIndexResult.createEmpty();
+ }
+
try {
Map<String, List<IndexFullTextSearchSplit>> rawSplitsByColumn = new HashMap<>();
for (Map.Entry<String, RawFullTextIndex> entry : rawIndexes.entrySet()) {
@@ -143,7 +168,8 @@ class RawFullTextReadImpl {
m ->
new MemorySeekableInputStream(
rawFileBytes(rawIndexes, m.filePath().getName())),
- executor)
+ executor,
+ matchingRows)
.topK(limit);
} finally {
IOUtils.closeAllQuietly(rawIndexes.values());
@@ -284,7 +310,8 @@ class RawFullTextReadImpl {
Map<String, List<IndexFullTextSearchSplit>> splitsByColumn,
IndexPathFactory indexPathFactory,
GlobalIndexFileReader indexFileReader,
- ExecutorService executor);
+ ExecutorService executor,
+ @Nullable RoaringNavigableMap64 includeRowIds);
}
private static class RawFullTextIndex implements Closeable {
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
index 0c4f2b5122..94f483ec6c 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
@@ -57,6 +57,10 @@ import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.Range;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import javax.annotation.Nullable;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -351,22 +355,554 @@ public class FullTextSearchBuilderTest extends TableTestBase {
}
@Test
- public void testHybridSearchRejectsDataFilterWithFullTextRoute() throws Exception {
+ public void testHybridSearchAppliesDataFilterToFullTextRoute() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndex(table, RANKED_DOCUMENTS);
+ buildAndCommitIdBTreeIndex(table, RANKED_DOCUMENTS.length);
+
+ Predicate idFilter = new PredicateBuilder(table.rowType()).greaterOrEqual(0, 3);
+ ScoredGlobalIndexResult result =
+ table.newHybridSearchBuilder()
+ .addFullTextRoute(TEXT_FIELD_NAME, matchQuery("paimon lake"), 2, 1.0f)
+ .withFilter(idFilter)
+ .withLimit(2)
+ .executeLocal();
+
+ // Rows 0-2 score higher, but the filter removes them before the route's top-k.
+ assertThat(result.results()).hasSize(2);
+ assertThat(result.results()).isSubsetOf(3L, 4L, 5L);
+ }
+
+ // ====================== Row filter tests ======================
+
+ /**
+ * Rows 0-2 match both query terms of {@code "paimon lake"} (score 1.0) while rows 3-5 match
+ * only {@code "paimon"} (score 0.5), so an unfiltered top-k is always taken from rows 0-2.
+ */
+ private static final String[] RANKED_DOCUMENTS = {
+ "paimon lake alpha",
+ "paimon lake beta",
+ "paimon lake gamma",
+ "paimon delta",
+ "paimon epsilon",
+ "paimon zeta"
+ };
+
+ @Test
+ public void testFullTextSearchWithFilterRanksOnlyMatchingRows() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndex(table, RANKED_DOCUMENTS);
+ buildAndCommitIdBTreeIndex(table, RANKED_DOCUMENTS.length);
+
+ GlobalIndexResult unfiltered = searchWithFilter(table, null, 2);
+ assertThat(unfiltered.results()).isSubsetOf(0L, 1L, 2L);
+
+ Predicate idFilter = new PredicateBuilder(table.rowType()).greaterOrEqual(0, 3);
+ GlobalIndexResult filtered = searchWithFilter(table, idFilter, 2);
+ assertThat(filtered.results()).hasSize(2);
+ assertThat(filtered.results()).isSubsetOf(3L, 4L, 5L);
+ assertThat(readIds(table, filtered)).allMatch(id -> id >= 3);
+
+ // The scores are the full-text scores of the surviving rows, not rescored.
+ ScoredGlobalIndexResult scored = (ScoredGlobalIndexResult) filtered;
+ for (long rowId : scored.results()) {
+ assertThat(scored.scoreGetter().score(rowId)).isEqualTo(0.5f);
+ }
+ }
+
+ @Test
+ public void testFullTextSearchWithCompoundFilters() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndex(table, RANKED_DOCUMENTS);
+ buildAndCommitIdBTreeIndex(table, RANKED_DOCUMENTS.length);
+
+ PredicateBuilder builder = new PredicateBuilder(table.rowType());
+ Predicate between =
+ PredicateBuilder.and(builder.greaterOrEqual(0, 2), builder.lessOrEqual(0, 4));
+ assertThat(searchWithFilter(table, between, 10).results())
+ .containsExactlyInAnyOrder(2L, 3L, 4L);
+
+ Predicate either = PredicateBuilder.or(builder.equal(0, 0), builder.equal(0, 5));
+ assertThat(searchWithFilter(table, either, 10).results()).containsExactlyInAnyOrder(0L, 5L);
+
+ Predicate in = builder.in(0, Arrays.asList(1, 4));
+ assertThat(searchWithFilter(table, in, 10).results()).containsExactlyInAnyOrder(1L, 4L);
+
+ // A filter that matches rows the query does not still yields only query matches.
+ Predicate all = builder.greaterOrEqual(0, 0);
+ assertThat(searchWithFilter(table, all, 10).results())
+ .containsExactlyInAnyOrder(0L, 1L, 2L, 3L, 4L, 5L);
+ }
+
+ @Test
+ public void testFullTextSearchWithFilterMatchingNoRows() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndex(table, RANKED_DOCUMENTS);
+ buildAndCommitIdBTreeIndex(table, RANKED_DOCUMENTS.length);
+
+ Predicate impossible = new PredicateBuilder(table.rowType()).greaterThan(0, 100);
+ GlobalIndexResult result = searchWithFilter(table, impossible, 10);
+ assertThat(result.results().isEmpty()).isTrue();
+ assertThat(readIds(table, result)).isEmpty();
+ }
+
+ @Test
+ public void testFullTextSearchFastModeExcludesRowsWithoutScalarIndex() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndex(table, RANKED_DOCUMENTS);
+ // No scalar index on "id": in fast mode nothing can be verified, so nothing is returned.
+
+ Predicate idFilter = new PredicateBuilder(table.rowType()).greaterOrEqual(0, 3);
+ FullTextSearchBuilder builder =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon"))
+ .withLimit(10)
+ .withFilter(idFilter);
+ FullTextScan.Plan plan = builder.newFullTextScan().scan();
+ assertThat(plan.splits()).noneMatch(RawFullTextSearchSplit.class::isInstance);
+ assertThat(builder.newFullTextRead().read(plan).results().isEmpty()).isTrue();
+ }
+
+ @Test
+ public void testFullTextSearchFullModeScansRowsWithoutScalarIndex() throws Exception {
+ FileStoreTable table = createTable("full_text_filter_full_mode", "full");
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndex(table, RANKED_DOCUMENTS);
+ // No scalar index on "id": scalar-index.search-mode=full routes every row to a raw
+ // scan where the predicate is evaluated on the data.
+
+ Predicate idFilter = new PredicateBuilder(table.rowType()).greaterOrEqual(0, 3);
+ FullTextSearchBuilder builder =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon lake"))
+ .withLimit(2)
+ .withFilter(idFilter);
+ FullTextScan.Plan plan = builder.newFullTextScan().scan();
+ // The rows stay in the full-text index; the filter is resolved by reading `id`.
+ assertThat(plan.splits()).noneMatch(RawFullTextSearchSplit.class::isInstance);
+
+ GlobalIndexResult result = builder.newFullTextRead().read(plan);
+ assertThat(result.results()).hasSize(2);
+ assertThat(result.results()).isSubsetOf(3L, 4L, 5L);
+ }
+
+ @Test
+ public void testFullTextSearchFullModeMixesIndexAndDataFilterEvaluation() throws Exception {
+ FileStoreTable table = createTable("full_text_filter_partial_scalar", "full");
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndex(table, RANKED_DOCUMENTS);
+ // The scalar index covers rows 0-2 only; rows 3-5 are decided by reading `id`.
+ buildAndCommitIdBTreeIndexRange(table, new Range(0, 2));
+
+ Predicate idFilter = new PredicateBuilder(table.rowType()).greaterOrEqual(0, 1);
+ FullTextSearchBuilder builder =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon"))
+ .withLimit(10)
+ .withFilter(idFilter);
+ FullTextScan.Plan plan = builder.newFullTextScan().scan();
+ assertThat(rawSplits(plan)).isEmpty();
+ assertThat(indexSplits(plan)).allMatch(split -> !split.scalarIndexFiles().isEmpty());
+
+ GlobalIndexResult result = builder.newFullTextRead().read(plan);
+ assertThat(result.results()).containsExactlyInAnyOrder(1L, 2L, 3L, 4L, 5L);
+ }
+
+ @Test
+ public void testFullTextSearchFastModeKeepsDataFilterInsideIndexedRanges() throws Exception {
+ // full-text mode fast, scalar mode full: rows outside the full-text coverage stay out
+ // even though their filter columns are unindexed; covered rows are filtered from data.
+ Identifier identifier = identifier("full_text_fast_scalar_full");
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column(TEXT_FIELD_NAME, DataTypes.STRING())
+ .option(CoreOptions.BUCKET.key(), "-1")
+ .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")
+ .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true")
+ .option(CoreOptions.SCALAR_INDEX_SEARCH_MODE.key(), "full")
+ .build();
+ catalog.createTable(identifier, schema, false);
+ FileStoreTable table = getTable(identifier);
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndexRange(
+ table,
+ Arrays.copyOfRange(RANKED_DOCUMENTS, 0, 4),
+ Collections.singletonList(table.rowType().getField(TEXT_FIELD_NAME)),
+ 0);
+
+ Predicate idFilter = new PredicateBuilder(table.rowType()).greaterOrEqual(0, 2);
+ FullTextSearchBuilder builder =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon"))
+ .withLimit(10)
+ .withFilter(idFilter);
+ FullTextScan.Plan plan = builder.newFullTextScan().scan();
+ assertThat(rawSplits(plan)).isEmpty();
+
+ assertThat(builder.newFullTextRead().read(plan).results())
+ .containsExactlyInAnyOrder(2L, 3L);
+ }
+
+ @Test
+ public void testFullTextSearchFilterCombinedWithDeletionVectors() throws Exception {
+ Identifier identifier = identifier("full_text_filter_dv");
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column(TEXT_FIELD_NAME, DataTypes.STRING())
+ .option(CoreOptions.BUCKET.key(), "-1")
+ .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")
+ .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true")
+ .option(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true")
+ .build();
+ catalog.createTable(identifier, schema, false);
+ FileStoreTable table = getTable(identifier);
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndex(table, RANKED_DOCUMENTS);
+ buildAndCommitIdBTreeIndex(table, RANKED_DOCUMENTS.length);
+ commitDeletionVectors(table, 3L);
+
+ Predicate idFilter = new PredicateBuilder(table.rowType()).greaterOrEqual(0, 3);
+ GlobalIndexResult result = searchWithFilter(table, idFilter, 10);
+ assertThat(result.results()).containsExactlyInAnyOrder(4L, 5L);
+ }
+
+ @Test
+ public void testFullTextSearchFilterAcrossMultipleIndexRanges() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ writeDocuments(table, RANKED_DOCUMENTS);
+ List<DataField> textFields =
+ Collections.singletonList(table.rowType().getField(TEXT_FIELD_NAME));
+ buildAndCommitIndexRange(table, Arrays.copyOfRange(RANKED_DOCUMENTS, 0, 3), textFields, 0);
+ buildAndCommitIndexRange(table, Arrays.copyOfRange(RANKED_DOCUMENTS, 3, 6), textFields, 3);
+ buildAndCommitIdBTreeIndexRange(table, new Range(0, 2));
+ buildAndCommitIdBTreeIndexRange(table, new Range(3, 5));
+
+ Predicate in = new PredicateBuilder(table.rowType()).in(0, Arrays.asList(1, 4));
+ FullTextSearchBuilder builder =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon"))
+ .withLimit(10)
+ .withFilter(in);
+ FullTextScan.Plan plan = builder.newFullTextScan().scan();
+ List<IndexFullTextSearchSplit> indexSplits = indexSplits(plan);
+ assertThat(indexSplits).hasSize(2);
+ // Each split only carries the scalar index file of its own range.
+ for (IndexFullTextSearchSplit split : indexSplits) {
+ assertThat(split.scalarIndexFiles()).hasSize(1);
+ GlobalIndexMeta scalarMeta = split.scalarIndexFiles().get(0).globalIndexMeta();
+ assertThat(scalarMeta.rowRangeStart()).isEqualTo(split.rowRangeStart());
+ assertThat(scalarMeta.rowRangeEnd()).isEqualTo(split.rowRangeEnd());
+ }
+
+ assertThat(builder.newFullTextRead().read(plan).results())
+ .containsExactlyInAnyOrder(1L, 4L);
+ }
+
+ @Test
+ public void testFullTextSearchFilterExtractsPartitionPredicate() throws Exception {
+ Identifier identifier = identifier("full_text_filter_partitioned");
+ Schema schema =
+ Schema.newBuilder()
+ .column("pt", DataTypes.INT())
+ .column("id", DataTypes.INT())
+ .column(TEXT_FIELD_NAME, DataTypes.STRING())
+ .partitionKeys("pt")
+ .option(CoreOptions.BUCKET.key(), "-1")
+ .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")
+ .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true")
+ .build();
+ catalog.createTable(identifier, schema, false);
+ FileStoreTable table = getTable(identifier);
+
+ RowType partitionType = RowType.of(DataTypes.INT());
+ InternalRowSerializer serializer = new InternalRowSerializer(partitionType);
+ BinaryRow partition1 = serializer.toBinaryRow(GenericRow.of(1)).copy();
+ BinaryRow partition2 = serializer.toBinaryRow(GenericRow.of(2)).copy();
+
+ String[] first = {"paimon one", "paimon two"};
+ String[] second = {"paimon three"};
+ writePartitionedDocuments(table, 1, first);
+ buildAndCommitIndexForColumn(table, TEXT_FIELD_NAME, first, partition1);
+ writePartitionedDocuments(table, 2, second);
+ buildAndCommitIndexForColumn(table, TEXT_FIELD_NAME, second, partition2, first.length);
+
+ // Only a partition predicate: no scalar index is needed, and no row is dropped.
+ Predicate partitionOnly = new PredicateBuilder(table.rowType()).equal(0, 2);
+ GlobalIndexResult result = searchWithFilter(table, partitionOnly, 10);
+ assertThat(result.results()).containsExactly(2L);
+
+ FullTextScan.Plan plan =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon"))
+ .withLimit(10)
+ .withFilter(partitionOnly)
+ .newFullTextScan()
+ .scan();
+ assertThat(indexSplits(plan)).hasSize(1);
+ assertThat(indexSplits(plan).get(0).scalarIndexFiles()).isEmpty();
+ }
+
+ @Test
+ public void testFullTextScanAttachesScalarIndexFilesOnlyForFilteredColumns() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndex(table, RANKED_DOCUMENTS);
+ buildAndCommitIdBTreeIndex(table, RANKED_DOCUMENTS.length);
+ buildAndCommitBTreeIndex(table, RANKED_DOCUMENTS);
+
+ FullTextSearchBuilder noFilter =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon"))
+ .withLimit(10);
+ for (IndexFullTextSearchSplit split : indexSplits(noFilter.newFullTextScan().scan())) {
+ assertThat(split.scalarIndexFiles()).isEmpty();
+ }
+
+ // A filter on "id" attaches the id btree, not the btree built on the text column.
+ Predicate idFilter = new PredicateBuilder(table.rowType()).greaterOrEqual(0, 3);
+ FullTextScan.Plan plan =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon"))
+ .withLimit(10)
+ .withFilter(idFilter)
+ .newFullTextScan()
+ .scan();
+ List<IndexFullTextSearchSplit> indexSplits = indexSplits(plan);
+ assertThat(indexSplits).hasSize(1);
+ assertThat(indexSplits.get(0).scalarIndexFiles()).hasSize(1);
+ assertThat(indexSplits.get(0).scalarIndexFiles().get(0).globalIndexMeta().indexFieldId())
+ .isEqualTo(table.rowType().getField("id").id());
+ assertThat(indexSplits.get(0).fullTextIndexFiles())
+ .allMatch(f -> f.indexType().equals(TestFullTextGlobalIndexerFactory.IDENTIFIER));
+ }
+
+ @Test
+ public void testFullTextSearchFilterOnTextColumnBTreeIsNotMistakenForFullText()
+ throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndex(table, RANKED_DOCUMENTS);
+ buildAndCommitBTreeIndex(table, RANKED_DOCUMENTS);
+
+ // Equality on the text column is evaluated by its btree, the full-text query by the
+ // full-text index; both are on the same column.
+ Predicate exact =
+ new PredicateBuilder(table.rowType())
+ .equal(1, BinaryString.fromString("paimon lake beta"));
+ assertThat(searchWithFilter(table, exact, 10).results()).containsExactly(1L);
+ }
+
+ @Test
+ public void testFullTextSearchAccumulatesFilters() throws Exception {
createTableDefault();
FileStoreTable table = getTableDefault();
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndex(table, RANKED_DOCUMENTS);
+ buildAndCommitIdBTreeIndex(table, RANKED_DOCUMENTS.length);
+
+ PredicateBuilder builder = new PredicateBuilder(table.rowType());
+ // Two withFilter calls are AND-ed, exactly like a single conjunction.
+ GlobalIndexResult result =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon"))
+ .withLimit(10)
+ .withFilter(builder.greaterOrEqual(0, 2))
+ .withFilter(builder.lessOrEqual(0, 4))
+ .executeLocal();
+ assertThat(result.results()).containsExactlyInAnyOrder(2L, 3L, 4L);
+ }
+
+ @Test
+ public void testFullTextSearchRawPathAppliesFilterWithoutShrinkingCorpus() throws Exception {
+ // full-text index covers rows 0-2 only, full-text mode full: rows 3-5 are searched raw.
+ // The temporary index is built over all raw rows and the filter is applied as a bitmap.
+ Identifier identifier = identifier("full_text_raw_prefilter");
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column(TEXT_FIELD_NAME, DataTypes.STRING())
+ .option(CoreOptions.BUCKET.key(), "-1")
+ .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")
+ .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true")
+ .option(CoreOptions.FULL_TEXT_INDEX_SEARCH_MODE.key(), "full")
+ .build();
+ catalog.createTable(identifier, schema, false);
+ FileStoreTable table = getTable(identifier);
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndexRange(
+ table,
+ Arrays.copyOfRange(RANKED_DOCUMENTS, 0, 3),
+ Collections.singletonList(table.rowType().getField(TEXT_FIELD_NAME)),
+ 0);
+ buildAndCommitIdBTreeIndex(table, RANKED_DOCUMENTS.length);
- Predicate idFilter = new PredicateBuilder(table.rowType()).equal(0, 1);
+ Predicate idFilter = new PredicateBuilder(table.rowType()).greaterOrEqual(0, 4);
+ FullTextSearchBuilder builder =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon"))
+ .withLimit(10)
+ .withFilter(idFilter);
+ FullTextScan.Plan plan = builder.newFullTextScan().scan();
+ List<RawFullTextSearchSplit> rawSplits = rawSplits(plan);
+ assertThat(rawSplits).hasSize(1);
+ assertThat(rawSplits.get(0).rowRanges()).containsExactly(new Range(3, 5));
+
+ assertThat(builder.newFullTextRead().read(plan).results())
+ .containsExactlyInAnyOrder(4L, 5L);
+
+ // A filter matching no raw row leaves the raw side empty without failing.
+ Predicate none = new PredicateBuilder(table.rowType()).greaterOrEqual(0, 100);
+ assertThat(searchWithFilter(table, none, 10).results().isEmpty()).isTrue();
+ }
- assertThatThrownBy(
- () ->
- table.newHybridSearchBuilder()
- .addFullTextRoute(
- TEXT_FIELD_NAME, matchQuery("Paimon"), 3, 1.0f)
- .withFilter(idFilter)
- .withLimit(3)
- .routeBuilders())
+ @Test
+ public void testFullTextSearchBuilderWithFilterIsOptionalForImplementations() {
+ FullTextSearchBuilder minimal =
+ new FullTextSearchBuilder() {
+ @Override
+ public FullTextSearchBuilder withLimit(int limit) {
+ return this;
+ }
+
+ @Override
+ public FullTextSearchBuilder withQuery(String fieldName, String query) {
+ return this;
+ }
+
+ @Override
+ public FullTextScan newFullTextScan() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FullTextRead newFullTextRead() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ assertThatThrownBy(() -> minimal.withFilter(PredicateBuilder.alwaysTrue()))
.isInstanceOf(UnsupportedOperationException.class)
- .hasMessageContaining("does not support non-partition filters");
+ .hasMessageContaining("does not support row filters");
+ }
+
+ @ParameterizedTest(name = "scalar-index.search-mode={0}")
+ @ValueSource(strings = {"fast", "full"})
+ public void testFullTextSearchRefinesCandidateOnlyIndexResultsBeforeTopK(String scalarMode)
+ throws Exception {
+ // BTree answers contains / endsWith / like with every non-null row (a candidate superset).
+ // Ranking that superset lets a non-matching, higher-scoring row take the single slot and
+ // the matching row is lost before any engine-side filter can run.
+ FileStoreTable table = createTable("full_text_candidate_only_" + scalarMode, scalarMode);
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndex(table, RANKED_DOCUMENTS);
+
+ PredicateBuilder builder = new PredicateBuilder(table.rowType());
+ Predicate containsZeta = builder.contains(1, BinaryString.fromString("zeta"));
+ Predicate endsWithZeta = builder.endsWith(1, BinaryString.fromString("zeta"));
+ Predicate likeZeta = builder.like(1, BinaryString.fromString("%zeta"));
+
+ // Without a scalar index, full mode evaluates the predicate on the data: exact.
+ if (scalarMode.equals("full")) {
+ assertThat(searchWithFilter(table, containsZeta, 1).results()).containsExactly(5L);
+ }
+
+ buildAndCommitBTreeIndex(table, RANKED_DOCUMENTS);
+ assertThat(searchWithFilter(table, containsZeta, 1).results()).containsExactly(5L);
+ assertThat(searchWithFilter(table, endsWithZeta, 1).results()).containsExactly(5L);
+ assertThat(searchWithFilter(table, likeZeta, 1).results()).containsExactly(5L);
+ assertThat(searchWithFilter(table, containsZeta, 10).results()).containsExactly(5L);
+
+ // Exact operators on the same btree stay exact.
+ assertThat(
+ searchWithFilter(
+ table,
+ builder.equal(1, BinaryString.fromString("paimon zeta")),
+ 1)
+ .results())
+ .containsExactly(5L);
+ }
+
+ @ParameterizedTest(name = "scalar-index.search-mode={0}")
+ @ValueSource(strings = {"fast", "full"})
+ public void testFullTextSearchPartiallyIndexedConjunctionIsExactBeforeTopK(String scalarMode)
+ throws Exception {
+ // Only `id` is indexed. The evaluator drops the conjunct it cannot evaluate, which makes
+ // the index result a superset (every row with id >= 0); ranking that superset lets row 0
+ // take the single slot although only row 5 satisfies the whole predicate.
+ FileStoreTable table = createTable("full_text_partial_and_" + scalarMode, scalarMode);
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndex(table, RANKED_DOCUMENTS);
+ buildAndCommitIdBTreeIndex(table, RANKED_DOCUMENTS.length);
+
+ PredicateBuilder builder = new PredicateBuilder(table.rowType());
+ Predicate partiallyIndexed =
+ PredicateBuilder.and(
+ builder.greaterOrEqual(0, 0),
+ builder.equal(1, BinaryString.fromString("paimon zeta")));
+ assertThat(searchWithFilter(table, partiallyIndexed, 1).results()).containsExactly(5L);
+ assertThat(searchWithFilter(table, partiallyIndexed, 10).results()).containsExactly(5L);
+ }
+
+ private GlobalIndexResult searchWithFilter(
+ FileStoreTable table, @Nullable Predicate filter, int limit) {
+ FullTextSearchBuilder builder =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon lake"))
+ .withLimit(limit);
+ if (filter != null) {
+ builder.withFilter(filter);
+ }
+ return builder.executeLocal();
+ }
+
+ private FileStoreTable createTable(String name, String scalarSearchMode) throws Exception {
+ Identifier identifier = identifier(name);
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column(TEXT_FIELD_NAME, DataTypes.STRING())
+ .option(CoreOptions.BUCKET.key(), "-1")
+ .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")
+ .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true")
+ .option(CoreOptions.SCALAR_INDEX_SEARCH_MODE.key(), scalarSearchMode)
+ .build();
+ catalog.createTable(identifier, schema, false);
+ return getTable(identifier);
+ }
+
+ private static List<IndexFullTextSearchSplit> indexSplits(FullTextScan.Plan plan) {
+ List<IndexFullTextSearchSplit> splits = new ArrayList<>();
+ for (FullTextSearchSplit split : plan.splits()) {
+ if (split instanceof IndexFullTextSearchSplit) {
+ splits.add((IndexFullTextSearchSplit) split);
+ }
+ }
+ return splits;
+ }
+
+ private static List<RawFullTextSearchSplit> rawSplits(FullTextScan.Plan plan) {
+ List<RawFullTextSearchSplit> splits = new ArrayList<>();
+ for (FullTextSearchSplit split : plan.splits()) {
+ if (split instanceof RawFullTextSearchSplit) {
+ splits.add((RawFullTextSearchSplit) split);
+ }
+ }
+ return splits;
}
@Test
@@ -848,11 +1384,13 @@ public class FullTextSearchBuilderTest extends TableTestBase {
String[] documents = {"Apache Paimon", "full-text search"};
writeDocuments(table, documents);
buildAndCommitIndex(table, documents);
+ buildAndCommitIdBTreeIndex(table, documents.length);
FullTextScan.Plan plan =
table.newFullTextSearchBuilder()
.withQuery(TEXT_FIELD_NAME, matchQuery("Paimon"))
.withLimit(2)
+ .withFilter(new PredicateBuilder(table.rowType()).greaterOrEqual(0, 0))
.newFullTextScan()
.scan();
@@ -879,6 +1417,9 @@ public class FullTextSearchBuilderTest extends TableTestBase {
assertThat(deserialized.fullTextIndexFiles().get(i).fileName())
.isEqualTo(original.fullTextIndexFiles().get(i).fileName());
}
+ assertThat(original.scalarIndexFiles()).hasSize(1);
+ assertThat(deserialized.scalarIndexFiles()).isEqualTo(original.scalarIndexFiles());
+ assertThat(deserialized).isEqualTo(original);
RawFullTextSearchSplit rawOriginal =
new RawFullTextSearchSplit(Collections.singletonList(new Range(2, 3)));
@@ -894,6 +1435,7 @@ public class FullTextSearchBuilderTest extends TableTestBase {
}
assertThat(rawDeserialized.rowRanges()).isEqualTo(rawOriginal.rowRanges());
+ assertThat(rawDeserialized).isEqualTo(rawOriginal);
}
// ====================== Helper methods ======================
@@ -1152,6 +1694,16 @@ public class FullTextSearchBuilderTest extends TableTestBase {
private void buildAndCommitIndexForColumn(
FileStoreTable table, String columnName, String[] documents, BinaryRow partition)
throws Exception {
+ buildAndCommitIndexForColumn(table, columnName, documents, partition, 0);
+ }
+
+ private void buildAndCommitIndexForColumn(
+ FileStoreTable table,
+ String columnName,
+ String[] documents,
+ BinaryRow partition,
+ long rowStart)
+ throws Exception {
Options options = table.coreOptions().toConfiguration();
DataField textField = table.rowType().getField(columnName);
@@ -1167,7 +1719,7 @@ public class FullTextSearchBuilderTest extends TableTestBase {
}
List<ResultEntry> entries = writer.finish();
- Range rowRange = new Range(0, documents.length - 1);
+ Range rowRange = new Range(rowStart, rowStart + documents.length - 1);
List<IndexFileMeta> indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
@@ -1187,6 +1739,47 @@ public class FullTextSearchBuilderTest extends TableTestBase {
}
}
+ private void buildAndCommitIdBTreeIndex(FileStoreTable table, int rowCount) throws Exception {
+ buildAndCommitIdBTreeIndexRange(table, new Range(0, rowCount - 1));
+ }
+
+ /** Builds a btree index on {@code id} for rows in {@code rowRange}; ids equal row ids. */
+ private void buildAndCommitIdBTreeIndexRange(FileStoreTable table, Range rowRange)
+ throws Exception {
+ Options options = table.coreOptions().toConfiguration();
+ DataField idField = table.rowType().getField("id");
+
+ GlobalIndexSingleColumnWriter writer =
+ (GlobalIndexSingleColumnWriter)
+ GlobalIndexBuilderUtils.createIndexWriter(
+ table, BTreeGlobalIndexerFactory.IDENTIFIER, idField, options);
+ for (long rowId = rowRange.from; rowId <= rowRange.to; rowId++) {
+ writer.write((int) rowId, rowId - rowRange.from);
+ }
+ List<ResultEntry> entries = writer.finish();
+
+ List<IndexFileMeta> indexFiles =
+ GlobalIndexBuilderUtils.toIndexFileMetas(
+ table.fileIO(),
+ table.store().pathFactory().globalIndexFileFactory(),
+ table.coreOptions(),
+ rowRange,
+ idField.id(),
+ BTreeGlobalIndexerFactory.IDENTIFIER,
+ entries);
+
+ CommitMessage message =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ null,
+ DataIncrement.indexIncrement(indexFiles),
+ CompactIncrement.emptyIncrement());
+ try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
+ commit.commit(Collections.singletonList(message));
+ }
+ }
+
private void buildAndCommitBTreeIndex(FileStoreTable table, String[] documents)
throws Exception {
Options options = table.coreOptions().toConfiguration();
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextSearchTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextSearchTest.java
index e3faa6a38d..400a609b38 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextSearchTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextSearchTest.java
@@ -19,6 +19,8 @@
package org.apache.paimon.table.source;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.types.DataField;
@@ -32,6 +34,7 @@ import java.util.HashMap;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -77,6 +80,35 @@ class PrimaryKeyFullTextSearchTest {
assertThat(scan).isInstanceOf(DataEvolutionFullTextScan.class);
}
+ @Test
+ void testPrimaryKeyFullTextRejectsRowFilter() {
+ FileStoreTable table = table(false);
+ when(table.partitionKeys()).thenReturn(Collections.emptyList());
+ Predicate rowFilter = new PredicateBuilder(table.rowType()).equal(0, 1);
+
+ FullTextSearchBuilder builder =
+ new FullTextSearchBuilderImpl(table)
+ .withQuery("content", "hello")
+ .withLimit(10)
+ .withFilter(rowFilter);
+
+ assertThatThrownBy(builder::newFullTextScan)
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessageContaining("Primary-key full-text search does not support");
+ assertThatThrownBy(builder::newFullTextRead)
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessageContaining("Primary-key full-text search does not support");
+
+ // The same filter on a column served by the global full-text path is accepted.
+ assertThat(
+ new FullTextSearchBuilderImpl(table)
+ .withQuery("other", "hello")
+ .withLimit(10)
+ .withFilter(rowFilter)
+ .newFullTextScan())
+ .isInstanceOf(DataEvolutionFullTextScan.class);
+ }
+
@Test
void testHybridRouteUsesPrimaryKeyFullText() {
FileStoreTable table = table(false);
diff --git a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextRowFilterTest.java b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextRowFilterTest.java
new file mode 100644
index 0000000000..e2898bdaee
--- /dev/null
+++ b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextRowFilterTest.java
@@ -0,0 +1,517 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fulltext.index;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.fs.FileIOFinder;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.globalindex.GlobalIndexBuilderUtils;
+import org.apache.paimon.globalindex.GlobalIndexResult;
+import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter;
+import org.apache.paimon.globalindex.ResultEntry;
+import org.apache.paimon.globalindex.ScoredGlobalIndexResult;
+import org.apache.paimon.globalindex.btree.BTreeGlobalIndexerFactory;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.io.CompactIncrement;
+import org.apache.paimon.io.DataIncrement;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateBuilder;
+import org.apache.paimon.schema.FileSystemSchemaManager;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.SchemaUtils;
+import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.AppendOnlyFileStoreTable;
+import org.apache.paimon.table.CatalogEnvironment;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Range;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.function.Supplier;
+
+import static org.apache.paimon.CoreOptions.DATA_EVOLUTION_ENABLED;
+import static org.apache.paimon.CoreOptions.GLOBAL_INDEX_ENABLED;
+import static org.apache.paimon.CoreOptions.PATH;
+import static org.apache.paimon.CoreOptions.ROW_TRACKING_ENABLED;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Row filters on full-text search against the native engine: correctness of filter-then-rank, and a
+ * manual benchmark comparing the index pre-filter with the over-fetch workaround.
+ */
+public class NativeFullTextRowFilterTest {
+
+ private static final String[] WORDS = {
+ "paimon", "lake", "stream", "batch", "index", "vector", "search", "table", "commit",
+ "snapshot", "manifest", "bucket", "partition", "schema", "column", "row", "query", "scan",
+ "read", "write"
+ };
+
+ @TempDir java.nio.file.Path tempDir;
+
+ @Test
+ public void testFilterThenRankMatchesFilteredUnfilteredRanking() throws Exception {
+ int rowCount = 2_000;
+ int categories = 8;
+ Dataset dataset = writeDataset("filter_rank", rowCount, categories, 42);
+ FileStoreTable table = dataset.table;
+ PredicateBuilder builder = new PredicateBuilder(table.rowType());
+
+ String query = matchQuery("paimon lake");
+ int limit = 20;
+
+ // Reference: rank every row without a filter, then keep the wanted category. BM25 scores
+ // do not depend on the filter, so filter-then-rank must equal rank-then-filter.
+ ScoredGlobalIndexResult all =
+ (ScoredGlobalIndexResult)
+ table.newFullTextSearchBuilder()
+ .withQuery("content", query)
+ .withLimit(rowCount)
+ .executeLocal();
+ for (int category = 0; category < categories; category++) {
+ List<Long> expected = new ArrayList<>();
+ for (long rowId : all.results()) {
+ if (dataset.categoryOf.get(rowId) == category) {
+ expected.add(rowId);
+ }
+ }
+ expected.sort(
+ (a, b) -> {
+ int byScore =
+ Float.compare(
+ all.scoreGetter().score(b), all.scoreGetter().score(a));
+ return byScore != 0 ? byScore : Long.compare(a, b);
+ });
+ List<Long> expectedTopK = expected.subList(0, Math.min(limit, expected.size()));
+ float cutoff =
+ expectedTopK.isEmpty()
+ ? 0f
+ : all.scoreGetter().score(expectedTopK.get(expectedTopK.size() - 1));
+
+ ScoredGlobalIndexResult filtered =
+ (ScoredGlobalIndexResult)
+ table.newFullTextSearchBuilder()
+ .withQuery("content", query)
+ .withLimit(limit)
+ .withFilter(builder.equal(1, category))
+ .executeLocal();
+
+ assertThat(filtered.results().getLongCardinality()).isEqualTo(expectedTopK.size());
+ for (long rowId : filtered.results()) {
+ assertThat(dataset.categoryOf.get(rowId)).isEqualTo(category);
+ assertThat(all.results().contains(rowId)).isTrue();
+ // Every returned row scores at least as high as the k-th expected row; ties at the
+ // cutoff may be broken differently by the engine.
+ assertThat(filtered.scoreGetter().score(rowId)).isGreaterThanOrEqualTo(cutoff);
+ assertThat(filtered.scoreGetter().score(rowId))
+ .isEqualTo(all.scoreGetter().score(rowId));
+ }
+ }
+
+ // A filter that matches nothing returns nothing.
+ GlobalIndexResult none =
+ table.newFullTextSearchBuilder()
+ .withQuery("content", query)
+ .withLimit(limit)
+ .withFilter(builder.equal(1, categories + 1))
+ .executeLocal();
+ assertThat(none.results().isEmpty()).isTrue();
+ }
+
+ @Test
+ public void testUnindexedFilterInFullModeKeepsOriginalRanking() throws Exception {
+ // `id` has no scalar index. In scalar-index.search-mode=full the filter must be resolved
+ // against the existing full-text index, not by rebuilding an index over the filtered
+ // rows: a smaller corpus changes BM25 document frequencies and average length, which
+ // changes which rows make the top-k, not just their reported scores.
+ int rowCount = 2_000;
+ Dataset dataset = writeDataset("unindexed_full_mode", rowCount, 8, 42);
+ FileStoreTable table =
+ (FileStoreTable)
+ dataset.table.copy(
+ Collections.singletonMap(
+ CoreOptions.SCALAR_INDEX_SEARCH_MODE.key(), "full"));
+ PredicateBuilder builder = new PredicateBuilder(table.rowType());
+ String query = matchQuery("paimon lake");
+ int limit = 20;
+ int idFrom = 1_800;
+
+ ScoredGlobalIndexResult all =
+ (ScoredGlobalIndexResult)
+ table.newFullTextSearchBuilder()
+ .withQuery("content", query)
+ .withLimit(rowCount)
+ .executeLocal();
+ List<Long> expected = new ArrayList<>();
+ for (long rowId : all.results()) {
+ if (rowId >= idFrom) {
+ expected.add(rowId);
+ }
+ }
+ expected.sort(
+ (a, b) -> {
+ int byScore =
+ Float.compare(all.scoreGetter().score(b), all.scoreGetter().score(a));
+ return byScore != 0 ? byScore : Long.compare(a, b);
+ });
+ List<Long> expectedTopK = expected.subList(0, Math.min(limit, expected.size()));
+ float cutoff = all.scoreGetter().score(expectedTopK.get(expectedTopK.size() - 1));
+
+ ScoredGlobalIndexResult filtered =
+ (ScoredGlobalIndexResult)
+ table.newFullTextSearchBuilder()
+ .withQuery("content", query)
+ .withLimit(limit)
+ .withFilter(builder.greaterOrEqual(0, idFrom))
+ .executeLocal();
+
+ List<Long> actual = new ArrayList<>();
+ filtered.results().forEach(actual::add);
+ // Membership first: every returned row must reach the cutoff of the original ranking.
+ for (long rowId : actual) {
+ assertThat(rowId).isGreaterThanOrEqualTo(idFrom);
+ assertThat(all.scoreGetter().score(rowId))
+ .as(
+ "row %s (original score %s) is below the filtered top-%s cutoff %s",
+ rowId, all.scoreGetter().score(rowId), limit, cutoff)
+ .isGreaterThanOrEqualTo(cutoff);
+ }
+ assertThat(actual).hasSize(expectedTopK.size());
+ for (long rowId : actual) {
+ assertThat(filtered.scoreGetter().score(rowId))
+ .as("row %s must keep its score from the original index", rowId)
+ .isEqualTo(all.scoreGetter().score(rowId));
+ }
+ }
+
+ @Test
+ public void testRawPathAppliesFilterOverTheWholeRawCorpus() throws Exception {
+ // Rows 0-999 are covered by the full-text index, rows 1000-1999 are not. In full-text
+ // mode full the uncovered rows are searched through a temporary index; a filter must not
+ // shrink that temporary corpus, so scores of the returned raw rows equal the scores of an
+ // unfiltered raw search.
+ int rowCount = 2_000;
+ int indexedRows = 1_000;
+ Dataset dataset = writeDataset("raw_corpus", rowCount, 8, 42, indexedRows);
+ Map<String, String> options = new HashMap<>();
+ options.put(CoreOptions.FULL_TEXT_INDEX_SEARCH_MODE.key(), "full");
+ options.put(CoreOptions.SCALAR_INDEX_SEARCH_MODE.key(), "full");
+ FileStoreTable table = (FileStoreTable) dataset.table.copy(options);
+ PredicateBuilder builder = new PredicateBuilder(table.rowType());
+ String query = matchQuery("paimon lake");
+ int limit = 20;
+ int idFrom = 1_800;
+
+ ScoredGlobalIndexResult unfiltered =
+ (ScoredGlobalIndexResult)
+ table.newFullTextSearchBuilder()
+ .withQuery("content", query)
+ .withLimit(rowCount)
+ .executeLocal();
+ ScoredGlobalIndexResult filtered =
+ (ScoredGlobalIndexResult)
+ table.newFullTextSearchBuilder()
+ .withQuery("content", query)
+ .withLimit(limit)
+ .withFilter(builder.greaterOrEqual(0, idFrom))
+ .executeLocal();
+
+ assertThat(filtered.results().getLongCardinality()).isEqualTo(limit);
+ for (long rowId : filtered.results()) {
+ assertThat(rowId).isGreaterThanOrEqualTo(idFrom);
+ assertThat(unfiltered.results().contains(rowId)).isTrue();
+ assertThat(filtered.scoreGetter().score(rowId))
+ .as("row %s must keep the score of the unfiltered raw corpus", rowId)
+ .isEqualTo(unfiltered.scoreGetter().score(rowId));
+ }
+ }
+
+ /**
+ * Run with {@code mvn -pl paimon-full-text test -Dtest=NativeFullTextRowFilterTest
+ * -DextraJavaTestArgs=-Dpaimon.benchmark=true}. Prints one line per strategy.
+ */
+ @Test
+ @EnabledIfSystemProperty(named = "paimon.benchmark", matches = "true")
+ public void benchmarkRowFilterStrategies() throws Exception {
+ int rowCount = 200_000;
+ int categories = 100; // 1% selectivity per category
+ Dataset dataset = writeDataset("benchmark", rowCount, categories, 7);
+ FileStoreTable table = dataset.table;
+ FileStoreTable fullModeTable =
+ (FileStoreTable)
+ table.copy(
+ Collections.singletonMap(
+ CoreOptions.SCALAR_INDEX_SEARCH_MODE.key(), "full"));
+ PredicateBuilder builder = new PredicateBuilder(table.rowType());
+ String query = matchQuery("paimon lake");
+ int limit = 10;
+ int category = 17;
+
+ System.out.printf(
+ "%nfull-text row filter benchmark: rows=%d, categories=%d, limit=%d%n",
+ rowCount, categories, limit);
+ long baseline =
+ time(
+ "no filter (baseline)",
+ () ->
+ table.newFullTextSearchBuilder()
+ .withQuery("content", query)
+ .withLimit(limit)
+ .executeLocal());
+ long preFilter =
+ time(
+ "withFilter via btree index (1% selective)",
+ () ->
+ table.newFullTextSearchBuilder()
+ .withQuery("content", query)
+ .withLimit(limit)
+ .withFilter(builder.equal(1, category))
+ .executeLocal());
+ time(
+ "withFilter via btree index (dense, ~99% selective)",
+ () ->
+ table.newFullTextSearchBuilder()
+ .withQuery("content", query)
+ .withLimit(limit)
+ .withFilter(builder.notEqual(1, category))
+ .executeLocal());
+ time(
+ "over-fetch limit*100 then client-side filter (old workaround)",
+ () -> {
+ ScoredGlobalIndexResult result =
+ (ScoredGlobalIndexResult)
+ table.newFullTextSearchBuilder()
+ .withQuery("content", query)
+ .withLimit(limit * 100)
+ .executeLocal();
+ List<Long> kept = new ArrayList<>();
+ for (long rowId : result.results()) {
+ if (dataset.categoryOf.get(rowId) == category) {
+ kept.add(rowId);
+ }
+ }
+ return kept;
+ });
+ Predicate unindexedFilter = builder.equal(2, 3);
+ time(
+ "withFilter on unindexed column, scalar mode full (raw scan)",
+ () ->
+ fullModeTable
+ .newFullTextSearchBuilder()
+ .withQuery("content", query)
+ .withLimit(limit)
+ .withFilter(unindexedFilter)
+ .executeLocal());
+ System.out.printf(
+ "pre-filter overhead over baseline: %.1fx%n", (double) preFilter / baseline);
+ }
+
+ private static long time(String name, Supplier<Object> action) {
+ for (int i = 0; i < 3; i++) {
+ action.get();
+ }
+ int iterations = 10;
+ long best = Long.MAX_VALUE;
+ long total = 0;
+ for (int i = 0; i < iterations; i++) {
+ long start = System.nanoTime();
+ action.get();
+ long elapsed = System.nanoTime() - start;
+ best = Math.min(best, elapsed);
+ total += elapsed;
+ }
+ System.out.printf(
+ " %-64s best %7.1f ms avg %7.1f ms%n",
+ name, best / 1_000_000.0, total / iterations / 1_000_000.0);
+ return best;
+ }
+
+ private static final class Dataset {
+ final FileStoreTable table;
+ final Map<Long, Integer> categoryOf;
+
+ Dataset(FileStoreTable table, Map<Long, Integer> categoryOf) {
+ this.table = table;
+ this.categoryOf = categoryOf;
+ }
+ }
+
+ /**
+ * Table (id INT, category INT, other INT, content STRING) with a native full-text index on
+ * {@code content} and a btree index on {@code category}; {@code other} stays unindexed.
+ */
+ private Dataset writeDataset(String tableName, int rowCount, int categories, long seed)
+ throws Exception {
+ return writeDataset(tableName, rowCount, categories, seed, rowCount);
+ }
+
+ private Dataset writeDataset(
+ String tableName, int rowCount, int categories, long seed, int fullTextIndexedRows)
+ throws Exception {
+ Path tablePath = new Path(tempDir.resolve(tableName).toUri());
+ LocalFileIO fileIO = LocalFileIO.create();
+
+ RowType rowType =
+ RowType.of(
+ new DataType[] {
+ DataTypes.INT(), DataTypes.INT(), DataTypes.INT(), DataTypes.STRING()
+ },
+ new String[] {"id", "category", "other", "content"});
+ Options options = new Options();
+ options.set(PATH, tablePath.toString());
+ options.set(ROW_TRACKING_ENABLED, true);
+ options.set(DATA_EVOLUTION_ENABLED, true);
+ options.set(GLOBAL_INDEX_ENABLED, true);
+ TableSchema tableSchema =
+ SchemaUtils.forceCommit(
+ new FileSystemSchemaManager(fileIO, tablePath),
+ new Schema(
+ rowType.getFields(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ options.toMap(),
+ ""));
+ FileStoreTable table =
+ new AppendOnlyFileStoreTable(
+ FileIOFinder.find(tablePath),
+ tablePath,
+ tableSchema,
+ CatalogEnvironment.empty());
+
+ Random random = new Random(seed);
+ Map<Long, Integer> categoryOf = new HashMap<>();
+ List<String> contents = new ArrayList<>(rowCount);
+ List<Integer> categoryValues = new ArrayList<>(rowCount);
+ BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder();
+ try (BatchTableWrite write = writeBuilder.newWrite();
+ BatchTableCommit commit = writeBuilder.newCommit()) {
+ for (int i = 0; i < rowCount; i++) {
+ int category = random.nextInt(categories);
+ StringBuilder content = new StringBuilder();
+ int words = 3 + random.nextInt(8);
+ for (int w = 0; w < words; w++) {
+ content.append(WORDS[random.nextInt(WORDS.length)]).append(' ');
+ }
+ contents.add(content.toString().trim());
+ categoryValues.add(category);
+ categoryOf.put((long) i, category);
+ write.write(
+ GenericRow.of(
+ i,
+ category,
+ random.nextInt(10),
+ BinaryString.fromString(contents.get(i))));
+ }
+ commit.commit(write.prepareCommit());
+ }
+
+ Range rowRange = new Range(0, rowCount - 1);
+ DataField contentField = table.rowType().getField("content");
+ GlobalIndexSingleColumnWriter textWriter =
+ (GlobalIndexSingleColumnWriter)
+ GlobalIndexBuilderUtils.createIndexWriter(
+ table,
+ NativeFullTextGlobalIndexerFactory.IDENTIFIER,
+ contentField,
+ table.coreOptions().toConfiguration());
+ for (int i = 0; i < fullTextIndexedRows; i++) {
+ textWriter.write(BinaryString.fromString(contents.get(i)), i);
+ }
+ List<IndexFileMeta> indexFiles =
+ new ArrayList<>(
+ GlobalIndexBuilderUtils.toIndexFileMetas(
+ table.fileIO(),
+ table.store().pathFactory().globalIndexFileFactory(),
+ table.coreOptions(),
+ new Range(0, fullTextIndexedRows - 1),
+ contentField.id(),
+ NativeFullTextGlobalIndexerFactory.IDENTIFIER,
+ textWriter.finish()));
+
+ DataField categoryField = table.rowType().getField("category");
+ GlobalIndexSingleColumnWriter categoryWriter =
+ (GlobalIndexSingleColumnWriter)
+ GlobalIndexBuilderUtils.createIndexWriter(
+ table,
+ BTreeGlobalIndexerFactory.IDENTIFIER,
+ categoryField,
+ table.coreOptions().toConfiguration());
+ // The btree writer is an SST writer: keys must arrive in sorted order.
+ List<Integer> rowIdsByCategory = new ArrayList<>(rowCount);
+ for (int i = 0; i < rowCount; i++) {
+ rowIdsByCategory.add(i);
+ }
+ rowIdsByCategory.sort(
+ (a, b) -> {
+ int byCategory = Integer.compare(categoryValues.get(a), categoryValues.get(b));
+ return byCategory != 0 ? byCategory : Integer.compare(a, b);
+ });
+ for (int rowId : rowIdsByCategory) {
+ categoryWriter.write(categoryValues.get(rowId), rowId);
+ }
+ List<ResultEntry> categoryEntries = categoryWriter.finish();
+ indexFiles.addAll(
+ GlobalIndexBuilderUtils.toIndexFileMetas(
+ table.fileIO(),
+ table.store().pathFactory().globalIndexFileFactory(),
+ table.coreOptions(),
+ rowRange,
+ categoryField.id(),
+ BTreeGlobalIndexerFactory.IDENTIFIER,
+ categoryEntries));
+
+ CommitMessage message =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ null,
+ DataIncrement.indexIncrement(indexFiles),
+ CompactIncrement.emptyIncrement());
+ try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
+ commit.commit(Collections.singletonList(message));
+ }
+ return new Dataset(table, categoryOf);
+ }
+
+ private static String matchQuery(String terms) {
+ return "{\"match\":{\"query\":\"" + terms + "\"}}";
+ }
+}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala
index dc71a9cfbf..ed3452d2c3 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala
@@ -131,9 +131,7 @@ abstract class PaimonBaseScan(table: InnerTable)
ftBuilder.withPartitionFilter(PartitionPredicate.and(pushedPartitionFilters.asJava))
}
if (pushedDataFilters.nonEmpty) {
- throw new UnsupportedOperationException(
- "Full-text search does not support non-partition filters because full-text indexes " +
- "cannot apply row-id pre-filters before top-k ranking.")
+ ftBuilder.withFilter(PredicateBuilder.and(pushedDataFilters.asJava))
}
ftBuilder.newFullTextRead().read(ftBuilder.newFullTextScan().scan())
}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/FullTextSearchTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/FullTextSearchTest.scala
index 4c131e1f03..dfdd820877 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/FullTextSearchTest.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/FullTextSearchTest.scala
@@ -325,4 +325,232 @@ class FullTextSearchTest extends PaimonSparkTestBase {
assert(searchResult.length == 10)
}
}
+
+ // ========== Row filter tests ==========
+
+ private def createRankedTable(extraProps: String = ""): Unit = {
+ spark.sql(s"""
+ |CREATE TABLE T (id INT, category STRING, content STRING)
+ |TBLPROPERTIES (
+ | 'bucket' = '-1',
+ | 'global-index.row-count-per-shard' = '10000',
+ | 'row-tracking.enabled' = 'true',
+ | 'data-evolution.enabled' = 'true'$extraProps)
+ |""".stripMargin)
+ // Rows 0-2 match both terms of "paimon lake" (score 1.0); rows 3-5 only "paimon" (0.5).
+ spark.sql("""
+ |INSERT INTO T VALUES
+ | (0, 'a', 'paimon lake alpha'),
+ | (1, 'a', 'paimon lake beta'),
+ | (2, 'b', 'paimon lake gamma'),
+ | (3, 'b', 'paimon delta'),
+ | (4, 'c', 'paimon epsilon'),
+ | (5, 'c', 'paimon zeta')
+ |""".stripMargin)
+ spark
+ .sql(
+ s"CALL sys.create_global_index(table => 'test.T', index_column => 'content', index_type => '$indexType')")
+ .collect()
+ }
+
+ private val rankedQuery = """{"match":{"column":"content","terms":"paimon lake"}}"""
+
+ test("full-text search - WHERE filter is applied before top-k") {
+ withTable("T") {
+ createRankedTable()
+ spark
+ .sql("CALL sys.create_global_index(table => 'test.T', index_column => 'id', index_type => 'btree')")
+ .collect()
+
+ val unfiltered = spark
+ .sql(s"SELECT id FROM full_text_search('T', 'content', '$rankedQuery', 2)")
+ .collect()
+ .map(_.getInt(0))
+ .toSet
+ assert(unfiltered.subsetOf(Set(0, 1, 2)))
+
+ val filtered = spark
+ .sql(s"""
+ |SELECT id, __paimon_search_score
+ |FROM full_text_search('T', 'content', '$rankedQuery', 2)
+ |WHERE id >= 3
+ |""".stripMargin)
+ .collect()
+ assert(filtered.length == 2)
+ assert(filtered.map(_.getInt(0)).toSet.subsetOf(Set(3, 4, 5)))
+ assert(filtered.forall(_.getFloat(1) == 0.5f))
+
+ // Equivalent to ranking the filtered subset without the index.
+ val expected = spark
+ .sql("SELECT id FROM T WHERE id >= 3 AND content LIKE '%paimon%' ORDER BY id LIMIT 3")
+ .collect()
+ .map(_.getInt(0))
+ .toSet
+ assert(filtered.map(_.getInt(0)).toSet.subsetOf(expected))
+ }
+ }
+
+ test("full-text search - WHERE filter on string column with bitmap index") {
+ withTable("T") {
+ createRankedTable()
+ spark
+ .sql("CALL sys.create_global_index(table => 'test.T', index_column => 'category', index_type => 'bitmap')")
+ .collect()
+
+ val result = spark
+ .sql(s"""
+ |SELECT id
+ |FROM full_text_search('T', 'content', '$rankedQuery', 10)
+ |WHERE category IN ('b', 'c')
+ |ORDER BY id
+ |""".stripMargin)
+ .collect()
+ .map(_.getInt(0))
+ .toSeq
+ assert(result == Seq(2, 3, 4, 5))
+
+ val none = spark
+ .sql(s"""
+ |SELECT id
+ |FROM full_text_search('T', 'content', '$rankedQuery', 10)
+ |WHERE category = 'z'
+ |""".stripMargin)
+ .collect()
+ assert(none.isEmpty)
+ }
+ }
+
+ test("full-text search - fast mode excludes rows whose filter column has no index") {
+ withTable("T") {
+ createRankedTable()
+
+ val result = spark
+ .sql(s"""
+ |SELECT id
+ |FROM full_text_search('T', 'content', '$rankedQuery', 10)
+ |WHERE id >= 3
+ |""".stripMargin)
+ .collect()
+ assert(result.isEmpty)
+ }
+ }
+
+ test("full-text search - full mode scans rows whose filter column has no index") {
+ withTable("T") {
+ createRankedTable(",\n 'scalar-index.search-mode' = 'full'")
+
+ val result = spark
+ .sql(s"""
+ |SELECT id
+ |FROM full_text_search('T', 'content', '$rankedQuery', 2)
+ |WHERE id >= 3
+ |""".stripMargin)
+ .collect()
+ .map(_.getInt(0))
+ .toSet
+ assert(result.size == 2)
+ assert(result.subsetOf(Set(3, 4, 5)))
+ }
+ }
+
+ test("full-text search - WHERE filter combined with partition filter") {
+ withTable("T") {
+ spark.sql("""
+ |CREATE TABLE T (id INT, content STRING, pt INT)
+ |PARTITIONED BY (pt)
+ |TBLPROPERTIES (
+ | 'bucket' = '-1',
+ | 'global-index.row-count-per-shard' = '10000',
+ | 'row-tracking.enabled' = 'true',
+ | 'data-evolution.enabled' = 'true')
+ |""".stripMargin)
+ spark.sql("""
+ |INSERT INTO T VALUES
+ | (0, 'paimon lake', 1),
+ | (1, 'paimon lake', 1),
+ | (2, 'paimon lake', 2),
+ | (3, 'paimon lake', 2)
+ |""".stripMargin)
+ spark
+ .sql(
+ s"CALL sys.create_global_index(table => 'test.T', index_column => 'content', index_type => '$indexType')")
+ .collect()
+ spark
+ .sql("CALL sys.create_global_index(table => 'test.T', index_column => 'id', index_type => 'btree')")
+ .collect()
+
+ val result = spark
+ .sql(s"""
+ |SELECT id
+ |FROM full_text_search('T', 'content', '$rankedQuery', 10)
+ |WHERE pt = 2 AND id >= 3
+ |""".stripMargin)
+ .collect()
+ .map(_.getInt(0))
+ .toSeq
+ assert(result == Seq(3))
+ }
+ }
+
+ test("full-text search - WHERE filter with deletion vectors") {
+ withTable("T") {
+ createRankedTable(",\n 'deletion-vectors.enabled' = 'true'")
+ spark
+ .sql("CALL sys.create_global_index(table => 'test.T', index_column => 'id', index_type => 'btree')")
+ .collect()
+ spark.sql("DELETE FROM T WHERE id = 3")
+
+ val result = spark
+ .sql(s"""
+ |SELECT id
+ |FROM full_text_search('T', 'content', '$rankedQuery', 10)
+ |WHERE id >= 3
+ |ORDER BY id
+ |""".stripMargin)
+ .collect()
+ .map(_.getInt(0))
+ .toSeq
+ assert(result == Seq(4, 5))
+ }
+ }
+
+ test("full-text search - predicate that cannot be pushed down is applied after the search") {
+ withTable("T") {
+ createRankedTable()
+ spark
+ .sql("CALL sys.create_global_index(table => 'test.T', index_column => 'id', index_type => 'btree')")
+ .collect()
+
+ // `id % 2 = 1` is not a pushable predicate, so Spark evaluates it after the top-k: the
+ // result is a subset of the unfiltered top-k, never rows outside it, and may be short.
+ val unfiltered = spark
+ .sql(s"SELECT id FROM full_text_search('T', 'content', '$rankedQuery', 3)")
+ .collect()
+ .map(_.getInt(0))
+ .toSet
+ val filtered = spark
+ .sql(s"""
+ |SELECT id
+ |FROM full_text_search('T', 'content', '$rankedQuery', 3)
+ |WHERE id % 2 = 1
+ |""".stripMargin)
+ .collect()
+ .map(_.getInt(0))
+ .toSet
+ assert(filtered.subsetOf(unfiltered))
+ assert(filtered.forall(_ % 2 == 1))
+
+ // Mixed: the pushable half narrows the candidates before top-k, the rest post-filters.
+ val mixed = spark
+ .sql(s"""
+ |SELECT id
+ |FROM full_text_search('T', 'content', '$rankedQuery', 3)
+ |WHERE id >= 3 AND id % 2 = 1
+ |""".stripMargin)
+ .collect()
+ .map(_.getInt(0))
+ .toSet
+ assert(mixed.subsetOf(Set(3, 5)))
+ }
+ }
}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/HybridSearchTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/HybridSearchTest.scala
index 254bf50c77..a9ec19a326 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/HybridSearchTest.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/HybridSearchTest.scala
@@ -315,7 +315,7 @@ class HybridSearchTest extends PaimonSparkTestBase {
}
}
- test("hybrid full-text route rejects non-partition filters") {
+ test("hybrid full-text route applies non-partition filters before ranking") {
withTable("T") {
spark.sql("""
|CREATE TABLE T (id INT, content STRING)
@@ -326,6 +326,8 @@ class HybridSearchTest extends PaimonSparkTestBase {
| 'data-evolution.enabled' = 'true')
|""".stripMargin)
+ // Row 0 scores higher for "paimon search"; the filter must remove it before the route's
+ // top-1 so that row 1 is returned instead of nothing.
spark.sql("""
|INSERT INTO T VALUES
| (0, 'paimon search'),
@@ -336,39 +338,30 @@ class HybridSearchTest extends PaimonSparkTestBase {
.sql("CALL sys.create_global_index(table => 'test.T', index_column => 'content', " +
"index_type => 'test-fulltext')")
.collect()
+ spark
+ .sql("CALL sys.create_global_index(table => 'test.T', index_column => 'id', " +
+ "index_type => 'btree')")
+ .collect()
- val error = intercept[Exception] {
- spark
- .sql("""
- |SELECT id
- |FROM hybrid_search(
- | 'T',
- | array(),
- | array(
- | named_struct(
- | 'column', 'content',
- | 'query', '{"match":{"column":"content","terms":"paimon"}}',
- | 'limit', 1,
- | 'weight', 1.0f,
- | 'options', map())),
- | 1)
- |WHERE id = 1
- |""".stripMargin)
- .collect()
- }
-
- assert(containsMessage(error, "does not support non-partition filters"))
- }
- }
+ val result = spark
+ .sql("""
+ |SELECT id
+ |FROM hybrid_search(
+ | 'T',
+ | array(),
+ | array(
+ | named_struct(
+ | 'column', 'content',
+ | 'query', '{"match":{"column":"content","terms":"paimon search"}}',
+ | 'limit', 1,
+ | 'weight', 1.0f,
+ | 'options', map())),
+ | 1)
+ |WHERE id = 1
+ |""".stripMargin)
+ .collect()
- private def containsMessage(error: Throwable, expected: String): Boolean = {
- var current = error
- while (current != null) {
- if (current.getMessage != null && current.getMessage.contains(expected)) {
- return true
- }
- current = current.getCause
+ assert(result.map(_.getInt(0)).toSeq == Seq(1))
}
- false
}
}
(paimon) 18/20: [python] Distribute vector search refinement with Ray (#9951)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit c2b6a0435ce36befb95fce4397db8d9ef9a58dea
Author: chaoyang <ch...@apache.org>
AuthorDate: Fri Sep 18 19:38:47 2026 +0800
[python] Distribute vector search refinement with Ray (#9951)
---
docs/docs/pypaimon/multimodal-search.md | 12 +-
paimon-python/pypaimon/multimodal/query.py | 2 +-
paimon-python/pypaimon/ray/vector_search.py | 14 +--
.../pypaimon/tests/ray_vector_refinement_test.py | 134 +++++++++++++++++++++
.../pypaimon/tests/ray_vector_search_test.py | 6 +-
5 files changed, 152 insertions(+), 16 deletions(-)
diff --git a/docs/docs/pypaimon/multimodal-search.md b/docs/docs/pypaimon/multimodal-search.md
index 9ec110a42c..73f0280b0e 100644
--- a/docs/docs/pypaimon/multimodal-search.md
+++ b/docs/docs/pypaimon/multimodal-search.md
@@ -143,8 +143,11 @@ Ray execution fails because NaN cannot be ranked consistently across tasks.
The driver fixes one read snapshot and plans the query. Workers search individual
index shards and, when required by the table's search mode, scan unindexed data.
-They return candidate row IDs and scores. The driver retains the local search
-rules for global candidate selection, refinement, filtering, and result lookup.
+They return candidate row IDs and scores. For single-vector queries with
+refinement enabled, the driver first selects the global candidate set, then
+workers read and score those candidate vectors using the index's persisted
+metric. The driver merges their results and retains the local rules for
+filtering and final row lookup.
In particular, `pre_filter` filters candidates before ranking; `where()` filters
the selected rows and can return fewer than the requested number of results.
@@ -158,8 +161,9 @@ All workers must be able to access the table's storage. Local filesystem paths
are suitable for a local Ray cluster; multiple nodes require shared storage.
Each index task searches one shard, while its native index I/O settings still
apply. Raw-scan parallelism is limited by the number of planned read splits,
-controlled by the table's `source.split.target-size` option. Refinement and final
-row lookup run on the driver. Candidate traffic grows
+controlled by the table's `source.split.target-size` option. Single-vector
+refinement also uses planned read splits; workers return scores for merging.
+Final row lookup runs on the driver. Candidate traffic grows
with the number of index shards and the configured refinement budget, so Ray
execution is most useful when shard search or raw scanning outweighs scheduling
and transfer costs. Small queries can be faster locally.
diff --git a/paimon-python/pypaimon/multimodal/query.py b/paimon-python/pypaimon/multimodal/query.py
index 60c3452dc5..da6d712713 100644
--- a/paimon-python/pypaimon/multimodal/query.py
+++ b/paimon-python/pypaimon/multimodal/query.py
@@ -537,7 +537,7 @@ class VectorQuery(_PreFilterQuery):
Ray execution supports data-evolution tables. ``concurrency`` bounds
in-flight tasks (defaults to 4); ``ray_remote_args`` configures their
- resources and retries. Refinement and result lookup run on the driver.
+ resources and retries. Result lookup runs on the driver.
"""
if execution == "local":
if concurrency is not None or ray_remote_args is not None:
diff --git a/paimon-python/pypaimon/ray/vector_search.py b/paimon-python/pypaimon/ray/vector_search.py
index 348df0f93c..dfd51f1f35 100644
--- a/paimon-python/pypaimon/ray/vector_search.py
+++ b/paimon-python/pypaimon/ray/vector_search.py
@@ -67,10 +67,6 @@ class _RayVectorSearchRead(DataEvolutionVectorRead):
self._concurrency = concurrency
self._remote_args = remote_args
- def _score_raw_batches(self, batches, query_vector, metric, score_candidates=None, reject_nan=True):
- # Refinement also runs through this scorer, before its local top-k.
- return super()._score_raw_batches(batches, query_vector, metric, score_candidates, reject_nan=True)
-
def _search_index_splits(self, splits, query, search_limit, pre_filters, batch=False):
# Filters are planned once on the driver, at the query snapshot. Each
# worker receives only its own include-row bitmap, not a table-wide set.
@@ -91,13 +87,11 @@ class _RayVectorSearchRead(DataEvolutionVectorRead):
def _read_raw_search(self, raw_row_ranges, pre_filter, query_vector,
index_type=None, include_filter=True,
score_candidates=None, snapshot=None):
- if score_candidates is not None:
- # Refinement uses the globally selected indexed candidates.
- return super()._read_raw_search(
- raw_row_ranges, pre_filter, query_vector, index_type,
- include_filter, score_candidates, snapshot)
-
ranges = _filtered_raw_row_ranges(raw_row_ranges, pre_filter)
+ if score_candidates is not None:
+ # Refine only the globally selected candidates. Intersect before
+ # planning so workers neither read nor score other rows.
+ ranges = _filtered_raw_row_ranges(ranges, score_candidates.to_range_list())
if not ranges:
return DictBasedScoredIndexResult({})
table_read, splits = self._plan_raw_read(ranges, include_filter, snapshot)
diff --git a/paimon-python/pypaimon/tests/ray_vector_refinement_test.py b/paimon-python/pypaimon/tests/ray_vector_refinement_test.py
new file mode 100644
index 0000000000..386dbaafbc
--- /dev/null
+++ b/paimon-python/pypaimon/tests/ray_vector_refinement_test.py
@@ -0,0 +1,134 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest.mock import patch
+
+import pytest
+
+pytest.importorskip("ray")
+
+from pypaimon.ray import vector_search as search_module
+from pypaimon.read.table_read import TableRead
+from pypaimon.tests import ray_vector_search_test as fixtures
+from pypaimon.tests.ray_vector_search_test import VECTORS, add_rows, build_index, ids
+from pypaimon.utils.range import Range
+from pypaimon.utils.roaring_bitmap import RoaringBitmap64
+
+
+ray_cluster = fixtures.ray_cluster
+table = fixtures.table
+
+
+@pytest.mark.parametrize("metric", ["l2", "cosine", "inner_product"])
+@pytest.mark.parametrize("concurrency", [1, 2])
+def test_refinement_reads_vectors_only_on_workers(table, ray_cluster, metric, concurrency):
+ add_rows(table, VECTORS[:3])
+ add_rows(table, VECTORS[3:], 3)
+ build_index(table, metric)
+ query = table.search([1., 1.], options={"refine_factor": "2"}).select(["id"]).limit(3)
+ expected = ids(query.to_arrow())
+ original_read = TableRead._new_arrow_batch_reader
+ original_map = search_module._map_tasks
+ dispatched = []
+
+ def no_driver_vectors(read, *args, **kwargs):
+ assert "embedding" not in [field.name for field in read.read_type]
+ return original_read(read, *args, **kwargs)
+
+ def record(worker, context, items, *args):
+ if worker is search_module._search_raw_split:
+ dispatched.append(len(items))
+ return original_map(worker, context, items, *args)
+
+ with patch.object(TableRead, "_new_arrow_batch_reader", no_driver_vectors), \
+ patch.object(search_module, "_map_tasks", record):
+ assert ids(query.to_arrow(execution="ray", concurrency=concurrency)) == expected
+ assert dispatched == [2]
+
+
+def test_candidate_intersection_excludes_nan_and_streams_real_splits(table, ray_cluster):
+ add_rows(table, [[0., 1.], [float("nan"), 1.]])
+ add_rows(table, [[float("nan"), 1.], [1., 1.]], 2)
+ query = table.search([1., 1.], options={"metric": "l2"}).limit(1)._for_execution()
+ reader = query._vector_search_builder(query).new_vector_search_read()
+ distributed = search_module._RayVectorSearchRead(reader, 2, {})
+ candidates = RoaringBitmap64()
+ candidates.add(0)
+ candidates.add(3)
+ result = distributed._read_raw_search(
+ [Range(0, 3)], None, [1., 1.], score_candidates=candidates,
+ snapshot=query._table._read_snapshot)
+ assert list(result.results()) == [3]
+ assert result.score_getter()(3) == 1.0
+ candidates.add(1)
+ with pytest.raises(ValueError, match="cannot rank NaN"):
+ distributed._read_raw_search(
+ [Range(0, 3)], None, [1., 1.], score_candidates=candidates,
+ snapshot=query._table._read_snapshot)
+
+
+def test_empty_refinement_does_not_plan_or_dispatch(table):
+ query = table.search([1., 1.]).limit(1)._for_execution()
+ reader = query._vector_search_builder(query).new_vector_search_read()
+ distributed = search_module._RayVectorSearchRead(reader, 2, {})
+ with patch.object(distributed, "_plan_raw_read", side_effect=AssertionError("empty read")), \
+ patch.object(search_module, "_map_tasks", side_effect=AssertionError("empty task")):
+ result = distributed._read_raw_search(
+ [Range(0, 100)], None, [1., 1.], score_candidates=RoaringBitmap64())
+ assert result.results().is_empty()
+
+
+@pytest.mark.parametrize("selector", ["snapshot", "tag"])
+def test_refinement_keeps_historical_filters_and_deletions(table, ray_cluster, selector):
+ add_rows(table, VECTORS)
+ build_index(table)
+ table.update("id = 2", {"category": "no"})
+ table.delete("id = 0")
+ snapshot = table.raw_table.snapshot_manager().get_latest_snapshot()
+ table.raw_table.create_tag("saved", snapshot_id=snapshot.id)
+ table.delete("id = 4")
+ if selector == "tag":
+ table.raw_table.file_io.delete(table.raw_table.snapshot_manager().get_snapshot_path(snapshot.id))
+ options = {"tag_name": "saved"}
+ else:
+ options = {"snapshot_id": snapshot.id}
+ query = table.search([1., 1.], pre_filter="category = 'yes'",
+ options={"refine_factor": "2"}, **options).select(["id"]).limit(3)
+ assert ids(query.to_arrow()) == [4]
+ assert ids(query.to_arrow(execution="ray")) == [4]
+
+
+def test_commit_between_candidate_selection_and_refinement_keeps_snapshot(table, ray_cluster):
+ add_rows(table, VECTORS)
+ build_index(table)
+ table.raw_table = table.raw_table.copy({"global-index.column-update-action": "DROP_PARTITION_INDEX"})
+ query = table.search([1., 1.], options={"refine_factor": "2"}).select(["id"]).limit(1)
+ expected = ids(query.to_arrow())
+ original = search_module._map_tasks
+ changed = []
+
+ def commit_before_refine(worker, *args):
+ if worker is search_module._search_raw_split and not changed:
+ table.update("id = 0", {"embedding": [100., 1.]})
+ changed.append(True)
+ return original(worker, *args)
+
+ with patch.object(search_module, "_map_tasks", commit_before_refine):
+ assert ids(query.to_arrow(execution="ray")) == expected
+ assert changed == [True]
+ assert ids(query.to_arrow()) != expected
+ assert ids(query.to_arrow(execution="ray")) == ids(query.to_arrow())
diff --git a/paimon-python/pypaimon/tests/ray_vector_search_test.py b/paimon-python/pypaimon/tests/ray_vector_search_test.py
index 06a7446863..6c251b039a 100644
--- a/paimon-python/pypaimon/tests/ray_vector_search_test.py
+++ b/paimon-python/pypaimon/tests/ray_vector_search_test.py
@@ -317,7 +317,11 @@ def test_refinement_uses_global_candidates_despite_reverse_completion_order(tabl
index_file = IndexFileMeta("ivf-flat", "fake", 1, 2, None)
splits = [IndexVectorSearchSplit(0, 1, [index_file]), IndexVectorSearchSplit(2, 3, [index_file])]
- def completed(*args):
+ def completed(worker, context, items, *args):
+ if worker is search_module._search_raw_split:
+ for ordinal, split in enumerate(items):
+ yield ordinal, worker(context, split)
+ return
# Row 3 is the true nearest, but is outside the GLOBAL approximate top-2.
# Refining independently per shard would incorrectly bring it back in.
yield 1, ("l2", {2: 8., 3: 7.})
(paimon) 06/20: [common] Convert variant timestamps with the requested zone instead of the JVM default (#9939)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 23633c3c44af931eb50fb06d41f885442678c1f4
Author: Xiangyi Zhu <82...@users.noreply.github.com>
AuthorDate: Fri Sep 18 12:00:16 2026 +0800
[common] Convert variant timestamps with the requested zone instead of the JVM default (#9939)
---
.../paimon/data/variant/BaseVariantReader.java | 3 +-
.../org/apache/paimon/data/variant/VariantGet.java | 88 +++++++++++++++++++++-
.../paimon/data/variant/GenericVariantTest.java | 81 ++++++++++++++++++++
.../data/variant/PaimonShreddingUtilsTest.java | 44 ++++++++++-
.../parquet/reader/VariantShreddingReadTest.java | 62 +++++++++++++++
.../apache/paimon/spark/sql/VariantTestBase.scala | 30 ++++++++
6 files changed, 303 insertions(+), 5 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java b/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java
index e5c5011d20..807f3a901c 100644
--- a/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java
+++ b/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java
@@ -543,7 +543,8 @@ public class BaseVariantReader {
if (noNeedCast) {
return i;
}
- Object result = VariantGet.castScalar(i, scalaType, targetType, resolve);
+ Object result =
+ VariantGet.castScalar(i, scalaType, targetType, resolve, castArgs.zoneId());
return result == null ? invalidCast(row, topLevelMetadata) : result;
}
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java b/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java
index 5a22fc1752..cf81143cc8 100644
--- a/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java
+++ b/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java
@@ -29,16 +29,21 @@ import org.apache.paimon.data.Timestamp;
import org.apache.paimon.data.variant.GenericVariantUtil.Type;
import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypeChecks;
+import org.apache.paimon.types.DataTypeRoot;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.MapType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.VariantType;
+import org.apache.paimon.utils.DateTimeUtils;
import javax.annotation.Nullable;
import java.math.BigDecimal;
import java.math.RoundingMode;
+import java.time.ZoneId;
import java.util.HashMap;
+import java.util.TimeZone;
/** Utils for variant get. */
public class VariantGet {
@@ -175,7 +180,7 @@ public class VariantGet {
CastExecutor<Object, Object> resolve =
(CastExecutor<Object, Object>) CastExecutors.resolve(inputType, dataType);
- Object result = castScalar(input, inputType, dataType, resolve);
+ Object result = castScalar(input, inputType, dataType, resolve, castArgs.zoneId());
return result == null ? invalidCast(v, dataType, castArgs) : result;
}
}
@@ -183,14 +188,21 @@ public class VariantGet {
/**
* Casts a non-null scalar read from a variant to {@code targetType}, returning null when the
* cast is invalid. The generic cast rules wrap a numeric value that does not fit the target, so
- * an out-of-range value is rejected here first, matching Spark's TRY cast semantics.
+ * an out-of-range value is rejected here first, matching Spark's TRY cast semantics. Casts that
+ * move between an instant and a local date or time use {@code zoneId}, the zone the query asked
+ * for, rather than the JVM default the generic rules fall back to.
*/
@Nullable
static Object castScalar(
Object input,
DataType inputType,
DataType targetType,
- @Nullable CastExecutor<Object, Object> executor) {
+ @Nullable CastExecutor<Object, Object> executor,
+ ZoneId zoneId) {
+ Object temporal = castTemporal(input, inputType, targetType, zoneId);
+ if (temporal != NOT_TEMPORAL) {
+ return temporal;
+ }
if (executor == null || !fitsIntegralTarget(input, inputType, targetType)) {
return null;
}
@@ -201,6 +213,76 @@ public class VariantGet {
}
}
+ private static final Object NOT_TEMPORAL = new Object();
+
+ /**
+ * The casts whose result depends on a time zone: between a timestamp with local time zone and a
+ * string, a timestamp without time zone or a date, and from a string to a timestamp with local
+ * time zone. Returns {@link #NOT_TEMPORAL} for every other pair, and null for a string that
+ * does not parse. A timestamp renders like Spark's cast, without trailing fraction zeros.
+ */
+ @Nullable
+ private static Object castTemporal(
+ Object input, DataType inputType, DataType targetType, ZoneId zoneId) {
+ TimeZone tz = TimeZone.getTimeZone(zoneId);
+ switch (inputType.getTypeRoot()) {
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ Timestamp instant = (Timestamp) input;
+ switch (targetType.getTypeRoot()) {
+ case CHAR:
+ case VARCHAR:
+ return BinaryString.fromString(
+ DateTimeUtils.formatTimestamp(
+ DateTimeUtils.timestampWithLocalZoneToTimestamp(
+ instant, tz),
+ 0));
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ return truncate(
+ DateTimeUtils.timestampWithLocalZoneToTimestamp(instant, tz),
+ targetType);
+ case DATE:
+ return DateTimeUtils.timestampWithLocalZoneToDate(instant, tz);
+ default:
+ return NOT_TEMPORAL;
+ }
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ Timestamp local = (Timestamp) input;
+ switch (targetType.getTypeRoot()) {
+ case CHAR:
+ case VARCHAR:
+ return BinaryString.fromString(DateTimeUtils.formatTimestamp(local, 0));
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ return truncate(
+ DateTimeUtils.timestampToTimestampWithLocalZone(local, tz),
+ targetType);
+ default:
+ return NOT_TEMPORAL;
+ }
+ case DATE:
+ if (targetType.is(DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE)) {
+ return DateTimeUtils.dateToTimestampWithLocalZone((Integer) input, tz);
+ }
+ return NOT_TEMPORAL;
+ case CHAR:
+ case VARCHAR:
+ if (targetType.is(DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE)) {
+ try {
+ return DateTimeUtils.parseTimestampData(
+ input.toString(), DataTypeChecks.getPrecision(targetType), tz);
+ } catch (Exception e) {
+ return null;
+ }
+ }
+ return NOT_TEMPORAL;
+ default:
+ return NOT_TEMPORAL;
+ }
+ }
+
+ private static Timestamp truncate(Timestamp timestamp, DataType targetType) {
+ return DateTimeUtils.truncate(timestamp, DataTypeChecks.getPrecision(targetType));
+ }
+
/** Whether a numeric {@code input} lies within the range of an integral {@code targetType}. */
private static boolean fitsIntegralTarget(
Object input, DataType inputType, DataType targetType) {
diff --git a/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java b/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java
index 46b5b8c992..12c54e54f8 100644
--- a/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java
@@ -23,6 +23,7 @@ import org.apache.paimon.data.Decimal;
import org.apache.paimon.data.GenericArray;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.Timestamp;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
@@ -34,7 +35,11 @@ import java.io.ObjectStreamClass;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.time.ZoneId;
import java.time.ZoneOffset;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TimeZone;
import static org.apache.paimon.data.variant.PaimonShreddingUtils.assembleVariant;
import static org.apache.paimon.data.variant.PaimonShreddingUtils.buildVariantSchema;
@@ -345,6 +350,82 @@ public class GenericVariantTest {
.isEqualTo(1.2345678901234567E19);
}
+ @Test
+ public void testVariantGetTemporalCastsUseRequestedZone() {
+ // 2023-11-14 22:13:20.5 UTC as an instant, the same wall time as a local timestamp,
+ // and the date 2024-10-04
+ Map<String, Object> values = new HashMap<>();
+ values.put("ts", 1700000000500000L);
+ values.put("ntz", 1700000000500000L);
+ values.put("d", 20000);
+ values.put("s", "2023-11-14 22:13:20");
+ RowType valueType =
+ RowType.of(
+ new DataType[] {
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(),
+ DataTypes.TIMESTAMP(),
+ DataTypes.DATE(),
+ DataTypes.STRING()
+ },
+ new String[] {"ts", "ntz", "d", "s"});
+ Variant variant = GenericVariantBuilderHelper.build(valueType, values);
+ VariantCastArgs shanghai = new VariantCastArgs(false, ZoneId.of("Asia/Shanghai"));
+ VariantCastArgs utc = new VariantCastArgs(false, ZoneOffset.UTC);
+
+ // The generic cast rules use the JVM default zone; the requested zone must win no
+ // matter what the JVM default is, so run the same assertions under two defaults.
+ TimeZone original = TimeZone.getDefault();
+ try {
+ for (String jvmZone : new String[] {"UTC", "America/Los_Angeles"}) {
+ TimeZone.setDefault(TimeZone.getTimeZone(jvmZone));
+
+ // instant rendered in the requested zone, without trailing fraction zeros
+ assertThat(variant.variantGet("$.ts", DataTypes.STRING(), shanghai))
+ .isEqualTo(BinaryString.fromString("2023-11-15 06:13:20.5"));
+ assertThat(variant.variantGet("$.ts", DataTypes.STRING(), utc))
+ .isEqualTo(BinaryString.fromString("2023-11-14 22:13:20.5"));
+ assertThat(variant.variantGet("$.ts", DataTypes.TIMESTAMP(), shanghai))
+ .isEqualTo(Timestamp.fromMicros(1700000000500000L + 8 * 3600_000_000L));
+ assertThat(variant.variantGet("$.ts", DataTypes.DATE(), shanghai)).isEqualTo(19676);
+ assertThat(variant.variantGet("$.ts", DataTypes.DATE(), utc)).isEqualTo(19675);
+
+ // a local timestamp has no zone to render, but gains one when it becomes an
+ // instant
+ assertThat(variant.variantGet("$.ntz", DataTypes.STRING(), shanghai))
+ .isEqualTo(BinaryString.fromString("2023-11-14 22:13:20.5"));
+ assertThat(
+ variant.variantGet(
+ "$.ntz",
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(),
+ shanghai))
+ .isEqualTo(Timestamp.fromMicros(1700000000500000L - 8 * 3600_000_000L));
+
+ // midnight of the date in the requested zone
+ assertThat(
+ variant.variantGet(
+ "$.d",
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(),
+ shanghai))
+ .isEqualTo(
+ Timestamp.fromMicros(20000L * 86400_000_000L - 8 * 3600_000_000L));
+
+ // a string is parsed as wall time in the requested zone
+ assertThat(
+ variant.variantGet(
+ "$.s",
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(),
+ shanghai))
+ .isEqualTo(Timestamp.fromMicros(1700000000000000L - 8 * 3600_000_000L));
+
+ // the target precision is still honoured
+ assertThat(variant.variantGet("$.ts", DataTypes.TIMESTAMP(0), utc))
+ .isEqualTo(Timestamp.fromMicros(1700000000000000L));
+ }
+ } finally {
+ TimeZone.setDefault(original);
+ }
+ }
+
@Test
public void testObjectFieldOrderingCompatibility() {
String bmpKey = "\uE000";
diff --git a/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java b/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java
index a65d88c6dc..01915ed422 100644
--- a/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java
@@ -46,6 +46,7 @@ import org.junit.jupiter.api.Test;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
+import java.time.ZoneId;
import java.time.ZoneOffset;
import java.util.Arrays;
import java.util.HashMap;
@@ -219,10 +220,11 @@ public class PaimonShreddingUtilsTest {
BinaryString.fromString("true"),
null,
BinaryString.fromString(DateTimeUtils.formatDate(20000)),
+ // rendered in the zone castArgs asks for, not the JVM default
BinaryString.fromString(
DateTimeUtils.formatTimestamp(
Timestamp.fromMicros(1_234_567_890_123_456L),
- TimeZone.getDefault(),
+ DateTimeUtils.UTC_ZONE,
6)),
BinaryString.fromString(
DateTimeUtils.formatTimestamp(
@@ -340,6 +342,46 @@ public class PaimonShreddingUtilsTest {
}
}
+ @Test
+ public void testShreddedTimestampCastsUseRequestedZone() {
+ // A shredded typed_value timestamp goes through the scalar reader, an unshredded one
+ // through VariantGet; both must convert with the requested zone, not the JVM default.
+ Map<String, Object> values = new HashMap<>();
+ values.put("ts", 1700000000500000L); // 2023-11-14 22:13:20.5 UTC
+ RowType valueType =
+ RowType.of(
+ new DataType[] {DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE()},
+ new String[] {"ts"});
+ GenericVariant v = GenericVariantBuilderHelper.build(valueType, values);
+ VariantCastArgs shanghai = new VariantCastArgs(true, ZoneId.of("Asia/Shanghai"));
+
+ TimeZone original = TimeZone.getDefault();
+ try {
+ TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"));
+ for (RowType shape : new RowType[] {valueType, RowType.of()}) {
+ VariantSchema variantSchema = buildVariantSchema(variantShreddingSchema(shape));
+ FieldToExtract[] fieldsToExtract = {
+ buildFieldsToExtract(DataTypes.STRING(), "$.ts", shanghai, variantSchema),
+ buildFieldsToExtract(DataTypes.TIMESTAMP(), "$.ts", shanghai, variantSchema),
+ buildFieldsToExtract(DataTypes.DATE(), "$.ts", shanghai, variantSchema)
+ };
+ assertThat(
+ assembleVariantStruct(
+ castShredded(v, variantSchema),
+ variantSchema,
+ fieldsToExtract))
+ .as("shape %s", shape)
+ .isEqualTo(
+ GenericRow.of(
+ BinaryString.fromString("2023-11-15 06:13:20.5"),
+ Timestamp.fromMicros(1700000000500000L + 8 * 3600_000_000L),
+ 19676));
+ }
+ } finally {
+ TimeZone.setDefault(original);
+ }
+ }
+
@Test
public void testAssembleDecimalWithScaleAbovePrecision() {
// the unshredded leg extracts through VariantGet, which used to build an invalid
diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/reader/VariantShreddingReadTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/reader/VariantShreddingReadTest.java
index 5b93ff3c73..12e5a60a1e 100644
--- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/reader/VariantShreddingReadTest.java
+++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/reader/VariantShreddingReadTest.java
@@ -23,8 +23,10 @@ import org.apache.paimon.data.Decimal;
import org.apache.paimon.data.GenericArray;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.Timestamp;
import org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.data.variant.GenericVariant;
+import org.apache.paimon.data.variant.GenericVariantBuilderHelper;
import org.apache.paimon.data.variant.VariantMetadataUtils;
import org.apache.paimon.format.FileFormatFactory;
import org.apache.paimon.format.FormatReaderContext;
@@ -38,6 +40,7 @@ import org.apache.paimon.fs.PositionOutputStream;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.options.Options;
import org.apache.paimon.reader.RecordReader;
+import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
@@ -50,7 +53,10 @@ import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.math.BigDecimal;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
@@ -594,6 +600,62 @@ public class VariantShreddingReadTest {
.hasMessageContaining("Invalid cast 99999999999 to INT");
}
+ @ParameterizedTest
+ @ValueSource(
+ strings = {
+ "null",
+ "{\"type\":\"ROW\",\"fields\":[{\"name\":\"v\",\"type\":{\"type\":\"ROW\",\"fields\":[{\"name\":\"ts\",\"type\":\"TIMESTAMP(6) WITH LOCAL TIME ZONE\"}]}}]}"
+ })
+ public void testReadTimestampInRequestedZone(String shreddingSchema) throws Exception {
+ // The extraction carries the zone the query runs in; a shredded typed_value and an
+ // unshredded value must both convert with it rather than with the JVM default.
+ Options options = new Options();
+ if (!shreddingSchema.equals("null")) {
+ options.set("parquet.variant.shreddingSchema", shreddingSchema);
+ }
+ ParquetFileFormat format =
+ new ParquetFileFormat(new FileFormatFactory.FormatContext(options, 1024, 1024));
+
+ RowType writeType = DataTypes.ROW(DataTypes.FIELD(0, "v", DataTypes.VARIANT()));
+ Map<String, Object> values = new HashMap<>();
+ values.put("ts", 1700000000500000L); // 2023-11-14 22:13:20.5 UTC
+ writeRows(
+ format.createWriterFactory(writeType),
+ GenericRow.of(
+ GenericVariantBuilderHelper.build(
+ RowType.of(
+ new DataType[] {DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE()},
+ new String[] {"ts"}),
+ values)));
+
+ RowType readType =
+ DataTypes.ROW(
+ DataTypes.FIELD(
+ 0,
+ "v",
+ VariantMetadataUtils.VariantRowTypeBuilder.builder()
+ .field(DataTypes.STRING(), "$.ts", true, "Asia/Shanghai")
+ .field(DataTypes.DATE(), "$.ts", true, "Asia/Shanghai")
+ .field(DataTypes.TIMESTAMP(), "$.ts", true, "UTC")
+ .build()));
+ TimeZone original = TimeZone.getDefault();
+ try {
+ for (String jvmZone : new String[] {"UTC", "America/Los_Angeles"}) {
+ TimeZone.setDefault(TimeZone.getTimeZone(jvmZone));
+ List<InternalRow> result = readRows(format, readType);
+ assertThat(result.get(0).getRow(0, 3))
+ .as("JVM zone %s", jvmZone)
+ .isEqualTo(
+ GenericRow.of(
+ BinaryString.fromString("2023-11-15 06:13:20.5"),
+ 19676,
+ Timestamp.fromMicros(1700000000500000L)));
+ }
+ } finally {
+ TimeZone.setDefault(original);
+ }
+ }
+
protected List<InternalRow> readRows(ParquetFileFormat format, RowType rowType)
throws IOException {
List<InternalRow> result = new ArrayList<>();
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
index 6bddc24fd4..fda16f8219 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
@@ -1130,6 +1130,36 @@ abstract class VariantTestBase extends PaimonSparkTestBase {
Seq(Row(1, null, null), Row(2, 7, 1L)))
}
+ test("Paimon Variant pushdown: timestamp extraction follows the session time zone") {
+ // The test JVM runs in America/Los_Angeles; the session zone below differs from it, so a
+ // pushed-down extraction must convert with the session zone Spark hands to the scan.
+ withSparkSQLConf("spark.sql.session.timeZone" -> "Asia/Shanghai") {
+ sql("CREATE TABLE T (id INT, v VARIANT)")
+ sql("""INSERT INTO T VALUES (1, to_variant_object(named_struct(
+ | 'ts', timestamp'2023-11-15 06:13:20.5',
+ | 'ntz', timestamp_ntz'2023-11-15 06:13:20.5',
+ | 'd', date'2024-10-04')))
+ |""".stripMargin)
+
+ checkAnswer(
+ sql("""SELECT
+ | variant_get(v, '$.ts', 'string'),
+ | CAST(try_variant_get(v, '$.ts', 'timestamp_ntz') AS STRING),
+ | CAST(try_variant_get(v, '$.ts', 'date') AS STRING),
+ | CAST(try_variant_get(v, '$.ntz', 'timestamp') AS STRING),
+ | CAST(try_variant_get(v, '$.d', 'timestamp') AS STRING)
+ |FROM T""".stripMargin),
+ Seq(
+ Row(
+ "2023-11-15 06:13:20.5",
+ "2023-11-15 06:13:20.5",
+ "2023-11-15",
+ "2023-11-15 06:13:20.5",
+ "2024-10-04 00:00:00"))
+ )
+ }
+ }
+
test("Paimon Variant pushdown: nested variant column inside a struct") {
assume(gteqSpark4_1)
sql("CREATE TABLE T (id INT, nested STRUCT<v: VARIANT, x: INT>)")
(paimon) 17/20: [core] Fix OOM error handling in MergeTreeWriter and improve resource cleanup (#9933)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 9d93f1bac81a367e31fb901cd1e69e029a0c61db
Author: Alex <16...@qq.com>
AuthorDate: Fri Sep 18 19:34:14 2026 +0800
[core] Fix OOM error handling in MergeTreeWriter and improve resource cleanup (#9933)
---
.../apache/paimon/io/RollingFileWriterImpl.java | 11 +++--
.../apache/paimon/io/RollingFileWriterTest.java | 57 ++++++++++++++++++++++
2 files changed, 65 insertions(+), 3 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriterImpl.java b/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriterImpl.java
index 11c332b90f..1d8e0fc158 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriterImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriterImpl.java
@@ -49,6 +49,7 @@ public class RollingFileWriterImpl<T, R> implements RollingFileWriter<T, R> {
private long recordCount = 0;
private long currentFileRecordCount = 0;
private boolean closed = false;
+ private boolean aborted = false;
public RollingFileWriterImpl(
Supplier<? extends SingleFileWriter<T, R>> writerFactory,
@@ -177,8 +178,12 @@ public class RollingFileWriterImpl<T, R> implements RollingFileWriter<T, R> {
@Override
public void abort() {
- if (currentWriter != null) {
- currentWriter.abort();
+ SingleFileWriter<T, R> writer = currentWriter;
+ currentWriter = null;
+ aborted = true;
+
+ if (writer != null) {
+ writer.abort();
}
for (FileWriterAbortExecutor abortExecutor : closedWriters) {
abortExecutor.abort();
@@ -201,7 +206,7 @@ public class RollingFileWriterImpl<T, R> implements RollingFileWriter<T, R> {
@Override
public void close() throws IOException {
- if (closed) {
+ if (closed || aborted) {
return;
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java
index 1676d7ebd4..175fc09a7e 100644
--- a/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java
@@ -24,6 +24,8 @@ import org.apache.paimon.data.InternalRow;
import org.apache.paimon.fileindex.FileIndexOptions;
import org.apache.paimon.format.FileFormat;
import org.apache.paimon.format.FormatReaderContext;
+import org.apache.paimon.format.FormatWriter;
+import org.apache.paimon.format.FormatWriterFactory;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.manifest.FileSource;
@@ -48,8 +50,11 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import java.util.function.Function;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Test for {@link RollingFileWriterImpl}. */
public class RollingFileWriterTest {
@@ -140,6 +145,44 @@ public class RollingFileWriterTest {
}
}
+ @Test
+ public void testCloseAfterWriteFailure() throws IOException {
+ Path path = new Path(tempDir.resolve("failed-file").toString());
+ OutOfMemoryError expected = new OutOfMemoryError("expected");
+ RollingFileWriterImpl<InternalRow, Void> writer =
+ new RollingFileWriterImpl<>(
+ () ->
+ new TestSingleFileWriter(
+ LocalFileIO.create(),
+ (out, compression) ->
+ new FormatWriter() {
+ @Override
+ public void addElement(InternalRow element) {
+ throw expected;
+ }
+
+ @Override
+ public boolean reachTargetSize(
+ boolean suggestedCheck,
+ long targetSize) {
+ return false;
+ }
+
+ @Override
+ public void close() {}
+ },
+ path),
+ Long.MAX_VALUE,
+ Long.MAX_VALUE);
+
+ assertThatThrownBy(() -> writer.write(GenericRow.of(1))).isSameAs(expected);
+ assertThat(LocalFileIO.create().exists(path)).isFalse();
+ assertThatCode(writer::abort).doesNotThrowAnyException();
+ assertThatCode(writer::close).doesNotThrowAnyException();
+ assertThatThrownBy(writer::result).isInstanceOf(IllegalStateException.class);
+ assertThatThrownBy(writer::drainAbortExecutors).isInstanceOf(IllegalStateException.class);
+ }
+
@Test
public void testRollingByRows() throws IOException {
// Huge byte target so only the row-count limit triggers rolling.
@@ -313,6 +356,20 @@ public class RollingFileWriterTest {
assertThat(file.valueStats().minValues().getFieldCount()).isEqualTo(SCHEMA.getFieldCount());
}
+ private static class TestSingleFileWriter extends SingleFileWriter<InternalRow, Void> {
+
+ private TestSingleFileWriter(
+ LocalFileIO fileIO, FormatWriterFactory writerFactory, Path path) {
+ super(fileIO, writerFactory, path, Function.identity(), null, false);
+ }
+
+ @Override
+ public Void result() throws IOException {
+ fileIO.getFileSize(path);
+ return null;
+ }
+ }
+
private static class SingleUseBundleRecords implements BundleRecords {
private final List<InternalRow> rows;
(paimon) 10/20: [python] Support distributed vector search with Ray (#9930)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit c0398c9d7e830f0218eb47bc0b5f7c276689b14d
Author: chaoyang <ch...@apache.org>
AuthorDate: Fri Sep 18 12:13:39 2026 +0800
[python] Support distributed vector search with Ray (#9930)
---
.github/workflows/ci-python.yml | 2 +-
docs/docs/pypaimon/multimodal-search.md | 64 ++++
.../lumina/lumina_vector_global_index_reader.py | 2 +-
.../vindex/vindex_vector_global_index_reader.py | 2 +-
paimon-python/pypaimon/multimodal/query.py | 27 +-
paimon-python/pypaimon/ray/vector_search.py | 182 +++++++++
.../pypaimon/table/source/vector_search_read.py | 17 +-
.../pypaimon/tests/ray_vector_search_test.py | 405 +++++++++++++++++++++
.../tests/vector_metric_consistency_test.py | 23 +-
paimon-python/pypaimon/tests/vindex_input_test.py | 11 +-
10 files changed, 721 insertions(+), 14 deletions(-)
diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml
index f7e127f074..157724ccec 100644
--- a/.github/workflows/ci-python.yml
+++ b/.github/workflows/ci-python.yml
@@ -381,7 +381,7 @@ jobs:
python -m pip install --no-cache-dir -q ray==$ray_version
python -c "import ray; print(f'Ray version: {ray.__version__}')"
python -c "from packaging.version import parse; import ray; assert parse(ray.__version__) == parse('$ray_version'), f'Expected Ray $ray_version, got {ray.__version__}'"
- tests="pypaimon/tests/ray_data_test.py::RayDataTest pypaimon/tests/ray_range_join_test.py::RayRangeJoinTest"
+ tests="pypaimon/tests/ray_data_test.py::RayDataTest pypaimon/tests/ray_range_join_test.py::RayRangeJoinTest pypaimon/tests/ray_vector_search_test.py::test_raw_search_matches_exact_top_k"
if [ "$ray_version" = "2.50.1" ]; then
tests="$tests pypaimon/tests/ray_update_by_row_id_test.py::RayUpdateByRowIdTest::test_empty_dataset_after_transform_is_noop"
fi
diff --git a/docs/docs/pypaimon/multimodal-search.md b/docs/docs/pypaimon/multimodal-search.md
index 034162016e..9ec110a42c 100644
--- a/docs/docs/pypaimon/multimodal-search.md
+++ b/docs/docs/pypaimon/multimodal-search.md
@@ -100,6 +100,70 @@ matches = (
)
```
+## Distributed Vector Search
+
+Use `execution="ray"` to execute a single vector query across Ray workers and
+return an Arrow table to the driver. This supports data-evolution tables;
+primary-key tables, batch vector queries, and hybrid queries are not supported
+by this execution mode.
+
+Install the same PyPaimon and index dependencies on the driver and workers:
+
+```shell
+python -m pip install 'pypaimon[ray,vindex]'
+```
+
+```python
+import ray
+
+ray.init() # Use address="auto" to connect to an existing cluster.
+
+neighbors = (
+ docs.search(
+ [0.1, 0.2, 0.3],
+ column="embedding",
+ pre_filter="category = 'lake'",
+ )
+ .select(["id", "content"])
+ .limit(10)
+ .to_arrow(
+ execution="ray",
+ concurrency=4,
+ ray_remote_args={"num_cpus": 1},
+ )
+)
+```
+
+`execution` defaults to `"local"`. `concurrency` is a positive integer limiting
+the number of in-flight Ray tasks, with a default of 4. `ray_remote_args` supplies
+Ray task options, including resources and retry settings; `num_returns` is
+managed by PyPaimon. These two arguments require `execution="ray"`.
+Query vectors must contain finite values. If stored vectors produce NaN scores,
+Ray execution fails because NaN cannot be ranked consistently across tasks.
+
+The driver fixes one read snapshot and plans the query. Workers search individual
+index shards and, when required by the table's search mode, scan unindexed data.
+They return candidate row IDs and scores. The driver retains the local search
+rules for global candidate selection, refinement, filtering, and result lookup.
+In particular, `pre_filter` filters candidates before ranking; `where()` filters
+the selected rows and can return fewer than the requested number of results.
+
+Index search, raw scans, refinement, lookup, and task retries use the same read
+snapshot. The existing `snapshot_id` and `tag_name` arguments to `search()` also
+work with Ray execution. A failed task fails the query rather than returning
+partial results. Snapshot pinning does not prevent file expiration; the files
+needed by the query must remain available for its duration.
+
+All workers must be able to access the table's storage. Local filesystem paths
+are suitable for a local Ray cluster; multiple nodes require shared storage.
+Each index task searches one shard, while its native index I/O settings still
+apply. Raw-scan parallelism is limited by the number of planned read splits,
+controlled by the table's `source.split.target-size` option. Refinement and final
+row lookup run on the driver. Candidate traffic grows
+with the number of index shards and the configured refinement budget, so Ray
+execution is most useful when shard search or raw scanning outweighs scheduling
+and transfer costs. Small queries can be faster locally.
+
## Search Hybrid
Use `search_hybrid` to combine vector and full-text routes, then rerank the
diff --git a/paimon-python/pypaimon/globalindex/lumina/lumina_vector_global_index_reader.py b/paimon-python/pypaimon/globalindex/lumina/lumina_vector_global_index_reader.py
index 2abba80617..0ddd1e3325 100644
--- a/paimon-python/pypaimon/globalindex/lumina/lumina_vector_global_index_reader.py
+++ b/paimon-python/pypaimon/globalindex/lumina/lumina_vector_global_index_reader.py
@@ -174,7 +174,7 @@ class LuminaVectorGlobalIndexReader(GlobalIndexReader):
self._searcher = LuminaSearcher(self._options)
self._searcher.open_stream(stream, self._io_meta.file_size)
self._stream = stream
- except Exception:
+ except BaseException:
stream.close()
raise
diff --git a/paimon-python/pypaimon/globalindex/vindex/vindex_vector_global_index_reader.py b/paimon-python/pypaimon/globalindex/vindex/vindex_vector_global_index_reader.py
index 3a7efb7d83..c3fc51f61a 100644
--- a/paimon-python/pypaimon/globalindex/vindex/vindex_vector_global_index_reader.py
+++ b/paimon-python/pypaimon/globalindex/vindex/vindex_vector_global_index_reader.py
@@ -223,7 +223,7 @@ class VindexVectorGlobalIndexReader(GlobalIndexReader):
self._reader = reader
self._search_params_type = SearchParams
self._stream = stream
- except Exception:
+ except BaseException:
try:
if reader is not None:
reader.close()
diff --git a/paimon-python/pypaimon/multimodal/query.py b/paimon-python/pypaimon/multimodal/query.py
index 9cfea57e6f..60c3452dc5 100644
--- a/paimon-python/pypaimon/multimodal/query.py
+++ b/paimon-python/pypaimon/multimodal/query.py
@@ -532,7 +532,32 @@ class VectorQuery(_PreFilterQuery):
super().__init__(
table, result_factory=self._execute_vector, pre_filter=pre_filter)
+ def to_arrow(self, *, execution="local", concurrency=None, ray_remote_args=None):
+ """Execute a vector query locally or on Ray workers, returning an Arrow table.
+
+ Ray execution supports data-evolution tables. ``concurrency`` bounds
+ in-flight tasks (defaults to 4); ``ray_remote_args`` configures their
+ resources and retries. Refinement and result lookup run on the driver.
+ """
+ if execution == "local":
+ if concurrency is not None or ray_remote_args is not None:
+ raise ValueError("Ray options require execution='ray'.")
+ return super().to_arrow()
+ if execution != "ray":
+ raise ValueError("execution must be 'local' or 'ray'.")
+
+ from pypaimon.ray.vector_search import _execute_vector_search
+
+ query = self._for_execution()
+ result = _execute_vector_search(
+ self._vector_search_builder(query),
+ concurrency=concurrency, ray_remote_args=ray_remote_args)
+ return query._read_global_index_result(result)
+
def _execute_vector(self, query):
+ return self._vector_search_builder(query).execute_local()
+
+ def _vector_search_builder(self, query):
limit = query._limit if query._limit is not None else 10
builder = (
query._table.new_vector_search_builder()
@@ -543,7 +568,7 @@ class VectorQuery(_PreFilterQuery):
)
if query._pre_filter is not None:
builder = builder.with_filter(query._pre_filter)
- return builder.execute_local()
+ return builder
class TextQuery(_PreFilterQuery):
diff --git a/paimon-python/pypaimon/ray/vector_search.py b/paimon-python/pypaimon/ray/vector_search.py
new file mode 100644
index 0000000000..348df0f93c
--- /dev/null
+++ b/paimon-python/pypaimon/ray/vector_search.py
@@ -0,0 +1,182 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Ray execution of snapshot-pinned, single-vector queries."""
+
+from contextlib import closing
+import math
+
+from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult
+from pypaimon.table.source.vector_search_read import (
+ DataEvolutionVectorRead,
+ _filtered_raw_row_ranges,
+ _offer_score,
+ _scored_result,
+)
+from pypaimon.utils.roaring_bitmap import RoaringBitmap64
+
+
+def _execute_vector_search(builder, *, concurrency=None, ray_remote_args=None):
+ """Execute the builder of an already snapshot-pinned ``VectorQuery``."""
+ concurrency = 4 if concurrency is None else concurrency
+ if isinstance(concurrency, bool) or not isinstance(concurrency, int) or concurrency < 1:
+ raise ValueError("concurrency must be a positive integer.")
+ remote_args = dict(ray_remote_args or {})
+ if "num_returns" in remote_args:
+ raise ValueError("Vector search manages num_returns; omit it from ray_remote_args.")
+
+ reader = builder.new_vector_search_read()
+ if (type(reader) is not DataEvolutionVectorRead
+ or not reader._table.options.data_evolution_enabled()):
+ raise ValueError("Ray vector search supports only data-evolution tables.")
+ if not all(math.isfinite(float(value)) for value in reader._query_vector):
+ raise ValueError("Ray vector search requires a finite query vector.")
+
+ try:
+ import ray # noqa: F401
+ except ModuleNotFoundError as error:
+ if error.name != "ray":
+ raise
+ raise ImportError("Ray vector search requires pypaimon[ray].") from error
+
+ plan = builder.new_vector_search_scan().scan()
+ return _RayVectorSearchRead(reader, concurrency, remote_args).read_plan(plan)
+
+
+class _RayVectorSearchRead(DataEvolutionVectorRead):
+ """Replace split execution while retaining local merge/refinement semantics."""
+
+ def __init__(self, reader, concurrency, remote_args):
+ super().__init__(
+ reader._table, reader._limit, reader._vector_column, reader._query_vector,
+ reader._filter, reader._partition_filter, reader._options)
+ self._concurrency = concurrency
+ self._remote_args = remote_args
+
+ def _score_raw_batches(self, batches, query_vector, metric, score_candidates=None, reject_nan=True):
+ # Refinement also runs through this scorer, before its local top-k.
+ return super()._score_raw_batches(batches, query_vector, metric, score_candidates, reject_nan=True)
+
+ def _search_index_splits(self, splits, query, search_limit, pre_filters, batch=False):
+ # Filters are planned once on the driver, at the query snapshot. Each
+ # worker receives only its own include-row bitmap, not a table-wide set.
+ items = [(split, None if not pre_filters or pre_filters[i] is None
+ else pre_filters[i].serialize()) for i, split in enumerate(splits)]
+ context = (self._table, self._vector_column, query, search_limit, self._options)
+ results = [None] * len(splits)
+ with closing(_map_tasks(
+ _search_index_split, context, items, self._concurrency, self._remote_args)) as tasks:
+ for i, (metric, scores) in tasks:
+ if metric is not None:
+ self._set_index_metric(metric)
+ # Keep plan order, including duplicate-row precedence, regardless
+ # of worker completion order. Do not refine or truncate per worker.
+ results[i] = DictBasedScoredIndexResult(scores)
+ return results
+
+ def _read_raw_search(self, raw_row_ranges, pre_filter, query_vector,
+ index_type=None, include_filter=True,
+ score_candidates=None, snapshot=None):
+ if score_candidates is not None:
+ # Refinement uses the globally selected indexed candidates.
+ return super()._read_raw_search(
+ raw_row_ranges, pre_filter, query_vector, index_type,
+ include_filter, score_candidates, snapshot)
+
+ ranges = _filtered_raw_row_ranges(raw_row_ranges, pre_filter)
+ if not ranges:
+ return DictBasedScoredIndexResult({})
+ table_read, splits = self._plan_raw_read(ranges, include_filter, snapshot)
+ context = (table_read, self._vector_column, query_vector,
+ self._limit, self._search_metric(index_type))
+ heap = []
+ # Keep each planned data split intact: its files may be vertical column
+ # groups that must be read together to reconstruct a logical row.
+ with closing(_map_tasks(
+ _search_raw_split, context, splits, self._concurrency, self._remote_args)) as tasks:
+ for _, scores in tasks:
+ for row_id, score in scores.items():
+ _offer_score(heap, self._limit, row_id, score)
+ return _scored_result(heap)
+
+
+def _search_index_split(context, item):
+ table, column, query, limit, options = context
+ split, include_bytes = item
+ reader = DataEvolutionVectorRead(table, limit, column, query, options=options)
+ include = None if include_bytes is None else RoaringBitmap64.deserialize(include_bytes)
+ result = reader._eval(
+ split.row_range_start, split.row_range_end,
+ split.vector_index_files, query, limit, include).result()
+ # The reader closes its native resources before returning a completed result.
+ # Return metric even for zero hits, so inconsistent indexes cannot be hidden
+ # by a scalar filter and raw fallback uses the persisted index metric.
+ return reader._index_metric, _scores(result)
+
+
+def _search_raw_split(context, split):
+ from pypaimon.read.table_read import _ClosableArrowBatchReader
+
+ table_read, column, query, limit, metric = context
+ scorer = DataEvolutionVectorRead(table_read.table, limit, column, query)
+ reader, batches = table_read._new_arrow_batch_reader([split])
+ with _ClosableArrowBatchReader(reader, batches) as batch_reader:
+ return _scores(scorer._score_raw_batches(batch_reader, query, metric, reject_nan=True))
+
+
+def _scores(result):
+ if result is None:
+ return {}
+ getter = result.score_getter()
+ scores = {row_id: getter(row_id) for row_id in result.results()}
+ if any(math.isnan(score) for score in scores.values()):
+ raise ValueError("Ray vector search cannot rank NaN scores.")
+ return scores
+
+
+def _map_tasks(worker, context, items, concurrency, remote_args):
+ """Yield completed tasks with their plan ordinal, bounding in-flight work."""
+ import ray
+
+ if not items:
+ return
+ # Share only serializable planning state, never open readers or closures.
+ context_ref = ray.put(context)
+ remote = ray.remote(worker).options(**remote_args)
+ remaining = iter(enumerate(items))
+ pending = {}
+ try:
+ while True:
+ while len(pending) < concurrency:
+ item = next(remaining, None)
+ if item is None:
+ break
+ ordinal, split = item
+ pending[remote.remote(context_ref, split)] = ordinal
+ if not pending:
+ break
+ ready, _ = ray.wait(list(pending), num_returns=1)
+ ref = ready[0]
+ result = ray.get(ref)
+ yield pending.pop(ref), result
+ finally:
+ for ref in pending:
+ try:
+ ray.cancel(ref)
+ except Exception:
+ # Preserve the search failure if the cluster is already gone.
+ pass
diff --git a/paimon-python/pypaimon/table/source/vector_search_read.py b/paimon-python/pypaimon/table/source/vector_search_read.py
index 178a7ea907..489022fe1a 100644
--- a/paimon-python/pypaimon/table/source/vector_search_read.py
+++ b/paimon-python/pypaimon/table/source/vector_search_read.py
@@ -273,7 +273,7 @@ class AbstractVectorSearchReadImpl:
try:
self._record_index_metric(reader, vector_index_files[0].index_type, metric_lock)
return reader, OffsetGlobalIndexReader(reader, row_range_start, row_range_end)
- except Exception:
+ except BaseException:
reader.close()
raise
@@ -314,11 +314,16 @@ class AbstractVectorSearchReadImpl:
if table is None or table.num_rows == 0:
return DictBasedScoredIndexResult({})
+ return self._score_raw_batches(
+ table.to_batches(max_chunksize=_score_block_size(query_vector)),
+ query_vector, self._search_metric(index_type), score_candidates)
+
+ def _score_raw_batches(self, batches, query_vector, metric, score_candidates=None, reject_nan=False):
+ """Score bounded Arrow batches, shared by local and distributed reads."""
+ import math
+
top_k_heap = []
- metric = self._search_metric(index_type)
- block_size = _score_block_size(query_vector)
- for start in range(0, table.num_rows, block_size):
- block = table.slice(start, block_size)
+ for block in batches:
row_ids = block.column(SpecialFields.ROW_ID.name).to_pylist()
vectors = block.column(self._vector_column.name)
if score_candidates is not None:
@@ -332,6 +337,8 @@ class AbstractVectorSearchReadImpl:
row_ids, _iter_arrow_scores(vectors, query_vector, metric)
):
if score is not None:
+ if reject_nan and math.isnan(score):
+ raise ValueError("Ray vector search cannot rank NaN scores.")
_offer_score(top_k_heap, self._limit, row_id, score)
return _scored_result(top_k_heap)
diff --git a/paimon-python/pypaimon/tests/ray_vector_search_test.py b/paimon-python/pypaimon/tests/ray_vector_search_test.py
new file mode 100644
index 0000000000..06a7446863
--- /dev/null
+++ b/paimon-python/pypaimon/tests/ray_vector_search_test.py
@@ -0,0 +1,405 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from contextlib import closing
+from unittest.mock import patch
+
+import numpy as np
+import pyarrow as pa
+import pytest
+
+ray = pytest.importorskip("ray")
+
+import pypaimon.multimodal as pm
+from pypaimon.multimodal.query import ScanQuery
+from pypaimon.ray import vector_search as search_module
+from pypaimon.snapshot.time_travel_util import TimeTravelUtil
+from pypaimon.table.source.vector_search_scan import DataEvolutionVectorScan
+
+
+SCHEMA = pa.schema([
+ ("id", pa.int64()), ("category", pa.string()), ("embedding", pa.list_(pa.float32(), 2))])
+VECTORS = [[0., 1.], [2., 1.], [1., 3.], [-2., 2.], [4., 1.], [1., -3.]]
+
+
+@pytest.fixture(scope="module")
+def ray_cluster():
+ started = not ray.is_initialized()
+ if started:
+ ray.init(address="local", num_cpus=2, include_dashboard=False,
+ object_store_memory=100 * 1024 * 1024)
+ yield
+ if started:
+ ray.shutdown()
+
+
+@pytest.fixture
+def table(tmp_path):
+ return pm.connect(options={"warehouse": str(tmp_path)}).create_table(
+ "vectors", schema=SCHEMA,
+ options={"file.format": "parquet", "vector.file.format": "parquet",
+ "vector-index.search-mode": "full", "read.batch-size": "2",
+ "source.split.target-size": "1 b", "source.split.open-file-cost": "1 b"})
+
+
+def add_rows(table, vectors, start=0):
+ ids = list(range(start, start + len(vectors)))
+ table.add(pa.table({"id": ids, "category": ["yes" if i % 2 == 0 else "no" for i in ids],
+ "embedding": vectors}, schema=SCHEMA))
+
+
+def build_index(table, metric="l2"):
+ pytest.importorskip("paimon_vindex")
+ # Build before any deletes. The index builder currently requires DVs off;
+ # subsequent reads/updates use the table's original DV-enabled options.
+ table.raw_table.copy({"deletion-vectors.enabled": "false"}).create_global_index(
+ "embedding", "ivf-flat", options={
+ "global-index.row-count-per-shard": "3",
+ "ivf-flat.nlist": "1", "ivf-flat.distance.metric": metric})
+
+
+def ids(result):
+ return sorted(result.column("id").to_pylist())
+
+
+@pytest.mark.parametrize("metric", ["l2", "cosine", "inner_product"])
+@pytest.mark.parametrize("concurrency", [1, 2])
+def test_raw_search_matches_exact_top_k(table, ray_cluster, metric, concurrency):
+ add_rows(table, VECTORS[:3])
+ add_rows(table, VECTORS[3:], 3)
+ query_vector = [1., 1.]
+ query = table.search(query_vector, options={"metric": metric}).select(["id"]).limit(3)
+ vectors = np.array(VECTORS)
+ if metric == "l2":
+ scores = -np.sum((vectors - query_vector) ** 2, axis=1)
+ elif metric == "cosine":
+ scores = vectors.dot(query_vector) / (np.linalg.norm(vectors, axis=1) * np.sqrt(2.))
+ else:
+ scores = vectors.dot(query_vector)
+ expected = sorted(sorted(range(len(VECTORS)), key=lambda i: (-scores[i], i))[:3])
+ assert ids(query.to_arrow()) == expected
+ original = search_module._map_tasks
+ dispatched = []
+
+ def record(worker, context, items, *args):
+ dispatched.append(len(items))
+ return original(worker, context, items, *args)
+
+ with patch.object(search_module, "_map_tasks", record):
+ assert ids(query.to_arrow(execution="ray", concurrency=concurrency)) == expected
+ assert dispatched == [2]
+
+
+@pytest.mark.parametrize("metric", ["l2", "cosine", "inner_product"])
+@pytest.mark.parametrize("refine", [False, True])
+def test_indexed_search_and_raw_fallback_use_persisted_metric(table, ray_cluster, metric, refine):
+ add_rows(table, VECTORS)
+ build_index(table, metric)
+ add_rows(table, [[1., 1.], [20., 1.]], 6)
+ # No metric override: raw workers must inherit it from the actual indexes.
+ options = {"ivf.nprobe": "1"}
+ if refine:
+ options["refine_factor"] = "2"
+ query = table.search([1., 1.], options=options).select(["id"]).limit(3)
+ expected = ids(query.to_arrow())
+ observed = []
+ original = search_module._map_tasks
+
+ def record(worker, context, items, *args):
+ observed.append((worker.__name__, len(items)))
+ return original(worker, context, items, *args)
+
+ with patch.object(search_module, "_map_tasks", record):
+ assert ids(query.to_arrow(execution="ray", concurrency=2)) == expected
+ assert ("_search_index_split", 2) in observed
+ assert any(name == "_search_raw_split" and count > 0 for name, count in observed)
+
+
+def test_filters_deletions_and_column_updates(table, ray_cluster):
+ add_rows(table, VECTORS)
+ build_index(table)
+ add_rows(table, [[1., 1.], [0., 1.]], 6)
+ table.update("id = 2", {"category": "no"})
+ table.delete("id = 0")
+ # The scalar filter has no index, so fallback overlaps vector-index coverage.
+ query = table.search([1., 1.], pre_filter="category = 'yes'").select(["id"]).limit(3)
+ assert ids(query.to_arrow()) == [4, 6]
+ assert ids(query.to_arrow(execution="ray", concurrency=2)) == [4, 6]
+ # Post-filtering keeps its existing meaning: it need not fill the requested K.
+ query = table.search([1., 1.]).where("id = 4").select(["id"]).limit(1)
+ assert query.to_arrow().num_rows == 0
+ assert query.to_arrow(execution="ray").num_rows == 0
+ assert table.search([1., 1.], pre_filter="category = 'missing'").to_arrow(execution="ray").num_rows == 0
+
+
+def test_partition_filter(tmp_path, ray_cluster):
+ table = pm.connect(options={"warehouse": str(tmp_path)}).create_table(
+ "partitioned", schema=SCHEMA, partitioned=["category"],
+ options={"file.format": "parquet", "vector.file.format": "parquet",
+ "vector-index.search-mode": "full"})
+ add_rows(table, VECTORS)
+ query = table.search([1., 1.], column="embedding", pre_filter="category = 'yes'").select(["id"]).limit(10)
+ assert ids(query.to_arrow()) == [0, 2, 4]
+ assert ids(query.to_arrow(execution="ray")) == [0, 2, 4]
+
+
+def test_fast_mode_does_not_scan_unindexed_rows(table, ray_cluster):
+ add_rows(table, VECTORS)
+ build_index(table)
+ add_rows(table, [[1., 1.]], 6)
+ table.raw_table = table.raw_table.copy({"vector-index.search-mode": "fast"})
+ query = table.search([1., 1.]).select(["id"]).limit(2)
+ with patch.object(search_module, "_search_raw_split", side_effect=AssertionError("raw scan")):
+ result = query.to_arrow(execution="ray")
+ assert ids(result) == ids(query.to_arrow())
+ assert 6 not in ids(result)
+
+
+@pytest.mark.parametrize("change", ["update", "delete"])
+def test_query_pins_worker_reads_and_lookup_but_can_be_reused(table, ray_cluster, change):
+ add_rows(table, [[1., 1.], [10., 1.]])
+ query = table.search([1., 1.]).select(["id", "category"]).limit(1)
+ expected = query.to_arrow().to_pylist()
+ original = search_module._map_tasks
+ changed = []
+
+ def change_before_dispatch(*args):
+ if not changed:
+ if change == "delete":
+ table.delete("id = 0")
+ else:
+ table.update("id = 0", {"category": "changed"})
+ changed.append(True)
+ return original(*args)
+
+ with patch.object(search_module, "_map_tasks", change_before_dispatch):
+ assert query.to_arrow(execution="ray").to_pylist() == expected
+ latest = query.to_arrow().to_pylist()
+ assert latest != expected
+ assert query.to_arrow(execution="ray").to_pylist() == latest
+
+
+def test_empty_snapshot_does_not_follow_first_commit(table, ray_cluster):
+ query = table.search([1., 1.]).select(["id"])
+ scan = DataEvolutionVectorScan.scan
+
+ def append_before_scan(search_scan):
+ add_rows(table, [[1., 1.]])
+ return scan(search_scan)
+
+ with patch.object(DataEvolutionVectorScan, "scan", append_before_scan), \
+ patch.object(search_module, "_map_tasks", side_effect=AssertionError("empty task")):
+ result = query.to_arrow(execution="ray")
+ assert result.schema.names == ["id"]
+ assert result.num_rows == 0
+ assert ids(query.to_arrow(execution="ray")) == [0]
+
+
+@pytest.mark.parametrize("selector", ["snapshot", "tag"])
+def test_snapshot_and_retained_tag_metadata_reach_workers(table, ray_cluster, selector):
+ add_rows(table, VECTORS)
+ snapshot = table.raw_table.snapshot_manager().get_latest_snapshot()
+ table.raw_table.create_tag("saved", snapshot_id=snapshot.id)
+ add_rows(table, [[1., 1.]], 6)
+ if selector == "tag":
+ table.raw_table.file_io.delete(table.raw_table.snapshot_manager().get_snapshot_path(snapshot.id))
+ query = table.search([1., 1.], tag_name="saved")
+ else:
+ query = table.search([1., 1.], snapshot_id=snapshot.id)
+ query.select(["id"]).limit(2)
+ assert ids(query.to_arrow(execution="ray")) == ids(query.to_arrow()) == [0, 1]
+
+
+def test_lookup_is_pinned_after_worker_completion(table, ray_cluster):
+ add_rows(table, [[1., 1.], [10., 1.]])
+ original = ScanQuery._read_global_index_result
+
+ def delete_before_lookup(query, result):
+ table.delete("id = 0")
+ return original(query, result)
+
+ with patch.object(ScanQuery, "_read_global_index_result", delete_before_lookup):
+ assert ids(table.search([1., 1.]).select(["id"]).limit(1).to_arrow(execution="ray")) == [0]
+
+
+def test_warmed_read_context_and_distinct_worker_processes(table, ray_cluster):
+ add_rows(table, VECTORS[:3])
+ add_rows(table, VECTORS[3:], 3)
+ query = table.search([1., 1.]).select(["id"]).limit(2)
+ query.to_arrow() # Warm filesystem and metadata state before serialization.
+ pinned = query._for_execution()._table
+
+ def inspect_context(table, ordinal):
+ import time
+ time.sleep(0.2)
+ return os.getpid(), TimeTravelUtil.resolve_snapshot(table).id
+
+ results = list(search_module._map_tasks(inspect_context, pinned, [0, 1], 2, {"num_cpus": 1}))
+ assert len({result[0] for _, result in results}) == 2
+ assert all(pid != os.getpid() and snapshot == 2 for _, (pid, snapshot) in results)
+ assert ids(query.to_arrow(execution="ray")) == [0, 1]
+
+
+def test_task_retry_keeps_original_snapshot(table, ray_cluster, tmp_path):
+ add_rows(table, [[1., 1.]])
+ pinned = table.search([1., 1.])._for_execution()._table
+ add_rows(table, [[2., 1.]], 1)
+ marker = str(tmp_path / "retry-marker")
+
+ def retry_once(table, marker):
+ try:
+ fd = os.open(marker, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+ except FileExistsError:
+ builder = table.new_read_builder().with_projection(["id"])
+ data = builder.new_read().to_arrow(builder.new_scan().plan().splits())
+ return TimeTravelUtil.resolve_snapshot(table).id, data.column("id").to_pylist()
+ os.close(fd)
+ raise RuntimeError("retry this read")
+
+ assert list(search_module._map_tasks(
+ retry_once, pinned, [marker], 1, {"max_retries": 1, "retry_exceptions": True})) == [(0, (1, [0]))]
+
+
+def test_failure_cancels_outstanding_tasks(ray_cluster):
+ def fail(context, item):
+ if item == 0:
+ raise ValueError("injected search failure")
+ import time
+ time.sleep(30)
+
+ with patch.object(ray, "cancel", wraps=ray.cancel) as cancel:
+ with pytest.raises(ValueError, match="injected search failure"):
+ list(search_module._map_tasks(fail, None, [0, 1, 2], 2, {}))
+ assert cancel.call_count == 2
+
+
+def test_map_tasks_bounds_submissions_and_cancels_on_early_close(ray_cluster):
+ def echo(context, item):
+ return item
+
+ wait = ray.wait
+ counts = []
+
+ def record_wait(refs, **kwargs):
+ counts.append(len(refs))
+ return wait(refs, **kwargs)
+
+ with patch.object(ray, "wait", record_wait), patch.object(ray, "cancel", wraps=ray.cancel) as cancel:
+ with closing(search_module._map_tasks(echo, None, list(range(10)), 2, {})) as tasks:
+ next(tasks)
+ assert counts == [2]
+ assert cancel.call_count == 1
+
+
+def test_refinement_uses_global_candidates_despite_reverse_completion_order(table):
+ from pypaimon.table.source.vector_search_split import IndexVectorSearchSplit
+ from pypaimon.index.index_file_meta import IndexFileMeta
+
+ add_rows(table, [[0., 1.], [10., 1.], [20., 1.], [1., 1.]])
+ query = table.search([1., 1.], options={"refine_factor": "2"}).limit(1)._for_execution()
+ reader = query._vector_search_builder(query).new_vector_search_read()
+ distributed = search_module._RayVectorSearchRead(reader, 2, {})
+ index_file = IndexFileMeta("ivf-flat", "fake", 1, 2, None)
+ splits = [IndexVectorSearchSplit(0, 1, [index_file]), IndexVectorSearchSplit(2, 3, [index_file])]
+
+ def completed(*args):
+ # Row 3 is the true nearest, but is outside the GLOBAL approximate top-2.
+ # Refining independently per shard would incorrectly bring it back in.
+ yield 1, ("l2", {2: 8., 3: 7.})
+ yield 0, ("l2", {0: 10., 1: 9.})
+
+ with patch.object(search_module, "_map_tasks", completed):
+ result = distributed._read_indexed(splits, [1., 1.], query._table._read_snapshot)
+ assert list(result.results()) == [0]
+ assert result.score_getter()(0) == 0.5 # L2 score is 1 / (1 + squared distance).
+
+
+def test_duplicate_scores_keep_plan_order(table):
+ from pypaimon.table.source.vector_search_split import IndexVectorSearchSplit
+ from pypaimon.index.index_file_meta import IndexFileMeta
+
+ query = table.search([1., 1.]).limit(2)._for_execution()
+ reader = query._vector_search_builder(query).new_vector_search_read()
+ distributed = search_module._RayVectorSearchRead(reader, 2, {})
+ index_file = IndexFileMeta("ivf-flat", "fake", 1, 2)
+ splits = [IndexVectorSearchSplit(0, 1, [index_file]), IndexVectorSearchSplit(0, 3, [index_file])]
+
+ def completed(*args):
+ yield 1, ("l2", {0: 100.})
+ yield 0, ("l2", {0: 1., 1: 2.})
+
+ with patch.object(search_module, "_map_tasks", completed):
+ result = distributed._read_indexed(splits, [1., 1.], None)
+ assert result.score_getter()(0) == 1.
+ assert list(result.top_k(1).results()) == [1]
+
+
+def test_metric_mismatch_fails_even_when_one_shard_has_no_hits(table):
+ from pypaimon.table.source.vector_search_split import IndexVectorSearchSplit
+
+ query = table.search([1., 1.])
+ reader = query._vector_search_builder(query).new_vector_search_read()
+ distributed = search_module._RayVectorSearchRead(reader, 2, {})
+ closed = []
+
+ def completed(*args):
+ try:
+ yield 0, ("l2", {})
+ yield 1, ("inner_product", {1: 10.})
+ finally:
+ closed.append(True)
+
+ with patch.object(search_module, "_map_tasks", completed), \
+ pytest.raises(ValueError, match="different metrics"):
+ distributed._search_index_splits(
+ [IndexVectorSearchSplit(0, 1, []), IndexVectorSearchSplit(2, 3, [])], [1., 1.], 1, [])
+ assert closed == [True]
+
+
+@pytest.mark.parametrize("kwargs, message", [
+ ({"execution": "unknown"}, "execution must"),
+ ({"concurrency": 2}, "require execution"),
+ ({"execution": "ray", "concurrency": 0}, "positive integer"),
+ ({"execution": "ray", "concurrency": True}, "positive integer"),
+ ({"execution": "ray", "concurrency": 1.5}, "positive integer"),
+ ({"execution": "ray", "ray_remote_args": {"num_returns": 2}}, "num_returns"),
+])
+def test_invalid_execution_options(table, kwargs, message):
+ with pytest.raises(ValueError, match=message):
+ table.search([1., 1.]).to_arrow(**kwargs)
+
+
+def test_rejects_non_data_evolution_table(table):
+ query = table.search([1., 1.])
+ with patch.object(query._table.options.__class__, "data_evolution_enabled", return_value=False), \
+ pytest.raises(ValueError, match="only data-evolution tables"):
+ query.to_arrow(execution="ray")
+
+
+@pytest.mark.parametrize("value", [float("nan"), float("inf"), -float("inf")])
+def test_rejects_nonfinite_query_before_dispatch(table, value):
+ with pytest.raises(ValueError, match="finite query vector"):
+ table.search([value, 1.]).to_arrow(execution="ray")
+
+
+def test_nan_scores_fail_before_worker_top_k(table, ray_cluster):
+ add_rows(table, [[1., 1.], [float("nan"), 1.], [10., 1.]])
+ # NaN may be discarded by a local heap. Validate before that truncation,
+ # otherwise a different split layout can silently change the answer.
+ with pytest.raises(ValueError, match="cannot rank NaN"):
+ table.search([1., 1.]).limit(1).to_arrow(execution="ray")
diff --git a/paimon-python/pypaimon/tests/vector_metric_consistency_test.py b/paimon-python/pypaimon/tests/vector_metric_consistency_test.py
index 8b761efada..bdb29735a3 100644
--- a/paimon-python/pypaimon/tests/vector_metric_consistency_test.py
+++ b/paimon-python/pypaimon/tests/vector_metric_consistency_test.py
@@ -178,22 +178,41 @@ class VectorMetricResolutionTest(unittest.TestCase):
def test_reader_closes_on_metadata_and_metric_errors(self):
entry = _entry(None, field_id=1, index_type='ivf-flat', file_name='vectors.index',
row_range_start=0, row_range_end=1)
- for failure in ('metadata', 'query', 'shard'):
+ for failure in ('metadata', 'query', 'shard', 'interrupt'):
with self.subTest(failure=failure):
native = Mock(spec=['vector_metric', 'close'])
native.vector_metric.return_value = 'inner_product'
reader = self._reader({'metric': 'l2'} if failure == 'query' else {})
if failure == 'metadata':
native.vector_metric.side_effect = RuntimeError('invalid index metadata')
+ if failure == 'interrupt':
+ native.vector_metric.side_effect = KeyboardInterrupt()
if failure == 'shard':
previous = Mock(spec=['vector_metric'])
previous.vector_metric.return_value = 'cosine'
reader._record_index_metric(previous, 'ivf-flat')
with patch('pypaimon.table.source.vector_search_read._create_vector_reader', return_value=native):
- with self.assertRaises((RuntimeError, ValueError)):
+ with self.assertRaises((RuntimeError, ValueError, KeyboardInterrupt)):
reader._open_offset_reader([entry.index_file], 0, 1)
native.close.assert_called_once_with()
+ def test_lumina_open_closes_stream_on_interrupt(self):
+ from pypaimon.globalindex.global_index_meta import GlobalIndexIOMeta
+ from pypaimon.globalindex.lumina.lumina_vector_global_index_reader import LuminaVectorGlobalIndexReader
+
+ stream, native, file_io = Mock(), Mock(), Mock()
+ file_io.new_input_stream.return_value = stream
+ native.open_stream.side_effect = KeyboardInterrupt()
+ meta = GlobalIndexIOMeta(
+ file_name='vectors.index', file_size=1,
+ metadata=b'{"index.dimension":"1","distance.metric":"l2"}')
+ with patch.dict('sys.modules', {'lumina_data': Mock(LuminaSearcher=Mock(return_value=native))}):
+ with self.assertRaises(KeyboardInterrupt):
+ with LuminaVectorGlobalIndexReader(file_io, '/index', [meta]) as reader:
+ reader._ensure_loaded()
+ stream.close.assert_called_once_with()
+ native.close.assert_called_once_with()
+
def test_metric_does_not_leak_between_read_calls(self):
_install_raw_vector_read_builder(self.table, 'embedding', {0: [2.0], 1: [3.0]})
split = RawVectorSearchSplit([Range(0, 1)], [], 'ivf-flat')
diff --git a/paimon-python/pypaimon/tests/vindex_input_test.py b/paimon-python/pypaimon/tests/vindex_input_test.py
index 892231462d..8e5935425b 100644
--- a/paimon-python/pypaimon/tests/vindex_input_test.py
+++ b/paimon-python/pypaimon/tests/vindex_input_test.py
@@ -182,7 +182,12 @@ class VindexInputTest(unittest.TestCase):
def test_reader_releases_workers_when_native_close_fails(self):
self._check_reader_cleanup("close")
- def _check_reader_cleanup(self, phase):
+ def test_reader_releases_workers_when_initialization_is_interrupted(self):
+ for phase in ("constructor", "initialize"):
+ with self.subTest(phase=phase):
+ self._check_reader_cleanup(phase, KeyboardInterrupt)
+
+ def _check_reader_cleanup(self, phase, error_type=OSError):
stream = mock.Mock(spec=["read_at", "close"])
stream.read_at.return_value = b"x"
io_ = mock.Mock()
@@ -199,7 +204,7 @@ class VindexInputTest(unittest.TestCase):
raise error
return native
- error = OSError("native reader failed")
+ error = error_type("native reader failed")
if phase == "initialize":
native.optimize_for_search.side_effect = error
if phase == "close":
@@ -210,7 +215,7 @@ class VindexInputTest(unittest.TestCase):
reader = VindexVectorGlobalIndexReader(
io_, "s3://bucket", [GlobalIndexIOMeta(file_name="index", file_size=2)])
if phase in ("constructor", "initialize"):
- with self.assertRaises(OSError) as raised:
+ with self.assertRaises(error_type) as raised:
reader._ensure_loaded()
self.assertIs(error, raised.exception)
else:
(paimon) 15/20: [core] Separate commit history bounds from strict mode (#9960)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 93b899b6b21966e22c343363a82521546ad6151d
Author: YeJunHao <41...@users.noreply.github.com>
AuthorDate: Fri Sep 18 19:05:45 2026 +0800
[core] Separate commit history bounds from strict mode (#9960)
---
docs/generated/core_configuration.html | 14 +++--
.../main/java/org/apache/paimon/CoreOptions.java | 41 +++++++++++--
.../paimon/operation/FileStoreCommitImpl.java | 9 +--
.../paimon/operation/commit/StrictModeChecker.java | 18 +++---
.../org/apache/paimon/table/PostponeUtils.java | 4 +-
.../java/org/apache/paimon/CoreOptionsTest.java | 32 +++++++++++
.../paimon/operation/FileStoreCommitTest.java | 39 +++++++++++++
.../org/apache/paimon/table/PostponeUtilsTest.java | 2 +-
.../apache/paimon/table/sink/TableCommitTest.java | 67 ++++++++++++++--------
.../flink/action/DataEvolutionMergeIntoAction.java | 2 +-
.../apache/paimon/flink/action/RescaleAction.java | 3 +-
.../dataevolution/DataEvolutionDeleteSink.java | 3 +-
.../action/DeleteActionDataEvolutionITCase.java | 2 +-
.../paimon/spark/procedure/RescaleProcedure.java | 2 +-
.../spark/SparkPostponeStagedCommitter.scala | 4 +-
.../paimon/spark/commands/PaimonSparkWriter.scala | 4 +-
16 files changed, 183 insertions(+), 63 deletions(-)
diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html
index 2fb5acc403..4c8538b4f1 100644
--- a/docs/generated/core_configuration.html
+++ b/docs/generated/core_configuration.html
@@ -326,6 +326,12 @@ under the License.
<td>Boolean</td>
<td>In streaming job, whether to force creating snapshot when there is no data in this write-commit phase.</td>
</tr>
+ <tr>
+ <td><h5>commit.last-safe-snapshot</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>Long</td>
+ <td>Snapshot preceding the earliest snapshot to inspect when committing. Only later snapshots are searched for this commit user's previous commits. This also provides the starting point for strict-mode checks when enabled. Keep this bound unchanged across retries and recovery.</td>
+ </tr>
<tr>
<td><h5>commit.max-retries</h5></td>
<td style="word-wrap: break-word;">10</td>
@@ -345,10 +351,10 @@ under the License.
<td>Min retry wait time when commit failed.</td>
</tr>
<tr>
- <td><h5>commit.strict-mode.last-safe-snapshot</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>Long</td>
- <td>If set, committer will check if there are other commit user's snapshot starting from the snapshot after this one. If found a COMPACT / OVERWRITE snapshot, or found a APPEND snapshot which committed files to fixed bucket, commit will be aborted.If the value of this option is -1, committer will not check for its first commit.</td>
+ <td><h5>commit.strict-mode.enabled</h5></td>
+ <td style="word-wrap: break-word;">true</td>
+ <td>Boolean</td>
+ <td>Whether to check concurrent snapshot changes after commit.last-safe-snapshot, when that bound is configured. Rejects COMPACT or OVERWRITE changes in the same partition, and fixed-bucket APPEND changes when committing OVERWRITE. Disabling this does not disable regular conflict detection or the history search bound.</td>
</tr>
<tr>
<td><h5>commit.timeout</h5></td>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index 60605bc74c..3842623860 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -2461,15 +2461,34 @@ public class CoreOptions implements Serializable {
+ "instead of at the end of the schema. "
+ "This only takes effect for partitioned tables.");
+ public static final ConfigOption<Long> COMMIT_LAST_SAFE_SNAPSHOT =
+ ConfigOptions.key("commit.last-safe-snapshot")
+ .longType()
+ .noDefaultValue()
+ .withFallbackKeys("commit.strict-mode.last-safe-snapshot")
+ .withDescription(
+ "Snapshot preceding the earliest snapshot to inspect when committing. "
+ + "Only later snapshots are searched for this commit user's previous commits. "
+ + "This also provides the starting point for strict-mode checks when enabled. "
+ + "Keep this bound unchanged across retries and recovery.");
+
+ public static final ConfigOption<Boolean> COMMIT_STRICT_MODE_ENABLED =
+ ConfigOptions.key("commit.strict-mode.enabled")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription(
+ "Whether to check concurrent snapshot changes after commit.last-safe-snapshot, "
+ + "when that bound is configured. Rejects COMPACT or OVERWRITE changes "
+ + "in the same partition, and fixed-bucket APPEND changes when committing OVERWRITE. "
+ + "Disabling this does not disable regular conflict detection or the history search bound.");
+
+ /** @deprecated Use {@link #COMMIT_LAST_SAFE_SNAPSHOT}. */
+ @Deprecated
public static final ConfigOption<Long> COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT =
ConfigOptions.key("commit.strict-mode.last-safe-snapshot")
.longType()
.noDefaultValue()
- .withDescription(
- "If set, committer will check if there are other commit user's snapshot starting from the "
- + "snapshot after this one. If found a COMPACT / OVERWRITE snapshot, or found a "
- + "APPEND snapshot which committed files to fixed bucket, commit will be aborted."
- + "If the value of this option is -1, committer will not check for its first commit.");
+ .withDescription("Deprecated alias for commit.last-safe-snapshot.");
public static final ConfigOption<String> CLUSTERING_COLUMNS =
key("clustering.columns")
@@ -4612,8 +4631,18 @@ public class CoreOptions implements Serializable {
return options.get(AGGREGATION_REMOVE_RECORD_ON_DELETE);
}
+ public Optional<Long> commitLastSafeSnapshot() {
+ return options.getOptional(COMMIT_LAST_SAFE_SNAPSHOT);
+ }
+
+ public boolean commitStrictModeEnabled() {
+ return options.get(COMMIT_STRICT_MODE_ENABLED);
+ }
+
+ /** @deprecated Use {@link #commitLastSafeSnapshot()}. */
+ @Deprecated
public Optional<Long> commitStrictModeLastSafeSnapshot() {
- return options.getOptional(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT);
+ return commitLastSafeSnapshot();
}
public List<String> clusteringColumns() {
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
index 0f60d6a3cb..e867a1cc85 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
@@ -222,7 +222,8 @@ public class FileStoreCommitImpl implements FileStoreCommit {
this.statsFileHandler = statsFileHandler;
this.bucketMode = bucketMode;
this.strictModeChecker =
- options.commitStrictModeLastSafeSnapshot()
+ options.commitLastSafeSnapshot()
+ .filter(ignored -> options.commitStrictModeEnabled())
.map(
id ->
new StrictModeChecker(
@@ -294,14 +295,14 @@ public class FileStoreCommitImpl implements FileStoreCommit {
"Committables must be sorted according to identifiers before filtering. This is unexpected.");
}
- Optional<Long> optionalStrictSnapshot = options.commitStrictModeLastSafeSnapshot();
+ Optional<Long> lastSafeSnapshot = options.commitLastSafeSnapshot();
Optional<Snapshot> latestSnapshot;
- if (optionalStrictSnapshot.isPresent()) {
+ if (lastSafeSnapshot.isPresent()) {
latestSnapshot =
snapshotManager.latestSnapshotOfUser(
commitUser,
snapshotManager.latestSnapshotId(),
- optionalStrictSnapshot.get() + 1);
+ lastSafeSnapshot.get() + 1);
} else {
latestSnapshot = snapshotManager.latestSnapshotOfUser(commitUser);
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/StrictModeChecker.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/StrictModeChecker.java
index 00d94845ea..dc93bf8dc9 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/StrictModeChecker.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/StrictModeChecker.java
@@ -44,7 +44,7 @@ public class StrictModeChecker {
private final IndexManifestFile indexManifestFile;
private final boolean dataEvolutionEnabled;
- private long strictModeLastSafeSnapshot;
+ private long lastSafeSnapshot;
public StrictModeChecker(
SnapshotManager snapshotManager,
@@ -52,19 +52,19 @@ public class StrictModeChecker {
Supplier<FileStoreScan> scanSupplier,
IndexManifestFile indexManifestFile,
boolean dataEvolutionEnabled,
- long strictModeLastSafeSnapshot) {
+ long lastSafeSnapshot) {
this.snapshotManager = snapshotManager;
this.commitUser = commitUser;
this.scanSupplier = scanSupplier;
this.indexManifestFile = indexManifestFile;
this.dataEvolutionEnabled = dataEvolutionEnabled;
- this.strictModeLastSafeSnapshot = strictModeLastSafeSnapshot;
+ this.lastSafeSnapshot = lastSafeSnapshot;
}
public void check(
long newSnapshotId, CommitKind newCommitKind, List<BinaryRow> newChangedPartitions) {
Set<BinaryRow> newPartitions = new HashSet<>(newChangedPartitions);
- for (long id = strictModeLastSafeSnapshot + 1; id < newSnapshotId; id++) {
+ for (long id = lastSafeSnapshot + 1; id < newSnapshotId; id++) {
Snapshot snapshot = snapshotManager.snapshot(id);
if (snapshot.commitUser().equals(commitUser)) {
continue;
@@ -84,13 +84,13 @@ public class StrictModeChecker {
String.format(
"When trying to commit snapshot %d, "
+ "commit user %s has found a %s snapshot (id: %d) by another user %s "
- + "which modified the same partition. Giving up committing as %s is set.",
+ + "which modified the same partition. Giving up committing as %s is true.",
newSnapshotId,
commitUser,
snapshot.commitKind().name(),
id,
snapshot.commitUser(),
- CoreOptions.COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key()));
+ CoreOptions.COMMIT_STRICT_MODE_ENABLED.key()));
}
}
if (snapshot.commitKind() == CommitKind.APPEND
@@ -109,12 +109,12 @@ public class StrictModeChecker {
"When trying to commit snapshot %d, "
+ "commit user %s has found a APPEND snapshot (id: %d) by another user %s "
+ "which committed files to fixed bucket on the same partition. "
- + "Giving up committing as %s is set.",
+ + "Giving up committing as %s is true.",
newSnapshotId,
commitUser,
id,
snapshot.commitUser(),
- CoreOptions.COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key()));
+ CoreOptions.COMMIT_STRICT_MODE_ENABLED.key()));
}
}
}
@@ -173,6 +173,6 @@ public class StrictModeChecker {
}
public void update(long newSafeSnapshot) {
- strictModeLastSafeSnapshot = newSafeSnapshot;
+ lastSafeSnapshot = newSafeSnapshot;
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/PostponeUtils.java b/paimon-core/src/main/java/org/apache/paimon/table/PostponeUtils.java
index 7ea59f78a9..1aeaf94096 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/PostponeUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/PostponeUtils.java
@@ -54,7 +54,7 @@ import java.util.Optional;
import java.util.Set;
import static org.apache.paimon.CoreOptions.BUCKET;
-import static org.apache.paimon.CoreOptions.COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT;
+import static org.apache.paimon.CoreOptions.COMMIT_LAST_SAFE_SNAPSHOT;
import static org.apache.paimon.CoreOptions.WRITE_ONLY;
import static org.apache.paimon.utils.Preconditions.checkArgument;
@@ -511,7 +511,7 @@ public class PostponeUtils {
Map<String, String> rewriteOptions = new HashMap<>();
rewriteOptions.put(BUCKET.key(), String.valueOf(numBuckets));
rewriteOptions.put(WRITE_ONLY.key(), "false");
- rewriteOptions.put(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(), String.valueOf(snapshotId));
+ rewriteOptions.put(COMMIT_LAST_SAFE_SNAPSHOT.key(), String.valueOf(snapshotId));
return table.copy(rewriteOptions);
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
index e83ca0d4ac..585dd718dd 100644
--- a/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
@@ -29,6 +29,38 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Tests for {@link org.apache.paimon.CoreOptions}. */
public class CoreOptionsTest {
+ @Test
+ void testCommitLastSafeSnapshotAndStrictMode() {
+ Options options = new Options();
+ CoreOptions core = new CoreOptions(options);
+ assertThat(core.commitLastSafeSnapshot()).isEmpty();
+ assertThat(core.commitStrictModeEnabled()).isTrue();
+
+ options.setString("commit.strict-mode.last-safe-snapshot", "7");
+ assertThat(core.commitLastSafeSnapshot()).contains(7L);
+ assertThat(core.commitStrictModeEnabled()).isTrue();
+
+ options.set(CoreOptions.COMMIT_LAST_SAFE_SNAPSHOT, 11L);
+ assertThat(core.commitLastSafeSnapshot()).contains(11L);
+
+ options.set(CoreOptions.COMMIT_STRICT_MODE_ENABLED, false);
+ assertThat(core.commitStrictModeEnabled()).isFalse();
+ assertThat(core.commitLastSafeSnapshot()).contains(11L);
+ }
+
+ @Test
+ @SuppressWarnings("deprecation")
+ void testLegacyCommitLastSafeSnapshotApi() {
+ Options options = new Options();
+ options.set(CoreOptions.COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT, 7L);
+ CoreOptions core = new CoreOptions(options);
+ assertThat(core.commitLastSafeSnapshot()).contains(7L);
+ assertThat(core.commitStrictModeLastSafeSnapshot()).contains(7L);
+
+ options.set(CoreOptions.COMMIT_LAST_SAFE_SNAPSHOT, 11L);
+ assertThat(core.commitStrictModeLastSafeSnapshot()).contains(11L);
+ }
+
@Test
void testManifestSidecarDefaultsToManifestSort() {
assertThat(CoreOptions.MANIFEST_SIDECAR_ENABLED.defaultValue()).isNull();
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
index 437944fae5..7a0d26b160 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
@@ -367,6 +367,45 @@ public class FileStoreCommitTest {
}
}
+ @ParameterizedTest
+ @ValueSource(strings = {"commit.last-safe-snapshot", "commit.strict-mode.last-safe-snapshot"})
+ public void testFilterCommittedWithStrictModeDisabled(String lastSafeKey) throws Exception {
+ Map<String, String> options = new HashMap<>();
+ options.put(lastSafeKey, "2");
+ options.put(CoreOptions.COMMIT_STRICT_MODE_ENABLED.key(), "false");
+ TestFileStore store = createStore(false, options);
+ try (FileStoreCommit commit = store.newCommit("older-user", null)) {
+ commit.ignoreEmptyCommit(false);
+ commit.commit(new ManifestCommittable(1), false);
+ commit.commit(new ManifestCommittable(2), false);
+ }
+
+ // A disabled strict checker must still honor the search bound. Fail if the lookup
+ // reaches old history, rather than relying on timing to detect a full history scan.
+ Path oldSnapshot = store.snapshotManager().snapshotPath(1);
+ store.fileIO().deleteQuietly(oldSnapshot);
+ store.fileIO().writeFile(oldSnapshot, "not a snapshot", false);
+ store.snapshotManager().invalidateCache();
+ ManifestCommittable pending = new ManifestCommittable(10);
+ try (FileStoreCommit commit = store.newCommit("new-user", null)) {
+ assertThat(commit.filterCommitted(Collections.singletonList(pending)))
+ .containsExactly(pending);
+ commit.ignoreEmptyCommit(false);
+ commit.commit(pending, false);
+ }
+ try (FileStoreCommit commit = store.newCommit("other-user", null)) {
+ commit.ignoreEmptyCommit(false);
+ commit.commit(new ManifestCommittable(1), false);
+ }
+
+ // Recovery after a successful commit with a lost response still deduplicates it.
+ try (FileStoreCommit recovered = store.newCommit("new-user", null)) {
+ ManifestCommittable next = new ManifestCommittable(11);
+ assertThat(recovered.filterCommitted(Arrays.asList(pending, next)))
+ .containsExactly(next);
+ }
+ }
+
protected void testRandomConcurrentNoConflict(
int numThreads, boolean failing, CoreOptions.ChangelogProducer changelogProducer)
throws Exception {
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/PostponeUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/table/PostponeUtilsTest.java
index d6c6ddc5d3..b117fa1bda 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/PostponeUtilsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/PostponeUtilsTest.java
@@ -253,7 +253,7 @@ public class PostponeUtilsTest {
assertThat(options.getValue())
.containsEntry("bucket", "4")
.containsEntry("write-only", "false")
- .containsEntry("commit.strict-mode.last-safe-snapshot", "5");
+ .containsEntry("commit.last-safe-snapshot", "5");
}
@Test
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
index 4c1a3a94e9..00eb30d2f9 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
@@ -52,6 +52,7 @@ import org.apache.paimon.utils.SnapshotManager;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
import java.util.Arrays;
@@ -68,7 +69,7 @@ import java.util.stream.Collectors;
import java.util.stream.LongStream;
import static java.util.Collections.singletonMap;
-import static org.apache.paimon.CoreOptions.COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT;
+import static org.apache.paimon.CoreOptions.COMMIT_LAST_SAFE_SNAPSHOT;
import static org.apache.paimon.utils.FileStorePathFactoryTest.createNonPartFactory;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
@@ -388,8 +389,15 @@ public class TableCommitTest {
commit1.close();
}
- @Test
- public void testStrictModeForCompact() throws Exception {
+ @ParameterizedTest
+ @CsvSource({
+ "commit.last-safe-snapshot, true",
+ "commit.last-safe-snapshot, false",
+ "commit.strict-mode.last-safe-snapshot, true",
+ "commit.strict-mode.last-safe-snapshot, false"
+ })
+ public void testStrictModeForCompact(String lastSafeKey, boolean strictEnabled)
+ throws Exception {
String path = tempDir.toString();
RowType rowType =
RowType.of(
@@ -428,7 +436,11 @@ public class TableCommitTest {
// test skip this commit check
String user2 = UUID.randomUUID().toString();
- table = table.copy(singletonMap(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(), "2"));
+ Map<String, String> commitOptions = new HashMap<>();
+ commitOptions.put(lastSafeKey, "2");
+ commitOptions.put(
+ CoreOptions.COMMIT_STRICT_MODE_ENABLED.key(), String.valueOf(strictEnabled));
+ table = table.copy(commitOptions);
TableWriteImpl<?> write2 = table.newWrite(user2);
TableCommitImpl commit2 = table.newCommit(user2);
@@ -444,16 +456,21 @@ public class TableCommitTest {
assertThatCode(() -> commit2.commit(2, write2.prepareCommit(false, 2)))
.doesNotThrowAnyException();
- // COMPACT on the same partition should be checked and fail
+ // COMPACT on the same partition is rejected only when strict mode is enabled.
write1.write(GenericRow.of(2, 6, 6L));
write1.compact(pt2, 0, true);
commit1.commit(3, write1.prepareCommit(true, 3));
write2.write(GenericRow.of(2, 7, 7L));
- assertThatThrownBy(() -> commit2.commit(3, write2.prepareCommit(false, 3)))
- .isInstanceOf(RuntimeException.class)
- .hasMessageContaining(
- "Giving up committing as commit.strict-mode.last-safe-snapshot is set.");
+ if (strictEnabled) {
+ assertThatThrownBy(() -> commit2.commit(3, write2.prepareCommit(false, 3)))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining(
+ "Giving up committing as commit.strict-mode.enabled is true.");
+ } else {
+ commit2.commit(3, write2.prepareCommit(false, 3));
+ assertThat(table.snapshotManager().latestSnapshot().commitUser()).isEqualTo(user2);
+ }
write1.close();
commit1.close();
@@ -502,7 +519,7 @@ public class TableCommitTest {
// test skip this commit check
String user2 = UUID.randomUUID().toString();
FileStoreTable tableWithStrict =
- table.copy(singletonMap(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(), "2"));
+ table.copy(singletonMap(COMMIT_LAST_SAFE_SNAPSHOT.key(), "2"));
TableWriteImpl<?> write2 = tableWithStrict.newWrite(user2);
TableCommitImpl commit2 = tableWithStrict.newCommit(user2);
@@ -532,7 +549,7 @@ public class TableCommitTest {
assertThatThrownBy(() -> commit2.commit(3, write2.prepareCommit(true, 3)))
.isInstanceOf(RuntimeException.class)
.hasMessageContaining(
- "Giving up committing as commit.strict-mode.last-safe-snapshot is set.");
+ "Giving up committing as commit.strict-mode.enabled is true.");
write1.close();
commit1.close();
@@ -585,7 +602,7 @@ public class TableCommitTest {
String user2 = UUID.randomUUID().toString();
FileStoreTable tableWithStrict =
- table.copy(singletonMap(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(), "1"));
+ table.copy(singletonMap(COMMIT_LAST_SAFE_SNAPSHOT.key(), "1"));
TableWriteImpl<?> write2 = tableWithStrict.newWrite(user2);
TableCommitImpl commit2 = tableWithStrict.newCommit(user2);
write2.write(GenericRow.of(1, 1, 1L));
@@ -620,7 +637,7 @@ public class TableCommitTest {
assertThatThrownBy(() -> commit2.commit(1, write2.prepareCommit(true, 1)))
.isInstanceOf(RuntimeException.class)
.hasMessageContaining(
- "Giving up committing as commit.strict-mode.last-safe-snapshot is set.");
+ "Giving up committing as commit.strict-mode.enabled is true.");
}
write1.close();
@@ -675,7 +692,7 @@ public class TableCommitTest {
// user2 with strict mode, last-safe-snapshot=2 (skip its own snapshot 2)
String user2 = UUID.randomUUID().toString();
FileStoreTable tableWithStrict =
- table.copy(singletonMap(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(), "2"));
+ table.copy(singletonMap(COMMIT_LAST_SAFE_SNAPSHOT.key(), "2"));
TableWriteImpl<?> write2 = tableWithStrict.newWrite(user2);
TableCommitImpl commit2 = tableWithStrict.newCommit(user2);
write2.write(GenericRow.of(1, 1, 1L));
@@ -699,7 +716,7 @@ public class TableCommitTest {
assertThatThrownBy(() -> commit2.commit(3, write2.prepareCommit(true, 3)))
.isInstanceOf(RuntimeException.class)
.hasMessageContaining(
- "Giving up committing as commit.strict-mode.last-safe-snapshot is set.");
+ "Giving up committing as commit.strict-mode.enabled is true.");
write1.close();
commit1.close();
@@ -764,7 +781,7 @@ public class TableCommitTest {
// and avoid throwing.
String user2 = UUID.randomUUID().toString();
FileStoreTable tableWithStrict =
- table.copy(singletonMap(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(), "2"));
+ table.copy(singletonMap(COMMIT_LAST_SAFE_SNAPSHOT.key(), "2"));
TableWriteImpl<?> write2 = tableWithStrict.newWrite(user2);
TableCommitImpl commit2 = tableWithStrict.newCommit(user2);
write2.write(GenericRow.of(3, 1, 1L));
@@ -852,7 +869,7 @@ public class TableCommitTest {
// test skip this commit check
String user2 = UUID.randomUUID().toString();
- table = table.copy(singletonMap(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(), "2"));
+ table = table.copy(singletonMap(COMMIT_LAST_SAFE_SNAPSHOT.key(), "2"));
TableWriteImpl<?> write2 = table.newWrite(user2);
TableCommitImpl commit2 = table.newCommit(user2).withOverwrite(singletonMap("pt", "1"));
@@ -898,7 +915,7 @@ public class TableCommitTest {
assertThatThrownBy(() -> commit2.commit(4, write2.prepareCommit(false, 4)))
.isInstanceOf(RuntimeException.class)
.hasMessageContaining(
- "Giving up committing as commit.strict-mode.last-safe-snapshot is set.");
+ "Giving up committing as commit.strict-mode.enabled is true.");
write1.close();
commit1.close();
@@ -983,7 +1000,7 @@ public class TableCommitTest {
// get filtered out and the conflict is silently missed.
String user2 = UUID.randomUUID().toString();
FileStoreTable tableWithStrict =
- fixedTable.copy(singletonMap(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(), "0"));
+ fixedTable.copy(singletonMap(COMMIT_LAST_SAFE_SNAPSHOT.key(), "0"));
TableWriteImpl<?> write2 = tableWithStrict.newWrite(user2);
TableCommitImpl commit2 =
tableWithStrict.newCommit(user2).withOverwrite(singletonMap("pt", "1"));
@@ -991,7 +1008,7 @@ public class TableCommitTest {
assertThatThrownBy(() -> commit2.commit(3, write2.prepareCommit(false, 3)))
.isInstanceOf(RuntimeException.class)
.hasMessageContaining(
- "Giving up committing as commit.strict-mode.last-safe-snapshot is set.");
+ "Giving up committing as commit.strict-mode.enabled is true.");
write2.close();
commit2.close();
@@ -1036,7 +1053,7 @@ public class TableCommitTest {
// test skip this commit check
String user2 = UUID.randomUUID().toString();
- table = table.copy(singletonMap(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(), "2"));
+ table = table.copy(singletonMap(COMMIT_LAST_SAFE_SNAPSHOT.key(), "2"));
TableWriteImpl<?> write2 = table.newWrite(user2);
TableCommitImpl commit2 = table.newCommit(user2);
@@ -1091,7 +1108,7 @@ public class TableCommitTest {
// test skip this commit check
String user2 = UUID.randomUUID().toString();
FileStoreTable tableWithStrict =
- table.copy(singletonMap(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(), "2"));
+ table.copy(singletonMap(COMMIT_LAST_SAFE_SNAPSHOT.key(), "2"));
TableWriteImpl<?> write2 = tableWithStrict.newWrite(user2);
TableCommitImpl commit2 = tableWithStrict.newCommit(user2);
@@ -1108,7 +1125,7 @@ public class TableCommitTest {
assertThatThrownBy(() -> commit2.commit(2, write2.prepareCommit(false, 2)))
.isInstanceOf(RuntimeException.class)
.hasMessageContaining(
- "Giving up committing as commit.strict-mode.last-safe-snapshot is set.");
+ "Giving up committing as commit.strict-mode.enabled is true.");
// APPEND with fixed bucket files should be caught when user2 commits OVERWRITE,
// since non-partitioned table entries share BinaryRow.EMPTY_ROW so they always
@@ -1119,7 +1136,7 @@ public class TableCommitTest {
commit1Append.commit(3, write1Append.prepareCommit(false, 3));
FileStoreTable tableWithStrict2 =
- table.copy(singletonMap(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(), "3"));
+ table.copy(singletonMap(COMMIT_LAST_SAFE_SNAPSHOT.key(), "3"));
TableWriteImpl<?> write2Ow = tableWithStrict2.newWrite(user2);
TableCommitImpl commit2Ow =
tableWithStrict2.newCommit(user2).withOverwrite(Collections.emptyMap());
@@ -1127,7 +1144,7 @@ public class TableCommitTest {
assertThatThrownBy(() -> commit2Ow.commit(4, write2Ow.prepareCommit(false, 4)))
.isInstanceOf(RuntimeException.class)
.hasMessageContaining(
- "Giving up committing as commit.strict-mode.last-safe-snapshot is set.");
+ "Giving up committing as commit.strict-mode.enabled is true.");
write1Append.close();
commit1Append.close();
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/DataEvolutionMergeIntoAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/DataEvolutionMergeIntoAction.java
index 4407ca3a85..5bda307641 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/DataEvolutionMergeIntoAction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/DataEvolutionMergeIntoAction.java
@@ -162,7 +162,7 @@ public class DataEvolutionMergeIntoAction extends TableActionBase {
table =
table.copy(
Collections.singletonMap(
- CoreOptions.COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(),
+ CoreOptions.COMMIT_LAST_SAFE_SNAPSHOT.key(),
latestSnapshotId.toString()));
this.coreOptions = ((FileStoreTable) table).coreOptions();
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/RescaleAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/RescaleAction.java
index 3855a5115d..ed60db636f 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/RescaleAction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/RescaleAction.java
@@ -91,8 +91,7 @@ public class RescaleAction extends TableActionBase {
// So we use strict mode to make sure nothing is lost.
Map<String, String> dynamicOptions = new HashMap<>();
dynamicOptions.put(
- CoreOptions.COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(),
- String.valueOf(snapshot.id()));
+ CoreOptions.COMMIT_LAST_SAFE_SNAPSHOT.key(), String.valueOf(snapshot.id()));
fileStoreTable = fileStoreTable.copy(dynamicOptions);
PartitionPredicate partitionPredicate =
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/dataevolution/DataEvolutionDeleteSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/dataevolution/DataEvolutionDeleteSink.java
index 55d6814c73..eddcf3bf91 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/dataevolution/DataEvolutionDeleteSink.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/dataevolution/DataEvolutionDeleteSink.java
@@ -81,8 +81,7 @@ public class DataEvolutionDeleteSink implements Serializable {
: (FileStoreTable)
table.copy(
Collections.singletonMap(
- CoreOptions.COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT
- .key(),
+ CoreOptions.COMMIT_LAST_SAFE_SNAPSHOT.key(),
String.valueOf(baseSnapshotId)));
this.baseSnapshotId = baseSnapshotId;
this.sinkParallelism = sinkParallelism;
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/DeleteActionDataEvolutionITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/DeleteActionDataEvolutionITCase.java
index f220f84808..eedc8bfadf 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/DeleteActionDataEvolutionITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/DeleteActionDataEvolutionITCase.java
@@ -326,7 +326,7 @@ public class DeleteActionDataEvolutionITCase extends ActionITCaseBase {
first.run();
assertThatThrownBy(stale::run)
- .hasStackTraceContaining(CoreOptions.COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key());
+ .hasStackTraceContaining(CoreOptions.COMMIT_STRICT_MODE_ENABLED.key());
testBatchRead(
"SELECT id, name, dt FROM T ORDER BY id",
Arrays.asList(
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RescaleProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RescaleProcedure.java
index 232f4370b5..1b549c6197 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RescaleProcedure.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RescaleProcedure.java
@@ -125,7 +125,7 @@ public class RescaleProcedure extends BaseProcedure {
// So we use strict mode to make sure nothing is lost.
Map<String, String> dynamicOptions = new HashMap<>();
dynamicOptions.put(
- CoreOptions.COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(),
+ CoreOptions.COMMIT_LAST_SAFE_SNAPSHOT.key(),
String.valueOf(snapshot.id()));
fileStoreTable = fileStoreTable.copy(dynamicOptions);
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkPostponeStagedCommitter.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkPostponeStagedCommitter.scala
index a4c5babc0d..da28f43d05 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkPostponeStagedCommitter.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkPostponeStagedCommitter.scala
@@ -60,9 +60,7 @@ private[spark] class SparkPostponeStagedCommitter(
private val fixedWriteCommitUser = coreOptions.createCommitUser()
private val fixedWriteTable = {
val options = new java.util.HashMap[String, String]()
- options.put(
- CoreOptions.COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(),
- baseSnapshotId.getOrElse(0L).toString)
+ options.put(CoreOptions.COMMIT_LAST_SAFE_SNAPSHOT.key(), baseSnapshotId.getOrElse(0L).toString)
// Every overwrite in this coordinator supplies its exact BinaryRow partitions. Disable the
// table-level dynamic rewrite so an empty rewritten partition is still removed, and implement
// INSERT OVERWRITE dynamic-partition semantics explicitly in commitCurrentBatch.
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala
index 91efc3d541..45f408fb78 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala
@@ -19,7 +19,7 @@
package org.apache.paimon.spark.commands
import org.apache.paimon.{CoreOptions, Snapshot}
-import org.apache.paimon.CoreOptions.{COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT, PartitionSinkStrategy, WRITE_ONLY}
+import org.apache.paimon.CoreOptions.{COMMIT_LAST_SAFE_SNAPSHOT, PartitionSinkStrategy, WRITE_ONLY}
import org.apache.paimon.codegen.CodeGenUtils
import org.apache.paimon.crosspartition.{IndexBootstrap, KeyPartOrRow}
import org.apache.paimon.data.BinaryRow
@@ -141,7 +141,7 @@ case class PaimonSparkWriter(
case Some(_) =>
val directWriteOptions = new java.util.HashMap[String, String]()
directWriteOptions.put(
- COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(),
+ COMMIT_LAST_SAFE_SNAPSHOT.key(),
postponeBaseSnapshotId.getOrElse(0L).toString)
val builder = table.copy(directWriteOptions).newPostponeFixedBucketWriteBuilder()
overwritePartitionSpec.foreach(spec => builder.withOverwrite(spec.asJava))
(paimon) 12/20: [format] Use buffered encoder for Avro writes (#9928)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit a27b548472658bb9311fe74fa4e088df44392029
Author: umi <55...@users.noreply.github.com>
AuthorDate: Fri Sep 18 12:55:49 2026 +0800
[format] Use buffered encoder for Avro writes (#9928)
---
.../java/org/apache/paimon/format/FileFormat.java | 9 +-
.../apache/paimon/format/FileFormatFactory.java | 25 ++++
.../apache/paimon/format/avro/AvroFileFormat.java | 21 ++-
.../paimon/format/avro/AvroFileFormatTest.java | 153 +++++++++++++++++++++
4 files changed, 205 insertions(+), 3 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
index f17c23b255..7a06e544ea 100644
--- a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
+++ b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
@@ -74,6 +74,10 @@ public abstract class FileFormat {
}
public static FileFormat fromIdentifier(String identifier, Options options) {
+ return fromIdentifier(identifier, options, false);
+ }
+
+ private static FileFormat fromIdentifier(String identifier, Options options, boolean manifest) {
return fromIdentifier(
normalizeFileFormat(identifier),
new FormatContext(
@@ -82,7 +86,8 @@ public abstract class FileFormat {
options.get(CoreOptions.WRITE_BATCH_SIZE),
options.get(CoreOptions.WRITE_BATCH_MEMORY),
options.get(CoreOptions.FILE_COMPRESSION_ZSTD_LEVEL),
- options.get(CoreOptions.FILE_BLOCK_SIZE)));
+ options.get(CoreOptions.FILE_BLOCK_SIZE),
+ manifest));
}
/** Create a {@link FileFormat} from format identifier and format options. */
@@ -120,6 +125,6 @@ public abstract class FileFormat {
// Manifest blocks retain the Avro default independently of data-file block sizing.
Options manifestOptions = new Options(options.toMap());
manifestOptions.remove(CoreOptions.FILE_BLOCK_SIZE);
- return FileFormat.fromIdentifier(CoreOptions.FILE_FORMAT_AVRO, manifestOptions);
+ return FileFormat.fromIdentifier(CoreOptions.FILE_FORMAT_AVRO, manifestOptions, true);
}
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java b/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java
index b5cba150fe..157f151ad2 100644
--- a/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java
+++ b/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java
@@ -41,6 +41,7 @@ public interface FileFormatFactory {
private final MemorySize writeBatchMemory;
private final int zstdLevel;
@Nullable private final MemorySize blockSize;
+ private final boolean manifest;
@VisibleForTesting
public FormatContext(Options options, int readBatchSize, int writeBatchSize) {
@@ -63,12 +64,31 @@ public interface FileFormatFactory {
MemorySize writeBatchMemory,
int zstdLevel,
@Nullable MemorySize blockSize) {
+ this(
+ options,
+ readBatchSize,
+ writeBatchSize,
+ writeBatchMemory,
+ zstdLevel,
+ blockSize,
+ false);
+ }
+
+ public FormatContext(
+ Options options,
+ int readBatchSize,
+ int writeBatchSize,
+ MemorySize writeBatchMemory,
+ int zstdLevel,
+ @Nullable MemorySize blockSize,
+ boolean manifest) {
this.options = options;
this.readBatchSize = readBatchSize;
this.writeBatchSize = writeBatchSize;
this.writeBatchMemory = writeBatchMemory;
this.zstdLevel = zstdLevel;
this.blockSize = blockSize;
+ this.manifest = manifest;
}
public Options options() {
@@ -96,6 +116,11 @@ public interface FileFormatFactory {
return blockSize;
}
+ /** Whether the format is used for manifest metadata rather than data files. */
+ public boolean isManifest() {
+ return manifest;
+ }
+
public boolean caseSensitive() {
return options.getOptional(CatalogOptions.CASE_SENSITIVE).orElse(true);
}
diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
index a23c66af21..845cce65d5 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
@@ -39,6 +39,7 @@ import org.apache.paimon.types.RowType;
import org.apache.avro.Schema;
import org.apache.avro.file.CodecFactory;
import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.io.EncoderFactory;
import javax.annotation.Nullable;
@@ -67,6 +68,7 @@ public class AvroFileFormat extends FileFormat {
private final Options options;
private final int zstdLevel;
@Nullable private final MemorySize blockSize;
+ private final boolean useBufferedEncoder;
public AvroFileFormat(FormatContext context) {
super(IDENTIFIER);
@@ -74,6 +76,7 @@ public class AvroFileFormat extends FileFormat {
this.options = getIdentifierPrefixOptions(context.options());
this.zstdLevel = context.zstdLevel();
this.blockSize = context.blockSize();
+ this.useBufferedEncoder = !context.isManifest();
}
@Override
@@ -91,10 +94,26 @@ public class AvroFileFormat extends FileFormat {
public AvroBlockWriter createBlockWriter(
PositionOutputStream out, RowType rowType, String compression) throws IOException {
+ // Retain Avro's direct encoder for pre-encoded manifest records. The buffered encoder
+ // copies array-backed ByteBuffers into a temporary byte array for each appendEncoded call.
+ return createBlockWriter(out, rowType, compression, false);
+ }
+
+ private AvroBlockWriter createBlockWriter(
+ PositionOutputStream out,
+ RowType rowType,
+ String compression,
+ boolean useBufferedEncoder)
+ throws IOException {
Schema schema =
AvroSchemaConverter.convertToSchema(rowType, options.get(AVRO_ROW_NAME_MAPPING));
AvroRowDatumWriter datumWriter = new AvroRowDatumWriter(rowType);
DataFileWriter<InternalRow> writer = new DataFileWriter<>(datumWriter);
+ if (useBufferedEncoder) {
+ // Batch data-file field encodings before writing them to the Avro block buffer.
+ writer.setEncoder(
+ outputStream -> EncoderFactory.get().binaryEncoder(outputStream, null));
+ }
writer.setCodec(createCodecFactory(compression));
if (blockSize != null) {
writer.setSyncInterval(Math.toIntExact(blockSize.getBytes()));
@@ -141,7 +160,7 @@ public class AvroFileFormat extends FileFormat {
@Override
public FormatWriter create(PositionOutputStream out, String compression)
throws IOException {
- return createBlockWriter(out, rowType, compression);
+ return createBlockWriter(out, rowType, compression, useBufferedEncoder);
}
}
}
diff --git a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
index 70780f8d9b..251f3ca974 100644
--- a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
+++ b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
@@ -44,7 +44,9 @@ import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.BinaryEncoder;
+import org.apache.avro.io.BufferedBinaryEncoder;
import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.io.DirectBinaryEncoder;
import org.apache.avro.io.EncoderFactory;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@ -176,6 +178,157 @@ public class AvroFileFormatTest {
}
}
+ @ParameterizedTest
+ @ValueSource(strings = {"null", "deflate", "snappy", "zstd"})
+ void testCloseFlushesBufferedRecords(String compression) throws IOException {
+ RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()).notNull();
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString());
+
+ try (PositionOutputStream out = fileIO.newOutputStream(file, false);
+ FormatWriter writer =
+ fileFormat.createWriterFactory(rowType).create(out, compression)) {
+ assertThat(writer)
+ .extracting("writer")
+ .extracting("bufOut")
+ .isInstanceOf(BufferedBinaryEncoder.class);
+ // Leave a partial record-encoder buffer as well as a partial Avro block.
+ writer.addElement(GenericRow.of(42));
+ }
+
+ List<Integer> result = new ArrayList<>();
+ try (RecordReader<InternalRow> reader =
+ fileFormat
+ .createReaderFactory(rowType, rowType, new ArrayList<>())
+ .createReader(
+ new FormatReaderContext(
+ fileIO, file, fileIO.getFileSize(file), null, null))) {
+ reader.forEachRemaining(row -> result.add(row.getInt(0)));
+ }
+ assertThat(result).containsExactly(42);
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "null, 16",
+ "null, 4096",
+ "deflate, 16",
+ "deflate, 4096",
+ "snappy, 16",
+ "snappy, 4096",
+ "zstd, 16",
+ "zstd, 4096"
+ })
+ void testBufferedRecordSerializationFailureRollsBack(String compression, int payloadSize)
+ throws IOException {
+ RowType rowType =
+ DataTypes.ROW(DataTypes.BYTES().notNull(), DataTypes.INT().notNull()).notNull();
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString());
+ GenericRow before = GenericRow.of(new byte[] {1, 2, 3}, 42);
+ GenericRow after = GenericRow.of(new byte[] {4, 5, 6}, 43);
+ byte[] failedPayload = new byte[payloadSize];
+ Arrays.fill(failedPayload, (byte) 7);
+
+ try (PositionOutputStream out = fileIO.newOutputStream(file, false);
+ FormatWriter writer =
+ fileFormat.createWriterFactory(rowType).create(out, compression)) {
+ assertThat(writer)
+ .extracting("writer")
+ .extracting("bufOut")
+ .isInstanceOf(BufferedBinaryEncoder.class);
+ writer.addElement(before);
+ // The first field is written before the second field fails. Small payloads stay in
+ // the encoder buffer; large payloads flush it and reach the Avro block buffer.
+ assertThatThrownBy(() -> writer.addElement(GenericRow.of(failedPayload, "not an int")))
+ .isInstanceOf(DataFileWriter.AppendWriteException.class)
+ .hasCauseInstanceOf(ClassCastException.class);
+ writer.addElement(after);
+ }
+
+ List<GenericRow> result = new ArrayList<>();
+ try (RecordReader<InternalRow> reader =
+ fileFormat
+ .createReaderFactory(rowType, rowType, new ArrayList<>())
+ .createReader(
+ new FormatReaderContext(
+ fileIO, file, fileIO.getFileSize(file), null, null))) {
+ reader.forEachRemaining(
+ row -> result.add(GenericRow.of(row.getBinary(0), row.getInt(1))));
+ }
+ assertThat(result).containsExactly(before, after);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"avro", "parquet", "orc"})
+ void testManifestWriterUsesDirectEncoder(String dataFileFormat) throws IOException {
+ Options options = new Options();
+ options.set(CoreOptions.FILE_FORMAT, dataFileFormat);
+ FileFormat manifestFormat = FileFormat.manifestFormat(new CoreOptions(options));
+ RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()).notNull();
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString());
+
+ try (PositionOutputStream out = fileIO.newOutputStream(file, false);
+ FormatWriter writer =
+ manifestFormat.createWriterFactory(rowType).create(out, "null")) {
+ assertThat(writer)
+ .extracting("writer")
+ .extracting("bufOut")
+ .isInstanceOf(DirectBinaryEncoder.class);
+ writer.addElement(GenericRow.of(42));
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"null", "deflate", "snappy", "zstd"})
+ void testRecordsAroundRawBlockCopy(String compression) throws IOException {
+ RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()).notNull();
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path source = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString());
+ Path target = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString());
+
+ try (PositionOutputStream out = fileIO.newOutputStream(source, false);
+ FormatWriter writer =
+ fileFormat.createWriterFactory(rowType).create(out, "deflate")) {
+ writer.addElement(GenericRow.of(2));
+ writer.addElement(GenericRow.of(3));
+ }
+
+ ByteArrayOutputStream encoded = new ByteArrayOutputStream();
+ BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(encoded, null);
+ encoder.writeInt(1);
+ encoder.flush();
+
+ try (AvroBlockReader reader = new AvroBlockReader(fileIO.newInputStream(source));
+ PositionOutputStream out = fileIO.newOutputStream(target, false);
+ AvroBlockWriter writer = fileFormat.createBlockWriter(out, rowType, compression)) {
+ assertThat(writer)
+ .extracting("writer")
+ .extracting("bufOut")
+ .isInstanceOf(DirectBinaryEncoder.class);
+ writer.addElement(GenericRow.of(0));
+ ByteBuffer record = ByteBuffer.wrap(encoded.toByteArray());
+ writer.addEncoded(record);
+ assertThat(record.position()).isZero();
+ assertThat(record.remaining()).isEqualTo(encoded.size());
+ // Copying a raw block must first flush pending rows and encoded records.
+ writer.addEncodedBlock(reader.nextBorrowedRawBlock());
+ writer.addElement(GenericRow.of(4));
+ }
+
+ List<Integer> result = new ArrayList<>();
+ try (RecordReader<InternalRow> reader =
+ fileFormat
+ .createReaderFactory(rowType, rowType, new ArrayList<>())
+ .createReader(
+ new FormatReaderContext(
+ fileIO, target, fileIO.getFileSize(target), null, null))) {
+ reader.forEachRemaining(row -> result.add(row.getInt(0)));
+ }
+ assertThat(result).containsExactly(0, 1, 2, 3, 4);
+ }
+
@Test
public void testSupportedDataTypes() {
ArrayList<DataField> dataFields = new ArrayList<>();
(paimon) 16/20: [core] Request only top-k candidates from data-evolution full-text indexes (#9955)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit e4f49ec969fc5acd4f86b4e5cefe8e0294490857
Author: Xiangyi Zhu <82...@users.noreply.github.com>
AuthorDate: Fri Sep 18 19:33:51 2026 +0800
[core] Request only top-k candidates from data-evolution full-text indexes (#9955)
---
.../TestFullTextGlobalIndexReader.java | 5 ++
.../table/source/DataEvolutionFullTextRead.java | 13 ++--
.../table/source/FullTextSearchBuilderTest.java | 70 ++++++++++++++++++++++
3 files changed, 79 insertions(+), 9 deletions(-)
diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexReader.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexReader.java
index b2cd4ca23e..c695c90950 100644
--- a/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexReader.java
+++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexReader.java
@@ -43,6 +43,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.PriorityQueue;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CopyOnWriteArrayList;
/**
* Test full-text index reader that performs brute-force text matching. Loads all documents into
@@ -60,6 +61,9 @@ public class TestFullTextGlobalIndexReader implements GlobalIndexReader {
private long[] rowIds;
private int count;
+ /** Limits requested from this reader, in call order; lets tests pin what the read asks for. */
+ public static final List<Integer> REQUESTED_LIMITS = new CopyOnWriteArrayList<>();
+
public TestFullTextGlobalIndexReader(
GlobalIndexFileReader fileReader, GlobalIndexIOMeta ioMeta) {
this.fileReader = fileReader;
@@ -76,6 +80,7 @@ public class TestFullTextGlobalIndexReader implements GlobalIndexReader {
}
int limit = fullTextSearch.limit();
+ REQUESTED_LIMITS.add(limit);
int effectiveK = Math.min(limit, count);
if (effectiveK <= 0) {
return CompletableFuture.completedFuture(Optional.empty());
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java
index 9a8227b1b9..8704145e25 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java
@@ -451,19 +451,14 @@ public class DataEvolutionFullTextRead implements FullTextRead {
indexIOMetaList,
rowRangeEnd - rowRangeStart + 1,
executor);
+ // Each split returns its own top-k; the union's top-k after merge equals the global
+ // top-k, and the native engine scores compound queries per document, so there is no
+ // need to request every candidate of the range.
FullTextSearch fullTextSearch =
- new FullTextSearch(
- textColumn.name(),
- query,
- candidateLimit(rowRangeStart, rowRangeEnd))
+ new FullTextSearch(textColumn.name(), query, limit)
.withIncludeRowIds(includeRowIds);
return new OffsetGlobalIndexReader(reader, rowRangeStart, rowRangeEnd)
.visitFullTextSearch(fullTextSearch)
.whenComplete((r, t) -> IOUtils.closeQuietly(reader));
}
-
- private static int candidateLimit(long rowRangeStart, long rowRangeEnd) {
- long size = rowRangeEnd - rowRangeStart + 1;
- return size > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) size;
- }
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
index 94f483ec6c..388d05700e 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
@@ -31,6 +31,7 @@ import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter;
import org.apache.paimon.globalindex.ResultEntry;
import org.apache.paimon.globalindex.ScoredGlobalIndexResult;
import org.apache.paimon.globalindex.btree.BTreeGlobalIndexerFactory;
+import org.apache.paimon.globalindex.testfulltext.TestFullTextGlobalIndexReader;
import org.apache.paimon.globalindex.testfulltext.TestFullTextGlobalIndexerFactory;
import org.apache.paimon.index.GlobalIndexMeta;
import org.apache.paimon.index.IndexFileMeta;
@@ -1045,6 +1046,75 @@ public class FullTextSearchBuilderTest extends TableTestBase {
assertThat(ids.size()).isLessThanOrEqualTo(5);
}
+ @Test
+ public void testFullTextSearchRequestsOnlyLimitCandidatesPerSplit() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ writeDocuments(table, RANKED_DOCUMENTS);
+ List<DataField> textFields =
+ Collections.singletonList(table.rowType().getField(TEXT_FIELD_NAME));
+ buildAndCommitIndexRange(table, Arrays.copyOfRange(RANKED_DOCUMENTS, 0, 3), textFields, 0);
+ buildAndCommitIndexRange(table, Arrays.copyOfRange(RANKED_DOCUMENTS, 3, 6), textFields, 3);
+
+ TestFullTextGlobalIndexReader.REQUESTED_LIMITS.clear();
+ GlobalIndexResult result =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon"))
+ .withLimit(2)
+ .executeLocal();
+
+ // Each of the two splits (3 rows each) is asked for the user limit, not its row count.
+ assertThat(TestFullTextGlobalIndexReader.REQUESTED_LIMITS).containsExactly(2, 2);
+ assertThat(result.results()).hasSize(2);
+ }
+
+ @Test
+ public void testFullTextSearchMergesPerSplitTopKIntoGlobalTopK() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ // Split 1 (rows 0-2) holds only 0.5-scored matches; split 2 (rows 3-5) holds the two
+ // 1.0-scored matches plus one 0.5. A global top-2 must come entirely from split 2.
+ String[] documents = {
+ "paimon alpha", "paimon beta", "paimon gamma",
+ "paimon lake delta", "paimon lake epsilon", "paimon zeta"
+ };
+ writeDocuments(table, documents);
+ List<DataField> textFields =
+ Collections.singletonList(table.rowType().getField(TEXT_FIELD_NAME));
+ buildAndCommitIndexRange(table, Arrays.copyOfRange(documents, 0, 3), textFields, 0);
+ buildAndCommitIndexRange(table, Arrays.copyOfRange(documents, 3, 6), textFields, 3);
+
+ ScoredGlobalIndexResult top2 =
+ (ScoredGlobalIndexResult)
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon lake"))
+ .withLimit(2)
+ .executeLocal();
+ assertThat(top2.results()).containsExactlyInAnyOrder(3L, 4L);
+
+ ScoredGlobalIndexResult top1 =
+ (ScoredGlobalIndexResult)
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon lake"))
+ .withLimit(1)
+ .executeLocal();
+ assertThat(top1.results()).hasSize(1);
+ assertThat(top1.results()).isSubsetOf(3L, 4L);
+
+ // A larger limit still ranks the 1.0 rows first and fills up with 0.5 rows.
+ ScoredGlobalIndexResult top4 =
+ (ScoredGlobalIndexResult)
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon lake"))
+ .withLimit(4)
+ .executeLocal();
+ assertThat(top4.results()).hasSize(4);
+ assertThat(top4.results()).contains(3L, 4L);
+ for (long rowId : top4.results()) {
+ assertThat(top4.scoreGetter().score(rowId)).isGreaterThanOrEqualTo(0.5f);
+ }
+ }
+
@Test
public void testFullTextSearchWithMultipleIndexFiles() throws Exception {
createTableDefault();
(paimon) 03/20: [flink] Fix rollback_to procedure throwing NullPointerException when neither snapshot_id nor tag is set (#9944)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 48d2fc60ef30b0f247cd44c8da6f9e9d32404f45
Author: Li Guo <al...@gmail.com>
AuthorDate: Thu Sep 17 20:58:58 2026 -0700
[flink] Fix rollback_to procedure throwing NullPointerException when neither snapshot_id nor tag is set (#9944)
---
docs/docs/flink/procedures/snapshots.md | 6 +-
.../flink/procedure/RollbackToProcedure.java | 7 +-
.../flink/procedure/RollbackProcedureITCase.java | 74 ++++++++++++++++++++++
3 files changed, 84 insertions(+), 3 deletions(-)
diff --git a/docs/docs/flink/procedures/snapshots.md b/docs/docs/flink/procedures/snapshots.md
index cb5902fef8..3f1e91b8b8 100644
--- a/docs/docs/flink/procedures/snapshots.md
+++ b/docs/docs/flink/procedures/snapshots.md
@@ -198,9 +198,11 @@ To rollback to a specific version of target table. Argument:
- `table`: the target table identifier. Cannot be empty.
-- snapshotId (Long): id of the snapshot that will roll back to.
+- `snapshot_id`: id of the snapshot that will roll back to.
-- `tagName`: name of the tag that will roll back to.
+- `tag`: name of the tag that will roll back to.
+
+Exactly one of `snapshot_id` and `tag` must be set.
**Syntax**
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RollbackToProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RollbackToProcedure.java
index 8320453f8e..7f57649e1b 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RollbackToProcedure.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RollbackToProcedure.java
@@ -66,8 +66,13 @@ public class RollbackToProcedure extends ProcedureBase {
Snapshot latestSnapshot = store.snapshotManager().latestSnapshot();
Preconditions.checkNotNull(latestSnapshot, "Latest snapshot is null, can not rollback.");
+ boolean hasTag = !StringUtils.isNullOrWhitespaceOnly(tagName);
+ boolean hasSnapshot = snapshotId != null;
+ Preconditions.checkArgument(
+ hasTag != hasSnapshot, "Must specify exactly one of tag and snapshot_id.");
+
long rollbackSnapshotId;
- if (!StringUtils.isNullOrWhitespaceOnly(tagName)) {
+ if (hasTag) {
table.rollbackTo(tagName);
rollbackSnapshotId = store.newTagManager().getOrThrow(tagName).trimToSnapshot().id();
} else {
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/RollbackProcedureITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/RollbackProcedureITCase.java
index 884237a16f..668dfe82ae 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/RollbackProcedureITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/RollbackProcedureITCase.java
@@ -26,6 +26,7 @@ import org.apache.flink.types.Row;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/** IT Case for {@link RollbackToProcedure} and {@link RollbackToTimestampProcedure}. */
@@ -59,6 +60,79 @@ public class RollbackProcedureITCase extends CatalogITCaseBase {
.containsExactly(Row.of(latestSnapshotId, 2L));
}
+ @Test
+ public void testRollbackToRequiresExactlyOneOfTagAndSnapshotId() throws Exception {
+ sql(
+ "CREATE TABLE T (id STRING, name STRING,"
+ + " PRIMARY KEY (id) NOT ENFORCED)"
+ + " WITH ('bucket'='1', 'write-only'='true')");
+
+ FileStoreTable table = paimonTable("T");
+ SnapshotManager snapshotManager = table.snapshotManager();
+
+ for (int i = 1; i <= 3; i++) {
+ sql("INSERT INTO T VALUES ('" + i + "', '" + i + "')");
+ }
+ assertEquals(3, snapshotManager.latestSnapshotId());
+ sql("CALL sys.create_tag(`table` => 'default.T', tag => 'tag-1', snapshot_id => 1)");
+
+ // neither tag nor snapshot_id
+ assertThatThrownBy(() -> sql("CALL sys.rollback_to(`table` => 'default.T')"))
+ .rootCause()
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Must specify exactly one of tag and snapshot_id.");
+
+ // both tag and snapshot_id
+ assertThatThrownBy(
+ () ->
+ sql(
+ "CALL sys.rollback_to(`table` => 'default.T', tag => 'tag-1', snapshot_id => 2)"))
+ .rootCause()
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Must specify exactly one of tag and snapshot_id.");
+
+ // nothing was rolled back
+ assertEquals(3, snapshotManager.latestSnapshotId());
+ }
+
+ @Test
+ public void testRollbackToTreatsBlankTagAsMissing() throws Exception {
+ sql(
+ "CREATE TABLE T (id STRING, name STRING,"
+ + " PRIMARY KEY (id) NOT ENFORCED)"
+ + " WITH ('bucket'='1', 'write-only'='true')");
+
+ FileStoreTable table = paimonTable("T");
+ SnapshotManager snapshotManager = table.snapshotManager();
+
+ for (int i = 1; i <= 3; i++) {
+ sql("INSERT INTO T VALUES ('" + i + "', '" + i + "')");
+ }
+ assertEquals(3, snapshotManager.latestSnapshotId());
+
+ // a whitespace-only tag alone: neither argument is usable
+ assertThatThrownBy(() -> sql("CALL sys.rollback_to(`table` => 'default.T', tag => ' ')"))
+ .rootCause()
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Must specify exactly one of tag and snapshot_id.");
+ assertEquals(3, snapshotManager.latestSnapshotId());
+
+ // explicit NULLs for both arguments behave like omitting them
+ assertThatThrownBy(
+ () ->
+ sql(
+ "CALL sys.rollback_to(`table` => 'default.T', tag => CAST(NULL AS STRING), snapshot_id => CAST(NULL AS BIGINT))"))
+ .rootCause()
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Must specify exactly one of tag and snapshot_id.");
+ assertEquals(3, snapshotManager.latestSnapshotId());
+
+ // an empty tag next to a snapshot_id keeps the old behaviour: the snapshot wins
+ assertThat(sql("CALL sys.rollback_to(`table` => 'default.T', tag => '', snapshot_id => 2)"))
+ .containsExactly(Row.of(3L, 2L));
+ assertEquals(2, snapshotManager.latestSnapshotId());
+ }
+
@Test
public void testRollbackToTimestamp() throws Exception {
sql(
(paimon) 04/20: [core] Reject a non-parquet per-level or changelog format for variant columns at DDL time (#9942)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit fa8ae00346866df0b006ca81322aaec2f5e0f958
Author: Xiangyi Zhu <82...@users.noreply.github.com>
AuthorDate: Fri Sep 18 11:59:15 2026 +0800
[core] Reject a non-parquet per-level or changelog format for variant columns at DDL time (#9942)
---
.../org/apache/paimon/schema/SchemaValidation.java | 46 ++++++++++++++++
.../apache/paimon/schema/SchemaValidationTest.java | 63 ++++++++++++++++++++++
2 files changed, 109 insertions(+)
diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
index 3a31f7dbba..ed316a7223 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
@@ -251,6 +251,7 @@ public class SchemaValidation {
FileFormat.fromIdentifier(options.formatType(), new Options(schema.options()));
RowType tableRowType = new RowType(schema.fields());
validateGeospatialTypes(schema, options, tableRowType);
+ validateVariantTypes(schema, options);
validateIcebergTimestampPrecisions(tableRowType, options);
validateIcebergTimePrecisions(tableRowType, options);
validateBlobFields(tableRowType, options);
@@ -492,6 +493,51 @@ public class SchemaValidation {
}
}
+ /**
+ * Only the Parquet format can store {@code VARIANT}, so every format the table may write data
+ * files with has to be parquet: the main format, each per-level override and the changelog
+ * format. Checking here reports the column and the option at DDL time, rather than the bare
+ * "unsupported type" a non-parquet writer raises on the first row.
+ */
+ private static void validateVariantTypes(TableSchema schema, CoreOptions options) {
+ List<String> variantColumns =
+ schema.fields().stream()
+ .filter(
+ field ->
+ containsType(
+ field.type(),
+ type -> type.is(DataTypeRoot.VARIANT)))
+ .map(DataField::name)
+ .collect(Collectors.toList());
+ if (variantColumns.isEmpty()) {
+ return;
+ }
+
+ checkArgument(
+ CoreOptions.FILE_FORMAT_PARQUET.equals(options.formatType()),
+ "Variant columns %s require '%s'='parquet', but was '%s'.",
+ variantColumns,
+ CoreOptions.FILE_FORMAT.key(),
+ options.formatType());
+ options.fileFormatPerLevel()
+ .forEach(
+ (level, format) ->
+ checkArgument(
+ CoreOptions.FILE_FORMAT_PARQUET.equals(format),
+ "Variant columns %s require parquet at every level, but '%s' contains '%s:%s'.",
+ variantColumns,
+ CoreOptions.FILE_FORMAT_PER_LEVEL.key(),
+ level,
+ format));
+ checkArgument(
+ options.changelogFileFormat() == null
+ || CoreOptions.FILE_FORMAT_PARQUET.equals(options.changelogFileFormat()),
+ "Variant columns %s require '%s' to be parquet, but was '%s'.",
+ variantColumns,
+ CoreOptions.CHANGELOG_FILE_FORMAT.key(),
+ options.changelogFileFormat());
+ }
+
private static void validateGeospatialTypes(
TableSchema schema, CoreOptions options, RowType rowType) {
boolean hasGeospatial =
diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
index 238dbef999..681edbf1da 100644
--- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
@@ -1920,6 +1920,69 @@ class SchemaValidationTest {
"Geometry and geography columns cannot be clustering columns: [nested].");
}
+ @Test
+ public void testVariantTypeValidation() {
+ // a top-level variant and one nested inside a struct
+ List<DataField> fields =
+ Arrays.asList(
+ new DataField(0, "id", DataTypes.INT()),
+ new DataField(1, "v", DataTypes.VARIANT()),
+ new DataField(
+ 2,
+ "nested",
+ DataTypes.ROW(DataTypes.FIELD(3, "inner", DataTypes.VARIANT()))));
+
+ assertThatNoException()
+ .isThrownBy(
+ () ->
+ validateTableSchema(
+ geospatialSchema(
+ fields,
+ emptyList(),
+ emptyList(),
+ new HashMap<>())));
+
+ // the main format used to be caught by the format's own type converter, with a bare
+ // "Unsupported type: VARIANT" that named neither the column nor the option
+ Map<String, String> orcOptions = new HashMap<>();
+ orcOptions.put(CoreOptions.FILE_FORMAT.key(), "orc");
+ assertThatThrownBy(
+ () ->
+ validateTableSchema(
+ geospatialSchema(
+ fields, emptyList(), emptyList(), orcOptions)))
+ .hasMessageContaining(
+ "Variant columns [v, nested] require 'file.format'='parquet', but was 'orc'");
+
+ // these two used to pass DDL and fail on the first row written with that format
+ Map<String, String> perLevelOptions = new HashMap<>();
+ perLevelOptions.put(CoreOptions.FILE_FORMAT_PER_LEVEL.key(), "0:orc");
+ assertThatThrownBy(
+ () ->
+ validateTableSchema(
+ geospatialSchema(
+ fields,
+ emptyList(),
+ singletonList("id"),
+ perLevelOptions)))
+ .hasMessageContaining(
+ "Variant columns [v, nested] require parquet at every level, but 'file.format.per.level' contains '0:orc'");
+
+ Map<String, String> changelogOptions = new HashMap<>();
+ changelogOptions.put(CoreOptions.CHANGELOG_PRODUCER.key(), "input");
+ changelogOptions.put(CoreOptions.CHANGELOG_FILE_FORMAT.key(), "avro");
+ assertThatThrownBy(
+ () ->
+ validateTableSchema(
+ geospatialSchema(
+ fields,
+ emptyList(),
+ singletonList("id"),
+ changelogOptions)))
+ .hasMessageContaining(
+ "Variant columns [v, nested] require 'changelog-file.format' to be parquet, but was 'avro'");
+ }
+
private TableSchema geospatialSchema(
List<DataField> fields,
List<String> partitionKeys,
(paimon) 05/20: [common] Infer a variant shredding schema regardless of the writer's key order (#9940)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 512f285d1c1c716a39b1b0939ee345a5fa379b03
Author: Xiangyi Zhu <82...@users.noreply.github.com>
AuthorDate: Fri Sep 18 11:59:38 2026 +0800
[common] Infer a variant shredding schema regardless of the writer's key order (#9940)
---
.../data/variant/InferVariantShreddingSchema.java | 24 +++++++----
.../variant/InferVariantShreddingSchemaTest.java | 47 ++++++++++++++++++++++
.../writer/InferVariantShreddingWriteTest.java | 41 +++++++++++++++++++
.../apache/paimon/spark/sql/VariantTestBase.scala | 17 ++++++++
4 files changed, 121 insertions(+), 8 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java b/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java
index 7c80064c24..cf4d52e9d5 100644
--- a/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java
+++ b/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java
@@ -18,6 +18,7 @@
package org.apache.paimon.data.variant;
+import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.DataField;
@@ -313,13 +314,11 @@ public class InferVariantShreddingSchema {
fields.add(dataField);
}
- // According to the variant spec, object fields must be sorted alphabetically
- for (int i = 1; i < fields.size(); i++) {
- if (fields.get(i - 1).name().compareTo(fields.get(i).name()) >= 0) {
- throw new RuntimeException(
- "Variant object fields must be sorted alphabetically");
- }
- }
+ // Writers disagree on how object keys are ordered: Paimon's builder sorts them
+ // by UTF-8 bytes, Spark's by UTF-16 code units, which differ once a supplementary
+ // character meets one from U+E000 upwards. mergeRowTypes merges sorted lists, so
+ // put every object into the same order here instead of trusting the writer's.
+ fields.sort((f1, f2) -> compareKeys(f1.name(), f2.name()));
return new RowType(fields);
@@ -391,6 +390,15 @@ public class InferVariantShreddingSchema {
}
}
+ /**
+ * The one key order inference works in: UTF-8 bytes, as Paimon's own builder sorts. {@link
+ * String#compareTo} orders by UTF-16 code units instead, which puts a supplementary character
+ * such as an emoji before every character from U+E000 upwards, the reverse of the byte order.
+ */
+ private static int compareKeys(String key1, String key2) {
+ return BinaryString.fromString(key1).compareTo(BinaryString.fromString(key2));
+ }
+
private double getFieldCount(DataField field) {
// Read count from description field
String desc = field.description();
@@ -470,7 +478,7 @@ public class InferVariantShreddingSchema {
DataField field2 = fields2.get(f2Idx);
String f1Name = field1.name();
String f2Name = field2.name();
- int comp = f1Name.compareTo(f2Name);
+ int comp = compareKeys(f1Name, f2Name);
if (comp == 0) {
DataType dataType = mergeSchema(field1.type(), field2.type());
diff --git a/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java b/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java
index 2b7835d7b1..51acc76f16 100644
--- a/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java
@@ -239,6 +239,53 @@ public class InferVariantShreddingSchemaTest {
.isEqualTo(variantShreddingSchema(DataTypes.VARIANT()));
}
+ @Test
+ void testInferSchemaWithKeysInEitherWriterOrder() {
+ RowType schema = RowType.of(new DataType[] {DataTypes.VARIANT()}, new String[] {"v"});
+
+ // Paimon's builder sorts keys by UTF-8 bytes, where U+FFE5 (EF BF A5) precedes the
+ // emoji U+1F600 (F0 9F 98 80); Spark's sorts by UTF-16 code units, where the emoji's
+ // surrogate pair (D83D) comes first. Inference used to reject the byte order as unsorted,
+ // and it must merge objects written in either order.
+ String yen = "\uFFE5";
+ String smile = new String(Character.toChars(0x1F600));
+ GenericVariant byteOrdered =
+ GenericVariant.fromJson("{\"" + yen + "\": 100, \"" + smile + "\": \"s\"}");
+ GenericVariant codeUnitOrdered = withSwappedFields(byteOrdered);
+ assertThat(byteOrdered.getFieldAtIndex(0).key).isEqualTo(yen);
+ assertThat(codeUnitOrdered.getFieldAtIndex(0).key).isEqualTo(smile);
+ GenericVariant third = GenericVariant.fromJson("{\"" + smile + "\": \"t\", \"a\": 2}");
+
+ // merged in byte order, with every key kept exactly once and both rows counted
+ RowType expectedType =
+ RowType.of(
+ new DataType[] {DataTypes.BIGINT(), DataTypes.BIGINT(), DataTypes.STRING()},
+ new String[] {"a", yen, smile});
+ for (GenericVariant first : new GenericVariant[] {byteOrdered, codeUnitOrdered}) {
+ List<InternalRow> rows = Arrays.asList(GenericRow.of(first), GenericRow.of(third));
+ RowType inferredSchema = defaultInferVariantShreddingSchema(schema).inferSchema(rows);
+ assertThat(inferredSchema.getField("v").type())
+ .isEqualTo(variantShreddingSchema(expectedType));
+ }
+ }
+
+ /**
+ * The same two-field object with its entries swapped, the order a UTF-16 sorting writer uses.
+ */
+ private static GenericVariant withSwappedFields(GenericVariant twoFields) {
+ byte[] value = twoFields.value().clone();
+ // header, size, then 2 one-byte ids and 3 one-byte offsets
+ int idStart = 2;
+ int offsetStart = idStart + 2;
+ byte temporary = value[idStart];
+ value[idStart] = value[idStart + 1];
+ value[idStart + 1] = temporary;
+ temporary = value[offsetStart];
+ value[offsetStart] = value[offsetStart + 1];
+ value[offsetStart + 1] = temporary;
+ return new GenericVariant(value, twoFields.metadata());
+ }
+
@Test
void testInferSchemaWithNullValues() {
// Schema: row<v: variant>
diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java
index f1187f2b54..95d72921dc 100644
--- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java
+++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java
@@ -496,6 +496,47 @@ public class InferVariantShreddingWriteTest {
assertThat(result2.get(1)).isEqualTo(GenericRow.of(GenericRow.of(5L)));
}
+ @Test
+ public void testInferSchemaWithKeysOrderedByUtf8Bytes() throws Exception {
+ ParquetFileFormat format = createFormat();
+ RowType writeType = DataTypes.ROW(DataTypes.FIELD(0, "v", DataTypes.VARIANT()));
+
+ // U+FFE5 sorts before the emoji U+1F600 by UTF-8 bytes but after it by UTF-16 code
+ // units; inference used to reject such an object as unsorted and fail the file write
+ String yen = "\uFFE5";
+ String smile = new String(Character.toChars(0x1F600));
+ FormatWriterFactory factory = format.createWriterFactory(writeType);
+ writeRows(
+ factory,
+ GenericRow.of(
+ GenericVariant.fromJson("{\"" + yen + "\":100,\"" + smile + "\":\"s\"}")),
+ GenericRow.of(GenericVariant.fromJson("{\"" + smile + "\":\"t\",\"a\":2}")));
+
+ RowType expectShreddedType =
+ RowType.of(
+ new DataType[] {DataTypes.BIGINT(), DataTypes.BIGINT(), DataTypes.STRING()},
+ new String[] {"a", yen, smile});
+ verifyShreddingSchema(expectShreddedType);
+
+ List<InternalRow> result = readRows(format, writeType);
+ assertThat(result.get(0).getVariant(0).toJson())
+ .isEqualTo("{\"" + yen + "\":100,\"" + smile + "\":\"s\"}");
+ assertThat(result.get(1).getVariant(0).toJson())
+ .isEqualTo("{\"a\":2,\"" + smile + "\":\"t\"}");
+
+ RowType variantRowType =
+ VariantMetadataUtils.VariantRowTypeBuilder.builder()
+ .field(DataTypes.STRING(), "$['" + smile + "']")
+ .field(DataTypes.BIGINT(), "$['" + yen + "']")
+ .build();
+ RowType readType = DataTypes.ROW(DataTypes.FIELD(0, "v", variantRowType));
+ List<InternalRow> result2 = readRows(format, readType);
+ assertThat(result2.get(0))
+ .isEqualTo(GenericRow.of(GenericRow.of(BinaryString.fromString("s"), 100L)));
+ assertThat(result2.get(1))
+ .isEqualTo(GenericRow.of(GenericRow.of(BinaryString.fromString("t"), null)));
+ }
+
@Test
public void testInferSchemaWithNullValues() throws Exception {
ParquetFileFormat format = createFormat();
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
index 4501713510..6bddc24fd4 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
@@ -459,6 +459,23 @@ abstract class VariantTestBase extends PaimonSparkTestBase {
)
}
+ test("Paimon Variant: keys that UTF-8 and UTF-16 order differently under inferred shredding") {
+ sql("CREATE TABLE T (id INT, v VARIANT)")
+ // Spark orders "\uFFE5" after the emoji (UTF-16 code units), Paimon's builder before it
+ // (UTF-8 bytes); inferred shredding used to reject one of the two orders as unsorted and
+ // fail the write. Key order is not asserted because each writer renders its own.
+ sql("""INSERT INTO T VALUES
+ | (1, parse_json('{"\uFFE5":100,"\uD83D\uDE00":"s"}')),
+ | (2, parse_json('{"\uD83D\uDE00":"t","a":2}'))
+ |""".stripMargin)
+
+ checkAnswer(
+ sql(
+ "SELECT id, variant_get(v, '$[\"\uD83D\uDE00\"]', 'string'), try_variant_get(v, '$[\"\uFFE5\"]', 'bigint'), try_variant_get(v, '$.a', 'int') FROM T ORDER BY id"),
+ Seq(Row(1, "s", 100L, null), Row(2, "t", null, 2))
+ )
+ }
+
test("Paimon Variant: read and write variant with null value") {
withTable("source_tbl", "target_tbl") {
sql("CREATE TABLE source_tbl (id INT, js STRING) USING paimon")
(paimon) 07/20: [python] Share final lookups across batch vector queries (#9935)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 1e9c720f9614175ae94f45a6dfd38ad2e40a34c7
Author: chaoyang <ch...@apache.org>
AuthorDate: Fri Sep 18 12:00:42 2026 +0800
[python] Share final lookups across batch vector queries (#9935)
---
docs/docs/pypaimon/multimodal-search.md | 3 +
paimon-python/pypaimon/multimodal/query.py | 40 ++++-
.../pypaimon/tests/batch_vector_lookup_test.py | 181 +++++++++++++++++++++
3 files changed, 220 insertions(+), 4 deletions(-)
diff --git a/docs/docs/pypaimon/multimodal-search.md b/docs/docs/pypaimon/multimodal-search.md
index 2a2940fe1c..034162016e 100644
--- a/docs/docs/pypaimon/multimodal-search.md
+++ b/docs/docs/pypaimon/multimodal-search.md
@@ -186,6 +186,9 @@ Dictionary vector routes also accept `anns_field`, `data`, and `param` aliases.
Use `search_vectors` for multiple query vectors against one vector column. It
returns one result set for each input vector, preserving input order.
+Batch results share a lookup of their combined row IDs, so overlapping results
+do not require reading the same row separately for each query. The limit and
+post-filter still apply to each query's result set.
```python
batch_neighbors = (
diff --git a/paimon-python/pypaimon/multimodal/query.py b/paimon-python/pypaimon/multimodal/query.py
index 9f5d92c19e..9cfea57e6f 100644
--- a/paimon-python/pypaimon/multimodal/query.py
+++ b/paimon-python/pypaimon/multimodal/query.py
@@ -634,10 +634,42 @@ class BatchVectorQuery(_PreFilterQuery):
def to_arrow(self):
query = self._for_execution()
- return [
- query._read_global_index_result(result)
- for result in query._execute_batch_vector(query)
- ]
+ return query._read_batch_results(query._execute_batch_vector(query))
+
+ def _read_batch_results(self, results):
+ from pypaimon.globalindex.global_index_result import GlobalIndexResult
+ from pypaimon.utils.roaring_bitmap import RoaringBitmap64
+
+ if len(results) <= 1 or not self._configured_read_builder().read_type():
+ return [self._read_global_index_result(result) for result in results]
+
+ row_ids = RoaringBitmap64()
+ for result in results:
+ row_ids = RoaringBitmap64.or_(row_ids, result.results())
+
+ lookup = copy(self)
+ # Each result is already top-k. A shared read must not apply that limit
+ # to the union; where() still filters the selected rows during lookup.
+ lookup._limit = None
+ projection = self._effective_projection()
+ lookup._projection = list(projection) if projection else [f.name for f in self._table.fields]
+ added_row_id = SpecialFields.ROW_ID.name not in lookup._projection
+ if added_row_id:
+ lookup._projection.append(SpecialFields.ROW_ID.name)
+ fields = lookup._configured_read_builder().read_type()
+ row_id_column = next(i for i, field in enumerate(fields) if field.id == SpecialFields.ROW_ID.id)
+ table = lookup._read_global_index_result(GlobalIndexResult.create(row_ids))
+ positions = {row_id: i for i, row_id in enumerate(table.column(row_id_column).to_pylist())}
+ if added_row_id:
+ table = table.select(list(range(table.num_columns - 1)))
+ output = []
+ for result in results:
+ # Keep the physical read order, rather than imposing score or row-id order.
+ selected = sorted(positions[row_id] for row_id in result.results() if row_id in positions)
+ if self._limit is not None:
+ selected = selected[:self._limit]
+ output.append(table.take(pa.array(selected, type=pa.int64())))
+ return output
def to_pandas(self):
return [table.to_pandas() for table in self.to_arrow()]
diff --git a/paimon-python/pypaimon/tests/batch_vector_lookup_test.py b/paimon-python/pypaimon/tests/batch_vector_lookup_test.py
new file mode 100644
index 0000000000..aa7877a749
--- /dev/null
+++ b/paimon-python/pypaimon/tests/batch_vector_lookup_test.py
@@ -0,0 +1,181 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest.mock import patch
+
+import pyarrow as pa
+import pytest
+
+import pypaimon.multimodal as pm
+from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult
+from pypaimon.multimodal.query import ScanQuery
+from pypaimon.read.table_read import TableRead
+
+
+@pytest.fixture(params=[1, 3])
+def docs(tmp_path, request):
+ schema = pa.schema([
+ ("id", pa.int64()), ("category", pa.string()), ("embedding", pa.list_(pa.float32(), 2)),
+ ("info", pa.struct([("label", pa.string()), ("value", pa.int32())])),
+ ("attrs", pa.map_(pa.string(), pa.string())), ("payload", pa.large_binary()),
+ ])
+ table = pm.connect(options={"warehouse": str(tmp_path)}).create_table(
+ "docs", schema=schema, partitioned=["category"], options={
+ "file.format": "parquet", "vector.file.format": "parquet",
+ "vector-index.search-mode": "full", "read.parallelism": str(request.param),
+ "read.batch-size": "2", "blob-as-descriptor": "false",
+ })
+ for start in (0, 4, 8):
+ table.add(pa.Table.from_pylist([{
+ "id": i, "category": "a" if i % 2 else "b", "embedding": [float(i), 0.0],
+ "info": {"label": str(i), "value": i}, "attrs": {"x": str(i)},
+ "payload": ("payload-%d" % i).encode(),
+ } for i in range(start, start + 4)], schema=schema))
+ return table
+
+
+def configure(query, projection, with_row_id=False, post_filter=None):
+ if projection is not None:
+ query.select(projection)
+ if with_row_id:
+ query.with_row_id()
+ if post_filter is not None:
+ query.where(post_filter)
+ return query.limit(3)
+
+
+@pytest.mark.parametrize("projection", [
+ None, [], ["id"], ["payload"], ["_ROW_ID", "id"], ["info", "attrs['x']"], ["missing"]])
+@pytest.mark.parametrize("with_row_id", [False, True])
+def test_batch_lookup_matches_individual_queries(docs, projection, with_row_id):
+ vectors = [[1.0, 0.0], [9.0, 0.0], [1.0, 0.0]]
+ expected = [configure(docs.search(vector), projection, with_row_id).to_arrow() for vector in vectors]
+ query = configure(docs.search_vectors(vectors), projection, with_row_id)
+ original = TableRead.to_arrow
+ reads = []
+
+ def read(table_read, splits, *args, **kwargs):
+ reads.append(len(splits))
+ return original(table_read, splits, *args, **kwargs)
+
+ with patch.object(TableRead, "to_arrow", read):
+ actual = query.to_arrow()
+ assert len(actual) == len(expected)
+ for left, right in zip(expected, actual):
+ assert left.schema == right.schema
+ assert left.to_pylist() == right.to_pylist()
+ # Preserve the existing zero-column behavior of invalid projections.
+ assert len(reads) == (3 if projection == ["missing"] and not with_row_id else 1)
+
+
+def test_shared_lookup_keeps_pre_and_post_filters_and_per_query_limits(docs):
+ vectors = [[1.0, 0.0], [9.0, 0.0], [1.0, 0.0]]
+ options = {"pre_filter": "category = 'b'"}
+ expected = [configure(docs.search(vector, **options), ["id"], post_filter="id >= 5").to_arrow()
+ for vector in vectors]
+ actual = configure(docs.search_vectors(vectors, **options), ["id"], post_filter="id >= 5").to_arrow()
+ assert [result.to_pylist() for result in actual] == [result.to_pylist() for result in expected]
+ assert actual[0].num_rows == 0
+ assert actual[1].num_rows == 3
+
+
+def test_empty_membership_missing_rows_and_scores_are_independent(docs):
+ row_ids = [row["_ROW_ID"] for row in docs.scan().select(["id"]).with_row_id().to_list()]
+ results = [DictBasedScoredIndexResult(values) for values in (
+ {row_ids[0]: 0.9, row_ids[1]: 0.7}, {}, {row_ids[0]: 0.1}, {10000: 1.0})]
+ query = docs.search_vectors([[0.0, 0.0]] * len(results)).select(["id"]).limit(3)._for_execution()
+ expected = [query._read_global_index_result(result) for result in results]
+ actual = query._read_batch_results(results)
+ assert [table.to_pylist() for table in actual] == [table.to_pylist() for table in expected]
+ assert all(table.schema == expected[0].schema for table in actual)
+ assert results[0].score_getter()(row_ids[0]) == 0.9
+ assert results[2].score_getter()(row_ids[0]) == 0.1
+
+
+def test_all_empty_results_preserve_projection_schema(docs):
+ query = docs.search_vectors([[0.0, 0.0], [11.0, 0.0]], pre_filter="category = 'missing'")
+ actual = query.select(["id", "payload"]).with_row_id().limit(2).to_arrow()
+ expected = docs.search([0.0, 0.0], pre_filter="category = 'missing'").select(
+ ["id", "payload"]).with_row_id().limit(2).to_arrow()
+ assert len(actual) == 2
+ assert all(table.equals(expected) for table in actual)
+
+
+@pytest.mark.parametrize("change", ["update", "delete"])
+def test_shared_lookup_uses_search_snapshot_and_query_can_be_reused(docs, change):
+ query = docs.search_vectors([[0.0, 0.0], [11.0, 0.0]]).select(["id", "info"]).limit(1)
+ expected = query.to_list()
+ lookup = ScanQuery._read_global_index_result
+ calls = []
+
+ def commit_before_lookup(execution, result):
+ calls.append(execution._table)
+ if change == "update":
+ docs.update("id = 0", {"info": {"label": "updated", "value": 99}})
+ else:
+ docs.delete("id = 0")
+ return lookup(execution, result)
+
+ with patch.object(ScanQuery, "_read_global_index_result", commit_before_lookup):
+ assert query.to_list() == expected
+ assert len(calls) == 1
+ assert query.to_list() != expected
+
+
+@pytest.mark.parametrize("selector", ["snapshot", "tag"])
+def test_shared_lookup_keeps_historical_views(docs, selector):
+ snapshot = docs.raw_table.snapshot_manager().get_latest_snapshot()
+ docs.raw_table.create_tag("before", snapshot_id=snapshot.id)
+ options = {"snapshot_id": snapshot.id} if selector == "snapshot" else {"tag_name": "before"}
+ query = docs.search_vectors([[0.0, 0.0], [11.0, 0.0]], **options).select(["id", "info"]).limit(1)
+ expected = query.to_list()
+ docs.update("id = 0", {"info": {"label": "new", "value": 99}})
+ if selector == "tag":
+ docs.raw_table.file_io.delete(docs.raw_table.snapshot_manager().get_snapshot_path(snapshot.id))
+ assert query.to_list() == expected
+
+
+def test_single_query_keeps_single_lookup(docs):
+ expected = docs.search([1.0, 0.0]).select(["id"]).limit(3).to_arrow()
+ result = docs.search_vectors([[1.0, 0.0]]).select(["id"]).limit(3).to_arrow()
+ assert result[0].equals(expected)
+
+
+def test_unsupported_nested_row_projection_still_raises(docs):
+ with pytest.raises(NotImplementedError, match="ROW nested-field projection"):
+ docs.search_vectors([[0.0, 0.0], [1.0, 0.0]]).select(["info.label"]).to_arrow()
+
+
+def test_native_index_batch_uses_shared_lookup(docs):
+ pytest.importorskip("paimon_vindex")
+ docs.raw_table.copy({"deletion-vectors.enabled": "false"}).create_global_index(
+ "embedding", index_type="ivf-flat", options={
+ "ivf-flat.dimension": "2", "ivf-flat.nlist": "1", "ivf-flat.distance.metric": "l2"})
+ vectors = [[1.0, 0.0], [10.0, 0.0], [1.0, 0.0]]
+ expected = [docs.search(vector).select(["id"]).limit(3).to_list() for vector in vectors]
+ query = docs.search_vectors(vectors).select(["id"]).limit(3)
+ lookup = ScanQuery._read_global_index_result
+ calls = []
+
+ def read(execution, result):
+ calls.append(result.results().cardinality())
+ return lookup(execution, result)
+
+ with patch.object(ScanQuery, "_read_global_index_result", read):
+ actual = query.to_list()
+ assert actual == expected
+ assert calls == [6]
(paimon) 08/20: [python] Keep batch vector scoring vectorized for null rows (#9934)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit b9e8d12f6d60526a13e7d1e803324ef8f13d25b9
Author: chaoyang <ch...@apache.org>
AuthorDate: Fri Sep 18 12:00:56 2026 +0800
[python] Keep batch vector scoring vectorized for null rows (#9934)
---
.../pypaimon/table/source/vector_search_read.py | 14 +++++-
.../pypaimon/tests/vector_scoring_test.py | 50 ++++++++++++++++++++++
2 files changed, 63 insertions(+), 1 deletion(-)
diff --git a/paimon-python/pypaimon/table/source/vector_search_read.py b/paimon-python/pypaimon/table/source/vector_search_read.py
index c473d36308..178a7ea907 100644
--- a/paimon-python/pypaimon/table/source/vector_search_read.py
+++ b/paimon-python/pypaimon/table/source/vector_search_read.py
@@ -1014,7 +1014,14 @@ def _iter_arrow_batch_scores(vectors, queries, metric):
for start in range(0, len(vectors), block_size):
block = vectors.slice(start, block_size)
- matrix = _arrow_score_matrix(block, dimension) if fast_queries else None
+ positions = None
+ score_vectors = block
+ if fast_queries and block.null_count:
+ # Compact only this block; null rows must not disable vectorized
+ # scoring for the remaining rows or change their result positions.
+ positions = [i for i, valid in enumerate(block.is_valid().to_pylist()) if valid]
+ score_vectors = block.take(np.asarray(positions, dtype=np.int64))
+ matrix = _arrow_score_matrix(score_vectors, dimension) if fast_queries else None
if matrix is not None and not np.isfinite(matrix).all():
matrix = None
if matrix is None:
@@ -1030,6 +1037,11 @@ def _iter_arrow_batch_scores(vectors, queries, metric):
working = matrix if query_index == last_query else matrix.copy()
scores = _compute_scores(query, working, metric)
assert scores is not None
+ if positions is not None:
+ restored = [None] * len(block)
+ for position, score in zip(positions, scores):
+ restored[position] = score
+ scores = restored
yield start, query_index, scores
diff --git a/paimon-python/pypaimon/tests/vector_scoring_test.py b/paimon-python/pypaimon/tests/vector_scoring_test.py
index b9ef51e0a3..5ad0c1fd7e 100644
--- a/paimon-python/pypaimon/tests/vector_scoring_test.py
+++ b/paimon-python/pypaimon/tests/vector_scoring_test.py
@@ -110,6 +110,56 @@ class VectorScoringTest(unittest.TestCase):
self.assertEqual(1.0, scores[1])
self.assertEqual([1.0], list(_iter_arrow_scores(pa.array([[]], type=pa.list_(pa.float32())), [], "l2")))
+ def test_batch_null_vectors_preserve_scores_without_scalar_fallback(self):
+ rng = np.random.default_rng(37)
+ values = rng.standard_normal((2053, 7)).astype(np.float32).tolist()
+ for i in (0, 2, 511, 1025, 2051, 2052):
+ values[i] = None
+ queries = [[1.0] * 7, [0.0] * 7, [-0.5] * 7]
+ for dtype in (pa.list_(pa.float32()), pa.large_list(pa.float32()), pa.list_(pa.float32(), 7)):
+ sliced = pa.array(values, type=dtype).slice(2, 2050)
+ chunked = pa.chunked_array([sliced.slice(0, 513), sliced.slice(513)])
+ for vectors in (sliced, chunked):
+ for metric in ("l2", "cosine", "inner_product"):
+ with self.subTest(dtype=dtype, chunked=isinstance(vectors, pa.ChunkedArray), metric=metric):
+ expected = [[None if row is None else _compute_score(query, row, metric)
+ for row in values[2:2052]] for query in queries]
+ actual = [[] for _ in queries]
+ with mock.patch("pypaimon.table.source.vector_search_read._compute_score",
+ side_effect=AssertionError("valid rows must use vectorized scoring")):
+ for start, query_index, scores in _iter_arrow_batch_scores(vectors, queries, metric):
+ self.assertEqual(start, len(actual[query_index]))
+ actual[query_index].extend(scores)
+ self.assertEqual(expected, actual)
+
+ def test_batch_all_null_vectors_and_invalid_children_keep_existing_behavior(self):
+ queries = [[1.0, 0.0], [0.0, 0.0]]
+ for dtype in (pa.list_(pa.float32()), pa.large_list(pa.float32()), pa.list_(pa.float32(), 2)):
+ for metric in ("l2", "cosine", "inner_product"):
+ result = list(_iter_arrow_batch_scores(pa.array([None, None], type=dtype), queries, metric))
+ self.assertEqual([(0, 0, [None, None]), (0, 1, [None, None])], result)
+ for rows, exception, message in (
+ ([None, [1.0]], ValueError, "dimension mismatch"),
+ ([None, [1.0, None]], TypeError, "NoneType")):
+ with self.subTest(rows=rows):
+ vectors = pa.array(rows, type=pa.list_(pa.float32()))
+ with self.assertRaisesRegex(exception, message):
+ list(_iter_arrow_batch_scores(vectors, queries, "l2"))
+ vectors = pa.array([None, [float("nan"), 0.0], [1.0, 0.0]], type=pa.list_(pa.float32()))
+ scores = list(_iter_arrow_batch_scores(vectors, queries, "l2"))[0][2]
+ self.assertIsNone(scores[0])
+ self.assertTrue(np.isnan(scores[1]))
+ self.assertEqual(1.0, scores[2])
+
+ def test_batch_null_parent_ignores_invalid_child_values(self):
+ vectors = pa.Array.from_buffers(
+ pa.list_(pa.float32(), 2), 2, [pa.py_buffer(b"\x02")],
+ children=[pa.array([float("nan"), None, 1.0, 2.0], type=pa.float32())])
+ with mock.patch("pypaimon.table.source.vector_search_read._compute_score",
+ side_effect=AssertionError("null children must not disable the fast path")):
+ result = list(_iter_arrow_batch_scores(vectors, [[1.0, 2.0]], "l2"))
+ self.assertEqual([(0, 0, [None, 1.0])], result)
+
def test_raw_search_filters_before_scoring_and_preserves_ties(self):
column = _field(1, "embedding", "FLOAT")
reader = DataEvolutionVectorRead(_StubTable([column], []), 2, column, [1, 0])
(paimon) 13/20: [api] Sign x-acs-action in DLFOpenApiV4Signer (#9937)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit f97545a0a6fba84f36f4b4f53a9d7d414a272023
Author: Dapeng Sun(孙大鹏) <da...@alibaba-inc.com>
AuthorDate: Fri Sep 18 13:01:21 2026 +0800
[api] Sign x-acs-action in DLFOpenApiV4Signer (#9937)
---
docs/docs/concepts/rest/dlf.md | 6 +-
.../apache/paimon/rest/auth/DLFOpenApiActions.java | 184 +++++++++++++++++++++
.../paimon/rest/auth/DLFOpenApiV4Signer.java | 13 ++
.../paimon/rest/auth/DLFOpenApiActionsTest.java | 141 ++++++++++++++++
.../paimon/rest/auth/DLFOpenApiV4SignerTest.java | 92 +++++++++++
.../pypaimon/api/auth/dlf_openapi_actions.py | 103 ++++++++++++
paimon-python/pypaimon/api/auth/dlf_signer.py | 8 +
.../pypaimon/tests/rest/dlf_signer_test.py | 127 ++++++++++++++
8 files changed, 672 insertions(+), 2 deletions(-)
diff --git a/docs/docs/concepts/rest/dlf.md b/docs/docs/concepts/rest/dlf.md
index 602555b4a8..dc8a79832d 100644
--- a/docs/docs/concepts/rest/dlf.md
+++ b/docs/docs/concepts/rest/dlf.md
@@ -145,7 +145,9 @@ The client selects a request signer from the configured endpoint unless
URIs containing `dlfnext` select the OpenAPI signer; other URIs select the default signer. Set
`dlf.signing-algorithm` to `openapi-v4` to sign with ACS4-HMAC-SHA256 instead, which hashes a
-canonical request and derives its key from the date, the region and the product. That signer is
-opt-in: no endpoint selects it on its own. The client also infers the region from the URI.
+canonical request and derives its key from the date, the region and the product. It also signs an
+`x-acs-action` header naming the DlfNext API that the request's method and path map to, which some
+gateways require before they route the call. That signer is opt-in: no endpoint selects it on its
+own. The client also infers the region from the URI.
Set `dlf.region` explicitly if the endpoint does not contain a recognizable region, for example when
using a custom hostname.
diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java
new file mode 100644
index 0000000000..4d72b9c991
--- /dev/null
+++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.rest.auth;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+
+import javax.annotation.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+/** Maps a REST request to the POP action it is registered under in DlfNext 2026-01-18. */
+final class DLFOpenApiActions {
+
+ /** Method, path template and action; a {@code {name}} segment matches any one segment. */
+ private static final String[][] OPERATIONS = {
+ {"GET", "/v1/config", "GetConfig"},
+ {"GET", "/v1/{prefix}/databases", "ListDatabases"},
+ {"POST", "/v1/{prefix}/databases", "CreateDatabase"},
+ {"GET", "/v1/{prefix}/databases/{database}", "GetDatabase"},
+ {"POST", "/v1/{prefix}/databases/{database}", "AlterDatabase"},
+ {"DELETE", "/v1/{prefix}/databases/{database}", "DropDatabase"},
+ {"POST", "/v1/{prefix}/databases/{database}/register", "RegisterTable"},
+ {"GET", "/v1/{prefix}/databases/{database}/tables", "ListTables"},
+ {"POST", "/v1/{prefix}/databases/{database}/tables", "CreateTable"},
+ {"GET", "/v1/{prefix}/databases/{database}/table-details", "ListTableDetails"},
+ {"GET", "/v1/{prefix}/databases/{database}/tables/{table}", "GetTable"},
+ {"POST", "/v1/{prefix}/databases/{database}/tables/{table}", "AlterTable"},
+ {"DELETE", "/v1/{prefix}/databases/{database}/tables/{table}", "DropTable"},
+ {"POST", "/v1/{prefix}/tables/rename", "RenameTable"},
+ {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/auth", "AuthTableQuery"},
+ {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/commit", "CommitTable"},
+ {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/token", "GetTableToken"},
+ {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/rollback", "RollbackToSnapshot"},
+ {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/snapshot", "GetTableSnapshot"},
+ {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/snapshots", "ListSnapshots"},
+ {
+ "GET",
+ "/v1/{prefix}/databases/{database}/tables/{table}/snapshots/{version}",
+ "GetVersionSnapshot"
+ },
+ {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/partitions", "ListPartitions"},
+ {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/partitions", "CreatePartitions"},
+ {
+ "POST",
+ "/v1/{prefix}/databases/{database}/tables/{table}/partitions/drop",
+ "DropPartitions"
+ },
+ {
+ "POST",
+ "/v1/{prefix}/databases/{database}/tables/{table}/partitions/mark",
+ "MarkDonePartitions"
+ },
+ {
+ "POST",
+ "/v1/{prefix}/databases/{database}/tables/{table}/partitions/list-by-names",
+ "ListPartitionsByNames"
+ },
+ {
+ "POST",
+ "/v1/{prefix}/databases/{database}/tables/{table}/partitions/list-by-filter",
+ "ListPartitionsByFilter"
+ },
+ {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/branches", "ListBranches"},
+ {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/branches", "CreateBranch"},
+ {
+ "DELETE",
+ "/v1/{prefix}/databases/{database}/tables/{table}/branches/{branch}",
+ "DropBranch"
+ },
+ {
+ "POST",
+ "/v1/{prefix}/databases/{database}/tables/{table}/branches/{branch}/forward",
+ "FastForwardBranch"
+ },
+ {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/tags", "ListTags"},
+ {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/tags", "CreateTag"},
+ {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/tags/{tag}", "GetTag"},
+ {"DELETE", "/v1/{prefix}/databases/{database}/tables/{table}/tags/{tag}", "DropTag"},
+ {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/policies", "ListPolicies"},
+ {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/policies", "CreatePolicy"},
+ {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/policies/drop", "DropPolicy"},
+ {"GET", "/v1/{prefix}/databases/{database}/views", "ListViews"},
+ {"POST", "/v1/{prefix}/databases/{database}/views", "CreateView"},
+ {"GET", "/v1/{prefix}/databases/{database}/view-details", "ListViewDetails"},
+ {"GET", "/v1/{prefix}/databases/{database}/views/{view}", "GetView"},
+ {"POST", "/v1/{prefix}/databases/{database}/views/{view}", "AlterView"},
+ {"DELETE", "/v1/{prefix}/databases/{database}/views/{view}", "DropView"},
+ {"POST", "/v1/{prefix}/views/rename", "RenameView"},
+ {"GET", "/v1/{prefix}/databases/{database}/functions", "ListFunctions"},
+ {"POST", "/v1/{prefix}/databases/{database}/functions", "CreateFunction"},
+ {"GET", "/v1/{prefix}/databases/{database}/functions/{function}", "GetFunction"},
+ {"POST", "/v1/{prefix}/databases/{database}/functions/{function}", "AlterFunction"},
+ {"DELETE", "/v1/{prefix}/databases/{database}/functions/{function}", "DropFunction"},
+ {"GET", "/v1/{prefix}/permissions", "ListPermissionAssignments"},
+ {"POST", "/v1/{prefix}/permissions/grant", "GrantPermissionAssignment"},
+ {"POST", "/v1/{prefix}/permissions/revoke", "RevokePermissionAssignment"},
+ };
+
+ private static final Map<String, List<Operation>> BY_METHOD = index();
+
+ private DLFOpenApiActions() {}
+
+ /** Returns the action for the request, or null when no operation is registered for it. */
+ @Nullable
+ static String resolve(@Nullable String method, @Nullable String resourcePath) {
+ if (method == null || resourcePath == null) {
+ return null;
+ }
+ List<Operation> candidates = BY_METHOD.get(method.toUpperCase(Locale.ROOT));
+ if (candidates == null) {
+ return null;
+ }
+ String[] segments = resourcePath.split("/", -1);
+ for (Operation operation : candidates) {
+ if (operation.matches(segments)) {
+ return operation.action;
+ }
+ }
+ return null;
+ }
+
+ /** Every registered operation as {method, path template, action}. */
+ @VisibleForTesting
+ static List<String[]> operations() {
+ List<String[]> copy = new ArrayList<>();
+ for (String[] operation : OPERATIONS) {
+ copy.add(operation.clone());
+ }
+ return copy;
+ }
+
+ private static Map<String, List<Operation>> index() {
+ Map<String, List<Operation>> byMethod = new HashMap<>();
+ for (String[] operation : OPERATIONS) {
+ byMethod.computeIfAbsent(operation[0], k -> new ArrayList<>())
+ .add(new Operation(operation[1].split("/", -1), operation[2]));
+ }
+ return Collections.unmodifiableMap(byMethod);
+ }
+
+ private static final class Operation {
+ private final String[] template;
+ private final String action;
+
+ private Operation(String[] template, String action) {
+ this.template = template;
+ this.action = action;
+ }
+
+ private boolean matches(String[] segments) {
+ if (segments.length != template.length) {
+ return false;
+ }
+ for (int i = 0; i < template.length; i++) {
+ boolean variable = template[i].startsWith("{");
+ if (variable ? segments[i].isEmpty() : !template[i].equals(segments[i])) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+}
diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiV4Signer.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiV4Signer.java
index d82103a9ec..4c090c7a8b 100644
--- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiV4Signer.java
+++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiV4Signer.java
@@ -46,6 +46,9 @@ import static org.apache.paimon.rest.RESTUtil.decodeString;
* Content-MD5} headers with HMAC-SHA1, this signer hashes a canonical request and signs it with a
* key derived from the date, the region and the product, the way the POP gateway does.
*
+ * <p>{@link #authorization} also puts the API name in the headers it is handed, because a POP
+ * gateway may route by that header and it has to be signed with the rest.
+ *
* <p>Reference: https://github.com/aliyun/alibabacloud-gateway/tree/master/alibabacloud-gateway-pop
*/
public class DLFOpenApiV4Signer implements DLFRequestSigner {
@@ -70,6 +73,7 @@ public class DLFOpenApiV4Signer implements DLFRequestSigner {
private static final String X_ACS_VERSION = "x-acs-version";
private static final String X_ACS_CONTENT_SHA256 = "x-acs-content-sha256";
private static final String X_ACS_SECURITY_TOKEN = "x-acs-security-token";
+ private static final String X_ACS_ACTION = "x-acs-action";
private static final String CONTENT_TYPE_VALUE = "application/json";
private static final String API_VERSION = "2026-01-18";
@@ -153,6 +157,15 @@ public class DLFOpenApiV4Signer implements DLFRequestSigner {
throw new IllegalArgumentException("Parameter 'signHeaders' cannot be null");
}
+ // POP gateways may route by this header, so it is added to the headers the caller sends
+ // and signed with them; the request names no API only when none is registered for it.
+ String action =
+ DLFOpenApiActions.resolve(
+ restAuthParameter.method(), restAuthParameter.resourcePath());
+ if (action != null) {
+ signHeaders.put(X_ACS_ACTION, action);
+ }
+
CanonicalHeadersResult canonicalHeadersResult = buildCanonicalHeaders(signHeaders);
String canonicalQueryString = buildCanonicalQueryString(restAuthParameter.parameters());
diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java
new file mode 100644
index 0000000000..d9a11067b3
--- /dev/null
+++ b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.rest.auth;
+
+import org.apache.paimon.management.PermissionResource;
+import org.apache.paimon.management.ResourceType;
+import org.apache.paimon.rest.ResourcePaths;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+/** Test for {@link DLFOpenApiActions}. */
+public class DLFOpenApiActionsTest {
+
+ private final ResourcePaths paths = new ResourcePaths("clg-paimon-1");
+
+ /** Each template, filled in, must come back to its own action rather than an earlier one. */
+ @Test
+ public void testEveryOperationResolvesToItself() {
+ Set<String> actions = new HashSet<>();
+ for (String[] operation : DLFOpenApiActions.operations()) {
+ String path = operation[1].replaceAll("\\{[^}]+}", "x1");
+ assertEquals(operation[2], DLFOpenApiActions.resolve(operation[0], path), path);
+ actions.add(operation[2]);
+ }
+ assertEquals(53, actions.size());
+ }
+
+ @Test
+ public void testClientPathsResolveToRegisteredActions() {
+ assertAction("GetConfig", "GET", ResourcePaths.config());
+ assertAction("ListDatabases", "GET", paths.databases());
+ assertAction("CreateDatabase", "POST", paths.databases());
+ assertAction("GetDatabase", "GET", paths.database("db"));
+ assertAction("AlterDatabase", "POST", paths.database("db"));
+ assertAction("DropDatabase", "DELETE", paths.database("db"));
+ assertAction("RegisterTable", "POST", paths.registerTable("db"));
+ assertAction("ListTables", "GET", paths.tables("db"));
+ assertAction("CreateTable", "POST", paths.tables("db"));
+ assertAction("ListTableDetails", "GET", paths.tableDetails("db"));
+ assertAction("GetTable", "GET", paths.table("db", "t"));
+ assertAction("AlterTable", "POST", paths.table("db", "t"));
+ assertAction("DropTable", "DELETE", paths.table("db", "t"));
+ assertAction("RenameTable", "POST", paths.renameTable());
+ assertAction("AuthTableQuery", "POST", paths.authTable("db", "t"));
+ assertAction("CommitTable", "POST", paths.commitTable("db", "t"));
+ assertAction("GetTableToken", "GET", paths.tableToken("db", "t"));
+ assertAction("RollbackToSnapshot", "POST", paths.rollbackTable("db", "t"));
+ assertAction("GetTableSnapshot", "GET", paths.tableSnapshot("db", "t"));
+ assertAction("ListSnapshots", "GET", paths.snapshots("db", "t"));
+ assertAction("GetVersionSnapshot", "GET", paths.tableSnapshot("db", "t", "3"));
+ assertAction("ListPartitions", "GET", paths.partitions("db", "t"));
+ assertAction("CreatePartitions", "POST", paths.partitions("db", "t"));
+ assertAction("DropPartitions", "POST", paths.dropPartitions("db", "t"));
+ assertAction("MarkDonePartitions", "POST", paths.markDonePartitions("db", "t"));
+ assertAction("ListPartitionsByNames", "POST", paths.listPartitionsByNames("db", "t"));
+ assertAction("ListPartitionsByFilter", "POST", paths.listPartitionsByFilter("db", "t"));
+ assertAction("ListBranches", "GET", paths.branches("db", "t"));
+ assertAction("CreateBranch", "POST", paths.branches("db", "t"));
+ assertAction("DropBranch", "DELETE", paths.branch("db", "t", "b"));
+ assertAction("FastForwardBranch", "POST", paths.forwardBranch("db", "t", "b"));
+ assertAction("ListTags", "GET", paths.tags("db", "t"));
+ assertAction("CreateTag", "POST", paths.tags("db", "t"));
+ assertAction("GetTag", "GET", paths.tag("db", "t", "tag"));
+ assertAction("DropTag", "DELETE", paths.tag("db", "t", "tag"));
+ PermissionResource table =
+ new PermissionResource(ResourceType.TABLE, "db", "t", null, null);
+ assertAction("ListPolicies", "GET", paths.policies(table));
+ assertAction("CreatePolicy", "POST", paths.policies(table));
+ assertAction("DropPolicy", "POST", paths.dropPolicy(table));
+ assertAction("ListViews", "GET", paths.views("db"));
+ assertAction("CreateView", "POST", paths.views("db"));
+ assertAction("ListViewDetails", "GET", paths.viewDetails("db"));
+ assertAction("GetView", "GET", paths.view("db", "v"));
+ assertAction("AlterView", "POST", paths.view("db", "v"));
+ assertAction("DropView", "DELETE", paths.view("db", "v"));
+ assertAction("RenameView", "POST", paths.renameView());
+ assertAction("ListFunctions", "GET", paths.functions("db"));
+ assertAction("CreateFunction", "POST", paths.functions("db"));
+ assertAction("GetFunction", "GET", paths.function("db", "f"));
+ assertAction("AlterFunction", "POST", paths.function("db", "f"));
+ assertAction("DropFunction", "DELETE", paths.function("db", "f"));
+ assertAction("ListPermissionAssignments", "GET", paths.permissions());
+ assertAction("GrantPermissionAssignment", "POST", paths.grantPermission());
+ assertAction("RevokePermissionAssignment", "POST", paths.revokePermission());
+ }
+
+ /** Names match whole segments, so one spelled like a literal still resolves. */
+ @Test
+ public void testNamesSpelledLikeLiterals() {
+ assertAction("GetDatabase", "GET", paths.database("tables"));
+ assertAction("GetTable", "GET", paths.table("db", "token"));
+ assertAction("DropBranch", "DELETE", paths.branch("db", "t", "forward"));
+ assertAction("GetTag", "GET", paths.tag("db", "t", "tags"));
+ assertAction("GetTableToken", "GET", paths.tableToken("config", "t$snapshots"));
+ assertAction("GetTable", "GET", paths.table("a/b", "c d"));
+ assertAction("ListDatabases", "GET", new ResourcePaths("rename").databases());
+ }
+
+ @Test
+ public void testUnregisteredRequestsHaveNoAction() {
+ assertNull(DLFOpenApiActions.resolve("GET", paths.tables()));
+ assertNull(DLFOpenApiActions.resolve("GET", paths.table("tbl-1")));
+ assertNull(DLFOpenApiActions.resolve("GET", paths.schemas("db", "t")));
+ assertNull(DLFOpenApiActions.resolve("PUT", paths.table("db", "t")));
+ assertNull(DLFOpenApiActions.resolve("GET", ResourcePaths.config() + "/"));
+ assertNull(DLFOpenApiActions.resolve("GET", "/v1//databases"));
+ assertNull(DLFOpenApiActions.resolve(null, ResourcePaths.config()));
+ assertNull(DLFOpenApiActions.resolve("GET", null));
+ }
+
+ @Test
+ public void testMethodIsCaseInsensitive() {
+ assertAction("GetConfig", "get", ResourcePaths.config());
+ }
+
+ private static void assertAction(String expected, String method, String path) {
+ assertEquals(expected, DLFOpenApiActions.resolve(method, path), method + " " + path);
+ }
+}
diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiV4SignerTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiV4SignerTest.java
index 540b2c82db..a81358aa88 100644
--- a/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiV4SignerTest.java
+++ b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiV4SignerTest.java
@@ -227,6 +227,98 @@ public class DLFOpenApiV4SignerTest {
authorization);
}
+ /** Known answer with a body and a resolved action; pypaimon pins the same string. */
+ @Test
+ public void testKnownSignatureWithAction() throws Exception {
+ DLFOpenApiV4Signer signer = new DLFOpenApiV4Signer("cn-hangzhou");
+ DLFToken token = new DLFToken("TestAKId", "TestAKSecret", null, null);
+ String host = "dlfnext.cn-hangzhou.aliyuncs.com";
+ String body = "{\"identifier\":{\"database\":\"db\",\"object\":\"t\"}}";
+ RESTAuthParameter restAuthParameter =
+ new RESTAuthParameter(
+ "/v1/clg-paimon-1/databases/db/tables", new HashMap<>(), "POST", body);
+
+ Map<String, String> signHeaders = signer.signHeaders(body, NOW, null, host);
+ signHeaders.put("x-acs-signature-nonce", "fixed-nonce-for-test");
+ String authorization = signer.authorization(restAuthParameter, token, host, signHeaders);
+
+ // signing adds the action to the headers the caller then sends
+ assertEquals("CreateTable", signHeaders.get("x-acs-action"));
+ assertEquals(
+ "ACS4-HMAC-SHA256 Credential=TestAKId/20250416/cn-hangzhou/DlfNext/aliyun_v4_request,"
+ + "SignedHeaders=content-type;host;x-acs-action;x-acs-content-sha256;"
+ + "x-acs-date;x-acs-signature-nonce;x-acs-version,"
+ + "Signature=420206b5263536e6bc271a7a32582b4a820e23c8c58ae692b708c69f9d19e51d",
+ authorization);
+ }
+
+ @Test
+ public void testUnregisteredPathIsSignedWithoutAnAction() throws Exception {
+ DLFOpenApiV4Signer signer = new DLFOpenApiV4Signer(REGION);
+ DLFToken token = new DLFToken("TestAKId", "TestAKSecret", null, null);
+ RESTAuthParameter restAuthParameter =
+ new RESTAuthParameter("/v1/clg-paimon-1/tables", new HashMap<>(), "GET", null);
+ Map<String, String> signHeaders = signer.signHeaders(null, NOW, null, HOST);
+
+ String authorization = signer.authorization(restAuthParameter, token, HOST, signHeaders);
+
+ assertFalse(signHeaders.containsKey("x-acs-action"));
+ assertFalse(authorization.contains("x-acs-action"));
+ }
+
+ /** Only the ACS4 signer names the API; the other schemes send what they always did. */
+ @Test
+ public void testOnlyOpenApiV4SendsAction() {
+ RESTAuthParameter restAuthParameter =
+ new RESTAuthParameter(
+ "/v1/clg-paimon-1/databases/db/tables/t/token",
+ new HashMap<>(),
+ "GET",
+ null);
+ for (String algorithm :
+ new String[] {DLFDefaultSigner.IDENTIFIER, DLFOpenApiSigner.IDENTIFIER}) {
+ DLFAuthProvider provider =
+ DLFAuthProvider.fromAccessKey(
+ "akId",
+ "akSecret",
+ null,
+ "https://dlfnext.cn-hangzhou.aliyuncs.com",
+ "cn-hangzhou",
+ algorithm);
+ Map<String, String> headers =
+ provider.mergeAuthHeader(new HashMap<>(), restAuthParameter);
+ assertFalse(headers.containsKey("x-acs-action"), algorithm);
+ }
+ }
+
+ @Test
+ public void testAuthProviderSendsSignedAction() {
+ DLFAuthProvider provider =
+ DLFAuthProvider.fromAccessKey(
+ "akId",
+ "akSecret",
+ "securityToken",
+ "https://dlfnext.cn-hangzhou.aliyuncs.com",
+ "cn-hangzhou",
+ DLFOpenApiV4Signer.IDENTIFIER);
+
+ Map<String, String> headers =
+ provider.mergeAuthHeader(
+ new HashMap<>(),
+ new RESTAuthParameter(
+ "/v1/clg-paimon-1/databases/db/tables/t/token",
+ new HashMap<>(),
+ "GET",
+ null));
+
+ assertEquals("GetTableToken", headers.get("x-acs-action"));
+ assertTrue(
+ headers.get("Authorization")
+ .contains(
+ "SignedHeaders=host;x-acs-action;x-acs-content-sha256;x-acs-date;"
+ + "x-acs-security-token;x-acs-signature-nonce;x-acs-version,"));
+ }
+
@Test
public void testSignedHeadersIncludeHostAndContentType() throws Exception {
DLFOpenApiV4Signer signer = new DLFOpenApiV4Signer(REGION);
diff --git a/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py b/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py
new file mode 100644
index 0000000000..8fd46e9f63
--- /dev/null
+++ b/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py
@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Maps a REST request to the POP action it is registered under in DlfNext 2026-01-18."""
+
+from typing import Dict, List, Optional, Tuple
+
+# Method, path template and action; a {name} segment matches any one segment.
+OPERATIONS: Tuple[Tuple[str, str, str], ...] = (
+ ("GET", "/v1/config", "GetConfig"),
+ ("GET", "/v1/{prefix}/databases", "ListDatabases"),
+ ("POST", "/v1/{prefix}/databases", "CreateDatabase"),
+ ("GET", "/v1/{prefix}/databases/{database}", "GetDatabase"),
+ ("POST", "/v1/{prefix}/databases/{database}", "AlterDatabase"),
+ ("DELETE", "/v1/{prefix}/databases/{database}", "DropDatabase"),
+ ("POST", "/v1/{prefix}/databases/{database}/register", "RegisterTable"),
+ ("GET", "/v1/{prefix}/databases/{database}/tables", "ListTables"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables", "CreateTable"),
+ ("GET", "/v1/{prefix}/databases/{database}/table-details", "ListTableDetails"),
+ ("GET", "/v1/{prefix}/databases/{database}/tables/{table}", "GetTable"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables/{table}", "AlterTable"),
+ ("DELETE", "/v1/{prefix}/databases/{database}/tables/{table}", "DropTable"),
+ ("POST", "/v1/{prefix}/tables/rename", "RenameTable"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/auth", "AuthTableQuery"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/commit", "CommitTable"),
+ ("GET", "/v1/{prefix}/databases/{database}/tables/{table}/token", "GetTableToken"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/rollback", "RollbackToSnapshot"),
+ ("GET", "/v1/{prefix}/databases/{database}/tables/{table}/snapshot", "GetTableSnapshot"),
+ ("GET", "/v1/{prefix}/databases/{database}/tables/{table}/snapshots", "ListSnapshots"),
+ ("GET", "/v1/{prefix}/databases/{database}/tables/{table}/snapshots/{version}", "GetVersionSnapshot"),
+ ("GET", "/v1/{prefix}/databases/{database}/tables/{table}/partitions", "ListPartitions"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/partitions", "CreatePartitions"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/partitions/drop", "DropPartitions"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/partitions/mark", "MarkDonePartitions"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/partitions/list-by-names",
+ "ListPartitionsByNames"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/partitions/list-by-filter",
+ "ListPartitionsByFilter"),
+ ("GET", "/v1/{prefix}/databases/{database}/tables/{table}/branches", "ListBranches"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/branches", "CreateBranch"),
+ ("DELETE", "/v1/{prefix}/databases/{database}/tables/{table}/branches/{branch}", "DropBranch"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/branches/{branch}/forward",
+ "FastForwardBranch"),
+ ("GET", "/v1/{prefix}/databases/{database}/tables/{table}/tags", "ListTags"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/tags", "CreateTag"),
+ ("GET", "/v1/{prefix}/databases/{database}/tables/{table}/tags/{tag}", "GetTag"),
+ ("DELETE", "/v1/{prefix}/databases/{database}/tables/{table}/tags/{tag}", "DropTag"),
+ ("GET", "/v1/{prefix}/databases/{database}/tables/{table}/policies", "ListPolicies"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/policies", "CreatePolicy"),
+ ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/policies/drop", "DropPolicy"),
+ ("GET", "/v1/{prefix}/databases/{database}/views", "ListViews"),
+ ("POST", "/v1/{prefix}/databases/{database}/views", "CreateView"),
+ ("GET", "/v1/{prefix}/databases/{database}/view-details", "ListViewDetails"),
+ ("GET", "/v1/{prefix}/databases/{database}/views/{view}", "GetView"),
+ ("POST", "/v1/{prefix}/databases/{database}/views/{view}", "AlterView"),
+ ("DELETE", "/v1/{prefix}/databases/{database}/views/{view}", "DropView"),
+ ("POST", "/v1/{prefix}/views/rename", "RenameView"),
+ ("GET", "/v1/{prefix}/databases/{database}/functions", "ListFunctions"),
+ ("POST", "/v1/{prefix}/databases/{database}/functions", "CreateFunction"),
+ ("GET", "/v1/{prefix}/databases/{database}/functions/{function}", "GetFunction"),
+ ("POST", "/v1/{prefix}/databases/{database}/functions/{function}", "AlterFunction"),
+ ("DELETE", "/v1/{prefix}/databases/{database}/functions/{function}", "DropFunction"),
+ ("GET", "/v1/{prefix}/permissions", "ListPermissionAssignments"),
+ ("POST", "/v1/{prefix}/permissions/grant", "GrantPermissionAssignment"),
+ ("POST", "/v1/{prefix}/permissions/revoke", "RevokePermissionAssignment"),
+)
+
+
+def _index() -> Dict[str, List[Tuple[List[str], str]]]:
+ by_method: Dict[str, List[Tuple[List[str], str]]] = {}
+ for method, template, action in OPERATIONS:
+ by_method.setdefault(method, []).append((template.split("/"), action))
+ return by_method
+
+
+_BY_METHOD = _index()
+
+
+def resolve_action(method: Optional[str], path: Optional[str]) -> Optional[str]:
+ """Returns the action for the request, or None when no operation is registered for it."""
+ if method is None or path is None:
+ return None
+ segments = path.split("/")
+ for template, action in _BY_METHOD.get(method.upper(), []):
+ if len(template) == len(segments) and all(
+ segment != "" if part.startswith("{") else part == segment
+ for part, segment in zip(template, segments)):
+ return action
+ return None
diff --git a/paimon-python/pypaimon/api/auth/dlf_signer.py b/paimon-python/pypaimon/api/auth/dlf_signer.py
index 9915fa012d..fa69c16f6b 100644
--- a/paimon-python/pypaimon/api/auth/dlf_signer.py
+++ b/paimon-python/pypaimon/api/auth/dlf_signer.py
@@ -27,6 +27,7 @@ from datetime import datetime, timezone
from typing import Dict, NamedTuple, Optional
from urllib.parse import quote, unquote
+from pypaimon.api.auth.dlf_openapi_actions import resolve_action
from pypaimon.api.token_loader import DLFToken
from pypaimon.api.typedef import RESTAuthParameter
@@ -536,6 +537,7 @@ class DLFOpenApiV4Signer(DLFRequestSigner):
X_ACS_VERSION = "x-acs-version"
X_ACS_CONTENT_SHA256 = "x-acs-content-sha256"
X_ACS_SECURITY_TOKEN = "x-acs-security-token"
+ X_ACS_ACTION = "x-acs-action"
# Values
ACS_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
@@ -604,6 +606,12 @@ class DLFOpenApiV4Signer(DLFRequestSigner):
raise ValueError("Parameter 'sign_headers' cannot be None")
try:
+ # POP gateways may route by this header, so it is added to the headers the caller
+ # sends and signed with them; it is absent only when no API is registered for it.
+ action = resolve_action(rest_auth_parameter.method, rest_auth_parameter.path)
+ if action is not None:
+ sign_headers[self.X_ACS_ACTION] = action
+
canonical = self._build_canonical_headers(sign_headers)
canonical_query_string = self._build_canonical_query_string(
rest_auth_parameter.parameters
diff --git a/paimon-python/pypaimon/tests/rest/dlf_signer_test.py b/paimon-python/pypaimon/tests/rest/dlf_signer_test.py
index 4b9ef61e95..53368b8f54 100644
--- a/paimon-python/pypaimon/tests/rest/dlf_signer_test.py
+++ b/paimon-python/pypaimon/tests/rest/dlf_signer_test.py
@@ -29,6 +29,8 @@ from pypaimon.api.auth import (
DLFOpenApiSigner,
DLFOpenApiV4Signer,
)
+from pypaimon.api.auth.dlf_openapi_actions import OPERATIONS, resolve_action
+from pypaimon.api.resource_paths import ResourcePaths
from pypaimon.api.token_loader import DLFToken
from pypaimon.api.typedef import RESTAuthParameter
@@ -479,6 +481,131 @@ class DLFSignerTest(unittest.TestCase):
self.assertEqual(
hashlib.sha256(b"").hexdigest(), DLFOpenApiV4Signer.EMPTY_BODY_SHA256)
+ def test_openapi_v4_every_operation_resolves_to_itself(self):
+ """Each template, filled in, must come back to its own action rather than an earlier one."""
+ actions = set()
+ for method, template, action in OPERATIONS:
+ path = re.sub(r"\{[^}]+\}", "x1", template)
+ self.assertEqual(action, resolve_action(method, path), path)
+ actions.add(action)
+ self.assertEqual(53, len(actions))
+
+ def test_openapi_v4_client_paths_resolve_to_registered_actions(self):
+ paths = ResourcePaths("clg-paimon-1")
+ cases = [
+ ("GetConfig", "GET", ResourcePaths.config()),
+ ("ListDatabases", "GET", paths.databases()),
+ ("CreateDatabase", "POST", paths.databases()),
+ ("GetDatabase", "GET", paths.database("db")),
+ ("DropDatabase", "DELETE", paths.database("db")),
+ ("ListTables", "GET", paths.tables("db")),
+ ("CreateTable", "POST", paths.tables("db")),
+ ("ListTableDetails", "GET", paths.table_details("db")),
+ ("GetTable", "GET", paths.table("db", "t")),
+ ("AlterTable", "POST", paths.table("db", "t")),
+ ("DropTable", "DELETE", paths.table("db", "t")),
+ ("RenameTable", "POST", paths.rename_table()),
+ ("GetTableToken", "GET", paths.table_token("db", "t")),
+ ("CommitTable", "POST", paths.commit_table("db", "t")),
+ ("RollbackToSnapshot", "POST", paths.rollback_table("db", "t")),
+ ("GetTableSnapshot", "GET", paths.table_snapshot("db", "t")),
+ ("ListPartitions", "GET", paths.partitions("db", "t")),
+ ("ListFunctions", "GET", paths.functions("db")),
+ ("GetFunction", "GET", paths.function("db", "f")),
+ ("ListTags", "GET", paths.tags("db", "t")),
+ ("GetTag", "GET", paths.tag("db", "t", "tag")),
+ ("ListBranches", "GET", paths.branches("db", "t")),
+ ("DropBranch", "DELETE", paths.branch("db", "t", "b")),
+ ("FastForwardBranch", "POST", paths.forward_branch("db", "t", "b")),
+ ("AuthTableQuery", "POST", paths.auth_table("db", "t")),
+ ]
+ for action, method, path in cases:
+ self.assertEqual(action, resolve_action(method, path), method + " " + path)
+
+ def test_openapi_v4_names_spelled_like_literals(self):
+ """Names match whole segments, so one spelled like a literal still resolves."""
+ paths = ResourcePaths("clg-paimon-1")
+ self.assertEqual("GetDatabase", resolve_action("GET", paths.database("tables")))
+ self.assertEqual("GetTable", resolve_action("GET", paths.table("db", "token")))
+ self.assertEqual("DropBranch", resolve_action("DELETE", paths.branch("db", "t", "forward")))
+ self.assertEqual("GetTableToken", resolve_action("GET", paths.table_token("config", "t$snapshots")))
+ self.assertEqual("GetTable", resolve_action("GET", "/v1/clg-paimon-1/databases/a%2Fb/tables/c%20d"))
+ self.assertEqual("ListDatabases", resolve_action("get", ResourcePaths("rename").databases()))
+
+ def test_openapi_v4_unregistered_requests_have_no_action(self):
+ paths = ResourcePaths("clg-paimon-1")
+ self.assertIsNone(resolve_action("GET", paths.tables()))
+ self.assertIsNone(resolve_action("POST", paths.rename_branch("db", "t", "b")))
+ self.assertIsNone(resolve_action("PUT", paths.table("db", "t")))
+ self.assertIsNone(resolve_action("GET", ResourcePaths.config() + "/"))
+ self.assertIsNone(resolve_action("GET", "/v1//databases"))
+ self.assertIsNone(resolve_action(None, ResourcePaths.config()))
+ self.assertIsNone(resolve_action("GET", None))
+
+ def test_openapi_v4_known_signature_with_action(self):
+ """Known answer with a body and a resolved action. The Java signer pins the same string."""
+ signer = DLFOpenApiV4Signer("cn-hangzhou")
+ token = DLFToken("TestAKId", "TestAKSecret", None, None)
+ host = "dlfnext.cn-hangzhou.aliyuncs.com"
+ now = datetime(2025, 4, 16, 3, 44, 46, tzinfo=timezone.utc)
+ body = '{"identifier":{"database":"db","object":"t"}}'
+ rest_param = RESTAuthParameter("POST", "/v1/clg-paimon-1/databases/db/tables", body, {})
+
+ headers = signer.sign_headers(body, now, None, host)
+ headers["x-acs-signature-nonce"] = "fixed-nonce-for-test"
+ authorization = signer.authorization(rest_param, token, host, headers)
+
+ # signing adds the action to the headers the caller then sends
+ self.assertEqual("CreateTable", headers["x-acs-action"])
+ self.assertEqual(
+ "ACS4-HMAC-SHA256 Credential=TestAKId/20250416/cn-hangzhou/DlfNext/aliyun_v4_request,"
+ "SignedHeaders=content-type;host;x-acs-action;x-acs-content-sha256;"
+ "x-acs-date;x-acs-signature-nonce;x-acs-version,"
+ "Signature=420206b5263536e6bc271a7a32582b4a820e23c8c58ae692b708c69f9d19e51d",
+ authorization)
+
+ def test_openapi_v4_unregistered_path_is_signed_without_an_action(self):
+ signer = DLFOpenApiV4Signer("cn-hangzhou")
+ token = DLFToken("TestAKId", "TestAKSecret", None, None)
+ host = "dlfnext.cn-hangzhou.aliyuncs.com"
+ now = datetime(2025, 4, 16, 3, 44, 46, tzinfo=timezone.utc)
+ rest_param = RESTAuthParameter("GET", "/v1/clg-paimon-1/tables", "", {})
+ headers = signer.sign_headers(None, now, None, host)
+
+ authorization = signer.authorization(rest_param, token, host, headers)
+
+ self.assertNotIn("x-acs-action", headers)
+ self.assertNotIn("x-acs-action", authorization)
+
+ def test_auth_provider_sends_signed_action(self):
+ provider = DLFAuthProvider(
+ uri="https://dlfnext.cn-hangzhou.aliyuncs.com",
+ region="cn-hangzhou",
+ signing_algorithm=DLFOpenApiV4Signer.IDENTIFIER,
+ token=DLFToken("akId", "akSecret", "securityToken", None)
+ )
+
+ header = provider.merge_auth_header(
+ {}, RESTAuthParameter("GET", "/v1/clg-paimon-1/databases/db/tables/t/token", "", {}))
+
+ self.assertEqual("GetTableToken", header["x-acs-action"])
+ self.assertIn(
+ "SignedHeaders=host;x-acs-action;x-acs-content-sha256;x-acs-date;"
+ "x-acs-security-token;x-acs-signature-nonce;x-acs-version,",
+ header["Authorization"])
+
+ def test_only_openapi_v4_sends_action(self):
+ """Only the ACS4 signer resolves an action; the other schemes send what they always did."""
+ rest_param = RESTAuthParameter("GET", "/v1/clg-paimon-1/databases/db/tables/t/token", "", {})
+ for algorithm in (DLFDefaultSigner.IDENTIFIER, DLFOpenApiSigner.IDENTIFIER):
+ provider = DLFAuthProvider(
+ uri="https://dlfnext.cn-hangzhou.aliyuncs.com",
+ region="cn-hangzhou",
+ signing_algorithm=algorithm,
+ token=DLFToken("akId", "akSecret", None, None)
+ )
+ self.assertNotIn("x-acs-action", provider.merge_auth_header({}, rest_param), algorithm)
+
if __name__ == '__main__':
unittest.main()
(paimon) 20/20: [core] Optimize tag reads during snapshot expiration (#9967)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 6bfd8418dc8fae6ae75c53dd0e04ece8be303a46
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Sat Sep 19 08:10:02 2026 +0800
[core] Optimize tag reads during snapshot expiration (#9967)
---
.../apache/paimon/manifest/ManifestAvroReader.java | 12 ++
.../org/apache/paimon/manifest/ManifestFile.java | 33 ++++-
.../apache/paimon/operation/FileDeletionBase.java | 145 +++++++++++++++++--
.../apache/paimon/operation/SnapshotDeletion.java | 6 +-
.../apache/paimon/table/ExpireSnapshotsImpl.java | 81 +++++++----
.../apache/paimon/manifest/ManifestFileTest.java | 44 ++++++
.../paimon/operation/ExpireSnapshotsTest.java | 153 ++++++++++++++++++++-
7 files changed, 427 insertions(+), 47 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java
index 95134f36a0..1c6c121eb1 100644
--- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java
+++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroReader.java
@@ -439,6 +439,18 @@ public final class ManifestAvroReader implements AutoCloseable {
return toRows(projectedType, null, null, true);
}
+ /**
+ * Lazily decompresses this block, applies manifest filters before decoding file metadata,
+ * and returns an iterator over one reusable row.
+ */
+ public RowIterator toRows(
+ RowType projectedType,
+ @Nullable PartitionPredicate partitionFilter,
+ @Nullable BucketFilter bucketFilter)
+ throws IOException {
+ return toRows(projectedType, partitionFilter, bucketFilter, true);
+ }
+
private RowIterator toRows(
RowType projectedType,
@Nullable PartitionPredicate partitionFilter,
diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java
index afc122c723..608c7dc576 100644
--- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java
+++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java
@@ -48,6 +48,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.function.Function;
+import java.util.function.Predicate;
/**
* This file includes several {@link ManifestEntry}s, representing the additional changes since last
@@ -303,17 +304,41 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
}
public List<ExpireFileEntry> readExpireFileEntries(String fileName) {
+ return readExpireFileEntries(fileName, null, entry -> true);
+ }
+
+ /**
+ * Reads only expiring entries accepted by the supplied filters.
+ *
+ * <p>The bucket filter is evaluated by the Avro reader before nested data-file metadata is
+ * decoded. The entry filter then runs on a reusable projected view, before an {@link
+ * ExpireFileEntry} is materialized. The entry filter must not retain its argument.
+ */
+ public List<ExpireFileEntry> readExpireFileEntries(
+ String fileName,
+ @Nullable BucketFilter bucketFilter,
+ Predicate<ProjectedManifestEntry> entryFilter) {
List<ExpireFileEntry> result = new ArrayList<>();
- try (CloseableIterator<ProjectedManifestEntry> entries =
- scan(fileName, EXPIRE_FILE_PROJECTION)) {
- while (entries.hasNext()) {
- result.add(ExpireFileEntry.from(entries.next()));
+ ProjectedManifestEntry entry = EXPIRE_FILE_PROJECTION.createEntry();
+ try (ManifestAvroReader reader = scanAvroBlocks(fileName, null)) {
+ while (reader.hasNext()) {
+ ManifestAvroReader.RowIterator rows =
+ reader.next()
+ .toRows(EXPIRE_FILE_PROJECTION.projectedType(), null, bucketFilter);
+ while (rows.hasNext()) {
+ entry.replace(rows.next());
+ if (entryFilter.test(entry)) {
+ result.add(ExpireFileEntry.from(entry));
+ }
+ }
}
} catch (Exception e) {
throw new RuntimeException(
String.format(
"Failed to scan expiring entries from manifest file '%s'.", fileName),
e);
+ } finally {
+ entry.clear();
}
return result;
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
index ad544af0ea..1b8b245e9b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
@@ -25,14 +25,17 @@ import org.apache.paimon.fs.Path;
import org.apache.paimon.index.IndexFileHandler;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.io.DataFilePathFactory;
+import org.apache.paimon.manifest.BucketFilter;
import org.apache.paimon.manifest.ExpireFileEntry;
import org.apache.paimon.manifest.FileEntry;
import org.apache.paimon.manifest.FileEntry.Identifier;
import org.apache.paimon.manifest.FileKind;
import org.apache.paimon.manifest.IndexManifestEntry;
+import org.apache.paimon.manifest.ManifestBucketFilter;
import org.apache.paimon.manifest.ManifestFile;
import org.apache.paimon.manifest.ManifestFileMeta;
import org.apache.paimon.manifest.ManifestList;
+import org.apache.paimon.manifest.ProjectedManifestEntry;
import org.apache.paimon.stats.StatsFileHandler;
import org.apache.paimon.utils.DataFilePathFactories;
import org.apache.paimon.utils.FileOperationThreadPool;
@@ -56,7 +59,9 @@ import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.NavigableSet;
import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
@@ -185,7 +190,7 @@ public abstract class FileDeletionBase<T extends Snapshot> {
}
/** Plan data files referenced by DELETE entries in the snapshot's delta manifest list. */
- public List<Path> planDeletedInDeltaManifest(T snapshot, Predicate<ExpireFileEntry> skipper) {
+ public DataFileDeletionPlan planDeletedInDeltaManifest(T snapshot) {
String deltaManifestList = snapshot.deltaManifestList();
// data file path -> (original manifest entry, extra file paths)
Map<Path, Pair<ExpireFileEntry, List<Path>>> dataFileToDelete = new HashMap<>();
@@ -218,12 +223,22 @@ public abstract class FileDeletionBase<T extends Snapshot> {
} catch (Exception e) {
// cancel deletion if any exception occurs
LOG.warn("Failed to read some manifest files. Cancel deletion.", e);
- return Collections.emptyList();
+ return DataFileDeletionPlan.empty();
}
+ return new DataFileDeletionPlan(dataFileToDelete);
+ }
+
+ /** Plan data files referenced by DELETE entries in the snapshot's delta manifest list. */
+ public List<Path> planDeletedInDeltaManifest(T snapshot, Predicate<ExpireFileEntry> skipper) {
+ return dataFilesToDelete(planDeletedInDeltaManifest(snapshot), skipper);
+ }
+
+ public List<Path> dataFilesToDelete(
+ DataFileDeletionPlan plan, Predicate<ExpireFileEntry> skipper) {
// apply skipper
List<Path> actualDataFileToDelete = new ArrayList<>();
- dataFileToDelete.forEach(
+ plan.dataFileToDelete.forEach(
(path, pair) -> {
ExpireFileEntry entry = pair.getLeft();
// check whether we should skip the data file
@@ -406,6 +421,38 @@ public abstract class FileDeletionBase<T extends Snapshot> {
return entry -> containsDataFile(tagDataFiles, entry);
}
+ /**
+ * Creates a tag skipper restricted to the files in the supplied deletion plans.
+ *
+ * <p>A tag may reference every data file in a large table. Building an index for all of them
+ * makes snapshot expiration use memory proportional to the table size (and, when tags are read
+ * concurrently, to the number of tags). Only files that are candidates for the current
+ * expiration batch can be deleted, so merge only matching tag entries instead.
+ */
+ public Predicate<ExpireFileEntry> createDataFileSkipperForTag(
+ Snapshot tag, Collection<DataFileDeletionPlan> plans) throws Exception {
+ Map<BinaryRow, Map<Integer, Set<String>>> candidates = new HashMap<>();
+ for (DataFileDeletionPlan plan : plans) {
+ for (Pair<ExpireFileEntry, List<Path>> pair : plan.dataFileToDelete.values()) {
+ addDataFile(candidates, pair.getLeft());
+ }
+ }
+ if (candidates.isEmpty()) {
+ return entry -> false;
+ }
+
+ Collection<ExpireFileEntry> matchingEntries =
+ readMergedDataFiles(
+ manifestList.readDataManifests(tag),
+ createCandidateBucketFilter(candidates),
+ entry -> containsDataFile(candidates, entry));
+ Map<BinaryRow, Map<Integer, Set<String>>> taggedCandidates = new HashMap<>();
+ for (ExpireFileEntry entry : matchingEntries) {
+ addDataFile(taggedCandidates, entry);
+ }
+ return entry -> containsDataFile(taggedCandidates, entry);
+ }
+
/**
* It is possible that a job was killed during expiration and some manifest files have been
* deleted, so if the clean methods need to get manifests of a snapshot to be cleaned, we should
@@ -430,10 +477,7 @@ public abstract class FileDeletionBase<T extends Snapshot> {
throws IOException {
for (ExpireFileEntry entry :
readMergedDataFiles(manifestList.readDataManifests(snapshot))) {
- dataFiles
- .computeIfAbsent(entry.partition(), p -> new HashMap<>())
- .computeIfAbsent(entry.bucket(), b -> new HashSet<>())
- .add(entry.fileName());
+ addDataFile(dataFiles, entry);
}
}
@@ -444,13 +488,79 @@ public abstract class FileDeletionBase<T extends Snapshot> {
return map.values();
}
+ protected Collection<ExpireFileEntry> readMergedDataFiles(
+ List<ManifestFileMeta> manifests,
+ BucketFilter bucketFilter,
+ Predicate<ProjectedManifestEntry> filter)
+ throws IOException {
+ Map<Identifier, ExpireFileEntry> map = new HashMap<>();
+ FileEntry.mergeEntries(
+ ManifestReadThreadPool.sequentialBatchedExecute(
+ manifest -> {
+ if (!bucketFilter.mayContain(manifest)) {
+ return Collections.emptyList();
+ }
+ return manifestFile.readExpireFileEntries(
+ manifest.fileName(), bucketFilter, filter);
+ },
+ manifests,
+ manifestReadParallelism),
+ map);
+ return map.values();
+ }
+
+ private BucketFilter createCandidateBucketFilter(
+ Map<BinaryRow, Map<Integer, Set<String>>> candidates) {
+ NavigableSet<Integer> candidateBuckets = new TreeSet<>();
+ for (Map<Integer, Set<String>> buckets : candidates.values()) {
+ candidateBuckets.addAll(buckets.keySet());
+ }
+
+ ManifestBucketFilter filter =
+ new ManifestBucketFilter() {
+ @Override
+ public boolean test(BinaryRow partition, Integer bucket, Integer totalBuckets) {
+ Map<Integer, Set<String>> buckets = candidates.get(partition);
+ return buckets != null && buckets.containsKey(bucket);
+ }
+
+ @Override
+ public boolean mayContain(int minBucket, int maxBucket, int totalBuckets) {
+ Integer firstCandidate = candidateBuckets.ceiling(minBucket);
+ return firstCandidate != null && firstCandidate <= maxBucket;
+ }
+ };
+ return new BucketFilter(false, null, null, filter);
+ }
+
+ private void addDataFile(
+ Map<BinaryRow, Map<Integer, Set<String>>> dataFiles, ExpireFileEntry entry) {
+ dataFiles
+ .computeIfAbsent(entry.partition(), p -> new HashMap<>())
+ .computeIfAbsent(entry.bucket(), b -> new HashSet<>())
+ .add(entry.fileName());
+ }
+
protected boolean containsDataFile(
Map<BinaryRow, Map<Integer, Set<String>>> dataFiles, ExpireFileEntry entry) {
- Map<Integer, Set<String>> buckets = dataFiles.get(entry.partition());
+ return containsDataFile(dataFiles, entry.partition(), entry.bucket(), entry.fileName());
+ }
+
+ private boolean containsDataFile(
+ Map<BinaryRow, Map<Integer, Set<String>>> dataFiles, ProjectedManifestEntry entry) {
+ return containsDataFile(dataFiles, entry.partition(), entry.bucket(), entry.fileName());
+ }
+
+ private boolean containsDataFile(
+ Map<BinaryRow, Map<Integer, Set<String>>> dataFiles,
+ BinaryRow partition,
+ int bucket,
+ String fileName) {
+ Map<Integer, Set<String>> buckets = dataFiles.get(partition);
if (buckets != null) {
- Set<String> fileNames = buckets.get(entry.bucket());
+ Set<String> fileNames = buckets.get(bucket);
if (fileNames != null) {
- return fileNames.contains(entry.fileName());
+ return fileNames.contains(fileName);
}
}
return false;
@@ -560,4 +670,19 @@ public abstract class FileDeletionBase<T extends Snapshot> {
throw new RuntimeException(e.getCause());
}
}
+
+ /** Candidate data files from one snapshot delta manifest. */
+ public static class DataFileDeletionPlan {
+
+ private final Map<Path, Pair<ExpireFileEntry, List<Path>>> dataFileToDelete;
+
+ private DataFileDeletionPlan(
+ Map<Path, Pair<ExpireFileEntry, List<Path>>> dataFileToDelete) {
+ this.dataFileToDelete = dataFileToDelete;
+ }
+
+ private static DataFileDeletionPlan empty() {
+ return new DataFileDeletionPlan(Collections.emptyMap());
+ }
+ }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java b/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java
index 00bb731d03..bbeb124627 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java
@@ -70,8 +70,8 @@ public class SnapshotDeletion extends FileDeletionBase<Snapshot> {
}
@Override
- public List<Path> planDeletedInDeltaManifest(
- Snapshot snapshot, Predicate<ExpireFileEntry> skipper) {
+ public List<Path> dataFilesToDelete(
+ DataFileDeletionPlan plan, Predicate<ExpireFileEntry> skipper) {
Predicate<ExpireFileEntry> enriched = skipper;
if (changelogDecoupled && !produceChangelog) {
// Skip clean the 'APPEND' data files.If we do not have the file source information
@@ -83,7 +83,7 @@ public class SnapshotDeletion extends FileDeletionBase<Snapshot> {
|| (manifestEntry.fileSource().orElse(FileSource.APPEND)
== FileSource.APPEND);
}
- return super.planDeletedInDeltaManifest(snapshot, enriched);
+ return super.dataFilesToDelete(plan, enriched);
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java
index b7cabb4d32..4b80085489 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java
@@ -24,6 +24,7 @@ import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.consumer.ConsumerManager;
import org.apache.paimon.fs.Path;
import org.apache.paimon.manifest.ExpireFileEntry;
+import org.apache.paimon.operation.FileDeletionBase.DataFileDeletionPlan;
import org.apache.paimon.operation.SnapshotDeletion;
import org.apache.paimon.options.ExpireConfig;
import org.apache.paimon.tag.Tag;
@@ -274,8 +275,7 @@ public class ExpireSnapshotsImpl implements ExpireSnapshots {
List<Snapshot> taggedSnapshots,
long beginInclusiveId)
throws ExecutionException, InterruptedException {
- Map<Long, Long> tagIdBySnapshotId = new HashMap<>();
- Map<Long, Snapshot> tags = new HashMap<>();
+ Map<Long, Snapshot> tagBySnapshotId = new HashMap<>();
int tagIndex = -1;
for (Snapshot snapshot : snapshotsIncludingEnd) {
long id = snapshot.id();
@@ -286,16 +286,11 @@ public class ExpireSnapshotsImpl implements ExpireSnapshots {
tagIndex = advancePreviousSnapshot(taggedSnapshots, tagIndex, id);
if (tagIndex >= 0) {
Snapshot tag = taggedSnapshots.get(tagIndex);
- tagIdBySnapshotId.put(id, tag.id());
- tags.put(tag.id(), tag);
+ tagBySnapshotId.put(id, tag);
}
}
- Map<Long, Optional<Predicate<ExpireFileEntry>>> skippers =
- collectTagSkippers(tags.values());
- Predicate<ExpireFileEntry> deleteAll = entry -> false;
- List<CompletableFuture<List<Path>>> futures = new ArrayList<>();
- int plannedSnapshots = 0;
+ List<Snapshot> batch = new ArrayList<>(snapshotExpireBatchSize);
for (Snapshot snapshot : snapshotsIncludingEnd) {
long id = snapshot.id();
if (id == beginInclusiveId) {
@@ -304,44 +299,76 @@ public class ExpireSnapshotsImpl implements ExpireSnapshots {
if (LOG.isDebugEnabled()) {
LOG.debug("Ready to delete merge tree files not used by snapshot #{}", id);
}
+ batch.add(snapshot);
+ if (batch.size() >= snapshotExpireBatchSize) {
+ cleanDataFileBatch(batch, tagBySnapshotId);
+ }
+ }
+ cleanDataFileBatch(batch, tagBySnapshotId);
+ }
+
+ private void cleanDataFileBatch(List<Snapshot> snapshots, Map<Long, Snapshot> tagBySnapshotId)
+ throws ExecutionException, InterruptedException {
+ if (snapshots.isEmpty()) {
+ return;
+ }
- Long tagId = tagIdBySnapshotId.get(id);
+ List<CompletableFuture<DataFileDeletionPlan>> planFutures = new ArrayList<>();
+ for (Snapshot snapshot : snapshots) {
+ planFutures.add(
+ CompletableFuture.supplyAsync(
+ () -> snapshotDeletion.planDeletedInDeltaManifest(snapshot),
+ fileExecutor));
+ }
+ List<DataFileDeletionPlan> plans = getAll(planFutures);
+
+ Map<Long, Snapshot> tags = new HashMap<>();
+ Map<Long, List<DataFileDeletionPlan>> plansByTag = new HashMap<>();
+ for (int i = 0; i < snapshots.size(); i++) {
+ Snapshot tag = tagBySnapshotId.get(snapshots.get(i).id());
+ if (tag != null) {
+ tags.put(tag.id(), tag);
+ plansByTag.computeIfAbsent(tag.id(), id -> new ArrayList<>()).add(plans.get(i));
+ }
+ }
+
+ Map<Long, Optional<Predicate<ExpireFileEntry>>> skippers =
+ collectTagSkippers(tags, plansByTag);
+ Predicate<ExpireFileEntry> deleteAll = entry -> false;
+ List<Path> paths = new ArrayList<>();
+ for (int i = 0; i < snapshots.size(); i++) {
+ Snapshot snapshot = snapshots.get(i);
+ Snapshot tag = tagBySnapshotId.get(snapshot.id());
Optional<Predicate<ExpireFileEntry>> skipper =
- tagId == null
+ tag == null
? Optional.of(deleteAll)
- : skippers.getOrDefault(tagId, Optional.empty());
+ : skippers.getOrDefault(tag.id(), Optional.empty());
if (!skipper.isPresent()) {
LOG.info(
"Skip cleaning data files of snapshot '{}' due to failed to build skipping set.",
- id);
+ snapshot.id());
continue;
}
- futures.add(
- CompletableFuture.supplyAsync(
- () ->
- snapshotDeletion.planDeletedInDeltaManifest(
- snapshot, skipper.get()),
- fileExecutor));
- if (++plannedSnapshots >= snapshotExpireBatchSize) {
- cleanBatch(futures);
- plannedSnapshots = 0;
- }
+ paths.addAll(snapshotDeletion.dataFilesToDelete(plans.get(i), skipper.get()));
}
- cleanBatch(futures);
+ snapshotDeletion.cleanDataFiles(paths);
+ snapshots.clear();
}
private Map<Long, Optional<Predicate<ExpireFileEntry>>> collectTagSkippers(
- Collection<Snapshot> tags) throws ExecutionException, InterruptedException {
+ Map<Long, Snapshot> tags, Map<Long, List<DataFileDeletionPlan>> plansByTag)
+ throws ExecutionException, InterruptedException {
Map<Long, CompletableFuture<Optional<Predicate<ExpireFileEntry>>>> futures =
new HashMap<>();
- for (Snapshot tag : tags) {
+ for (Snapshot tag : tags.values()) {
futures.put(
tag.id(),
CompletableFuture.supplyAsync(
() -> {
try {
return Optional.of(
- snapshotDeletion.createDataFileSkipperForTag(tag));
+ snapshotDeletion.createDataFileSkipperForTag(
+ tag, plansByTag.get(tag.id())));
} catch (Exception e) {
LOG.info(
"Failed to build data file skipping set for tag snapshot '{}'.",
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
index 5172c30521..24f84659d1 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
@@ -1084,6 +1084,50 @@ public class ManifestFileTest {
}
}
+ @Test
+ void testReadExpireFileEntriesPushesFiltersIntoStreamingScan() {
+ List<ManifestEntry> entries = Arrays.asList(gen.next(), gen.next(), gen.next());
+ ManifestFile manifestFile = createManifestFile(tempDir.toString(), Long.MAX_VALUE);
+ ManifestFileMeta manifest = writeSingleManifest(manifestFile, entries);
+
+ int[] exactFilterCalls = {0};
+ List<ExpireFileEntry> bucketRejected =
+ manifestFile.readExpireFileEntries(
+ manifest.fileName(),
+ new BucketFilter(false, null, bucket -> false, null),
+ entry -> {
+ exactFilterCalls[0]++;
+ return true;
+ });
+ assertThat(bucketRejected).isEmpty();
+ assertThat(exactFilterCalls[0]).isZero();
+
+ String selectedFile = entries.get(1).fileName();
+ ProjectedManifestEntry[] reusableView = {null};
+ List<ExpireFileEntry> selected =
+ manifestFile.readExpireFileEntries(
+ manifest.fileName(),
+ null,
+ entry -> {
+ if (reusableView[0] == null) {
+ reusableView[0] = entry;
+ } else {
+ assertThat(entry).isSameAs(reusableView[0]);
+ }
+ exactFilterCalls[0]++;
+ return entry.fileName().equals(selectedFile);
+ });
+
+ assertThat(exactFilterCalls[0]).isEqualTo(entries.size());
+ assertThat(selected)
+ .containsExactly(
+ ExpireFileEntry.from(
+ entries.stream()
+ .filter(entry -> entry.fileName().equals(selectedFile))
+ .findFirst()
+ .orElseThrow(AssertionError::new)));
+ }
+
@Test
void testScanProjectedManifestCreatesDistinctEntryWrappers() throws Exception {
List<ManifestEntry> entries = Arrays.asList(gen.next(), gen.next(), gen.next());
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java
index 957e4b7317..7fac929cdd 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java
@@ -33,6 +33,7 @@ import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.io.DataFilePathFactory;
+import org.apache.paimon.manifest.BucketFilter;
import org.apache.paimon.manifest.ExpireFileEntry;
import org.apache.paimon.manifest.FileKind;
import org.apache.paimon.manifest.FileSource;
@@ -40,7 +41,9 @@ import org.apache.paimon.manifest.ManifestEntry;
import org.apache.paimon.manifest.ManifestFileMeta;
import org.apache.paimon.manifest.ManifestIndexTestUtils;
import org.apache.paimon.manifest.ManifestSidecar;
+import org.apache.paimon.manifest.ProjectedManifestEntry;
import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction;
+import org.apache.paimon.operation.FileDeletionBase.DataFileDeletionPlan;
import org.apache.paimon.options.ExpireConfig;
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.schema.FileSystemSchemaManager;
@@ -74,6 +77,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -966,6 +970,85 @@ public class ExpireSnapshotsTest {
assertSnapshot(tagManager.getOrThrow("tag6").trimToSnapshot(), allData, snapshotPositions);
}
+ @Test
+ public void testExpireFiltersTagFilesByDeletionCandidates() throws Exception {
+ List<KeyValue> candidateData = FileStoreTestUtils.partitionedData(5, gen, "0401", 8);
+ List<KeyValue> unrelatedData = FileStoreTestUtils.partitionedData(5, gen, "0402", 8);
+ BinaryRow candidatePartition = gen.getPartition(candidateData.get(0));
+ BinaryRow unrelatedPartition = gen.getPartition(unrelatedData.get(0));
+ Map<BinaryRow, Map<Integer, RecordWriter<KeyValue>>> writers = new HashMap<>();
+ writers.put(
+ candidatePartition,
+ Collections.singletonMap(
+ 0,
+ FileStoreTestUtils.writeData(store, candidateData, candidatePartition, 0)));
+ writers.put(
+ unrelatedPartition,
+ Collections.singletonMap(
+ 0,
+ FileStoreTestUtils.writeData(store, unrelatedData, unrelatedPartition, 0)));
+ FileStoreTestUtils.commitData(store, 0, writers);
+
+ Snapshot taggedSnapshot = snapshotManager.latestSnapshot();
+ TagManager tagManager = store.newTagManager();
+ tagManager.createTag(
+ taggedSnapshot,
+ "tag1",
+ store.options().tagDefaultTimeRetained(),
+ Collections.emptyList(),
+ false);
+
+ List<ManifestEntry> delete =
+ store.newScan().plan().files().stream()
+ .filter(entry -> entry.partition().equals(candidatePartition))
+ .map(
+ entry ->
+ ManifestEntry.create(
+ FileKind.DELETE,
+ entry.partition(),
+ entry.bucket(),
+ entry.totalBuckets(),
+ entry.file()))
+ .collect(Collectors.toList());
+ try (FileStoreCommitImpl commit = store.newCommit()) {
+ commit.tryCommitOnce(
+ null,
+ delete,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ 1,
+ null,
+ Collections.emptyMap(),
+ Snapshot.CommitKind.APPEND,
+ false,
+ taggedSnapshot,
+ true,
+ null);
+ }
+
+ CandidateTrackingSnapshotDeletion snapshotDeletion =
+ new CandidateTrackingSnapshotDeletion(store);
+ ExpireSnapshotsImpl expire =
+ newExpireWithSnapshotDeletion(store, snapshotManager, snapshotDeletion);
+ expire.config(
+ ExpireConfig.builder()
+ .snapshotRetainMin(1)
+ .snapshotRetainMax(1)
+ .snapshotTimeRetain(Duration.ofMillis(Long.MAX_VALUE))
+ .build());
+ expire.expire();
+
+ assertThat(snapshotDeletion.candidateTagSkipperCalls()).isGreaterThan(0);
+ assertThat(snapshotDeletion.filteredTagReads()).isGreaterThan(0);
+ assertThat(snapshotDeletion.materializedTagEntries()).isOne();
+ List<KeyValue> taggedData = new ArrayList<>(candidateData);
+ taggedData.addAll(unrelatedData);
+ assertSnapshot(
+ tagManager.getOrThrow("tag1").trimToSnapshot(),
+ taggedData,
+ Collections.singletonList(taggedData.size()));
+ }
+
@Test
public void testExpireReadsTagsConcurrentlyWithObjectStoreFileIO() throws Exception {
TestFileStore slowStore = createSlowStore();
@@ -1567,12 +1650,11 @@ public class ExpireSnapshotsTest {
}
@Override
- public List<Path> planDeletedInDeltaManifest(
- Snapshot snapshot, Predicate<ExpireFileEntry> skipper) {
+ public DataFileDeletionPlan planDeletedInDeltaManifest(Snapshot snapshot) {
if (blockDataFilePlans && shouldBlock(snapshot.id())) {
dataFilePlans.awaitConcurrentCall();
}
- return super.planDeletedInDeltaManifest(snapshot, skipper);
+ return super.planDeletedInDeltaManifest(snapshot);
}
@Override
@@ -1608,6 +1690,71 @@ public class ExpireSnapshotsTest {
}
}
+ private static class CandidateTrackingSnapshotDeletion extends SnapshotDeletion {
+
+ private final AtomicInteger candidateTagSkipperCalls = new AtomicInteger();
+ private final AtomicInteger filteredTagReads = new AtomicInteger();
+ private final AtomicInteger materializedTagEntries = new AtomicInteger();
+
+ private CandidateTrackingSnapshotDeletion(TestFileStore store) {
+ super(
+ store.fileIO(),
+ store.pathFactory(),
+ store.manifestFileFactory().create(),
+ store.manifestListFactory().create(),
+ store.newIndexFileHandler(),
+ store.newStatsFileHandler(),
+ store.options().changelogProducer() != CoreOptions.ChangelogProducer.NONE,
+ store.options().cleanEmptyDirectories(),
+ store.options().fileOperationThreadNum(),
+ store.options().scanManifestParallelism());
+ }
+
+ @Override
+ public Predicate<ExpireFileEntry> createDataFileSkipperForTag(Snapshot tag)
+ throws Exception {
+ throw new AssertionError("Snapshot expiration must not index all files in a tag.");
+ }
+
+ @Override
+ public Predicate<ExpireFileEntry> createDataFileSkipperForTag(
+ Snapshot tag, Collection<DataFileDeletionPlan> plans) throws Exception {
+ candidateTagSkipperCalls.incrementAndGet();
+ return super.createDataFileSkipperForTag(tag, plans);
+ }
+
+ @Override
+ protected Collection<ExpireFileEntry> readMergedDataFiles(List<ManifestFileMeta> manifests)
+ throws IOException {
+ throw new AssertionError("Snapshot expiration must not merge all files in a tag.");
+ }
+
+ @Override
+ protected Collection<ExpireFileEntry> readMergedDataFiles(
+ List<ManifestFileMeta> manifests,
+ BucketFilter bucketFilter,
+ Predicate<ProjectedManifestEntry> filter)
+ throws IOException {
+ filteredTagReads.incrementAndGet();
+ Collection<ExpireFileEntry> entries =
+ super.readMergedDataFiles(manifests, bucketFilter, filter);
+ materializedTagEntries.addAndGet(entries.size());
+ return entries;
+ }
+
+ private int candidateTagSkipperCalls() {
+ return candidateTagSkipperCalls.get();
+ }
+
+ private int filteredTagReads() {
+ return filteredTagReads.get();
+ }
+
+ private int materializedTagEntries() {
+ return materializedTagEntries.get();
+ }
+ }
+
private static class CapturingSnapshotDeletion extends SnapshotDeletion {
private final List<List<Object>> deleteBatches = new ArrayList<>();
(paimon) 09/20: [core] Preserve query authorization when narrowing chain table splits (#9929)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 0952d147bd4edf190af6cda35b48bfe43fc57fea
Author: Jiajia Li <pl...@alibaba-inc.com>
AuthorDate: Fri Sep 18 12:02:30 2026 +0800
[core] Preserve query authorization when narrowing chain table splits (#9929)
---
.../paimon/catalog/TableQueryAuthResult.java | 33 +++
.../apache/paimon/table/ChainGroupReadTable.java | 93 ++++--
.../paimon/table/ChainTableFileStoreTable.java | 5 +-
.../apache/paimon/table/ChainTableStreamScan.java | 49 +++-
.../paimon/table/source/AbstractDataTableScan.java | 22 +-
.../apache/paimon/table/source/DataTableScan.java | 12 +
.../apache/paimon/table/source/QueryAuthSplit.java | 10 +
.../paimon/catalog/TableQueryAuthResultTest.java | 85 ++++++
.../paimon/table/ChainTableFileStoreTableTest.java | 314 ++++++++++++++++++++-
9 files changed, 585 insertions(+), 38 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java
index 6ab47711dd..cf3bfdf049 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java
@@ -55,6 +55,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
@@ -95,6 +96,38 @@ public class TableQueryAuthResult implements Serializable {
return extractPredicate() != null || !extractColumnMasking().isEmpty();
}
+ /**
+ * A catalog builds a fresh instance per authorization call and deserialization builds another,
+ * so identity says nothing about whether two results agree.
+ *
+ * <p>Compared as the rules parse out rather than as they arrive, the filter as the set of its
+ * conjuncts, so neither the JSON text nor the order the catalog lists them in matters.
+ */
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof TableQueryAuthResult)) {
+ return false;
+ }
+ TableQueryAuthResult that = (TableQueryAuthResult) o;
+ return parsedFilterConjuncts().equals(that.parsedFilterConjuncts())
+ && extractColumnMasking().equals(that.extractColumnMasking());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(parsedFilterConjuncts(), extractColumnMasking());
+ }
+
+ private Set<Predicate> parsedFilterConjuncts() {
+ Predicate predicate = extractPredicate();
+ return predicate == null
+ ? Collections.emptySet()
+ : new HashSet<>(PredicateBuilder.splitAnd(predicate));
+ }
+
/**
* Rejects a search on a query-auth table. Called from the methods that produce a scan or a
* read, not from the builder constructors: the builders are serializable, and deserialization
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java b/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java
index b0f4278666..c141d9b8e2 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java
@@ -19,6 +19,7 @@
package org.apache.paimon.table;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.catalog.TableQueryAuthResult;
import org.apache.paimon.codegen.CodeGenUtils;
import org.apache.paimon.codegen.RecordComparator;
import org.apache.paimon.data.BinaryRow;
@@ -36,6 +37,7 @@ import org.apache.paimon.table.source.DataFilePlan;
import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.table.source.DataTableScan;
import org.apache.paimon.table.source.InnerTableRead;
+import org.apache.paimon.table.source.QueryAuthSplit;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.table.source.TableRead;
import org.apache.paimon.types.RowType;
@@ -45,6 +47,8 @@ import org.apache.paimon.utils.Filter;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.RowDataToObjectArrayConverter;
+import javax.annotation.Nullable;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -405,8 +409,11 @@ public class ChainGroupReadTable extends FallbackReadFileStoreTable {
tableSchema.logicalPartitionType(), snapshotSearchPred);
// List snapshot partitions for this group, sorted by chain dimension.
+ // The anchor of an admitted logical partition sits in an earlier one, which a
+ // rule on a chain partition key excludes.
List<BinaryRow> snapshotPartitionsInGroup =
newChainPartitionListingScan(true, snapshotAnchorPredicate)
+ .withoutAuthPartitionPushdown()
.listPartitions();
// Find delta → snapshot mapping (for each delta partition, find the nearest
@@ -452,29 +459,25 @@ public class ChainGroupReadTable extends FallbackReadFileStoreTable {
deltaScan.withPartitionFilter(selectedDeltaPartitions);
}
- List<DataSplit> deltaSubSplits =
- deltaScan.plan().splits().stream()
- .map(s -> (DataSplit) s)
- .collect(Collectors.toList());
+ ChainAuth auth = new ChainAuth();
+ List<DataSplit> deltaSubSplits = auth.unwrap(deltaScan.plan().splits());
List<DataSplit> snapshotSubSplits = new ArrayList<>();
if (partitionPairs.getValue() != null) {
snapshotScan.withPartitionFilter(
Collections.singletonList(partitionPairs.getValue()));
- snapshotSubSplits =
- snapshotScan.plan().splits().stream()
- .map(s -> (DataSplit) s)
- .collect(Collectors.toList());
+ snapshotSubSplits = auth.unwrap(snapshotScan.plan().splits());
}
splits.addAll(
- ChainTableUtils.buildChainSplits(
- partitionPairs.getKey(),
- snapshotSubSplits,
- deltaSubSplits,
- options.scanFallbackSnapshotBranch(),
- options.scanFallbackDeltaBranch(),
- keyComparator,
- targetSplitSize,
- openFileCost));
+ auth.reapply(
+ ChainTableUtils.buildChainSplits(
+ partitionPairs.getKey(),
+ snapshotSubSplits,
+ deltaSubSplits,
+ options.scanFallbackSnapshotBranch(),
+ options.scanFallbackDeltaBranch(),
+ keyComparator,
+ targetSplitSize,
+ openFileCost)));
}
}
}
@@ -518,10 +521,12 @@ public class ChainGroupReadTable extends FallbackReadFileStoreTable {
return snapshotPartitions;
}
- for (Split split : mainScan.plan().splits()) {
- DataSplit dataSplit = (DataSplit) split;
- splits.add(ChainSplit.from(dataSplit, options.scanFallbackSnapshotBranch()));
+ ChainAuth auth = new ChainAuth();
+ List<Split> chainSplits = new ArrayList<>();
+ for (DataSplit dataSplit : auth.unwrap(mainScan.plan().splits())) {
+ chainSplits.add(ChainSplit.from(dataSplit, options.scanFallbackSnapshotBranch()));
}
+ splits.addAll(auth.reapply(chainSplits));
snapshotPartitions.addAll(
newChainPartitionListingScan(true, getMainPartitionPredicate())
@@ -531,9 +536,10 @@ public class ChainGroupReadTable extends FallbackReadFileStoreTable {
private DataTableScan newFilteredScan(boolean snapshot) {
DataTableScan scan =
- snapshot
- ? chainGroupReadTable.newSnapshotScan(scanCreator)
- : chainGroupReadTable.newDeltaScan(scanCreator);
+ (snapshot
+ ? chainGroupReadTable.newSnapshotScan(scanCreator)
+ : chainGroupReadTable.newDeltaScan(scanCreator))
+ .withoutAuthPartitionPushdown();
if (dataPredicate != null) {
scan.withFilter(dataPredicate);
}
@@ -603,11 +609,48 @@ public class ChainGroupReadTable extends FallbackReadFileStoreTable {
@Override
public RecordReader<InternalRow> createReader(Split split) throws IOException {
- if (split instanceof ChainSplit || split instanceof DataSplit) {
+ // fallbackRead unwraps and applies the rules itself, so it gets the wrapper untouched.
+ Split inner = QueryAuthSplit.unwrap(split);
+ if (inner instanceof ChainSplit || inner instanceof DataSplit) {
return fallbackRead.createReader(split);
}
throw new IllegalArgumentException(
- "Unsupported split type for chain table read: " + split.getClass().getName());
+ "Unsupported split type for chain table read: " + inner.getClass().getName());
+ }
+ }
+
+ /**
+ * Carries the branch scans' authorization across chain split building; dropping it would turn
+ * the failed cast into unrestricted reads. Both branches authorize the same logical table, so
+ * one result covers the splits built from either side.
+ */
+ private static final class ChainAuth {
+
+ @Nullable private TableQueryAuthResult authResult;
+
+ List<DataSplit> unwrap(List<Split> splits) {
+ List<DataSplit> dataSplits = new ArrayList<>(splits.size());
+ for (Split split : splits) {
+ if (split instanceof QueryAuthSplit) {
+ TableQueryAuthResult next = ((QueryAuthSplit) split).authResult();
+ // Enforced rather than assumed: a divergence would apply one branch's rules to
+ // the other's rows.
+ checkArgument(
+ authResult == null || authResult.equals(next),
+ "Branch scans of the same chain table returned different authorization rules.");
+ authResult = next;
+ }
+ dataSplits.add((DataSplit) QueryAuthSplit.unwrap(split));
+ }
+ return dataSplits;
+ }
+
+ List<Split> reapply(List<? extends Split> splits) {
+ List<Split> result = new ArrayList<>(splits.size());
+ for (Split split : splits) {
+ result.add(authResult == null ? split : new QueryAuthSplit(split, authResult));
+ }
+ return result;
}
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ChainTableFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/ChainTableFileStoreTable.java
index ba8f8bf48c..8daed52edc 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/ChainTableFileStoreTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/ChainTableFileStoreTable.java
@@ -29,6 +29,7 @@ import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.source.ChainSplit;
import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.table.source.InnerTableRead;
+import org.apache.paimon.table.source.QueryAuthSplit;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.table.source.StreamDataTableScan;
import org.apache.paimon.table.source.TableRead;
@@ -217,7 +218,9 @@ public class ChainTableFileStoreTable extends FallbackReadFileStoreTable {
// FallbackSplit (including FallbackDataSplit): use inherited fallback read logic
return fallbackRead.createReader(split);
}
- if (split instanceof ChainSplit || split instanceof DataSplit) {
+ // Route on what the wrapper carries, so the wrapper still reaches the read.
+ Split inner = QueryAuthSplit.unwrap(split);
+ if (inner instanceof ChainSplit || inner instanceof DataSplit) {
return chainGroupRead.createReader(split);
}
// Other split types: use inherited fallback read logic
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ChainTableStreamScan.java b/paimon-core/src/main/java/org/apache/paimon/table/ChainTableStreamScan.java
index 5812a2611b..e9f181f47f 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/ChainTableStreamScan.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/ChainTableStreamScan.java
@@ -20,6 +20,7 @@ package org.apache.paimon.table;
import org.apache.paimon.CoreOptions;
import org.apache.paimon.Snapshot;
+import org.apache.paimon.catalog.TableQueryAuthResult;
import org.apache.paimon.codegen.CodeGenUtils;
import org.apache.paimon.codegen.RecordComparator;
import org.apache.paimon.data.BinaryRow;
@@ -33,6 +34,7 @@ import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.table.source.DataTableScan;
import org.apache.paimon.table.source.DataTableStreamScan;
import org.apache.paimon.table.source.InnerTableScan;
+import org.apache.paimon.table.source.QueryAuthSplit;
import org.apache.paimon.table.source.SnapshotNotExistPlan;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.table.source.StreamDataTableScan;
@@ -57,6 +59,8 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
/**
* Streaming scan for chain tables with a two-phase design:
*
@@ -104,6 +108,9 @@ public class ChainTableStreamScan implements StreamDataTableScan {
/** Whether the starting plan (Phase 1) has been completed. */
private boolean startingDone = false;
+ /** Authorization removed while grouping the starting splits; re-applied before returning. */
+ @Nullable private TableQueryAuthResult startingAuthResult;
+
/**
* Predicates, shard, and bucket filter for applying to local scans in {@link #planStarting()}.
*/
@@ -190,6 +197,9 @@ public class ChainTableStreamScan implements StreamDataTableScan {
* allowing streaming readers to see cross-branch deletions and updates.
*/
private TableScan.Plan planStarting() {
+ // restore() can re-run this on the same instance; a stale result would re-apply rules
+ // revoked in between.
+ startingAuthResult = null;
FileStoreTable deltaTable = chainGroupReadTable.other();
String deltaBranch = deltaTable.coreOptions().branch();
String snapshotBranch = chainGroupReadTable.wrapped.coreOptions().branch();
@@ -241,7 +251,7 @@ public class ChainTableStreamScan implements StreamDataTableScan {
Map<BinaryRow, List<DataSplit>> deltaSplitsByPartition;
if (deltaLatestId != null) {
FileStoreTable pinnedDelta = deltaTable.copy(pinnedOptions(deltaLatestId));
- DataTableScan pinnedDeltaScan = pinnedDelta.newScan();
+ DataTableScan pinnedDeltaScan = pinnedDelta.newScan().withoutAuthPartitionPushdown();
applyPredicatesShardAndBucket(pinnedDeltaScan);
deltaSplitsByPartition = groupByPartition(pinnedDeltaScan);
} else {
@@ -256,7 +266,8 @@ public class ChainTableStreamScan implements StreamDataTableScan {
FileStoreTable pinnedSnapshot = null;
if (snapshotLatestId != null) {
pinnedSnapshot = chainGroupReadTable.wrapped.copy(pinnedOptions(snapshotLatestId));
- DataTableScan partitionListingScan = pinnedSnapshot.newScan();
+ DataTableScan partitionListingScan =
+ pinnedSnapshot.newScan().withoutAuthPartitionPushdown();
for (BinaryRow partition : partitionListingScan.listPartitions()) {
Object groupKey = toGroupKey(partition);
BinaryRow existingLatest = latestChainPartitionPerGroup.get(groupKey);
@@ -275,7 +286,7 @@ public class ChainTableStreamScan implements StreamDataTableScan {
List<BinaryRow> latestPartitions = new ArrayList<>(latestChainPartitionPerGroup.values());
Map<BinaryRow, List<DataSplit>> snapshotSplitsByPartition;
if (!latestPartitions.isEmpty() && pinnedSnapshot != null) {
- DataTableScan snapshotScan = pinnedSnapshot.newScan();
+ DataTableScan snapshotScan = pinnedSnapshot.newScan().withoutAuthPartitionPushdown();
snapshotScan.withPartitionFilter(latestPartitions);
applyPredicatesShardAndBucket(snapshotScan);
snapshotSplitsByPartition = groupByPartition(snapshotScan);
@@ -316,7 +327,7 @@ public class ChainTableStreamScan implements StreamDataTableScan {
allSplits.size());
startingDone = true;
- return new DataFilePlan<>(allSplits);
+ return new DataFilePlan<>(reapplyStartingAuth(allSplits));
}
/**
@@ -476,16 +487,42 @@ public class ChainTableStreamScan implements StreamDataTableScan {
return latestId;
}
- /** Plans a scan and groups the resulting splits by partition. */
- private static Map<BinaryRow, List<DataSplit>> groupByPartition(DataTableScan scan) {
+ /**
+ * Authorization is stripped here and kept for {@link #reapplyStartingAuth}; dropping it would
+ * silently discard the rules.
+ */
+ private Map<BinaryRow, List<DataSplit>> groupByPartition(DataTableScan scan) {
Map<BinaryRow, List<DataSplit>> grouped = new LinkedHashMap<>();
for (Split s : scan.plan().splits()) {
+ if (s instanceof QueryAuthSplit) {
+ QueryAuthSplit authSplit = (QueryAuthSplit) s;
+ TableQueryAuthResult next = authSplit.authResult();
+ // The delta and the snapshot scan authorize separately. Replacing one result with
+ // the other would widen what the reader sees.
+ checkArgument(
+ startingAuthResult == null || startingAuthResult.equals(next),
+ "Branch scans of the same chain table returned different authorization rules.");
+ startingAuthResult = next;
+ s = authSplit.split();
+ }
DataSplit ds = (DataSplit) s;
grouped.computeIfAbsent(ds.partition(), k -> new ArrayList<>()).add(ds);
}
return grouped;
}
+ /** Puts back the authorization removed by {@link #groupByPartition}. */
+ private List<Split> reapplyStartingAuth(List<Split> splits) {
+ if (startingAuthResult == null) {
+ return splits;
+ }
+ List<Split> wrapped = new ArrayList<>(splits.size());
+ for (Split split : splits) {
+ wrapped.add(new QueryAuthSplit(split, startingAuthResult));
+ }
+ return wrapped;
+ }
+
/**
* Extracts a stable group key from a full partition row. When there is no group partition (all
* fields are chain keys), returns a shared singleton to avoid zero-field {@link BinaryRow}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java
index 8be73caff6..9cea4edfe4 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java
@@ -103,6 +103,9 @@ abstract class AbstractDataTableScan implements DataTableScan {
// Whether the auth predicate has a non-partition part (enforced only at read time). Used by
// AbstractBatchTableScan to disable limit push down; not pushed through withFilter.
protected boolean authHasNonPartitionFilter;
+ private boolean authPartitionPushdown = true;
+ // Turning the pushdown off after a filter was pushed has to rebuild it, rules unchanged.
+ private boolean appliedAuthPartitionPushdown = true;
// auth state, refreshed each plan(). The filter is pushed once, without the conjuncts on
// masked columns, whose raw statistics a mask invalidates; the partition fields are
// replaced on each push, as ManifestsReader#withPartitionFilter overwrites rather than ands.
@@ -160,10 +163,12 @@ abstract class AbstractDataTableScan implements DataTableScan {
protected abstract TableScan.Plan planWithoutAuth();
private void applyAuthFilter(@Nullable Predicate authPredicate) {
- if (Objects.equals(authPredicate, appliedAuthPredicate)) {
+ if (Objects.equals(authPredicate, appliedAuthPredicate)
+ && authPartitionPushdown == appliedAuthPartitionPushdown) {
return;
}
appliedAuthPredicate = authPredicate;
+ appliedAuthPartitionPushdown = authPartitionPushdown;
PartitionPredicate authPartitionFilter = null;
boolean hasNonPartitionPart = false;
@@ -183,11 +188,20 @@ abstract class AbstractDataTableScan implements DataTableScan {
// Push only the partition part, to a dedicated slot overwritten/cleared each plan() so a
// changed/removed auth leaves no stale pruning. The full filter is enforced at read time.
- snapshotReader.manifestsReader().withAuthPartitionFilter(authPartitionFilter);
+ snapshotReader
+ .manifestsReader()
+ .withAuthPartitionFilter(authPartitionPushdown ? authPartitionFilter : null);
// A non-partition auth part is enforced only at read time, so limit push down is unsafe
// (AbstractBatchTableScan reads this). Kept off SnapshotReader since it is not a pushed
- // filter.
- this.authHasNonPartitionFilter = hasNonPartitionPart;
+ // filter. Without the partition push down the whole rule is read time.
+ this.authHasNonPartitionFilter =
+ authPartitionPushdown ? hasNonPartitionPart : authPredicate != null;
+ }
+
+ @Override
+ public AbstractDataTableScan withoutAuthPartitionPushdown() {
+ this.authPartitionPushdown = false;
+ return this;
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableScan.java
index 29c825b0be..173e21e34c 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableScan.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableScan.java
@@ -23,4 +23,16 @@ public interface DataTableScan extends InnerTableScan {
/** Specify the shard to be read, and allocate sharded files to read records. */
DataTableScan withShard(int indexOfThisSubtask, int numberOfParallelSubtasks);
+
+ /**
+ * Stops the partition part of an authorization rule from pruning here, for a scan whose
+ * physical partitions are not the ones the rule speaks about — a chain table maps a branch
+ * partition onto a different logical one.
+ *
+ * <p>Only for a scan whose splits are then read, which applies the whole rule anyway. {@link
+ * #listPartitionEntries} answers the caller directly and must keep the pruning.
+ */
+ default DataTableScan withoutAuthPartitionPushdown() {
+ return this;
+ }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/QueryAuthSplit.java b/paimon-core/src/main/java/org/apache/paimon/table/source/QueryAuthSplit.java
index 3f6106d015..f38fbc500f 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/QueryAuthSplit.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/QueryAuthSplit.java
@@ -54,6 +54,16 @@ public class QueryAuthSplit implements Split {
return split;
}
+ /**
+ * The split this wrapper carries, or {@code split} itself when it is not one. Narrowing a
+ * planned split to a concrete type must go through this, or the cast fails once the table
+ * carries a rule. A caller handing the split to a read passes the wrapper on instead, or the
+ * authorization never reaches the reader.
+ */
+ public static Split unwrap(Split split) {
+ return split instanceof QueryAuthSplit ? ((QueryAuthSplit) split).split() : split;
+ }
+
@Nullable
public TableQueryAuthResult authResult() {
return authResult;
diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java
index 5b7f64fe8a..6ff67de361 100644
--- a/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java
@@ -24,7 +24,9 @@ import org.apache.paimon.predicate.Equal;
import org.apache.paimon.predicate.FieldRef;
import org.apache.paimon.predicate.FieldTransform;
import org.apache.paimon.predicate.LeafPredicate;
+import org.apache.paimon.predicate.LowerTransform;
import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.UpperTransform;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.JsonSerdeUtil;
@@ -162,4 +164,87 @@ public class TableQueryAuthResultTest {
Map<String, String> masking = Collections.singletonMap("display", maskJson());
assertThat(new TableQueryAuthResult(null, masking).hasRules()).isTrue();
}
+
+ /**
+ * Chain table planning aborts a query whose branches disagree, so a difference that is not a
+ * difference in the rules would fail a query it should have served.
+ */
+ @Test
+ public void testEqualsComparesRulesNotTheTransportShapeTheyArriveIn() {
+ Map<String, String> masking = Collections.singletonMap("display", maskJson());
+
+ // the same conjuncts listed in the other order
+ assertThat(
+ new TableQueryAuthResult(
+ Arrays.asList(filterJson(), otherFilterJson()), masking))
+ .isEqualTo(
+ new TableQueryAuthResult(
+ Arrays.asList(otherFilterJson(), filterJson()), masking))
+ .hasSameHashCodeAs(
+ new TableQueryAuthResult(
+ Arrays.asList(otherFilterJson(), filterJson()), masking));
+
+ // an absent rule and an empty one
+ assertThat(new TableQueryAuthResult(Collections.singletonList(filterJson()), null))
+ .isEqualTo(
+ new TableQueryAuthResult(
+ Collections.singletonList(filterJson()), Collections.emptyMap()));
+ assertThat(new TableQueryAuthResult(null, masking))
+ .isEqualTo(new TableQueryAuthResult(Collections.emptyList(), masking));
+
+ assertThat(new TableQueryAuthResult(null, masking))
+ .isEqualTo(
+ new TableQueryAuthResult(
+ null, Collections.singletonMap("display", maskJson())));
+
+ // JSON spaced out by a different serializer
+ assertThat(new TableQueryAuthResult(Collections.singletonList(filterJson()), masking))
+ .isEqualTo(
+ new TableQueryAuthResult(
+ Collections.singletonList(spacedOut(filterJson())),
+ Collections.singletonMap("display", spacedOut(maskJson()))));
+
+ // same shape, same column, still two different masks
+ assertThat(new TableQueryAuthResult(null, Collections.singletonMap("display", upperJson())))
+ .isNotEqualTo(
+ new TableQueryAuthResult(
+ null, Collections.singletonMap("display", lowerJson())));
+
+ // rules that really do differ
+ assertThat(new TableQueryAuthResult(Collections.singletonList(filterJson()), null))
+ .isNotEqualTo(
+ new TableQueryAuthResult(
+ Collections.singletonList(otherFilterJson()), null))
+ .isNotEqualTo(new TableQueryAuthResult(null, null));
+ assertThat(new TableQueryAuthResult(null, masking))
+ .isNotEqualTo(new TableQueryAuthResult(null, null))
+ .isNotEqualTo(
+ new TableQueryAuthResult(
+ null, Collections.singletonMap("other", maskJson())));
+ }
+
+ /** The same JSON, with another serializer's whitespace. */
+ private static String spacedOut(String json) {
+ return json.replace(",", ", ").replace(":", ": ");
+ }
+
+ private static String upperJson() {
+ return JsonSerdeUtil.toFlatJson(
+ new UpperTransform(
+ Collections.singletonList(new FieldRef(1, "extra", DataTypes.STRING()))));
+ }
+
+ private static String lowerJson() {
+ return JsonSerdeUtil.toFlatJson(
+ new LowerTransform(
+ Collections.singletonList(new FieldRef(1, "extra", DataTypes.STRING()))));
+ }
+
+ private static String otherFilterJson() {
+ return JsonSerdeUtil.toFlatJson(
+ LeafPredicate.of(
+ new FieldTransform(new FieldRef(1, "extra", DataTypes.STRING())),
+ Equal.INSTANCE,
+ Collections.singletonList(BinaryString.fromString("y"))));
+ }
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/ChainTableFileStoreTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/ChainTableFileStoreTableTest.java
index 66b82195f0..b65b2e6ec0 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/ChainTableFileStoreTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/ChainTableFileStoreTableTest.java
@@ -19,6 +19,9 @@
package org.apache.paimon.table;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.catalog.TableQueryAuthResult;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
@@ -27,6 +30,11 @@ import org.apache.paimon.disk.IOManagerImpl;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.options.Options;
+import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.FieldTransform;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateBuilder;
+import org.apache.paimon.predicate.Transform;
import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.schema.FileSystemSchemaManager;
import org.apache.paimon.schema.Schema;
@@ -37,13 +45,17 @@ import org.apache.paimon.table.sink.CommitMessage;
import org.apache.paimon.table.sink.InnerTableWrite;
import org.apache.paimon.table.sink.StreamTableCommit;
import org.apache.paimon.table.source.ChainSplit;
+import org.apache.paimon.table.source.DataTableScan;
import org.apache.paimon.table.source.DeletionFile;
+import org.apache.paimon.table.source.InnerTableRead;
+import org.apache.paimon.table.source.QueryAuthSplit;
import org.apache.paimon.table.source.ReadBuilder;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.table.source.TableScan;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowKind;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.JsonSerdeUtil;
import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap;
@@ -53,6 +65,7 @@ import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.Mockito;
import java.util.ArrayList;
import java.util.Arrays;
@@ -63,6 +76,8 @@ import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.function.Consumer;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.paimon.CoreOptions.BUCKET;
@@ -350,7 +365,303 @@ public class ChainTableFileStoreTableTest {
assertThat(foundChainSplit).as("Should have found at least one ChainSplit").isTrue();
}
+ @Test
+ public void testQueryAuthBatchPlanKeepsChainSplitsWrapped() throws Exception {
+ createChainTableWithQueryAuth();
+ TableQueryAuthResult authResult = maskValueAndFilterKey();
+
+ FileStoreTable table = loadTable(queryAuthEnvironment(this::maskValueAndFilterKey));
+ List<Split> splits = planQueriedPartition(table.newScan());
+
+ assertChainSplitsCarryAuth(splits, authResult);
+ }
+
+ @Test
+ public void testQueryAuthScanWithSnapshotReaderFactoryKeepsChainSplitsWrapped()
+ throws Exception {
+ createChainTableWithQueryAuth();
+ TableQueryAuthResult authResult = maskValueAndFilterKey();
+
+ FileStoreTable table = loadTable(queryAuthEnvironment(this::maskValueAndFilterKey));
+ List<Split> splits = planQueriedPartition(table.newScan(FileStoreTable::newSnapshotReader));
+
+ assertChainSplitsCarryAuth(splits, authResult);
+ }
+
+ @Test
+ public void testQueryAuthRulesAreAppliedWhenReadingChainSplits() throws Exception {
+ createChainTableWithQueryAuth();
+
+ // the same query without any rule: raw values, no row dropped
+ assertThat(getResult(loadTable(), QUERIED_PARTITION))
+ .containsExactlyInAnyOrder(
+ row(1L, 2L, "1-1", "CN", "20250810", "21"),
+ row(2L, 1L, "2", "CN", "20250810", "21"),
+ row(4L, 1L, "4", "CN", "20250810", "21"));
+
+ FileStoreTable table = loadTable(queryAuthEnvironment(this::maskValueAndFilterKey));
+
+ // 'v' comes back as the region it is masked with, and k=4 is dropped by the row filter
+ assertThat(getResult(table, QUERIED_PARTITION))
+ .containsExactlyInAnyOrder(
+ row(1L, 2L, "CN", "CN", "20250810", "21"),
+ row(2L, 1L, "CN", "CN", "20250810", "21"));
+ }
+
+ /**
+ * A chain partition key names the partition a row is reported under, not the branch partition
+ * it is stored in: k=2 sits in the snapshot branch under hour 20, and the chain merge hands it
+ * to the reader under the queried hour 21. A rule on 'hour' therefore cannot prune branch
+ * partitions, or the row it does admit disappears.
+ */
+ @Test
+ public void testQueryAuthChainPartitionRuleKeepsRowsOfAnEarlierBranchPartition()
+ throws Exception {
+ createChainTableWithQueryAuth();
+
+ FileStoreTable table = loadTable(queryAuthEnvironment(this::keepQueriedChainPartition));
+
+ assertThat(getResult(table, QUERIED_PARTITION))
+ .containsExactlyInAnyOrder(
+ row(1L, 2L, "1-1", "CN", "20250810", "21"),
+ row(2L, 1L, "2", "CN", "20250810", "21"),
+ row(4L, 1L, "4", "CN", "20250810", "21"));
+ }
+
+ /**
+ * The other direction: a partition the snapshot branch holds in full is read from it directly,
+ * branch partition and queried partition being the same row of values. A rule excluding that
+ * partition still has to prune it away, not hand the reader a split it filters down to nothing.
+ */
+ @Test
+ public void testQueryAuthPartitionRuleStillPrunesASnapshotBranchPartition() throws Exception {
+ createChainTableWithQueryAuth();
+ Map<String, String> snapshotPartition = ImmutableMap.of("dt", "20250810", "hour", "20");
+
+ assertThat(planPartition(loadTable().newScan(), snapshotPartition)).isNotEmpty();
+
+ // the rule admits hour 21 only, so nothing of the queried hour 20 is left to read
+ FileStoreTable table = loadTable(queryAuthEnvironment(this::keepQueriedChainPartition));
+ assertThat(planPartition(table.newScan(), snapshotPartition)).isEmpty();
+ assertThat(getResult(table, snapshotPartition)).isEmpty();
+ }
+
+ /**
+ * Partition listing has no read to follow it, so an excluded partition would be reported — with
+ * its row and file counts — to a caller that cannot read a row of it.
+ */
+ @Test
+ public void testQueryAuthPartitionRulePrunesTheListedPartitions() throws Exception {
+ createChainTableWithQueryAuth();
+
+ assertThat(listedHours(loadTable())).containsExactlyInAnyOrder("20", "21");
+
+ FileStoreTable table = loadTable(queryAuthEnvironment(this::keepQueriedChainPartition));
+ assertThat(listedHours(table)).containsExactly("21");
+ }
+
+ private List<String> listedHours(FileStoreTable table) {
+ return table.newScan().listPartitions().stream()
+ .map(partition -> partition.getString(2).toString())
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * The authorization has not changed when the pushdown is turned off, so the cheap way out of
+ * reapplying it would leave the pruning the caller just asked to drop.
+ */
+ @Test
+ public void testQueryAuthPartitionPushdownIsDroppedAfterOneWasPushed() throws Exception {
+ createChainTableWithQueryAuth();
+ FileStoreTable snapshotBranch =
+ loadBranchTable(
+ SNAPSHOT_BRANCH, queryAuthEnvironment(this::keepQueriedChainPartition));
+
+ // the rule admits hour 21 and this branch holds only hour 20, so the push prunes it away
+ DataTableScan scan = snapshotBranch.newScan();
+ assertThat(scan.listPartitions()).isEmpty();
+
+ assertThat(scan.withoutAuthPartitionPushdown().plan().splits()).isNotEmpty();
+ }
+
+ /** The branch as an ordinary table. switchToBranch would build another chain table. */
+ private FileStoreTable loadBranchTable(String branch, CatalogEnvironment catalogEnvironment) {
+ Path tablePath = new Path(tempDir.toUri().toString(), tableName);
+ LocalFileIO fileIO = LocalFileIO.create();
+ Optional<TableSchema> schemaOpt =
+ new FileSystemSchemaManager(fileIO, tablePath, branch).latest();
+ assertThat(schemaOpt.isPresent()).isTrue();
+ Options options = new Options(schemaOpt.get().options());
+ options.set(CHAIN_TABLE_ENABLED, false);
+ options.set(CoreOptions.BRANCH, branch);
+ return FileStoreTableFactory.create(
+ fileIO, tablePath, schemaOpt.get().copy(options.toMap()), catalogEnvironment);
+ }
+
+ /** Keeps only the rows reported under the chain partition hour = '21'. */
+ private TableQueryAuthResult keepQueriedChainPartition() {
+ RowType rowType = loadTable().schema().logicalRowType();
+ Predicate rowFilter = new PredicateBuilder(rowType).equal(5, BinaryString.fromString("21"));
+ return new TableQueryAuthResult(
+ Collections.singletonList(JsonSerdeUtil.toFlatJson(rowFilter)), null);
+ }
+
+ @Test
+ public void testQueryAuthWithoutRulesLeavesPlanAndReadUntouched() throws Exception {
+ createChainTableWithQueryAuth();
+ List<GenericRow> withoutQueryAuth = getResult(loadTable(), QUERIED_PARTITION);
+
+ // authorization answers with neither a row filter nor a column mask
+ FileStoreTable table =
+ loadTable(queryAuthEnvironment(() -> new TableQueryAuthResult(null, null)));
+ List<Split> splits = planQueriedPartition(table.newScan());
+
+ assertThat(splits).isNotEmpty();
+ assertThat(splits)
+ .allSatisfy(
+ split ->
+ assertThat(
+ ((FallbackReadFileStoreTable.FallbackSplit) split)
+ .wrapped())
+ .isInstanceOf(ChainSplit.class));
+ assertThat(getResult(table, QUERIED_PARTITION))
+ .containsExactlyInAnyOrderElementsOf(withoutQueryAuth);
+ }
+
+ @Test
+ public void testQueryAuthStreamingStartingPlanKeepsChainSplitsWrapped() throws Exception {
+ createChainTableWithQueryAuth();
+ TableQueryAuthResult authResult = maskValueAndFilterKey();
+
+ FileStoreTable table = loadTable(queryAuthEnvironment(this::maskValueAndFilterKey));
+ List<Split> splits = table.newStreamScan().plan().splits();
+
+ assertThat(splits).isNotEmpty();
+ assertThat(splits)
+ .allSatisfy(
+ split -> {
+ assertThat(split).isInstanceOf(QueryAuthSplit.class);
+ QueryAuthSplit authSplit = (QueryAuthSplit) split;
+ assertThat(authSplit.authResult()).isEqualTo(authResult);
+ assertThat(authSplit.split()).isInstanceOf(ChainSplit.class);
+ });
+
+ // phase 1 reads the latest snapshot partition plus the later delta partitions, so the
+ // same key shows up under both; what matters here is that both carry the rules
+ List<GenericRow> rows = readSplits(table, splits);
+ assertThat(rows).isNotEmpty();
+ assertThat(rows)
+ .allSatisfy(
+ r -> {
+ assertThat(r.getLong(0)).isLessThan(3L);
+ assertThat(r.getString(2)).isEqualTo(r.getString(3));
+ });
+ }
+
+ private static final Map<String, String> QUERIED_PARTITION =
+ ImmutableMap.of("dt", "20250810", "hour", "21");
+
+ private void createChainTableWithQueryAuth() throws Exception {
+ createChainTable(
+ options -> {
+ options.set(CoreOptions.QUERY_AUTH_ENABLED, true);
+ // a chain read that cannot handle its own split otherwise falls through to the
+ // main branch with nothing but a log line, which would hide a routing mistake
+ options.set(CoreOptions.SCAN_FALLBACK_BRANCH_READ_FAIL_FAST, true);
+ });
+ FileStoreTable chainTable = loadTable();
+ writeWithCommit(
+ chainTable.switchToBranch(SNAPSHOT_BRANCH),
+ row(1L, 1L, "1", "CN", "20250810", "20"),
+ row(2L, 1L, "2", "CN", "20250810", "20"));
+ writeWithCommit(
+ chainTable.switchToBranch(DELTA_BRANCH),
+ row(1L, 2L, "1-1", "CN", "20250810", "21"),
+ row(4L, 1L, "4", "CN", "20250810", "21"));
+ }
+
+ /** Masks column 'v' with the value of column 'region' and keeps only the rows with k < 3. */
+ private TableQueryAuthResult maskValueAndFilterKey() {
+ RowType rowType = loadTable().schema().logicalRowType();
+ Predicate rowFilter = new PredicateBuilder(rowType).lessThan(0, 3L);
+ Transform mask = new FieldTransform(new FieldRef(3, "region", DataTypes.STRING()));
+ return new TableQueryAuthResult(
+ Collections.singletonList(JsonSerdeUtil.toFlatJson(rowFilter)),
+ Collections.singletonMap("v", JsonSerdeUtil.toFlatJson(mask)));
+ }
+
+ /**
+ * A catalog environment whose authorization answers with {@code authResult}. The table, its
+ * branch scans and its reads are all real; only the catalog call fetching the rules is stubbed.
+ * {@link CatalogEnvironment#empty()} cannot express this: without a catalog loader it short
+ * circuits to "no rules", so no split is ever wrapped.
+ */
+ private CatalogEnvironment queryAuthEnvironment(Supplier<TableQueryAuthResult> authResult) {
+ Catalog catalog = Mockito.mock(Catalog.class);
+ try {
+ // a fresh instance per call, as RESTCatalog#authTableQuery builds one per scan
+ Mockito.when(catalog.authTableQuery(Mockito.any(), Mockito.any()))
+ .thenAnswer(invocation -> authResult.get());
+ // the snapshot manager takes the same catalog; let it fall back to the file system
+ // instead of answering "no snapshot" for every branch
+ Mockito.when(catalog.loadSnapshot(Mockito.any(Identifier.class)))
+ .thenThrow(new UnsupportedOperationException());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return new CatalogEnvironment(
+ Identifier.create("default", tableName),
+ null,
+ () -> catalog,
+ null,
+ null,
+ null,
+ false,
+ false);
+ }
+
+ private List<Split> planQueriedPartition(DataTableScan scan) {
+ return planPartition(scan, QUERIED_PARTITION);
+ }
+
+ private List<Split> planPartition(DataTableScan scan, Map<String, String> partition) {
+ return scan.withPartitionFilter(partition).plan().splits();
+ }
+
+ private void assertChainSplitsCarryAuth(List<Split> splits, TableQueryAuthResult authResult) {
+ assertThat(splits).isNotEmpty();
+ assertThat(splits)
+ .allSatisfy(
+ split -> {
+ assertThat(split)
+ .isInstanceOf(FallbackReadFileStoreTable.FallbackSplit.class);
+ Split wrapped =
+ ((FallbackReadFileStoreTable.FallbackSplit) split).wrapped();
+ assertThat(wrapped).isInstanceOf(QueryAuthSplit.class);
+ QueryAuthSplit authSplit = (QueryAuthSplit) wrapped;
+ assertThat(authSplit.authResult()).isEqualTo(authResult);
+ assertThat(authSplit.split()).isInstanceOf(ChainSplit.class);
+ });
+ }
+
+ private List<GenericRow> readSplits(FileStoreTable table, List<Split> splits) throws Exception {
+ InternalRowSerializer serializer =
+ new InternalRowSerializer(table.schema().logicalRowType());
+ InnerTableRead read = table.newRead();
+ List<GenericRow> result = new ArrayList<>();
+ for (Split split : splits) {
+ try (RecordReader<InternalRow> reader = read.createReader(split)) {
+ reader.forEachRemaining(row -> result.add((GenericRow) serializer.copy(row)));
+ }
+ }
+ return result;
+ }
+
private FileStoreTable loadTable() {
+ return loadTable(CatalogEnvironment.empty());
+ }
+
+ private FileStoreTable loadTable(CatalogEnvironment catalogEnvironment) {
Path tablePath = new Path(tempDir.toUri().toString(), tableName);
LocalFileIO fileIO = LocalFileIO.create();
Options options = new Options();
@@ -359,8 +670,7 @@ public class ChainTableFileStoreTableTest {
Optional<TableSchema> schemaOpt =
new FileSystemSchemaManager(fileIO, tablePath, branchName).latest();
assertThat(schemaOpt.isPresent()).isTrue();
- return FileStoreTableFactory.create(
- fileIO, tablePath, schemaOpt.get(), CatalogEnvironment.empty());
+ return FileStoreTableFactory.create(fileIO, tablePath, schemaOpt.get(), catalogEnvironment);
}
private void createChainTable(Consumer<Options> optionCustomizer) throws Exception {
(paimon) 11/20: [core] Apply deletion vectors during data evolution file index pushdown (#9922)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 2af7567e5fdad5b40cdd760e1120588d9a604850
Author: sanshi <43...@users.noreply.github.com>
AuthorDate: Fri Sep 18 12:26:40 2026 +0800
[core] Apply deletion vectors during data evolution file index pushdown (#9922)
---
.../org/apache/paimon/io/FileIndexEvaluator.java | 47 ++++++++++--
.../paimon/operation/DataEvolutionSplitRead.java | 25 +++++--
.../apache/paimon/io/FileIndexEvaluatorTest.java | 83 ++++++++++++++--------
.../paimon/table/DataEvolutionFileIndexTest.java | 68 +++++++++++++++++-
4 files changed, 177 insertions(+), 46 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/io/FileIndexEvaluator.java b/paimon-core/src/main/java/org/apache/paimon/io/FileIndexEvaluator.java
index a82194cc76..42e62d7b99 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/FileIndexEvaluator.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/FileIndexEvaluator.java
@@ -34,6 +34,7 @@ import org.apache.paimon.utils.RoaringBitmap32;
import javax.annotation.Nullable;
import java.io.IOException;
+import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
@@ -53,6 +54,27 @@ public class FileIndexEvaluator {
DataFileMeta file,
@Nullable DeletionVector dv)
throws IOException {
+ return evaluate(
+ fileIO, dataSchema, dataFilter, topN, limit, dataFilePathFactory, file, dv, 0L);
+ }
+
+ /**
+ * Evaluates a file index with a deletion vector whose positions may be relative to a containing
+ * file range.
+ *
+ * @param fileOffset offset from this file's local position to the deletion vector position
+ */
+ public static FileIndexResult evaluate(
+ FileIO fileIO,
+ TableSchema dataSchema,
+ List<Predicate> dataFilter,
+ @Nullable TopN topN,
+ @Nullable Integer limit,
+ DataFilePathFactory dataFilePathFactory,
+ DataFileMeta file,
+ @Nullable DeletionVector dv,
+ long fileOffset)
+ throws IOException {
// File index selections use 32-bit positions. Fall back when they cannot safely represent
// the file or its deletion vector.
if (file.rowCount() > RoaringBitmap32.MAX_VALUE || dv instanceof Bitmap64DeletionVector) {
@@ -64,7 +86,7 @@ public class FileIndexEvaluator {
return FileIndexResult.REMAIN;
} else {
// limit can not work with other predicates.
- return createBaseSelection(file, dv).limit(limit);
+ return createBaseSelection(file, dv, fileOffset).limit(limit);
}
}
@@ -74,7 +96,7 @@ public class FileIndexEvaluator {
return FileIndexResult.REMAIN;
}
- BitmapIndexResult selection = createBaseSelection(file, dv);
+ BitmapIndexResult selection = createBaseSelection(file, dv, fileOffset);
FileIndexResult result;
if (!isNullOrEmpty(dataFilter)) {
Predicate filter = PredicateBuilder.and(dataFilter.toArray(new Predicate[0]));
@@ -90,26 +112,37 @@ public class FileIndexEvaluator {
return FileIndexResult.REMAIN;
}
+ if (!result.remain()) {
+ return FileIndexResult.SKIP;
+ }
+
// if all position selected, or if only and not the deletion
// the effect will not obvious, just return REMAIN.
if (Objects.equals(result, selection)) {
return FileIndexResult.REMAIN;
}
- if (!result.remain()) {
- return FileIndexResult.SKIP;
- }
-
return result;
}
}
private static BitmapIndexResult createBaseSelection(
- DataFileMeta file, @Nullable DeletionVector dv) {
+ DataFileMeta file, @Nullable DeletionVector dv, long fileOffset) {
BitmapIndexResult selection =
new BitmapIndexResult(() -> RoaringBitmap32.bitmapOfRange(0, file.rowCount()));
if (dv instanceof BitmapDeletionVector) {
RoaringBitmap32 deletion = ((BitmapDeletionVector) dv).get();
+ if (fileOffset != 0) {
+ RoaringBitmap32 localDeletion = new RoaringBitmap32();
+ Iterator<Integer> iterator = deletion.iterator();
+ while (iterator.hasNext()) {
+ long localPosition = (long) iterator.next() - fileOffset;
+ if (localPosition >= 0 && localPosition < file.rowCount()) {
+ localDeletion.add((int) localPosition);
+ }
+ }
+ deletion = localDeletion;
+ }
selection = selection.andNot(deletion);
}
return selection;
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
index de0f8fd5ee..0b05ee835e 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
@@ -670,6 +670,12 @@ public class DataEvolutionSplitRead implements SplitRead<InternalRow> {
FileIndexResult fileIndexResult = null;
if (fileIndexReadEnabled) {
+ DeletionVector dv = deletionVector == null ? null : deletionVector.deletionVector;
+ long fileOffset =
+ dv == null || dv.isEmpty()
+ ? 0L
+ : deletionVectorOffset(
+ file.nonNullRowIdRange(), rowRanges, deletionVector);
fileIndexResult =
FileIndexEvaluator.evaluate(
fileIO,
@@ -679,7 +685,8 @@ public class DataEvolutionSplitRead implements SplitRead<InternalRow> {
null,
dataFilePathFactory,
file,
- null);
+ dv,
+ fileOffset);
if (!fileIndexResult.remain()) {
return new EmptyFileRecordReader<>();
}
@@ -778,6 +785,15 @@ public class DataEvolutionSplitRead implements SplitRead<InternalRow> {
return reader;
}
+ return new ApplyDeletionVectorReader(
+ reader,
+ deletionVector.deletionVector,
+ // Convert anchor-range DV positions to this reader's local returned positions.
+ deletionVectorOffset(readerRange, rowRanges, deletionVector));
+ }
+
+ private long deletionVectorOffset(
+ Range readerRange, List<Range> rowRanges, DeletionVectorWithRange deletionVector) {
checkArgument(
selectedRangesContainedByDeletionVector(
readerRange, rowRanges, deletionVector.range),
@@ -785,12 +801,7 @@ public class DataEvolutionSplitRead implements SplitRead<InternalRow> {
deletionVector.range,
rowRanges,
readerRange);
-
- return new ApplyDeletionVectorReader(
- reader,
- deletionVector.deletionVector,
- // Convert anchor-range DV positions to this reader's local returned positions.
- readerRange.from - deletionVector.range.from);
+ return readerRange.from - deletionVector.range.from;
}
private boolean selectedRangesContainedByDeletionVector(
diff --git a/paimon-core/src/test/java/org/apache/paimon/io/FileIndexEvaluatorTest.java b/paimon-core/src/test/java/org/apache/paimon/io/FileIndexEvaluatorTest.java
index 891ae8380a..bd7f781855 100644
--- a/paimon-core/src/test/java/org/apache/paimon/io/FileIndexEvaluatorTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/io/FileIndexEvaluatorTest.java
@@ -112,37 +112,60 @@ public class FileIndexEvaluatorTest {
BitmapDeletionVector deletionVector = new BitmapDeletionVector();
deletionVector.delete(0);
- FileIndexResult result =
- FileIndexEvaluator.evaluate(
- null,
- tableSchema(),
- Collections.singletonList(
- new PredicateBuilder(
- RowType.of(
- new DataType[] {DataTypes.INT()},
- new String[] {"a"}))
- .equal(0, 1)),
- null,
- null,
- null,
- DataFileMeta.forAppend(
- "file",
- 0,
- 2,
- SimpleStats.EMPTY_STATS,
- 0,
- 0,
- 0,
- Collections.emptyList(),
- embeddedBitmapIndex(),
- null,
- null,
- null,
- null,
- null),
- deletionVector);
+ assertThat(evaluateEmbeddedIndex(deletionVector, 0L)).isSameAs(FileIndexResult.SKIP);
+ }
+
+ @Test
+ public void testFullyDeletedFileIsSkipped() throws Exception {
+ BitmapDeletionVector deletionVector = new BitmapDeletionVector();
+ deletionVector.delete(0);
+ deletionVector.delete(1);
- assertThat(result).isSameAs(FileIndexResult.SKIP);
+ assertThat(evaluateEmbeddedIndex(deletionVector, 0L)).isSameAs(FileIndexResult.SKIP);
+ }
+
+ @Test
+ public void testDataFilterIntersectsOffsetDeletionVector() throws Exception {
+ BitmapDeletionVector hit = new BitmapDeletionVector();
+ hit.delete(10);
+ assertThat(evaluateEmbeddedIndex(hit, 10L)).isSameAs(FileIndexResult.SKIP);
+
+ BitmapDeletionVector neighbour = new BitmapDeletionVector();
+ neighbour.delete(11);
+ assertThat(evaluateEmbeddedIndex(neighbour, 10L).remain()).isTrue();
+ }
+
+ private static FileIndexResult evaluateEmbeddedIndex(
+ BitmapDeletionVector deletionVector, long fileOffset) throws IOException {
+ return FileIndexEvaluator.evaluate(
+ null,
+ tableSchema(),
+ Collections.singletonList(
+ new PredicateBuilder(
+ RowType.of(
+ new DataType[] {DataTypes.INT()},
+ new String[] {"a"}))
+ .equal(0, 1)),
+ null,
+ null,
+ null,
+ DataFileMeta.forAppend(
+ "file",
+ 0,
+ 2,
+ SimpleStats.EMPTY_STATS,
+ 0,
+ 0,
+ 0,
+ Collections.emptyList(),
+ embeddedBitmapIndex(),
+ null,
+ null,
+ null,
+ null,
+ null),
+ deletionVector,
+ fileOffset);
}
private static TableSchema tableSchema() {
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
index caa3e7d42b..9b0b7da89f 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
@@ -33,6 +33,7 @@ import org.apache.paimon.fileindex.FileIndexOptions;
import org.apache.paimon.fileindex.bitmap.BitmapFileIndexFactory;
import org.apache.paimon.fileindex.bloomfilter.BloomFilterFileIndexFactory;
import org.apache.paimon.fileindex.bsi.BitSliceIndexBitmapFileIndexFactory;
+import org.apache.paimon.fs.Path;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.io.CompactIncrement;
import org.apache.paimon.io.DataFileMeta;
@@ -477,6 +478,48 @@ public class DataEvolutionFileIndexTest extends DataEvolutionTestBase {
assertRow(assertSingleRow(rows), 50);
}
+ @Test
+ public void testBitmapSelectionComposesWithDeletionVectorForNonZeroRowId() throws Exception {
+ Map<String, String> options = bitmapOptions("f1");
+ options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true");
+ FileStoreTable table = createTable("bitmap_dv_non_zero_row_id", options);
+ writeAllColumns(table, ROW_COUNT);
+ writeAllColumns(table, ROW_COUNT);
+
+ deleteRowsFrom(table, ROW_COUNT, 50);
+
+ FileStoreTable latest = getTable(identifier(table.name()));
+ DataSplit targetSplit =
+ latest.newReadBuilder().newScan().plan().splits().stream()
+ .map(split -> (DataSplit) split)
+ .filter(
+ split ->
+ split.dataFiles().stream()
+ .anyMatch(
+ file ->
+ file.nonNullFirstRowId()
+ == ROW_COUNT))
+ .findFirst()
+ .orElseThrow(IllegalStateException::new);
+ DataFileMeta targetFile =
+ targetSplit.dataFiles().stream()
+ .filter(file -> file.nonNullFirstRowId() == ROW_COUNT)
+ .findFirst()
+ .orElseThrow(IllegalStateException::new);
+ Path targetPath =
+ latest.store()
+ .pathFactory()
+ .createDataFilePathFactory(targetSplit.partition(), targetSplit.bucket())
+ .toPath(targetFile);
+ assertThat(latest.fileIO().delete(targetPath, false)).isTrue();
+
+ // The second group can only succeed now if the bitmap hit is intersected with its DV
+ // before opening the missing data file. The first group remains visible.
+ RowType readType = rowTypeWithRowId(rowType()).project(SpecialFields.ROW_ID.name(), "f1");
+ List<InternalRow> rows = readWithFilter(table, equalF1(f1(50)), readType);
+ assertThat(rowIds(rows)).containsExactly(50L);
+ }
+
@Test
public void testMergedGroupKeptWhenFilterColumnOverwritten() throws Exception {
FileStoreTable table = createTable("overwritten", Collections.emptyMap());
@@ -503,7 +546,28 @@ public class DataEvolutionFileIndexTest extends DataEvolutionTestBase {
FileStoreTable latest = getTable(identifier(table.name()));
List<DataFileMeta> dataFiles =
((DataSplit) latest.newReadBuilder().newScan().plan().splits().get(0)).dataFiles();
- String anchor = retrieveAnchorFile(dataFiles, file -> file).fileName();
+ deleteRows(latest, retrieveAnchorFile(dataFiles, file -> file), positions);
+ }
+
+ private void deleteRowsFrom(FileStoreTable table, long firstRowId, long... positions)
+ throws Exception {
+ FileStoreTable latest = getTable(identifier(table.name()));
+ DataFileMeta anchor =
+ latest.newReadBuilder().newScan().plan().splits().stream()
+ .map(split -> (DataSplit) split)
+ .flatMap(split -> split.dataFiles().stream())
+ .filter(file -> file.nonNullFirstRowId() == firstRowId)
+ .findFirst()
+ .orElseThrow(
+ () ->
+ new IllegalArgumentException(
+ "Cannot find data file with first row id "
+ + firstRowId));
+ deleteRows(latest, anchor, positions);
+ }
+
+ private void deleteRows(FileStoreTable latest, DataFileMeta anchor, long... positions)
+ throws Exception {
BaseAppendDeleteFileMaintainer maintainer =
BaseAppendDeleteFileMaintainer.forUnawareAppend(
@@ -514,7 +578,7 @@ public class DataEvolutionFileIndexTest extends DataEvolutionTestBase {
for (long position : positions) {
deletionVector.delete(position);
}
- maintainer.notifyNewDeletionVector(anchor, deletionVector);
+ maintainer.notifyNewDeletionVector(anchor.fileName(), deletionVector);
List<IndexFileMeta> newIndexFiles = new ArrayList<>();
for (IndexManifestEntry entry : maintainer.persist()) {